diff --git a/.eslintrc.json b/.eslintrc.json index 75584b8163..f6f03c6523 100644 --- a/.eslintrc.json +++ b/.eslintrc.json @@ -19,6 +19,7 @@ "bundle.js" ], "extends": ["eslint:recommended"], + "plugins": ["import", "eslint-plugin-local-rules"], "overrides": [ { "files": ["**/*.svelte"], @@ -30,7 +31,6 @@ "sourceType": "module", "allowImportExportEverywhere": true } - }, { "files": ["**/*.ts"], @@ -42,13 +42,22 @@ "no-case-declarations": "off", "no-useless-escape": "off", "no-undef": "off", - "no-prototype-builtins": "off" + "no-prototype-builtins": "off", + "local-rules/no-budibase-imports": "error" } } ], "rules": { "no-self-assign": "off", - "no-unused-vars": ["error", { "varsIgnorePattern": "^_", "argsIgnorePattern": "^_", "destructuredArrayIgnorePattern": "^_" }] + "no-unused-vars": [ + "error", + { + "varsIgnorePattern": "^_", + "argsIgnorePattern": "^_", + "destructuredArrayIgnorePattern": "^_" + } + ], + "import/no-relative-packages": "error" }, "globals": { "GeolocationPositionError": true diff --git a/.github/workflows/budibase_ci.yml b/.github/workflows/budibase_ci.yml index c73d013b78..cb71d0702b 100644 --- a/.github/workflows/budibase_ci.yml +++ b/.github/workflows/budibase_ci.yml @@ -18,51 +18,42 @@ env: BASE_BRANCH: ${{ github.event.pull_request.base.ref}} PERSONAL_ACCESS_TOKEN: ${{ secrets.PERSONAL_ACCESS_TOKEN }} NX_BASE_BRANCH: origin/${{ github.base_ref }} - USE_NX_AFFECTED: ${{ github.event_name == 'pull_request' && github.base_ref != 'master'}} + USE_NX_AFFECTED: ${{ github.event_name == 'pull_request' }} + IS_OSS_CONTRIBUTOR: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != 'Budibase/budibase' }} jobs: lint: runs-on: ubuntu-latest steps: - - name: Checkout repo and submodules + - name: Checkout repo uses: actions/checkout@v3 - if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == 'Budibase/budibase' with: - submodules: true + submodules: ${{ env.IS_OSS_CONTRIBUTOR == 'false' }} token: ${{ secrets.PERSONAL_ACCESS_TOKEN || github.token }} - - name: Checkout repo only - uses: actions/checkout@v3 - if: github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != 'Budibase/budibase' - name: Use Node.js 18.x uses: actions/setup-node@v3 with: node-version: 18.x - cache: "yarn" + cache: yarn - run: yarn --frozen-lockfile - run: yarn lint build: runs-on: ubuntu-latest steps: - - name: Checkout repo and submodules + - name: Checkout repo uses: actions/checkout@v3 - if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == 'Budibase/budibase' with: - submodules: true + submodules: ${{ env.IS_OSS_CONTRIBUTOR == 'false' }} token: ${{ secrets.PERSONAL_ACCESS_TOKEN || github.token }} fetch-depth: 0 - - name: Checkout repo only - uses: actions/checkout@v3 - if: github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != 'Budibase/budibase' - with: - fetch-depth: 0 - name: Use Node.js 18.x uses: actions/setup-node@v3 with: node-version: 18.x - cache: "yarn" + cache: yarn - run: yarn --frozen-lockfile # Run build all the projects @@ -81,24 +72,18 @@ jobs: test-libraries: runs-on: ubuntu-latest steps: - - name: Checkout repo and submodules + - name: Checkout repo uses: actions/checkout@v3 - if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == 'Budibase/budibase' with: - submodules: true + submodules: ${{ env.IS_OSS_CONTRIBUTOR == 'false' }} token: ${{ secrets.PERSONAL_ACCESS_TOKEN || github.token }} fetch-depth: 0 - - name: Checkout repo only - uses: actions/checkout@v3 - if: github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != 'Budibase/budibase' - with: - fetch-depth: 0 - name: Use Node.js 18.x uses: actions/setup-node@v3 with: node-version: 18.x - cache: "yarn" + cache: yarn - run: yarn --frozen-lockfile - name: Test run: | @@ -116,24 +101,18 @@ jobs: test-worker: runs-on: ubuntu-latest steps: - - name: Checkout repo and submodules + - name: Checkout repo uses: actions/checkout@v3 - if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == 'Budibase/budibase' with: - submodules: true + submodules: ${{ env.IS_OSS_CONTRIBUTOR == 'false' }} token: ${{ secrets.PERSONAL_ACCESS_TOKEN || github.token }} fetch-depth: 0 - - name: Checkout repo only - uses: actions/checkout@v3 - if: github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != 'Budibase/budibase' - with: - fetch-depth: 0 - name: Use Node.js 18.x uses: actions/setup-node@v3 with: node-version: 18.x - cache: "yarn" + cache: yarn - run: yarn --frozen-lockfile - name: Test worker run: | @@ -152,24 +131,18 @@ jobs: test-server: runs-on: ubuntu-latest steps: - - name: Checkout repo and submodules + - name: Checkout repo uses: actions/checkout@v3 - if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == 'Budibase/budibase' with: - submodules: true + submodules: ${{ env.IS_OSS_CONTRIBUTOR == 'false' }} token: ${{ secrets.PERSONAL_ACCESS_TOKEN || github.token }} fetch-depth: 0 - - name: Checkout repo only - uses: actions/checkout@v3 - if: github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != 'Budibase/budibase' - with: - fetch-depth: 0 - name: Use Node.js 18.x uses: actions/setup-node@v3 with: node-version: 18.x - cache: "yarn" + cache: yarn - run: yarn --frozen-lockfile - name: Test server run: | @@ -200,7 +173,7 @@ jobs: uses: actions/setup-node@v3 with: node-version: 18.x - cache: "yarn" + cache: yarn - run: yarn --frozen-lockfile - name: Test run: | @@ -213,21 +186,17 @@ jobs: integration-test: runs-on: ubuntu-latest steps: - - name: Checkout repo and submodules + - name: Checkout repo uses: actions/checkout@v3 - if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == 'Budibase/budibase' with: - submodules: true + submodules: ${{ env.IS_OSS_CONTRIBUTOR == 'false' }} token: ${{ secrets.PERSONAL_ACCESS_TOKEN || github.token }} - - name: Checkout repo only - uses: actions/checkout@v3 - if: github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != 'Budibase/budibase' - name: Use Node.js 18.x uses: actions/setup-node@v3 with: node-version: 18.x - cache: "yarn" + cache: yarn - run: yarn --frozen-lockfile - name: Build packages run: yarn build --scope @budibase/server --scope @budibase/worker diff --git a/.github/workflows/deploy-cloud.yaml b/.github/workflows/deploy-cloud.yaml deleted file mode 100644 index 389b10f7d3..0000000000 --- a/.github/workflows/deploy-cloud.yaml +++ /dev/null @@ -1,48 +0,0 @@ -name: Budibase Deploy Production - -on: - workflow_dispatch: - inputs: - version: - description: Budibase release version. For example - 1.0.0 - required: false - -jobs: - release: - runs-on: ubuntu-latest - - steps: - - name: Fail if not a tag - run: | - if [[ $GITHUB_REF != refs/tags/* ]]; then - echo "Workflow Dispatch can only be run on tags" - exit 1 - fi - - uses: actions/checkout@v2 - with: - fetch-depth: 0 - - - name: Fail if tag is not in master - run: | - if ! git merge-base --is-ancestor ${{ github.sha }} origin/master; then - echo "Tag is not in master. This pipeline can only execute tags that are present on the master branch" - exit 1 - fi - - - name: Get the latest budibase release version - id: version - run: | - if [ -z "${{ github.event.inputs.version }}" ]; then - release_version=$(cat lerna.json | jq -r '.version') - else - release_version=${{ github.event.inputs.version }} - fi - echo "RELEASE_VERSION=$release_version" >> $GITHUB_ENV - - - uses: passeidireto/trigger-external-workflow-action@main - env: - PAYLOAD_VERSION: ${{ env.RELEASE_VERSION }} - with: - repository: budibase/budibase-deploys - event: budicloud-prod-deploy - github_pat: ${{ secrets.GH_ACCESS_TOKEN }} diff --git a/.github/workflows/deploy-qa.yml b/.github/workflows/deploy-qa.yml index a3fff65f35..1339ad2eb9 100644 --- a/.github/workflows/deploy-qa.yml +++ b/.github/workflows/deploy-qa.yml @@ -4,7 +4,6 @@ on: push: branches: - master - - BUDI-7641/push_v2_images_to_qa workflow_dispatch: jobs: @@ -12,10 +11,12 @@ jobs: runs-on: ubuntu-latest steps: - uses: peter-evans/repository-dispatch@v2 - env: - PAYLOAD_VERSION: ${{ github.sha }} - REF_NAME: ${{ github.ref_name}} with: repository: budibase/budibase-deploys event-type: budicloud-qa-deploy token: ${{ secrets.GH_ACCESS_TOKEN }} + client-payload: |- + { + "VERSION": "${{ github.sha }}", + "REF_NAME": "${{ github.ref_name}}" + } diff --git a/.github/workflows/release-master.yml b/.github/workflows/release-master.yml deleted file mode 100644 index 9ab8530341..0000000000 --- a/.github/workflows/release-master.yml +++ /dev/null @@ -1,130 +0,0 @@ -name: Budibase Release -concurrency: - group: release - cancel-in-progress: false - -on: - push: - tags: - - "[0-9]+.[0-9]+.[0-9]+" - # Exclude all pre-releases - - "!*[0-9]+.[0-9]+.[0-9]+-*" - -env: - # Posthog token used by ui at build time - POSTHOG_TOKEN: phc_bIjZL7oh2GEUd2vqvTBH8WvrX0fWTFQMs6H5KQxiUxU - INTERCOM_TOKEN: ${{ secrets.INTERCOM_TOKEN }} - PERSONAL_ACCESS_TOKEN: ${{ secrets.PERSONAL_ACCESS_TOKEN }} - -jobs: - release-images: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - with: - submodules: true - token: ${{ secrets.PERSONAL_ACCESS_TOKEN }} - fetch-depth: 0 - - - name: Fail if tag is not in master - run: | - if ! git merge-base --is-ancestor ${{ github.sha }} origin/master; then - echo "Tag is not in master. This pipeline can only execute tags that are present on the master branch" - exit 1 - fi - - - uses: actions/setup-node@v1 - with: - node-version: 18.x - - - run: yarn install --frozen-lockfile - - name: Update versions - run: ./scripts/updateVersions.sh - - run: yarn lint - - run: yarn build - - run: yarn build:sdk - - - name: Publish budibase packages to NPM - env: - NPM_TOKEN: ${{ secrets.NPM_TOKEN }} - run: | - # setup the username and email. I tend to use 'GitHub Actions Bot' with no email by default - git config --global user.name "Budibase Release Bot" - git config --global user.email "<>" - git submodule foreach git commit -a -m 'Release process' - git commit -a -m 'Release process' - echo //registry.npmjs.org/:_authToken=${NPM_TOKEN} >> .npmrc - yarn release - - - name: "Get Current tag" - id: currenttag - run: | - version=$(./scripts/getCurrentVersion.sh) - echo "Using tag $version" - echo "version=$version" >> "$GITHUB_OUTPUT" - - - name: Build/release Docker images - run: | - docker login -u $DOCKER_USER -p $DOCKER_PASSWORD - yarn build:docker - env: - DOCKER_USER: ${{ secrets.DOCKER_USERNAME }} - DOCKER_PASSWORD: ${{ secrets.DOCKER_API_KEY }} - BUDIBASE_RELEASE_VERSION: ${{ steps.currenttag.outputs.version }} - - release-helm-chart: - needs: [release-images] - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - - name: Setup Helm - uses: azure/setup-helm@v1 - id: helm-install - - - name: Get the latest budibase release version - id: version - run: | - release_version=$(cat lerna.json | jq -r '.version') - echo "RELEASE_VERSION=$release_version" >> $GITHUB_ENV - - # due to helm repo index issue: https://github.com/helm/helm/issues/7363 - # we need to create new package in a different dir, merge the index and move the package back - - name: Build and release helm chart - run: | - git config user.name "Budibase Helm Bot" - git config user.email "<>" - git reset --hard - git fetch - mkdir sync - echo "Packaging chart to sync dir" - helm package charts/budibase --version 0.0.0-master --app-version "$RELEASE_VERSION" --destination sync - echo "Packaging successful" - git checkout gh-pages - echo "Indexing helm repo" - helm repo index --merge docs/index.yaml sync - mv -f sync/* docs - rm -rf sync - echo "Pushing new helm release" - git add -A - git commit -m "Helm Release: ${{ env.RELEASE_VERSION }}" - git push - - trigger-deploy-to-qa-env: - needs: [release-helm-chart] - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - - name: Get the current budibase release version - id: version - run: | - release_version=$(cat lerna.json | jq -r '.version') - echo "RELEASE_VERSION=$release_version" >> $GITHUB_ENV - - - uses: passeidireto/trigger-external-workflow-action@main - env: - PAYLOAD_VERSION: ${{ env.RELEASE_VERSION }} - REF_NAME: ${{ github.ref_name}} - with: - repository: budibase/budibase-deploys - event: budicloud-qa-deploy - github_pat: ${{ secrets.GH_ACCESS_TOKEN }} diff --git a/.github/workflows/release-selfhost.yml b/.github/workflows/release-selfhost.yml deleted file mode 100644 index d2689a0ea0..0000000000 --- a/.github/workflows/release-selfhost.yml +++ /dev/null @@ -1,125 +0,0 @@ -name: Budibase Release Selfhost - -on: - workflow_dispatch: - -jobs: - release: - runs-on: ubuntu-latest - - steps: - - name: Fail if not a tag - run: | - if [[ $GITHUB_REF != refs/tags/* ]]; then - echo "Workflow Dispatch can only be run on tags" - exit 1 - fi - - - uses: actions/checkout@v2 - with: - submodules: true - token: ${{ secrets.PERSONAL_ACCESS_TOKEN }} - fetch-depth: 0 - - - name: Fail if tag is not in master - run: | - if ! git merge-base --is-ancestor ${{ github.sha }} origin/master; then - echo "Tag is not in master. This pipeline can only execute tags that are present on the master branch" - exit 1 - fi - - - name: Use Node.js 18.x - uses: actions/setup-node@v1 - with: - node-version: 18.x - - - name: Get the latest budibase release version - id: version - run: | - release_version=$(cat lerna.json | jq -r '.version') - echo "RELEASE_VERSION=$release_version" >> $GITHUB_ENV - - - name: Tag and release Docker images (Self Host) - run: | - docker login -u $DOCKER_USER -p $DOCKER_PASSWORD - - release_tag=${{ env.RELEASE_VERSION }} - - # Pull apps and worker images - docker pull budibase/apps:$release_tag - docker pull budibase/worker:$release_tag - docker pull budibase/proxy:$release_tag - - # Tag apps and worker images - docker tag budibase/apps:$release_tag budibase/apps:$SELFHOST_TAG - docker tag budibase/worker:$release_tag budibase/worker:$SELFHOST_TAG - docker tag budibase/proxy:$release_tag budibase/proxy:$SELFHOST_TAG - - # Push images - docker push budibase/apps:$SELFHOST_TAG - docker push budibase/worker:$SELFHOST_TAG - docker push budibase/proxy:$SELFHOST_TAG - env: - DOCKER_USER: ${{ secrets.DOCKER_USERNAME }} - DOCKER_PASSWORD: ${{ secrets.DOCKER_API_KEY }} - SELFHOST_TAG: latest - - - name: Bootstrap and build (CLI) - run: | - yarn - yarn build - - - name: Build OpenAPI spec - run: | - pushd packages/server - yarn - yarn specs - popd - - - name: Setup Helm - uses: azure/setup-helm@v1 - id: helm-install - - # due to helm repo index issue: https://github.com/helm/helm/issues/7363 - # we need to create new package in a different dir, merge the index and move the package back - - name: Build and release helm chart - run: | - git config user.name "Budibase Helm Bot" - git config user.email "<>" - git reset --hard - git fetch - mkdir sync - echo "Packaging chart to sync dir" - helm package charts/budibase --version "$RELEASE_VERSION" --app-version "$RELEASE_VERSION" --destination sync - echo "Packaging successful" - git checkout gh-pages - echo "Indexing helm repo" - helm repo index --merge docs/index.yaml sync - mv -f sync/* docs - rm -rf sync - echo "Pushing new helm release" - git add -A - git commit -m "Helm Release: ${{ env.RELEASE_VERSION }}" - git push - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - - name: Perform Github Release - uses: softprops/action-gh-release@v1 - with: - name: ${{ env.RELEASE_VERSION }} - tag_name: ${{ env.RELEASE_VERSION }} - generate_release_notes: true - files: | - packages/cli/build/cli-win.exe - packages/cli/build/cli-linux - packages/cli/build/cli-macos - packages/server/specs/openapi.yaml - packages/server/specs/openapi.json - - - name: Discord Webhook Action - uses: tsickert/discord-webhook@v4.0.0 - with: - webhook-url: ${{ secrets.PROD_DEPLOY_WEBHOOK_URL }} - content: "Self Host Deployment Complete: ${{ env.RELEASE_VERSION }} deployed to Self Host." - embed-title: ${{ env.RELEASE_VERSION }} diff --git a/.github/workflows/release-singleimage-test.yml b/.github/workflows/release-singleimage-test.yml deleted file mode 100644 index 79b9afdd44..0000000000 --- a/.github/workflows/release-singleimage-test.yml +++ /dev/null @@ -1,69 +0,0 @@ -name: Test - -on: - workflow_dispatch: - -env: - CI: true - PERSONAL_ACCESS_TOKEN: ${{ secrets.PERSONAL_ACCESS_TOKEN }} - REGISTRY_URL: registry.hub.docker.com - NX_CLOUD_ACCESS_TOKEN: ${{ secrets.NX_CLOUD_ACCESS_TOKEN }} -jobs: - build: - name: "build" - runs-on: ubuntu-latest - strategy: - matrix: - node-version: [18.x] - steps: - - name: "Checkout" - uses: actions/checkout@v4 - with: - submodules: true - token: ${{ secrets.PERSONAL_ACCESS_TOKEN }} - - - name: Use Node.js ${{ matrix.node-version }} - uses: actions/setup-node@v3 - with: - node-version: ${{ matrix.node-version }} - cache: "yarn" - - name: Setup QEMU - uses: docker/setup-qemu-action@v3 - - name: Setup Docker Buildx - id: buildx - uses: docker/setup-buildx-action@v3 - - name: Run Yarn - run: yarn - - name: Run Yarn Build - run: yarn build --scope @budibase/server --scope @budibase/worker - - name: Login to Docker Hub - uses: docker/login-action@v2 - with: - username: ${{ secrets.DOCKER_USERNAME }} - password: ${{ secrets.DOCKER_API_KEY }} - - name: Get the latest release version - id: version - run: | - release_version=$(cat lerna.json | jq -r '.version') - echo $release_version - echo "RELEASE_VERSION=$release_version" >> $GITHUB_ENV - - name: Tag and release Budibase service docker image - uses: docker/build-push-action@v5 - with: - context: . - push: true - pull: true - platforms: linux/amd64,linux/arm64 - tags: budibase/budibase-test:test - file: ./hosting/single/Dockerfile.v2 - cache-from: type=registry,ref=budibase/budibase-test:test - cache-to: type=inline - - name: Tag and release Budibase Azure App Service docker image - uses: docker/build-push-action@v2 - with: - context: . - push: true - platforms: linux/amd64 - build-args: TARGETBUILD=aas - tags: budibase/budibase-test:aas - file: ./hosting/single/Dockerfile.v2 diff --git a/.github/workflows/release-singleimage.yml b/.github/workflows/release-singleimage.yml deleted file mode 100644 index f7f87f6e4c..0000000000 --- a/.github/workflows/release-singleimage.yml +++ /dev/null @@ -1,79 +0,0 @@ -name: Deploy Budibase Single Container Image to DockerHub - -on: - workflow_dispatch: - -env: - CI: true - PERSONAL_ACCESS_TOKEN: ${{ secrets.PERSONAL_ACCESS_TOKEN }} - REGISTRY_URL: registry.hub.docker.com -jobs: - build: - name: "build" - runs-on: ubuntu-latest - strategy: - matrix: - node-version: [18.x] - steps: - - name: Maximize build space - uses: easimon/maximize-build-space@master - with: - root-reserve-mb: 30000 - swap-size-mb: 1024 - remove-android: "true" - remove-dotnet: "true" - - name: Fail if not a tag - run: | - if [[ $GITHUB_REF != refs/tags/* ]]; then - echo "Workflow Dispatch can only be run on tags" - exit 1 - fi - - name: "Checkout" - uses: actions/checkout@v2 - with: - submodules: true - token: ${{ secrets.PERSONAL_ACCESS_TOKEN }} - - - name: Use Node.js ${{ matrix.node-version }} - uses: actions/setup-node@v1 - with: - node-version: ${{ matrix.node-version }} - - name: Setup QEMU - uses: docker/setup-qemu-action@v1 - - name: Setup Docker Buildx - id: buildx - uses: docker/setup-buildx-action@v1 - - name: Run Yarn - run: yarn - - name: Update versions - run: ./scripts/updateVersions.sh - - name: Run Yarn Build - run: yarn build - - name: Login to Docker Hub - uses: docker/login-action@v2 - with: - username: ${{ secrets.DOCKER_USERNAME }} - password: ${{ secrets.DOCKER_API_KEY }} - - name: Get the latest release version - id: version - run: | - release_version=$(cat lerna.json | jq -r '.version') - echo $release_version - echo "RELEASE_VERSION=$release_version" >> $GITHUB_ENV - - name: Tag and release Budibase service docker image - uses: docker/build-push-action@v2 - with: - context: . - push: true - platforms: linux/amd64,linux/arm64 - tags: budibase/budibase,budibase/budibase:${{ env.RELEASE_VERSION }} - file: ./hosting/single/Dockerfile - - name: Tag and release Budibase Azure App Service docker image - uses: docker/build-push-action@v2 - with: - context: . - push: true - platforms: linux/amd64 - build-args: TARGETBUILD=aas - tags: budibase/budibase-aas,budibase/budibase-aas:${{ env.RELEASE_VERSION }} - file: ./hosting/single/Dockerfile diff --git a/.github/workflows/tag-release.yml b/.github/workflows/tag-release.yml index eaf71ae61a..13d59d1019 100644 --- a/.github/workflows/tag-release.yml +++ b/.github/workflows/tag-release.yml @@ -1,4 +1,4 @@ -name: Tag release +name: Release concurrency: group: tag-release cancel-in-progress: false @@ -19,6 +19,8 @@ on: jobs: tag-release: runs-on: ubuntu-latest + outputs: + version: ${{ steps.tag-release.outputs.version }} steps: - name: Fail if branch is not master @@ -33,6 +35,7 @@ jobs: - run: cd scripts && yarn - name: Tag release + id: tag-release run: | cd scripts # setup the username and email. @@ -41,3 +44,23 @@ jobs: BUMP_TYPE_INPUT=${{ github.event.inputs.versioning }} BUMP_TYPE=${BUMP_TYPE_INPUT:-"patch"} ./versionCommit.sh $BUMP_TYPE + + cd .. + new_version=$(./scripts/getCurrentVersion.sh) + echo "version=$new_version" >> $GITHUB_OUTPUT + + trigger-release: + needs: [tag-release] + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + + - uses: peter-evans/repository-dispatch@v2 + with: + repository: budibase/budibase-deploys + event-type: release-prod + token: ${{ secrets.GH_ACCESS_TOKEN }} + client-payload: |- + { + "TAG": "${{ needs.tag-release.outputs.version }}" + } diff --git a/README.md b/README.md index 9deb16cd4f..35b84a8816 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@

- Budibase + Budibase

@@ -126,13 +126,6 @@ You can learn more about the Budibase API at the following places: - [Build an app with Budibase and Next.js](https://budibase.com/blog/building-a-crud-app-with-budibase-and-next.js/) -

- Budibase data -

-

- -


- ## 🏁 Get started Deploy Budibase self-hosted in your existing infrastructure, using Docker, Kubernetes, and Digital Ocean. diff --git a/eslint-local-rules/index.js b/eslint-local-rules/index.js new file mode 100644 index 0000000000..af02599c90 --- /dev/null +++ b/eslint-local-rules/index.js @@ -0,0 +1,21 @@ +module.exports = { + "no-budibase-imports": { + create: function (context) { + return { + ImportDeclaration(node) { + const importPath = node.source.value + + if ( + /^@budibase\/[^/]+\/.*$/.test(importPath) && + importPath !== "@budibase/backend-core/tests" + ) { + context.report({ + node, + message: `Importing from @budibase is not allowed, except for @budibase/backend-core/tests.`, + }) + } + }, + } + }, + }, +} diff --git a/hosting/couchdb/build-target-paths.sh b/hosting/couchdb/build-target-paths.sh index 67e1765ca8..34227011f4 100644 --- a/hosting/couchdb/build-target-paths.sh +++ b/hosting/couchdb/build-target-paths.sh @@ -2,8 +2,8 @@ echo ${TARGETBUILD} > /buildtarget.txt if [[ "${TARGETBUILD}" = "aas" ]]; then - # Azure AppService uses /home for persisent data & SSH on port 2222 - DATA_DIR=/home + # Azure AppService uses /home for persistent data & SSH on port 2222 + DATA_DIR="${DATA_DIR:-/home}" WEBSITES_ENABLE_APP_SERVICE_STORAGE=true mkdir -p $DATA_DIR/{search,minio,couch} mkdir -p $DATA_DIR/couch/{dbs,views} diff --git a/hosting/docker-compose.build.yaml b/hosting/docker-compose.build.yaml index e192620b59..7ead001a1c 100644 --- a/hosting/docker-compose.build.yaml +++ b/hosting/docker-compose.build.yaml @@ -7,6 +7,8 @@ services: build: context: .. dockerfile: packages/server/Dockerfile.v2 + args: + - BUDIBASE_VERSION=0.0.0+dev-docker container_name: build-bbapps environment: SELF_HOSTED: 1 @@ -30,13 +32,13 @@ services: depends_on: - worker-service - redis-service - # volumes: - # - /some/path/to/plugins:/plugins worker-service: build: context: .. dockerfile: packages/worker/Dockerfile.v2 + args: + - BUDIBASE_VERSION=0.0.0+dev-docker container_name: build-bbworker environment: SELF_HOSTED: 1 diff --git a/hosting/letsencrypt/nginx-ssl.conf b/hosting/letsencrypt/nginx-ssl.conf index 50c5e0198a..b3f51e5cc5 100644 --- a/hosting/letsencrypt/nginx-ssl.conf +++ b/hosting/letsencrypt/nginx-ssl.conf @@ -2,16 +2,18 @@ server { listen 443 ssl default_server; listen [::]:443 ssl default_server; server_name _; - ssl_certificate /etc/letsencrypt/live/CUSTOM_DOMAIN/fullchain.pem; - ssl_certificate_key /etc/letsencrypt/live/CUSTOM_DOMAIN/privkey.pem; - include /etc/letsencrypt/options-ssl-nginx.conf; - ssl_dhparam /etc/letsencrypt/ssl-dhparams.pem; - + error_log /dev/stderr warn; + access_log /dev/stdout main; client_max_body_size 1000m; ignore_invalid_headers off; proxy_buffering off; # port_in_redirect off; + ssl_certificate /etc/letsencrypt/live/CUSTOM_DOMAIN/fullchain.pem; + ssl_certificate_key /etc/letsencrypt/live/CUSTOM_DOMAIN/privkey.pem; + include /etc/letsencrypt/options-ssl-nginx.conf; + ssl_dhparam /etc/letsencrypt/ssl-dhparams.pem; + location ^~ /.well-known/acme-challenge/ { default_type "text/plain"; root /var/www/html; @@ -47,6 +49,24 @@ server { rewrite ^/worker/(.*)$ /$1 break; } + location /api/backups/ { + # calls to export apps are limited + limit_req zone=ratelimit burst=20 nodelay; + + # 1800s timeout for app export requests + proxy_read_timeout 1800s; + proxy_connect_timeout 1800s; + proxy_send_timeout 1800s; + + proxy_http_version 1.1; + proxy_set_header Connection $connection_upgrade; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + + proxy_pass http://127.0.0.1:4001; + } + location /api/ { # calls to the API are rate limited with bursting limit_req zone=ratelimit burst=20 nodelay; @@ -70,18 +90,49 @@ server { rewrite ^/db/(.*)$ /$1 break; } + location /socket/ { + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection 'upgrade'; + proxy_set_header Host $host; + proxy_cache_bypass $http_upgrade; + proxy_pass http://127.0.0.1:4001; + } + location / { proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header Host $http_host; proxy_connect_timeout 300; proxy_http_version 1.1; proxy_set_header Connection ""; chunked_transfer_encoding off; + proxy_pass http://127.0.0.1:9000; } + location /files/signed/ { + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + + # IMPORTANT: Signed urls will inspect the host header of the request. + # Normally a signed url will need to be generated with a specified client host in mind. + # To support dynamic hosts, e.g. some unknown self-hosted installation url, + # use a predefined host header. The host 'minio-service' is also used at the time of url signing. + proxy_set_header Host minio-service; + + proxy_connect_timeout 300; + proxy_http_version 1.1; + proxy_set_header Connection ""; + chunked_transfer_encoding off; + + proxy_pass http://127.0.0.1:9000; + rewrite ^/files/signed/(.*)$ /$1 break; + } + client_header_timeout 60; client_body_timeout 60; keepalive_timeout 60; diff --git a/hosting/proxy/error.html b/hosting/proxy/error.html index 023c1ebaff..545d6c7f6d 100644 --- a/hosting/proxy/error.html +++ b/hosting/proxy/error.html @@ -57,8 +57,8 @@ --spectrum-global-color-gray-600: rgb(144,144,144); --spectrum-global-color-gray-900: rgb(255,255,255); --spectrum-global-color-gray-800: rgb(227,227,227); - --spectrum-global-color-static-blue-600: rgb(20,115,230); - --spectrum-global-color-static-blue-hover: rgb( 18, 103, 207); + --bb-indigo: #6E56FF; + --bb-indigo-light: #9F8FFF; } html, body { @@ -90,15 +90,8 @@ .info { display: flex; flex-direction: column; - align-items: left; + align-items: flex-start; } - - @media only screen and (max-width: 600px) { - .info { - align-items: center; - } - } - .status { color: var(--spectrum-global-color-gray-600) } @@ -113,13 +106,14 @@ .buttons { display: flex; flex-direction: row; + justify-content: flex-start; margin-top: 15px; } .homeButton { - background-color: var(--spectrum-global-color-static-blue-600); + background-color: var(--bb-indigo); } .homeButton:hover { - background-color: var(--spectrum-global-color-static-blue-hover); + background-color: var(--bb-indigo-light); } .statusButton { background-color: transparent; @@ -127,20 +121,30 @@ border: none; } .hero { - height: 160px; - width: 160px; - margin-right: 80px; + height: 60px; + margin: 10px 40px 10px 0; + } + .hero img { + height: 100%; } .content { display: flex; flex-direction: row; - align-items: flex-end; + align-items: center; justify-content: center; + padding: 0 40px; + } + h1 { + margin-bottom: 10px; + } + h3 { + margin-top: 0; } @media only screen and (max-width: 600px) { .content { flex-direction: column; + align-items: flex-start; } } @@ -152,16 +156,15 @@
- Budibase Logo + Budibase Logo
-

+

 

Houston we have a problem!

-

-

+

 

diff --git a/hosting/proxy/nginx.prod.conf b/hosting/proxy/nginx.prod.conf index 365765ccbb..6da2e4a1c3 100644 --- a/hosting/proxy/nginx.prod.conf +++ b/hosting/proxy/nginx.prod.conf @@ -51,7 +51,7 @@ http { proxy_buffering off; set $csp_default "default-src 'self'"; - set $csp_script "script-src 'self' 'unsafe-inline' 'unsafe-eval' https://*.budibase.net https://cdn.budi.live https://js.intercomcdn.com https://widget.intercom.io"; + set $csp_script "script-src 'self' 'unsafe-inline' 'unsafe-eval' https://*.budibase.net https://cdn.budi.live https://js.intercomcdn.com https://widget.intercom.io https://d2l5prqdbvm3op.cloudfront.net"; set $csp_style "style-src 'self' 'unsafe-inline' https://cdn.jsdelivr.net https://fonts.googleapis.com https://rsms.me https://maxcdn.bootstrapcdn.com"; set $csp_object "object-src 'none'"; set $csp_base_uri "base-uri 'self'"; diff --git a/hosting/scripts/build-target-paths.sh b/hosting/scripts/build-target-paths.sh index 67e1765ca8..34227011f4 100644 --- a/hosting/scripts/build-target-paths.sh +++ b/hosting/scripts/build-target-paths.sh @@ -2,8 +2,8 @@ echo ${TARGETBUILD} > /buildtarget.txt if [[ "${TARGETBUILD}" = "aas" ]]; then - # Azure AppService uses /home for persisent data & SSH on port 2222 - DATA_DIR=/home + # Azure AppService uses /home for persistent data & SSH on port 2222 + DATA_DIR="${DATA_DIR:-/home}" WEBSITES_ENABLE_APP_SERVICE_STORAGE=true mkdir -p $DATA_DIR/{search,minio,couch} mkdir -p $DATA_DIR/couch/{dbs,views} diff --git a/hosting/scripts/install-minio.sh b/hosting/scripts/install-minio.sh deleted file mode 100755 index 8297593599..0000000000 --- a/hosting/scripts/install-minio.sh +++ /dev/null @@ -1,10 +0,0 @@ -#!/bin/bash -if [[ $TARGETARCH == arm* ]] ; -then - echo "INSTALLING ARM64 MINIO" - wget https://dl.min.io/server/minio/release/linux-arm64/minio -else - echo "INSTALLING AMD64 MINIO" - wget https://dl.min.io/server/minio/release/linux-amd64/minio -fi -chmod +x minio diff --git a/hosting/scripts/linux/release-to-docker-hub.sh b/hosting/scripts/linux/release-to-docker-hub.sh deleted file mode 100755 index 599a10f914..0000000000 --- a/hosting/scripts/linux/release-to-docker-hub.sh +++ /dev/null @@ -1,18 +0,0 @@ -#!/bin/bash - -tag=$1 - -if [[ ! "$tag" ]]; then - echo "No tag present. You must pass a tag to this script" - exit 1 -fi - -echo "Tagging images with tag: $tag" - -docker tag proxy-service budibase/proxy:$tag -docker tag app-service budibase/apps:$tag -docker tag worker-service budibase/worker:$tag - -docker push --all-tags budibase/apps -docker push --all-tags budibase/worker -docker push --all-tags budibase/proxy diff --git a/hosting/single/Dockerfile.v2 b/hosting/single/Dockerfile.v2 index ad11545a22..ec03a1b5a2 100644 --- a/hosting/single/Dockerfile.v2 +++ b/hosting/single/Dockerfile.v2 @@ -26,6 +26,7 @@ RUN ./scripts/removeWorkspaceDependencies.sh packages/worker/package.json # We will never want to sync pro, but the script is still required RUN echo '' > scripts/syncProPackage.js RUN jq 'del(.scripts.postinstall)' package.json > temp.json && mv temp.json package.json +RUN ./scripts/removeWorkspaceDependencies.sh package.json RUN --mount=type=cache,target=/root/.yarn YARN_CACHE_FOLDER=/root/.yarn yarn install --production # copy the actual code @@ -41,6 +42,7 @@ COPY packages/string-templates packages/string-templates FROM budibase/couchdb as runner ARG TARGETARCH ENV TARGETARCH $TARGETARCH +ENV NODE_MAJOR 18 #TARGETBUILD can be set to single (for single docker image) or aas (for azure app service) # e.g. docker build --build-arg TARGETBUILD=aas .... ARG TARGETBUILD=single @@ -48,10 +50,10 @@ ENV TARGETBUILD $TARGETBUILD # install base dependencies RUN apt-get update && \ - apt-get install -y --no-install-recommends software-properties-common nginx uuid-runtime redis-server + apt-get install -y --no-install-recommends software-properties-common nginx uuid-runtime redis-server libaio1 # Install postgres client for pg_dump utils -RUN apt install software-properties-common apt-transport-https gpg -y \ +RUN apt install -y software-properties-common apt-transport-https ca-certificates gnupg \ && curl -fsSl https://www.postgresql.org/media/keys/ACCC4CF8.asc | gpg --dearmor | tee /usr/share/keyrings/postgresql.gpg > /dev/null \ && echo deb [arch=amd64,arm64,ppc64el signed-by=/usr/share/keyrings/postgresql.gpg] http://apt.postgresql.org/pub/repos/apt/ $(lsb_release -cs)-pgdg main | tee /etc/apt/sources.list.d/postgresql.list \ && apt update -y \ @@ -60,10 +62,8 @@ RUN apt install software-properties-common apt-transport-https gpg -y \ # install other dependencies, nodejs, oracle requirements, jdk8, redis, nginx WORKDIR /nodejs -RUN curl -sL https://deb.nodesource.com/setup_18.x -o /tmp/nodesource_setup.sh && \ - bash /tmp/nodesource_setup.sh && \ - apt-get install -y --no-install-recommends libaio1 nodejs && \ - npm install --global yarn pm2 +COPY scripts/install-node.sh ./install.sh +RUN chmod +x install.sh && ./install.sh # setup nginx COPY hosting/single/nginx/nginx.conf /etc/nginx @@ -117,6 +117,10 @@ EXPOSE 443 EXPOSE 2222 VOLUME /data +ARG BUDIBASE_VERSION +# Ensuring the version argument is sent +RUN test -n "$BUDIBASE_VERSION" +ENV BUDIBASE_VERSION=$BUDIBASE_VERSION HEALTHCHECK --interval=15s --timeout=15s --start-period=45s CMD "/healthcheck.sh" diff --git a/hosting/single/runner.sh b/hosting/single/runner.sh index 9dc7aa25d8..87201c95c0 100644 --- a/hosting/single/runner.sh +++ b/hosting/single/runner.sh @@ -22,7 +22,7 @@ declare -a DOCKER_VARS=("APP_PORT" "APPS_URL" "ARCHITECTURE" "BUDIBASE_ENVIRONME # Azure App Service customisations if [[ "${TARGETBUILD}" = "aas" ]]; then - DATA_DIR=/home + DATA_DIR="${DATA_DIR:-/home}" WEBSITES_ENABLE_APP_SERVICE_STORAGE=true /etc/init.d/ssh start else diff --git a/i18n/README.de.md b/i18n/README.de.md index a2f4c3afb9..17d3d1ebbe 100644 --- a/i18n/README.de.md +++ b/i18n/README.de.md @@ -1,6 +1,6 @@

- Budibase + Budibase

diff --git a/i18n/README.es.md b/i18n/README.es.md index 21eb8caef7..227d5d5d5f 100644 --- a/i18n/README.es.md +++ b/i18n/README.es.md @@ -1,6 +1,6 @@

- Budibase + Budibase

diff --git a/i18n/README.fr.md b/i18n/README.fr.md index 12abd4d073..f5f9fbb25e 100644 --- a/i18n/README.fr.md +++ b/i18n/README.fr.md @@ -1,6 +1,6 @@

- Budibase + Budibase

diff --git a/i18n/README.id.md b/i18n/README.id.md index d4a25f569c..c2077f3922 100644 --- a/i18n/README.id.md +++ b/i18n/README.id.md @@ -1,6 +1,6 @@

- Budibase + Budibase

diff --git a/i18n/README.jp.md b/i18n/README.jp.md index 6fea497d53..62d0b1d3aa 100644 --- a/i18n/README.jp.md +++ b/i18n/README.jp.md @@ -1,6 +1,6 @@

- Budibase + Budibase

diff --git a/i18n/README.zh.md b/i18n/README.zh.md index 7e4dffd387..a6a9575029 100644 --- a/i18n/README.zh.md +++ b/i18n/README.zh.md @@ -1,6 +1,6 @@

- Budibase + Budibase

diff --git a/lerna.json b/lerna.json index e01e5ae03e..a12b1238b3 100644 --- a/lerna.json +++ b/lerna.json @@ -1,5 +1,5 @@ { - "version": "2.11.39", + "version": "2.13.10", "npmClient": "yarn", "packages": [ "packages/*" diff --git a/package.json b/package.json index 100a306a35..2978483448 100644 --- a/package.json +++ b/package.json @@ -2,11 +2,17 @@ "name": "root", "private": true, "devDependencies": { + "@babel/core": "^7.22.5", + "@babel/eslint-parser": "^7.22.5", + "@babel/preset-env": "^7.22.5", "@esbuild-plugins/tsconfig-paths": "^0.1.2", "@typescript-eslint/parser": "6.7.2", "esbuild": "^0.18.17", "esbuild-node-externals": "^1.8.0", "eslint": "^8.44.0", + "eslint-plugin-import": "^2.29.0", + "eslint-plugin-local-rules": "^2.0.0", + "eslint-plugin-svelte": "^2.32.2", "husky": "^8.0.3", "kill-port": "^1.6.1", "lerna": "7.1.1", @@ -17,23 +23,18 @@ "prettier": "2.8.8", "prettier-plugin-svelte": "^2.3.0", "svelte": "3.49.0", - "typescript": "5.2.2", - "@babel/core": "^7.22.5", - "@babel/eslint-parser": "^7.22.5", - "@babel/preset-env": "^7.22.5", - "eslint-plugin-svelte": "^2.32.2", - "svelte-eslint-parser": "^0.32.0" + "svelte-eslint-parser": "^0.32.0", + "typescript": "5.2.2" }, "scripts": { "preinstall": "node scripts/syncProPackage.js", "setup": "git config submodule.recurse true && git submodule update && node ./hosting/scripts/setup.js && yarn && yarn build && yarn dev", - "build": "lerna run build --stream", + "build": "NODE_OPTIONS=--max-old-space-size=1500 lerna run build --stream", "build:dev": "lerna run --stream prebuild && yarn nx run-many --target=build --output-style=dynamic --watch --preserveWatchOutput", "check:types": "lerna run check:types", "build:sdk": "lerna run --stream build:sdk", "deps:circular": "madge packages/server/dist/index.js packages/worker/src/index.ts packages/backend-core/dist/src/index.js packages/cli/src/index.js --circular", "release": "lerna publish from-package --yes --force-publish --no-git-tag-version --no-push --no-git-reset", - "release:develop": "yarn release --dist-tag develop", "restore": "yarn run clean && yarn && yarn run build", "nuke": "yarn run nuke:packages && yarn run nuke:docker", "nuke:packages": "yarn run restore", @@ -55,10 +56,6 @@ "lint:fix:prettier": "prettier --write \"packages/**/*.{js,ts,svelte}\" && prettier --write \"examples/**/*.{js,ts,svelte}\" && prettier --write \"qa-core/**/*.{js,ts,svelte}\"", "lint:fix": "yarn run lint:fix:prettier && yarn run lint:fix:eslint", "build:specs": "lerna run --stream specs", - "build:docker": "lerna run --stream build:docker && yarn build:docker:proxy && cd hosting/scripts/linux/ && ./release-to-docker-hub.sh $BUDIBASE_RELEASE_VERSION && cd -", - "build:docker:proxy": "docker build hosting/proxy -t proxy-service", - "build:docker:selfhost": "lerna run --stream build:docker && cd hosting/scripts/linux/ && ./release-to-docker-hub.sh latest && cd -", - "build:docker:develop": "node scripts/pinVersions && lerna run --stream build:docker && yarn build:docker:proxy && cd hosting/scripts/linux/ && ./release-to-docker-hub.sh develop && cd -", "build:docker:airgap": "node hosting/scripts/airgapped/airgappedDockerBuild", "build:docker:airgap:single": "SINGLE_IMAGE=1 node hosting/scripts/airgapped/airgappedDockerBuild", "build:digitalocean": "cd hosting/digitalocean && ./build.sh && cd -", diff --git a/packages/backend-core/__mocks__/aws-sdk.ts b/packages/backend-core/__mocks__/aws-sdk.ts index b8d91dbaa9..e3be511d08 100644 --- a/packages/backend-core/__mocks__/aws-sdk.ts +++ b/packages/backend-core/__mocks__/aws-sdk.ts @@ -3,6 +3,7 @@ const mockS3 = { deleteObject: jest.fn().mockReturnThis(), deleteObjects: jest.fn().mockReturnThis(), createBucket: jest.fn().mockReturnThis(), + getObject: jest.fn().mockReturnThis(), listObject: jest.fn().mockReturnThis(), getSignedUrl: jest.fn((operation: string, params: any) => { return `http://s3.example.com/${params.Bucket}/${params.Key}` diff --git a/packages/backend-core/package.json b/packages/backend-core/package.json index b23cd8e5b1..dc8d71b52c 100644 --- a/packages/backend-core/package.json +++ b/packages/backend-core/package.json @@ -21,7 +21,7 @@ "test:watch": "jest --watchAll" }, "dependencies": { - "@budibase/nano": "10.1.2", + "@budibase/nano": "10.1.3", "@budibase/pouchdb-replication-stream": "1.2.10", "@budibase/shared-core": "0.0.0", "@budibase/types": "0.0.0", diff --git a/packages/backend-core/src/cache/appMetadata.ts b/packages/backend-core/src/cache/appMetadata.ts index bd3efc20db..d442511fb8 100644 --- a/packages/backend-core/src/cache/appMetadata.ts +++ b/packages/backend-core/src/cache/appMetadata.ts @@ -19,7 +19,7 @@ async function populateFromDB(appId: string) { return doWithDB( appId, (db: Database) => { - return db.get(DocumentType.APP_METADATA) + return db.get(DocumentType.APP_METADATA) }, { skip_setup: true } ) diff --git a/packages/backend-core/src/cache/generic.ts b/packages/backend-core/src/cache/generic.ts index 7cd5d6227f..7a2be5a0f0 100644 --- a/packages/backend-core/src/cache/generic.ts +++ b/packages/backend-core/src/cache/generic.ts @@ -1,6 +1,6 @@ -const BaseCache = require("./base") +import BaseCache from "./base" -const GENERIC = new BaseCache.default() +const GENERIC = new BaseCache() export enum CacheKey { CHECKLIST = "checklist", @@ -19,6 +19,7 @@ export enum TTL { } function performExport(funcName: string) { + // @ts-ignore return (...args: any) => GENERIC[funcName](...args) } diff --git a/packages/backend-core/src/cache/index.ts b/packages/backend-core/src/cache/index.ts index 58928c271a..4fa986e4e2 100644 --- a/packages/backend-core/src/cache/index.ts +++ b/packages/backend-core/src/cache/index.ts @@ -2,4 +2,6 @@ export * as generic from "./generic" export * as user from "./user" export * as app from "./appMetadata" export * as writethrough from "./writethrough" +export * as invite from "./invite" +export * as passwordReset from "./passwordReset" export * from "./generic" diff --git a/packages/backend-core/src/cache/invite.ts b/packages/backend-core/src/cache/invite.ts new file mode 100644 index 0000000000..e43ebc4aa8 --- /dev/null +++ b/packages/backend-core/src/cache/invite.ts @@ -0,0 +1,86 @@ +import * as utils from "../utils" +import { Duration, DurationType } from "../utils" +import env from "../environment" +import { getTenantId } from "../context" +import * as redis from "../redis/init" + +const TTL_SECONDS = Duration.fromDays(7).toSeconds() + +interface Invite { + email: string + info: any +} + +interface InviteWithCode extends Invite { + code: string +} + +/** + * Given an invite code and invite body, allow the update an existing/valid invite in redis + * @param code The invite code for an invite in redis + * @param value The body of the updated user invitation + */ +export async function updateCode(code: string, value: Invite) { + const client = await redis.getInviteClient() + await client.store(code, value, TTL_SECONDS) +} + +/** + * Generates an invitation code and writes it to redis - which can later be checked for user creation. + * @param email the email address which the code is being sent to (for use later). + * @param info Information to be carried along with the invitation. + * @return returns the code that was stored to redis. + */ +export async function createCode(email: string, info: any): Promise { + const code = utils.newid() + const client = await redis.getInviteClient() + await client.store(code, { email, info }, TTL_SECONDS) + return code +} + +/** + * Checks that the provided invite code is valid - will return the email address of user that was invited. + * @param code the invite code that was provided as part of the link. + * @return If the code is valid then an email address will be returned. + */ +export async function getCode(code: string): Promise { + const client = await redis.getInviteClient() + const value = (await client.get(code)) as Invite | undefined + if (!value) { + throw "Invitation is not valid or has expired, please request a new one." + } + return value +} + +export async function deleteCode(code: string) { + const client = await redis.getInviteClient() + await client.delete(code) +} + +/** + Get all currently available user invitations for the current tenant. + **/ +export async function getInviteCodes(): Promise { + const client = await redis.getInviteClient() + const invites: { key: string; value: Invite }[] = await client.scan() + + const results: InviteWithCode[] = invites.map(invite => { + return { + ...invite.value, + code: invite.key, + } + }) + if (!env.MULTI_TENANCY) { + return results + } + const tenantId = getTenantId() + return results.filter(invite => tenantId === invite.info.tenantId) +} + +export async function getExistingInvites( + emails: string[] +): Promise { + return (await getInviteCodes()).filter(invite => + emails.includes(invite.email) + ) +} diff --git a/packages/backend-core/src/cache/passwordReset.ts b/packages/backend-core/src/cache/passwordReset.ts new file mode 100644 index 0000000000..7f5a93f149 --- /dev/null +++ b/packages/backend-core/src/cache/passwordReset.ts @@ -0,0 +1,38 @@ +import * as redis from "../redis/init" +import * as utils from "../utils" +import { Duration, DurationType } from "../utils" + +const TTL_SECONDS = Duration.fromHours(1).toSeconds() + +interface PasswordReset { + userId: string + info: any +} + +/** + * Given a user ID this will store a code (that is returned) for an hour in redis. + * The user can then return this code for resetting their password (through their reset link). + * @param userId the ID of the user which is to be reset. + * @param info Info about the user/the reset process. + * @return returns the code that was stored to redis. + */ +export async function createCode(userId: string, info: any): Promise { + const code = utils.newid() + const client = await redis.getPasswordResetClient() + await client.store(code, { userId, info }, TTL_SECONDS) + return code +} + +/** + * Given a reset code this will lookup to redis, check if the code is valid. + * @param code The code provided via the email link. + * @return returns the user ID if it is found + */ +export async function getCode(code: string): Promise { + const client = await redis.getPasswordResetClient() + const value = (await client.get(code)) as PasswordReset | undefined + if (!value) { + throw "Provided information is not valid, cannot reset password - please try again." + } + return value +} diff --git a/packages/backend-core/src/constants/db.ts b/packages/backend-core/src/constants/db.ts index 9577e3bbfb..eeff1b6f93 100644 --- a/packages/backend-core/src/constants/db.ts +++ b/packages/backend-core/src/constants/db.ts @@ -28,7 +28,7 @@ export enum ViewName { APP_BACKUP_BY_TRIGGER = "by_trigger", } -export const DeprecatedViews = { +export const DeprecatedViews: Record = { [ViewName.USER_BY_EMAIL]: [ // removed due to inaccuracy in view doc filter logic "by_email", diff --git a/packages/backend-core/src/context/Context.ts b/packages/backend-core/src/context/Context.ts index d29b6935a8..a59f5c6503 100644 --- a/packages/backend-core/src/context/Context.ts +++ b/packages/backend-core/src/context/Context.ts @@ -4,7 +4,7 @@ import { ContextMap } from "./types" export default class Context { static storage = new AsyncLocalStorage() - static run(context: ContextMap, func: any) { + static run(context: ContextMap, func: () => T) { return Context.storage.run(context, () => func()) } diff --git a/packages/backend-core/src/context/mainContext.ts b/packages/backend-core/src/context/mainContext.ts index 609c18abb5..d2259cfcab 100644 --- a/packages/backend-core/src/context/mainContext.ts +++ b/packages/backend-core/src/context/mainContext.ts @@ -98,17 +98,17 @@ function updateContext(updates: ContextMap): ContextMap { return context } -async function newContext(updates: ContextMap, task: any) { +async function newContext(updates: ContextMap, task: () => T) { // see if there already is a context setup let context: ContextMap = updateContext(updates) return Context.run(context, task) } -export async function doInAutomationContext(params: { +export async function doInAutomationContext(params: { appId: string automationId: string - task: any -}): Promise { + task: () => T +}): Promise { const tenantId = getTenantIDFromAppID(params.appId) return newContext( { @@ -144,10 +144,10 @@ export async function doInTenant( return newContext(updates, task) } -export async function doInAppContext( +export async function doInAppContext( appId: string | null, - task: any -): Promise { + task: () => T +): Promise { if (!appId && !env.isTest()) { throw new Error("appId is required") } @@ -165,10 +165,10 @@ export async function doInAppContext( return newContext(updates, task) } -export async function doInIdentityContext( +export async function doInIdentityContext( identity: IdentityContext, - task: any -): Promise { + task: () => T +): Promise { if (!identity) { throw new Error("identity is required") } @@ -276,6 +276,9 @@ export function getAuditLogsDB(): Database { */ export function getAppDB(opts?: any): Database { const appId = getAppId() + if (!appId) { + throw new Error("Unable to retrieve app DB - no app ID.") + } return getDB(appId, opts) } diff --git a/packages/backend-core/src/db/constants.ts b/packages/backend-core/src/db/constants.ts index aea485e3e3..bfa7595d62 100644 --- a/packages/backend-core/src/db/constants.ts +++ b/packages/backend-core/src/db/constants.ts @@ -8,3 +8,7 @@ export const CONSTANT_INTERNAL_ROW_COLS = [ ] as const export const CONSTANT_EXTERNAL_ROW_COLS = ["_id", "_rev", "tableId"] as const + +export function isInternalColumnName(name: string): boolean { + return (CONSTANT_INTERNAL_ROW_COLS as readonly string[]).includes(name) +} diff --git a/packages/backend-core/src/db/couch/DatabaseImpl.ts b/packages/backend-core/src/db/couch/DatabaseImpl.ts index a1f8696af2..d54e995fe8 100644 --- a/packages/backend-core/src/db/couch/DatabaseImpl.ts +++ b/packages/backend-core/src/db/couch/DatabaseImpl.ts @@ -10,6 +10,7 @@ import { DatabaseDeleteIndexOpts, Document, isDocument, + RowResponse, } from "@budibase/types" import { getCouchInfo } from "./connections" import { directCouchUrlCall } from "./utils" @@ -49,10 +50,7 @@ export class DatabaseImpl implements Database { private readonly couchInfo = getCouchInfo() - constructor(dbName?: string, opts?: DatabaseOpts, connection?: string) { - if (dbName == null) { - throw new Error("Database name cannot be undefined.") - } + constructor(dbName: string, opts?: DatabaseOpts, connection?: string) { this.name = dbName this.pouchOpts = opts || {} if (connection) { @@ -113,7 +111,7 @@ export class DatabaseImpl implements Database { } } - async get(id?: string): Promise { + async get(id?: string): Promise { const db = await this.checkSetup() if (!id) { throw new Error("Unable to get doc without a valid _id.") @@ -121,6 +119,35 @@ export class DatabaseImpl implements Database { return this.updateOutput(() => db.get(id)) } + async getMultiple( + ids: string[], + opts?: { allowMissing?: boolean } + ): Promise { + // get unique + ids = [...new Set(ids)] + const response = await this.allDocs({ + keys: ids, + include_docs: true, + }) + const rowUnavailable = (row: RowResponse) => { + // row is deleted - key lookup can return this + if (row.doc == null || ("deleted" in row.value && row.value.deleted)) { + return true + } + return row.error === "not_found" + } + + const rows = response.rows.filter(row => !rowUnavailable(row)) + const someMissing = rows.length !== response.rows.length + // some were filtered out - means some missing + if (!opts?.allowMissing && someMissing) { + const missing = response.rows.filter(row => rowUnavailable(row)) + const missingIds = missing.map(row => row.key).join(", ") + throw new Error(`Unable to get documents: ${missingIds}`) + } + return rows.map(row => row.doc!) + } + async remove(idOrDoc: string | Document, rev?: string) { const db = await this.checkSetup() let _id: string @@ -176,7 +203,9 @@ export class DatabaseImpl implements Database { return this.updateOutput(() => db.bulk({ docs: documents })) } - async allDocs(params: DatabaseQueryOpts): Promise> { + async allDocs( + params: DatabaseQueryOpts + ): Promise> { const db = await this.checkSetup() return this.updateOutput(() => db.list(params)) } @@ -196,7 +225,7 @@ export class DatabaseImpl implements Database { return (await response.json()) as T } - async query( + async query( viewName: string, params: DatabaseQueryOpts ): Promise> { diff --git a/packages/backend-core/src/db/db.ts b/packages/backend-core/src/db/db.ts index 9aae64b892..3e69d49f0e 100644 --- a/packages/backend-core/src/db/db.ts +++ b/packages/backend-core/src/db/db.ts @@ -1,10 +1,7 @@ -import env from "../environment" import { directCouchQuery, DatabaseImpl } from "./couch" -import { CouchFindOptions, Database } from "@budibase/types" +import { CouchFindOptions, Database, DatabaseOpts } from "@budibase/types" -const dbList = new Set() - -export function getDB(dbName?: string, opts?: any): Database { +export function getDB(dbName: string, opts?: DatabaseOpts): Database { return new DatabaseImpl(dbName, opts) } @@ -14,7 +11,7 @@ export function getDB(dbName?: string, opts?: any): Database { export async function doWithDB( dbName: string, cb: (db: Database) => Promise, - opts = {} + opts?: DatabaseOpts ) { const db = getDB(dbName, opts) // need this to be async so that we can correctly close DB after all @@ -22,13 +19,6 @@ export async function doWithDB( return await cb(db) } -export function allDbs() { - if (!env.isTest()) { - throw new Error("Cannot be used outside test environment.") - } - return [...dbList] -} - export async function directCouchAllDbs(queryString?: string) { let couchPath = "/_all_dbs" if (queryString) { diff --git a/packages/backend-core/src/db/views.ts b/packages/backend-core/src/db/views.ts index f0980ad217..5d9c5b74d3 100644 --- a/packages/backend-core/src/db/views.ts +++ b/packages/backend-core/src/db/views.ts @@ -7,12 +7,19 @@ import { } from "../constants" import { getGlobalDB } from "../context" import { doWithDB } from "./" -import { AllDocsResponse, Database, DatabaseQueryOpts } from "@budibase/types" +import { + AllDocsResponse, + Database, + DatabaseQueryOpts, + Document, + DesignDocument, + DBView, +} from "@budibase/types" import env from "../environment" const DESIGN_DB = "_design/database" -function DesignDoc() { +function DesignDoc(): DesignDocument { return { _id: DESIGN_DB, // view collation information, read before writing any complex views: @@ -21,20 +28,14 @@ function DesignDoc() { } } -interface DesignDocument { - views: any -} - async function removeDeprecated(db: Database, viewName: ViewName) { - // @ts-ignore if (!DeprecatedViews[viewName]) { return } try { const designDoc = await db.get(DESIGN_DB) - // @ts-ignore for (let deprecatedNames of DeprecatedViews[viewName]) { - delete designDoc.views[deprecatedNames] + delete designDoc.views?.[deprecatedNames] } await db.put(designDoc) } catch (err) { @@ -43,18 +44,18 @@ async function removeDeprecated(db: Database, viewName: ViewName) { } export async function createView( - db: any, + db: Database, viewJs: string, viewName: string ): Promise { let designDoc try { - designDoc = (await db.get(DESIGN_DB)) as DesignDocument + designDoc = await db.get(DESIGN_DB) } catch (err) { // no design doc, make one designDoc = DesignDoc() } - const view = { + const view: DBView = { map: viewJs, } designDoc.views = { @@ -109,7 +110,7 @@ export interface QueryViewOptions { arrayResponse?: boolean } -export async function queryViewRaw( +export async function queryViewRaw( viewName: ViewName, params: DatabaseQueryOpts, db: Database, @@ -137,18 +138,16 @@ export async function queryViewRaw( } } -export const queryView = async ( +export const queryView = async ( viewName: ViewName, params: DatabaseQueryOpts, db: Database, createFunc: any, opts?: QueryViewOptions -): Promise => { +): Promise => { const response = await queryViewRaw(viewName, params, db, createFunc, opts) const rows = response.rows - const docs = rows.map((row: any) => - params.include_docs ? row.doc : row.value - ) + const docs = rows.map(row => (params.include_docs ? row.doc! : row.value)) // if arrayResponse has been requested, always return array regardless of length if (opts?.arrayResponse) { @@ -198,11 +197,11 @@ export const createPlatformUserView = async () => { await createPlatformView(viewJs, ViewName.PLATFORM_USERS_LOWERCASE) } -export const queryPlatformView = async ( +export const queryPlatformView = async ( viewName: ViewName, params: DatabaseQueryOpts, opts?: QueryViewOptions -): Promise => { +): Promise => { const CreateFuncByName: any = { [ViewName.ACCOUNT_BY_EMAIL]: createPlatformAccountEmailView, [ViewName.PLATFORM_USERS_LOWERCASE]: createPlatformUserView, @@ -220,7 +219,7 @@ const CreateFuncByName: any = { [ViewName.USER_BY_APP]: createUserAppView, } -export const queryGlobalView = async ( +export const queryGlobalView = async ( viewName: ViewName, params: DatabaseQueryOpts, db?: Database, @@ -231,10 +230,10 @@ export const queryGlobalView = async ( db = getGlobalDB() } const createFn = CreateFuncByName[viewName] - return queryView(viewName, params, db!, createFn, opts) + return queryView(viewName, params, db!, createFn, opts) } -export async function queryGlobalViewRaw( +export async function queryGlobalViewRaw( viewName: ViewName, params: DatabaseQueryOpts, opts?: QueryViewOptions diff --git a/packages/backend-core/src/docIds/params.ts b/packages/backend-core/src/docIds/params.ts index 36fd75622b..d9baee3dc6 100644 --- a/packages/backend-core/src/docIds/params.ts +++ b/packages/backend-core/src/docIds/params.ts @@ -6,6 +6,7 @@ import { ViewName, } from "../constants" import { getProdAppID } from "./conversions" +import { DatabaseQueryOpts } from "@budibase/types" /** * If creating DB allDocs/query params with only a single top level ID this can be used, this @@ -22,8 +23,8 @@ import { getProdAppID } from "./conversions" export function getDocParams( docType: string, docId?: string | null, - otherProps: any = {} -) { + otherProps: Partial = {} +): DatabaseQueryOpts { if (docId == null) { docId = "" } @@ -45,8 +46,8 @@ export function getDocParams( export function getRowParams( tableId?: string | null, rowId?: string | null, - otherProps = {} -) { + otherProps: Partial = {} +): DatabaseQueryOpts { if (tableId == null) { return getDocParams(DocumentType.ROW, null, otherProps) } @@ -88,7 +89,10 @@ export const isDatasourceId = (id: string) => { /** * Gets parameters for retrieving workspaces. */ -export function getWorkspaceParams(id = "", otherProps = {}) { +export function getWorkspaceParams( + id = "", + otherProps: Partial = {} +): DatabaseQueryOpts { return { ...otherProps, startkey: `${DocumentType.WORKSPACE}${SEPARATOR}${id}`, @@ -99,7 +103,10 @@ export function getWorkspaceParams(id = "", otherProps = {}) { /** * Gets parameters for retrieving users. */ -export function getGlobalUserParams(globalId: any, otherProps: any = {}) { +export function getGlobalUserParams( + globalId: any, + otherProps: Partial = {} +): DatabaseQueryOpts { if (!globalId) { globalId = "" } @@ -117,11 +124,17 @@ export function getGlobalUserParams(globalId: any, otherProps: any = {}) { /** * Gets parameters for retrieving users, this is a utility function for the getDocParams function. */ -export function getUserMetadataParams(userId?: string | null, otherProps = {}) { +export function getUserMetadataParams( + userId?: string | null, + otherProps: Partial = {} +): DatabaseQueryOpts { return getRowParams(InternalTable.USER_METADATA, userId, otherProps) } -export function getUsersByAppParams(appId: any, otherProps: any = {}) { +export function getUsersByAppParams( + appId: any, + otherProps: Partial = {} +): DatabaseQueryOpts { const prodAppId = getProdAppID(appId) return { ...otherProps, diff --git a/packages/backend-core/src/environment.ts b/packages/backend-core/src/environment.ts index 510d580f28..138dbbd9e0 100644 --- a/packages/backend-core/src/environment.ts +++ b/packages/backend-core/src/environment.ts @@ -75,12 +75,12 @@ function getPackageJsonFields(): { const content = readFileSync(packageJsonFile!, "utf-8") const parsedContent = JSON.parse(content) return { - VERSION: parsedContent.version, + VERSION: process.env.BUDIBASE_VERSION || parsedContent.version, SERVICE_NAME: parsedContent.name, } } catch { // throwing an error here is confusing/causes backend-core to be hard to import - return { VERSION: "", SERVICE_NAME: "" } + return { VERSION: process.env.BUDIBASE_VERSION || "", SERVICE_NAME: "" } } } diff --git a/packages/backend-core/src/index.ts b/packages/backend-core/src/index.ts index ffffd8240a..2cfd517941 100644 --- a/packages/backend-core/src/index.ts +++ b/packages/backend-core/src/index.ts @@ -30,7 +30,9 @@ export * as timers from "./timers" export { default as env } from "./environment" export * as blacklist from "./blacklist" export * as docUpdates from "./docUpdates" +export * from "./utils/Duration" export { SearchParams } from "./db" +export * as docIds from "./docIds" // Add context to tenancy for backwards compatibility // only do this for external usages to prevent internal // circular dependencies @@ -49,6 +51,7 @@ export * from "./constants" // expose package init function import * as db from "./db" + export const init = (opts: any = {}) => { db.init(opts.db) } diff --git a/packages/backend-core/src/objectStore/buckets/app.ts b/packages/backend-core/src/objectStore/buckets/app.ts index be9fddeaa6..43bc965c65 100644 --- a/packages/backend-core/src/objectStore/buckets/app.ts +++ b/packages/backend-core/src/objectStore/buckets/app.ts @@ -1,37 +1,50 @@ import env from "../../environment" import * as objectStore from "../objectStore" import * as cloudfront from "../cloudfront" +import qs from "querystring" +import { DEFAULT_TENANT_ID, getTenantId } from "../../context" + +export function clientLibraryPath(appId: string) { + return `${objectStore.sanitizeKey(appId)}/budibase-client.js` +} /** - * In production the client library is stored in the object store, however in development - * we use the symlinked version produced by lerna, located in node modules. We link to this - * via a specific endpoint (under /api/assets/client). - * @param appId In production we need the appId to look up the correct bucket, as the - * version of the client lib may differ between apps. - * @param version The version to retrieve. - * @return The URL to be inserted into appPackage response or server rendered - * app index file. + * Previously we used to serve the client library directly from Cloudfront, however + * due to issues with the domain we were unable to continue doing this - keeping + * incase we are able to switch back to CDN path again in future. */ -export const clientLibraryUrl = (appId: string, version: string) => { - if (env.isProd()) { - let file = `${objectStore.sanitizeKey(appId)}/budibase-client.js` - if (env.CLOUDFRONT_CDN) { - // append app version to bust the cache - if (version) { - file += `?v=${version}` - } - // don't need to use presigned for client with cloudfront - // file is public - return cloudfront.getUrl(file) - } else { - return objectStore.getPresignedUrl(env.APPS_BUCKET_NAME, file) +export function clientLibraryCDNUrl(appId: string, version: string) { + let file = clientLibraryPath(appId) + if (env.CLOUDFRONT_CDN) { + // append app version to bust the cache + if (version) { + file += `?v=${version}` } + // don't need to use presigned for client with cloudfront + // file is public + return cloudfront.getUrl(file) } else { - return `/api/assets/client` + return objectStore.getPresignedUrl(env.APPS_BUCKET_NAME, file) } } -export const getAppFileUrl = (s3Key: string) => { +export function clientLibraryUrl(appId: string, version: string) { + let tenantId, qsParams: { appId: string; version: string; tenantId?: string } + try { + tenantId = getTenantId() + } finally { + qsParams = { + appId, + version, + } + } + if (tenantId && tenantId !== DEFAULT_TENANT_ID) { + qsParams.tenantId = tenantId + } + return `/api/assets/client?${qs.encode(qsParams)}` +} + +export function getAppFileUrl(s3Key: string) { if (env.CLOUDFRONT_CDN) { return cloudfront.getPresignedUrl(s3Key) } else { diff --git a/packages/backend-core/src/objectStore/buckets/plugins.ts b/packages/backend-core/src/objectStore/buckets/plugins.ts index f7721afb23..6f1b7116ae 100644 --- a/packages/backend-core/src/objectStore/buckets/plugins.ts +++ b/packages/backend-core/src/objectStore/buckets/plugins.ts @@ -6,7 +6,7 @@ import { Plugin } from "@budibase/types" // URLS -export const enrichPluginURLs = (plugins: Plugin[]) => { +export function enrichPluginURLs(plugins: Plugin[]) { if (!plugins || !plugins.length) { return [] } @@ -17,12 +17,12 @@ export const enrichPluginURLs = (plugins: Plugin[]) => { }) } -const getPluginJSUrl = (plugin: Plugin) => { +function getPluginJSUrl(plugin: Plugin) { const s3Key = getPluginJSKey(plugin) return getPluginUrl(s3Key) } -const getPluginIconUrl = (plugin: Plugin): string | undefined => { +function getPluginIconUrl(plugin: Plugin): string | undefined { const s3Key = getPluginIconKey(plugin) if (!s3Key) { return @@ -30,7 +30,7 @@ const getPluginIconUrl = (plugin: Plugin): string | undefined => { return getPluginUrl(s3Key) } -const getPluginUrl = (s3Key: string) => { +function getPluginUrl(s3Key: string) { if (env.CLOUDFRONT_CDN) { return cloudfront.getPresignedUrl(s3Key) } else { @@ -40,11 +40,11 @@ const getPluginUrl = (s3Key: string) => { // S3 KEYS -export const getPluginJSKey = (plugin: Plugin) => { +export function getPluginJSKey(plugin: Plugin) { return getPluginS3Key(plugin, "plugin.min.js") } -export const getPluginIconKey = (plugin: Plugin) => { +export function getPluginIconKey(plugin: Plugin) { // stored iconUrl is deprecated - hardcode to icon.svg in this case const iconFileName = plugin.iconUrl ? "icon.svg" : plugin.iconFileName if (!iconFileName) { @@ -53,12 +53,12 @@ export const getPluginIconKey = (plugin: Plugin) => { return getPluginS3Key(plugin, iconFileName) } -const getPluginS3Key = (plugin: Plugin, fileName: string) => { +function getPluginS3Key(plugin: Plugin, fileName: string) { const s3Key = getPluginS3Dir(plugin.name) return `${s3Key}/${fileName}` } -export const getPluginS3Dir = (pluginName: string) => { +export function getPluginS3Dir(pluginName: string) { let s3Key = `${pluginName}` if (env.MULTI_TENANCY) { const tenantId = context.getTenantId() diff --git a/packages/backend-core/src/objectStore/buckets/tests/app.spec.ts b/packages/backend-core/src/objectStore/buckets/tests/app.spec.ts index aaa07ec9d3..cbbbee6255 100644 --- a/packages/backend-core/src/objectStore/buckets/tests/app.spec.ts +++ b/packages/backend-core/src/objectStore/buckets/tests/app.spec.ts @@ -1,5 +1,4 @@ import * as app from "../app" -import { getAppFileUrl } from "../app" import { testEnv } from "../../../../tests/extra" describe("app", () => { @@ -7,6 +6,15 @@ describe("app", () => { testEnv.nodeJest() }) + function baseCheck(url: string, tenantId?: string) { + expect(url).toContain("/api/assets/client") + if (tenantId) { + expect(url).toContain(`tenantId=${tenantId}`) + } + expect(url).toContain("appId=app_123") + expect(url).toContain("version=2.0.0") + } + describe("clientLibraryUrl", () => { function getClientUrl() { return app.clientLibraryUrl("app_123/budibase-client.js", "2.0.0") @@ -20,31 +28,19 @@ describe("app", () => { it("gets url in dev", () => { testEnv.nodeDev() const url = getClientUrl() - expect(url).toBe("/api/assets/client") - }) - - it("gets url with embedded minio", () => { - testEnv.withMinio() - const url = getClientUrl() - expect(url).toBe( - "/files/signed/prod-budi-app-assets/app_123/budibase-client.js/budibase-client.js" - ) + baseCheck(url) }) it("gets url with custom S3", () => { testEnv.withS3() const url = getClientUrl() - expect(url).toBe( - "http://s3.example.com/prod-budi-app-assets/app_123/budibase-client.js/budibase-client.js" - ) + baseCheck(url) }) it("gets url with cloudfront + s3", () => { testEnv.withCloudfront() const url = getClientUrl() - expect(url).toBe( - "http://cf.example.com/app_123/budibase-client.js/budibase-client.js?v=2.0.0" - ) + baseCheck(url) }) }) @@ -57,7 +53,7 @@ describe("app", () => { testEnv.nodeDev() await testEnv.withTenant(tenantId => { const url = getClientUrl() - expect(url).toBe("/api/assets/client") + baseCheck(url, tenantId) }) }) @@ -65,9 +61,7 @@ describe("app", () => { await testEnv.withTenant(tenantId => { testEnv.withMinio() const url = getClientUrl() - expect(url).toBe( - "/files/signed/prod-budi-app-assets/app_123/budibase-client.js/budibase-client.js" - ) + baseCheck(url, tenantId) }) }) @@ -75,9 +69,7 @@ describe("app", () => { await testEnv.withTenant(tenantId => { testEnv.withS3() const url = getClientUrl() - expect(url).toBe( - "http://s3.example.com/prod-budi-app-assets/app_123/budibase-client.js/budibase-client.js" - ) + baseCheck(url, tenantId) }) }) @@ -85,9 +77,7 @@ describe("app", () => { await testEnv.withTenant(tenantId => { testEnv.withCloudfront() const url = getClientUrl() - expect(url).toBe( - "http://cf.example.com/app_123/budibase-client.js/budibase-client.js?v=2.0.0" - ) + baseCheck(url, tenantId) }) }) }) diff --git a/packages/backend-core/src/objectStore/objectStore.ts b/packages/backend-core/src/objectStore/objectStore.ts index c36a09915e..76d2dd6689 100644 --- a/packages/backend-core/src/objectStore/objectStore.ts +++ b/packages/backend-core/src/objectStore/objectStore.ts @@ -1,6 +1,6 @@ const sanitize = require("sanitize-s3-objectkey") import AWS from "aws-sdk" -import stream from "stream" +import stream, { Readable } from "stream" import fetch from "node-fetch" import tar from "tar-fs" import zlib from "zlib" @@ -66,10 +66,10 @@ export function sanitizeBucket(input: string) { * @return an S3 object store object, check S3 Nodejs SDK for usage. * @constructor */ -export const ObjectStore = ( +export function ObjectStore( bucket: string, opts: { presigning: boolean } = { presigning: false } -) => { +) { const config: any = { s3ForcePathStyle: true, signatureVersion: "v4", @@ -104,7 +104,7 @@ export const ObjectStore = ( * Given an object store and a bucket name this will make sure the bucket exists, * if it does not exist then it will create it. */ -export const makeSureBucketExists = async (client: any, bucketName: string) => { +export async function makeSureBucketExists(client: any, bucketName: string) { bucketName = sanitizeBucket(bucketName) try { await client @@ -139,13 +139,13 @@ export const makeSureBucketExists = async (client: any, bucketName: string) => { * Uploads the contents of a file given the required parameters, useful when * temp files in use (for example file uploaded as an attachment). */ -export const upload = async ({ +export async function upload({ bucket: bucketName, filename, path, type, metadata, -}: UploadParams) => { +}: UploadParams) { const extension = filename.split(".").pop() const fileBytes = fs.readFileSync(path) @@ -180,12 +180,12 @@ export const upload = async ({ * Similar to the upload function but can be used to send a file stream * through to the object store. */ -export const streamUpload = async ( +export async function streamUpload( bucketName: string, filename: string, stream: any, extra = {} -) => { +) { const objectStore = ObjectStore(bucketName) await makeSureBucketExists(objectStore, bucketName) @@ -215,7 +215,7 @@ export const streamUpload = async ( * retrieves the contents of a file from the object store, if it is a known content type it * will be converted, otherwise it will be returned as a buffer stream. */ -export const retrieve = async (bucketName: string, filepath: string) => { +export async function retrieve(bucketName: string, filepath: string) { const objectStore = ObjectStore(bucketName) const params = { Bucket: sanitizeBucket(bucketName), @@ -230,7 +230,7 @@ export const retrieve = async (bucketName: string, filepath: string) => { } } -export const listAllObjects = async (bucketName: string, path: string) => { +export async function listAllObjects(bucketName: string, path: string) { const objectStore = ObjectStore(bucketName) const list = (params: ListParams = {}) => { return objectStore @@ -261,11 +261,11 @@ export const listAllObjects = async (bucketName: string, path: string) => { /** * Generate a presigned url with a default TTL of 1 hour */ -export const getPresignedUrl = ( +export function getPresignedUrl( bucketName: string, key: string, durationSeconds: number = 3600 -) => { +) { const objectStore = ObjectStore(bucketName, { presigning: true }) const params = { Bucket: sanitizeBucket(bucketName), @@ -291,7 +291,7 @@ export const getPresignedUrl = ( /** * Same as retrieval function but puts to a temporary file. */ -export const retrieveToTmp = async (bucketName: string, filepath: string) => { +export async function retrieveToTmp(bucketName: string, filepath: string) { bucketName = sanitizeBucket(bucketName) filepath = sanitizeKey(filepath) const data = await retrieve(bucketName, filepath) @@ -300,7 +300,7 @@ export const retrieveToTmp = async (bucketName: string, filepath: string) => { return outputPath } -export const retrieveDirectory = async (bucketName: string, path: string) => { +export async function retrieveDirectory(bucketName: string, path: string) { let writePath = join(budibaseTempDir(), v4()) fs.mkdirSync(writePath) const objects = await listAllObjects(bucketName, path) @@ -324,7 +324,7 @@ export const retrieveDirectory = async (bucketName: string, path: string) => { /** * Delete a single file. */ -export const deleteFile = async (bucketName: string, filepath: string) => { +export async function deleteFile(bucketName: string, filepath: string) { const objectStore = ObjectStore(bucketName) await makeSureBucketExists(objectStore, bucketName) const params = { @@ -334,7 +334,7 @@ export const deleteFile = async (bucketName: string, filepath: string) => { return objectStore.deleteObject(params).promise() } -export const deleteFiles = async (bucketName: string, filepaths: string[]) => { +export async function deleteFiles(bucketName: string, filepaths: string[]) { const objectStore = ObjectStore(bucketName) await makeSureBucketExists(objectStore, bucketName) const params = { @@ -349,10 +349,10 @@ export const deleteFiles = async (bucketName: string, filepaths: string[]) => { /** * Delete a path, including everything within. */ -export const deleteFolder = async ( +export async function deleteFolder( bucketName: string, folder: string -): Promise => { +): Promise { bucketName = sanitizeBucket(bucketName) folder = sanitizeKey(folder) const client = ObjectStore(bucketName) @@ -383,11 +383,11 @@ export const deleteFolder = async ( } } -export const uploadDirectory = async ( +export async function uploadDirectory( bucketName: string, localPath: string, bucketPath: string -) => { +) { bucketName = sanitizeBucket(bucketName) let uploads = [] const files = fs.readdirSync(localPath, { withFileTypes: true }) @@ -404,11 +404,11 @@ export const uploadDirectory = async ( return files } -export const downloadTarballDirect = async ( +export async function downloadTarballDirect( url: string, path: string, headers = {} -) => { +) { path = sanitizeKey(path) const response = await fetch(url, { headers }) if (!response.ok) { @@ -418,11 +418,11 @@ export const downloadTarballDirect = async ( await streamPipeline(response.body, zlib.createUnzip(), tar.extract(path)) } -export const downloadTarball = async ( +export async function downloadTarball( url: string, bucketName: string, path: string -) => { +) { bucketName = sanitizeBucket(bucketName) path = sanitizeKey(path) const response = await fetch(url) @@ -438,3 +438,17 @@ export const downloadTarball = async ( // return the temporary path incase there is a use for it return tmpPath } + +export async function getReadStream( + bucketName: string, + path: string +): Promise { + bucketName = sanitizeBucket(bucketName) + path = sanitizeKey(path) + const client = ObjectStore(bucketName) + const params = { + Bucket: bucketName, + Key: path, + } + return client.getObject(params).createReadStream() +} diff --git a/packages/backend-core/src/objectStore/utils.ts b/packages/backend-core/src/objectStore/utils.ts index dba5f3d1c2..4c3a84ba91 100644 --- a/packages/backend-core/src/objectStore/utils.ts +++ b/packages/backend-core/src/objectStore/utils.ts @@ -18,8 +18,12 @@ export const ObjectStoreBuckets = { } const bbTmp = join(tmpdir(), ".budibase") -if (!fs.existsSync(bbTmp)) { +try { fs.mkdirSync(bbTmp) +} catch (e: any) { + if (e.code !== "EEXIST") { + throw e + } } export function budibaseTempDir() { diff --git a/packages/backend-core/src/queue/inMemoryQueue.ts b/packages/backend-core/src/queue/inMemoryQueue.ts index af2ec6dbaa..a8add7ecb6 100644 --- a/packages/backend-core/src/queue/inMemoryQueue.ts +++ b/packages/backend-core/src/queue/inMemoryQueue.ts @@ -36,7 +36,7 @@ class InMemoryQueue { * @param opts This is not used by the in memory queue as there is no real use * case when in memory, but is the same API as Bull */ - constructor(name: string, opts = null) { + constructor(name: string, opts?: any) { this._name = name this._opts = opts this._messages = [] diff --git a/packages/backend-core/src/queue/queue.ts b/packages/backend-core/src/queue/queue.ts index 0658147709..0657437a3b 100644 --- a/packages/backend-core/src/queue/queue.ts +++ b/packages/backend-core/src/queue/queue.ts @@ -2,11 +2,17 @@ import env from "../environment" import { getRedisOptions } from "../redis/utils" import { JobQueue } from "./constants" import InMemoryQueue from "./inMemoryQueue" -import BullQueue from "bull" +import BullQueue, { QueueOptions } from "bull" import { addListeners, StalledFn } from "./listeners" +import { Duration } from "../utils" import * as timers from "../timers" -const CLEANUP_PERIOD_MS = 60 * 1000 +// the queue lock is held for 5 minutes +const QUEUE_LOCK_MS = Duration.fromMinutes(5).toMs() +// queue lock is refreshed every 30 seconds +const QUEUE_LOCK_RENEW_INTERNAL_MS = Duration.fromSeconds(30).toMs() +// cleanup the queue every 60 seconds +const CLEANUP_PERIOD_MS = Duration.fromSeconds(60).toMs() let QUEUES: BullQueue.Queue[] | InMemoryQueue[] = [] let cleanupInterval: NodeJS.Timeout @@ -20,8 +26,15 @@ export function createQueue( jobQueue: JobQueue, opts: { removeStalledCb?: StalledFn } = {} ): BullQueue.Queue { - const { opts: redisOpts, redisProtocolUrl } = getRedisOptions() - const queueConfig: any = redisProtocolUrl || { redis: redisOpts } + const redisOpts = getRedisOptions() + const queueConfig: QueueOptions = { + redis: redisOpts, + settings: { + maxStalledCount: 0, + lockDuration: QUEUE_LOCK_MS, + lockRenewTime: QUEUE_LOCK_RENEW_INTERNAL_MS, + }, + } let queue: any if (!env.isTest()) { queue = new BullQueue(jobQueue, queueConfig) diff --git a/packages/backend-core/src/redis/init.ts b/packages/backend-core/src/redis/init.ts index 55ffe3dd12..f3bcee3209 100644 --- a/packages/backend-core/src/redis/init.ts +++ b/packages/backend-core/src/redis/init.ts @@ -7,15 +7,19 @@ let userClient: Client, cacheClient: Client, writethroughClient: Client, lockClient: Client, - socketClient: Client + socketClient: Client, + inviteClient: Client, + passwordResetClient: Client -async function init() { +export async function init() { userClient = await new Client(utils.Databases.USER_CACHE).init() sessionClient = await new Client(utils.Databases.SESSIONS).init() appClient = await new Client(utils.Databases.APP_METADATA).init() cacheClient = await new Client(utils.Databases.GENERIC_CACHE).init() lockClient = await new Client(utils.Databases.LOCKS).init() writethroughClient = await new Client(utils.Databases.WRITE_THROUGH).init() + inviteClient = await new Client(utils.Databases.INVITATIONS).init() + passwordResetClient = await new Client(utils.Databases.PW_RESETS).init() socketClient = await new Client( utils.Databases.SOCKET_IO, utils.SelectableDatabase.SOCKET_IO @@ -29,6 +33,8 @@ export async function shutdown() { if (cacheClient) await cacheClient.finish() if (writethroughClient) await writethroughClient.finish() if (lockClient) await lockClient.finish() + if (inviteClient) await inviteClient.finish() + if (passwordResetClient) await passwordResetClient.finish() if (socketClient) await socketClient.finish() } @@ -84,3 +90,17 @@ export async function getSocketClient() { } return socketClient } + +export async function getInviteClient() { + if (!inviteClient) { + await init() + } + return inviteClient +} + +export async function getPasswordResetClient() { + if (!passwordResetClient) { + await init() + } + return passwordResetClient +} diff --git a/packages/backend-core/src/redis/redis.ts b/packages/backend-core/src/redis/redis.ts index d1e2d8989e..701e262091 100644 --- a/packages/backend-core/src/redis/redis.ts +++ b/packages/backend-core/src/redis/redis.ts @@ -16,6 +16,7 @@ import { getRedisOptions, SEPARATOR, SelectableDatabase, + getRedisConnectionDetails, } from "./utils" import * as timers from "../timers" @@ -27,7 +28,6 @@ const DEFAULT_SELECT_DB = SelectableDatabase.DEFAULT // for testing just generate the client once let CLOSED = false let CLIENTS: { [key: number]: any } = {} -0 let CONNECTED = false // mock redis always connected @@ -91,12 +91,11 @@ function init(selectDb = DEFAULT_SELECT_DB) { if (client) { client.disconnect() } - const { redisProtocolUrl, opts, host, port } = getRedisOptions() + const { host, port } = getRedisConnectionDetails() + const opts = getRedisOptions() if (CLUSTERED) { client = new RedisCore.Cluster([{ host, port }], opts) - } else if (redisProtocolUrl) { - client = new RedisCore(redisProtocolUrl) } else { client = new RedisCore(opts) } diff --git a/packages/backend-core/src/redis/utils.ts b/packages/backend-core/src/redis/utils.ts index 34b7275a2b..5187fe13f8 100644 --- a/packages/backend-core/src/redis/utils.ts +++ b/packages/backend-core/src/redis/utils.ts @@ -1,4 +1,5 @@ import env from "../environment" +import * as Redis from "ioredis" const SLOT_REFRESH_MS = 2000 const CONNECT_TIMEOUT_MS = 10000 @@ -42,7 +43,7 @@ export enum Databases { export enum SelectableDatabase { DEFAULT = 0, SOCKET_IO = 1, - UNUSED_1 = 2, + RATE_LIMITING = 2, UNUSED_2 = 3, UNUSED_3 = 4, UNUSED_4 = 5, @@ -58,7 +59,7 @@ export enum SelectableDatabase { UNUSED_14 = 15, } -export function getRedisOptions() { +export function getRedisConnectionDetails() { let password = env.REDIS_PASSWORD let url: string[] | string = env.REDIS_URL.split("//") // get rid of the protocol @@ -74,28 +75,34 @@ export function getRedisOptions() { } const [host, port] = url.split(":") - let redisProtocolUrl - - // fully qualified redis URL - if (/rediss?:\/\//.test(env.REDIS_URL)) { - redisProtocolUrl = env.REDIS_URL + return { + host, + password, + port: parseInt(port), } +} - const opts: any = { +export function getRedisOptions() { + const { host, password, port } = getRedisConnectionDetails() + let redisOpts: Redis.RedisOptions = { connectTimeout: CONNECT_TIMEOUT_MS, + port: port, + host, + password, } + let opts: Redis.ClusterOptions | Redis.RedisOptions = redisOpts if (env.REDIS_CLUSTERED) { - opts.redisOptions = {} - opts.redisOptions.tls = {} - opts.redisOptions.password = password - opts.slotsRefreshTimeout = SLOT_REFRESH_MS - opts.dnsLookup = (address: string, callback: any) => callback(null, address) - } else { - opts.host = host - opts.port = port - opts.password = password + opts = { + connectTimeout: CONNECT_TIMEOUT_MS, + redisOptions: { + ...redisOpts, + tls: {}, + }, + slotsRefreshTimeout: SLOT_REFRESH_MS, + dnsLookup: (address: string, callback: any) => callback(null, address), + } as Redis.ClusterOptions } - return { opts, host, port: parseInt(port), redisProtocolUrl } + return opts } export function addDbPrefix(db: string, key: string) { diff --git a/packages/backend-core/src/security/roles.ts b/packages/backend-core/src/security/roles.ts index b05cf79c8c..0d33031de5 100644 --- a/packages/backend-core/src/security/roles.ts +++ b/packages/backend-core/src/security/roles.ts @@ -122,7 +122,9 @@ export async function roleToNumber(id?: string) { if (isBuiltin(id)) { return builtinRoleToNumber(id) } - const hierarchy = (await getUserRoleHierarchy(id)) as RoleDoc[] + const hierarchy = (await getUserRoleHierarchy(id, { + defaultPublic: true, + })) as RoleDoc[] for (let role of hierarchy) { if (isBuiltin(role?.inherits)) { return builtinRoleToNumber(role.inherits) + 1 @@ -192,12 +194,15 @@ export async function getRole( /** * Simple function to get all the roles based on the top level user role ID. */ -async function getAllUserRoles(userRoleId?: string): Promise { +async function getAllUserRoles( + userRoleId?: string, + opts?: { defaultPublic?: boolean } +): Promise { // admins have access to all roles if (userRoleId === BUILTIN_IDS.ADMIN) { return getAllRoles() } - let currentRole = await getRole(userRoleId) + let currentRole = await getRole(userRoleId, opts) let roles = currentRole ? [currentRole] : [] let roleIds = [userRoleId] // get all the inherited roles @@ -226,12 +231,16 @@ export async function getUserRoleIdHierarchy( * Returns an ordered array of the user's inherited role IDs, this can be used * to determine if a user can access something that requires a specific role. * @param userRoleId The user's role ID, this can be found in their access token. + * @param opts optional - if want to default to public use this. * @returns returns an ordered array of the roles, with the first being their * highest level of access and the last being the lowest level. */ -export async function getUserRoleHierarchy(userRoleId?: string) { +export async function getUserRoleHierarchy( + userRoleId?: string, + opts?: { defaultPublic?: boolean } +) { // special case, if they don't have a role then they are a public user - return getAllUserRoles(userRoleId) + return getAllUserRoles(userRoleId, opts) } // this function checks that the provided permissions are in an array format diff --git a/packages/backend-core/src/users/db.ts b/packages/backend-core/src/users/db.ts index 8bb6300d4e..bd85097bbd 100644 --- a/packages/backend-core/src/users/db.ts +++ b/packages/backend-core/src/users/db.ts @@ -164,14 +164,10 @@ export class UserDB { } } - static async getUsersByAppAccess(appId?: string) { - const opts: any = { - include_docs: true, - limit: 50, - } + static async getUsersByAppAccess(opts: { appId?: string; limit?: number }) { let response: User[] = await usersCore.searchGlobalUsersByAppAccess( - appId, - opts + opts.appId, + { limit: opts.limit || 50 } ) return response } @@ -307,7 +303,7 @@ export class UserDB { static async bulkCreate( newUsersRequested: User[], - groups: string[] + groups?: string[] ): Promise { const tenantId = getTenantId() @@ -332,7 +328,7 @@ export class UserDB { }) continue } - newUser.userGroups = groups + newUser.userGroups = groups || [] newUsers.push(newUser) if (isCreator(newUser)) { newCreators.push(newUser) @@ -417,15 +413,13 @@ export class UserDB { } // Get users and delete - const allDocsResponse: AllDocsResponse = await db.allDocs({ + const allDocsResponse = await db.allDocs({ include_docs: true, keys: userIds, }) - const usersToDelete: User[] = allDocsResponse.rows.map( - (user: RowResponse) => { - return user.doc - } - ) + const usersToDelete = allDocsResponse.rows.map(user => { + return user.doc! + }) // Delete from DB const toDelete = usersToDelete.map(user => ({ diff --git a/packages/backend-core/src/users/lookup.ts b/packages/backend-core/src/users/lookup.ts index 17d0e91d88..355be74dab 100644 --- a/packages/backend-core/src/users/lookup.ts +++ b/packages/backend-core/src/users/lookup.ts @@ -6,6 +6,7 @@ import { } from "@budibase/types" import * as dbUtils from "../db" import { ViewName } from "../constants" +import { getExistingInvites } from "../cache/invite" /** * Apply a system-wide search on emails: @@ -26,6 +27,9 @@ export async function searchExistingEmails(emails: string[]) { const existingAccounts = await getExistingAccounts(emails) matchedEmails.push(...existingAccounts.map(account => account.email)) + const invitedEmails = await getExistingInvites(emails) + matchedEmails.push(...invitedEmails.map(invite => invite.email)) + return [...new Set(matchedEmails.map(email => email.toLowerCase()))] } diff --git a/packages/backend-core/src/users/users.ts b/packages/backend-core/src/users/users.ts index a64997224e..9f4a41f6df 100644 --- a/packages/backend-core/src/users/users.ts +++ b/packages/backend-core/src/users/users.ts @@ -19,6 +19,8 @@ import { SearchUsersRequest, User, ContextUser, + DatabaseQueryOpts, + CouchFindOptions, } from "@budibase/types" import { getGlobalDB } from "../context" import * as context from "../context" @@ -139,7 +141,7 @@ export const getGlobalUserByEmail = async ( export const searchGlobalUsersByApp = async ( appId: any, - opts: any, + opts: DatabaseQueryOpts, getOpts?: GetOpts ) => { if (typeof appId !== "string") { @@ -149,7 +151,7 @@ export const searchGlobalUsersByApp = async ( include_docs: true, }) params.startkey = opts && opts.startkey ? opts.startkey : params.startkey - let response = await queryGlobalView(ViewName.USER_BY_APP, params) + let response = await queryGlobalView(ViewName.USER_BY_APP, params) if (!response) { response = [] @@ -165,7 +167,10 @@ export const searchGlobalUsersByApp = async ( Return any user who potentially has access to the application Admins, developers and app users with the explicitly role. */ -export const searchGlobalUsersByAppAccess = async (appId: any, opts: any) => { +export const searchGlobalUsersByAppAccess = async ( + appId: any, + opts?: { limit?: number } +) => { const roleSelector = `roles.${appId}` let orQuery: any[] = [ @@ -186,7 +191,7 @@ export const searchGlobalUsersByAppAccess = async (appId: any, opts: any) => { orQuery.push(roleCheck) } - let searchOptions = { + let searchOptions: CouchFindOptions = { selector: { $or: orQuery, _id: { @@ -197,7 +202,7 @@ export const searchGlobalUsersByAppAccess = async (appId: any, opts: any) => { } const resp = await directCouchFind(context.getGlobalDBName(), searchOptions) - return resp?.rows + return resp.rows } export const getGlobalUserByAppPage = (appId: string, user: User) => { @@ -241,12 +246,15 @@ export const paginatedUsers = async ({ bookmark, query, appId, + limit, }: SearchUsersRequest = {}) => { const db = getGlobalDB() + const pageSize = limit ?? PAGE_LIMIT + const pageLimit = pageSize + 1 // get one extra document, to have the next page - const opts: any = { + const opts: DatabaseQueryOpts = { include_docs: true, - limit: PAGE_LIMIT + 1, + limit: pageLimit, } // add a startkey if the page was specified (anchor) if (bookmark) { @@ -269,7 +277,7 @@ export const paginatedUsers = async ({ const response = await db.allDocs(getGlobalUserParams(null, opts)) userList = response.rows.map((row: any) => row.doc) } - return pagination(userList, PAGE_LIMIT, { + return pagination(userList, pageSize, { paginate: true, property, getKey, diff --git a/packages/backend-core/src/utils/Duration.ts b/packages/backend-core/src/utils/Duration.ts new file mode 100644 index 0000000000..3c7ef23b11 --- /dev/null +++ b/packages/backend-core/src/utils/Duration.ts @@ -0,0 +1,52 @@ +export enum DurationType { + MILLISECONDS = "milliseconds", + SECONDS = "seconds", + MINUTES = "minutes", + HOURS = "hours", + DAYS = "days", +} + +const conversion: Record = { + milliseconds: 1, + seconds: 1000, + minutes: 60 * 1000, + hours: 60 * 60 * 1000, + days: 24 * 60 * 60 * 1000, +} + +export class Duration { + static convert(from: DurationType, to: DurationType, duration: number) { + const milliseconds = duration * conversion[from] + return milliseconds / conversion[to] + } + + static from(from: DurationType, duration: number) { + return { + to: (to: DurationType) => { + return Duration.convert(from, to, duration) + }, + toMs: () => { + return Duration.convert(from, DurationType.MILLISECONDS, duration) + }, + toSeconds: () => { + return Duration.convert(from, DurationType.SECONDS, duration) + }, + } + } + + static fromSeconds(duration: number) { + return Duration.from(DurationType.SECONDS, duration) + } + + static fromMinutes(duration: number) { + return Duration.from(DurationType.MINUTES, duration) + } + + static fromHours(duration: number) { + return Duration.from(DurationType.HOURS, duration) + } + + static fromDays(duration: number) { + return Duration.from(DurationType.DAYS, duration) + } +} diff --git a/packages/backend-core/src/utils/index.ts b/packages/backend-core/src/utils/index.ts index 318a7f13ba..ac17227459 100644 --- a/packages/backend-core/src/utils/index.ts +++ b/packages/backend-core/src/utils/index.ts @@ -1,3 +1,4 @@ export * from "./hashing" export * from "./utils" export * from "./stringUtils" +export * from "./Duration" diff --git a/packages/backend-core/src/utils/tests/Duration.spec.ts b/packages/backend-core/src/utils/tests/Duration.spec.ts new file mode 100644 index 0000000000..46b996f788 --- /dev/null +++ b/packages/backend-core/src/utils/tests/Duration.spec.ts @@ -0,0 +1,19 @@ +import { Duration, DurationType } from "../Duration" + +describe("duration", () => { + it("should convert minutes to milliseconds", () => { + expect(Duration.fromMinutes(5).toMs()).toBe(300000) + }) + + it("should convert seconds to milliseconds", () => { + expect(Duration.fromSeconds(30).toMs()).toBe(30000) + }) + + it("should convert days to milliseconds", () => { + expect(Duration.fromDays(1).toMs()).toBe(86400000) + }) + + it("should convert minutes to days", () => { + expect(Duration.fromMinutes(1440).to(DurationType.DAYS)).toBe(1) + }) +}) diff --git a/packages/backend-core/src/utils/tests/utils.spec.ts b/packages/backend-core/src/utils/tests/utils.spec.ts index 5a0ac4f283..7b411e801c 100644 --- a/packages/backend-core/src/utils/tests/utils.spec.ts +++ b/packages/backend-core/src/utils/tests/utils.spec.ts @@ -188,4 +188,17 @@ describe("utils", () => { expectResult(false) }) }) + + describe("hasCircularStructure", () => { + it("should detect a circular structure", () => { + const a: any = { b: "b" } + const b = { a } + a.b = b + expect(utils.hasCircularStructure(b)).toBe(true) + }) + + it("should allow none circular structures", () => { + expect(utils.hasCircularStructure({ a: "b" })).toBe(false) + }) + }) }) diff --git a/packages/backend-core/src/utils/utils.ts b/packages/backend-core/src/utils/utils.ts index b92471a7a4..1c1ca8473b 100644 --- a/packages/backend-core/src/utils/utils.ts +++ b/packages/backend-core/src/utils/utils.ts @@ -237,3 +237,17 @@ export function timeout(timeMs: number) { export function isAudited(event: Event) { return !!AuditedEventFriendlyName[event] } + +export function hasCircularStructure(json: any) { + if (typeof json !== "object") { + return false + } + try { + JSON.stringify(json) + } catch (err) { + if (err instanceof Error && err?.message.includes("circular structure")) { + return true + } + } + return false +} diff --git a/packages/backend-core/tests/core/utilities/mocks/date.ts b/packages/backend-core/tests/core/utilities/mocks/date.ts index f580b68349..1e6d105d93 100644 --- a/packages/backend-core/tests/core/utilities/mocks/date.ts +++ b/packages/backend-core/tests/core/utilities/mocks/date.ts @@ -1,2 +1,3 @@ export const MOCK_DATE = new Date("2020-01-01T00:00:00.000Z") + export const MOCK_DATE_TIMESTAMP = 1577836800000 diff --git a/packages/backend-core/tests/core/utilities/structures/users.ts b/packages/backend-core/tests/core/utilities/structures/users.ts index 66d23696e0..68ee29686c 100644 --- a/packages/backend-core/tests/core/utilities/structures/users.ts +++ b/packages/backend-core/tests/core/utilities/structures/users.ts @@ -12,7 +12,7 @@ import { generator } from "./generator" import { tenant } from "." export const newEmail = () => { - return `${uuid()}@test.com` + return `${uuid()}@example.com` } export const user = (userProps?: Partial>): User => { diff --git a/packages/bbui/src/Form/Core/Checkbox.svelte b/packages/bbui/src/Form/Core/Checkbox.svelte index 3efc737bfb..e24f5669eb 100644 --- a/packages/bbui/src/Form/Core/Checkbox.svelte +++ b/packages/bbui/src/Form/Core/Checkbox.svelte @@ -8,6 +8,7 @@ export let id = null export let text = null export let disabled = false + export let readonly = false export let size export let indeterminate = false @@ -24,6 +25,7 @@ class:is-invalid={!!error} class:checked={value} class:is-indeterminate={indeterminate} + class:readonly > diff --git a/packages/bbui/src/Form/Core/CheckboxGroup.svelte b/packages/bbui/src/Form/Core/CheckboxGroup.svelte index 2b8a1e438a..66ac55561b 100644 --- a/packages/bbui/src/Form/Core/CheckboxGroup.svelte +++ b/packages/bbui/src/Form/Core/CheckboxGroup.svelte @@ -8,6 +8,7 @@ export let options = [] export let error = null export let disabled = false + export let readonly = false export let getOptionLabel = option => option export let getOptionValue = option => option @@ -34,6 +35,7 @@ title={getOptionLabel(option)} class="spectrum-Checkbox spectrum-FieldGroup-item" class:is-invalid={!!error} + class:readonly >

diff --git a/packages/bbui/src/Form/Core/TextArea.svelte b/packages/bbui/src/Form/Core/TextArea.svelte index 465212cd44..be7eed466d 100644 --- a/packages/bbui/src/Form/Core/TextArea.svelte +++ b/packages/bbui/src/Form/Core/TextArea.svelte @@ -5,6 +5,7 @@ export let value = "" export let placeholder = null export let disabled = false + export let readonly = false export let error = null export let id = null export let height = null @@ -61,6 +62,7 @@ class="spectrum-Textfield-input" style={align ? `text-align: ${align}` : ""} {disabled} + {readonly} {id} on:focus={() => (focus = true)} on:blur={onChange} diff --git a/packages/bbui/src/Form/DatePicker.svelte b/packages/bbui/src/Form/DatePicker.svelte index 04ce8b5467..f17871a576 100644 --- a/packages/bbui/src/Form/DatePicker.svelte +++ b/packages/bbui/src/Form/DatePicker.svelte @@ -7,6 +7,7 @@ export let label = null export let labelPosition = "above" export let disabled = false + export let readonly = false export let error = null export let enableTime = true export let timeOnly = false @@ -33,6 +34,7 @@ { if (mde && val !== latestValue) { @@ -54,6 +58,7 @@ easyMDEOptions={{ initialValue: value, placeholder, + toolbar: disabled || readonly ? false : undefined, ...easyMDEOptions, }} /> diff --git a/packages/bbui/src/Table/Table.svelte b/packages/bbui/src/Table/Table.svelte index 529d1144ee..2610d6106c 100644 --- a/packages/bbui/src/Table/Table.svelte +++ b/packages/bbui/src/Table/Table.svelte @@ -106,6 +106,13 @@ name: fieldName, } } + + // Delete numeric only widths as these are grid widths and should be + // ignored + const width = fixedSchema[fieldName].width + if (width != null && `${width}`.trim().match(/^[0-9]+$/)) { + delete fixedSchema[fieldName].width + } }) return fixedSchema } diff --git a/packages/bbui/src/bbui.css b/packages/bbui/src/bbui.css index 343aa77b27..9b5d89f61c 100644 --- a/packages/bbui/src/bbui.css +++ b/packages/bbui/src/bbui.css @@ -2,6 +2,15 @@ --background: #ffffff; --ink: #000000; + /* Brand colours */ + --bb-coral: #FF4E4E; + --bb-coral-light: #F97777; + --bb-indigo: #6E56FF; + --bb-indigo-light: #9F8FFF; + --bb-lime: #ECFFB5; + --bb-forest-green: #053835; + --bb-beige: #F6EFEA; + --grey-1: #fafafa; --grey-2: #f5f5f5; --grey-3: #eeeeee; diff --git a/packages/builder/.gitignore b/packages/builder/.gitignore index e5c961d509..abc5671984 100644 --- a/packages/builder/.gitignore +++ b/packages/builder/.gitignore @@ -5,4 +5,5 @@ package-lock.json release/ dist/ routify -.routify/ \ No newline at end of file +.routify/ +svelte.config.js \ No newline at end of file diff --git a/packages/builder/assets/bb-emblem.svg b/packages/builder/assets/bb-emblem.svg index 7d499e4862..26d09cc97f 100644 --- a/packages/builder/assets/bb-emblem.svg +++ b/packages/builder/assets/bb-emblem.svg @@ -1,80 +1,13 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + diff --git a/packages/builder/assets/bb-space-black.svg b/packages/builder/assets/bb-space-black.svg deleted file mode 100644 index fa1743f90c..0000000000 --- a/packages/builder/assets/bb-space-black.svg +++ /dev/null @@ -1,18 +0,0 @@ - - - - - - - - - - - - - \ No newline at end of file diff --git a/packages/builder/assets/bb-space-purple.svg b/packages/builder/assets/bb-space-purple.svg deleted file mode 100644 index ccfb8b220d..0000000000 --- a/packages/builder/assets/bb-space-purple.svg +++ /dev/null @@ -1,25 +0,0 @@ - - - - - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/packages/builder/public/bblogo.png b/packages/builder/public/bblogo.png index 8c89c12f19..aa5ee4466e 100644 Binary files a/packages/builder/public/bblogo.png and b/packages/builder/public/bblogo.png differ diff --git a/packages/builder/src/builderStore/componentUtils.js b/packages/builder/src/builderStore/componentUtils.js index 16b972058e..522dbae416 100644 --- a/packages/builder/src/builderStore/componentUtils.js +++ b/packages/builder/src/builderStore/componentUtils.js @@ -5,6 +5,7 @@ import { encodeJSBinding, findHBSBlocks, } from "@budibase/string-templates" +import { capitalise } from "helpers" /** * Recursively searches for a specific component ID @@ -235,3 +236,13 @@ export const makeComponentUnique = component => { // Recurse on all children return JSON.parse(definition) } + +export const getComponentText = component => { + if (component?._instanceName) { + return component._instanceName + } + const type = + component._component.replace("@budibase/standard-components/", "") || + "component" + return capitalise(type) +} diff --git a/packages/builder/src/builderStore/store/automation/index.js b/packages/builder/src/builderStore/store/automation/index.js index c22240370b..ba2458f414 100644 --- a/packages/builder/src/builderStore/store/automation/index.js +++ b/packages/builder/src/builderStore/store/automation/index.js @@ -3,6 +3,7 @@ import { API } from "api" import { cloneDeep } from "lodash/fp" import { generate } from "shortid" import { selectedAutomation } from "builderStore" +import { notifications } from "@budibase/bbui" const initialAutomationState = { automations: [], @@ -21,6 +22,37 @@ export const getAutomationStore = () => { return store } +const updateReferencesInObject = (obj, modifiedIndex, action) => { + const regex = /{{\s*steps\.(\d+)\./g + for (const key in obj) { + if (typeof obj[key] === "string") { + let matches + while ((matches = regex.exec(obj[key])) !== null) { + const referencedStep = parseInt(matches[1]) + if (action === "add" && referencedStep >= modifiedIndex) { + obj[key] = obj[key].replace( + `{{ steps.${referencedStep}.`, + `{{ steps.${referencedStep + 1}.` + ) + } else if (action === "delete" && referencedStep > modifiedIndex) { + obj[key] = obj[key].replace( + `{{ steps.${referencedStep}.`, + `{{ steps.${referencedStep - 1}.` + ) + } + } + } else if (typeof obj[key] === "object" && obj[key] !== null) { + updateReferencesInObject(obj[key], modifiedIndex, action) + } + } +} + +const updateStepReferences = (steps, modifiedIndex, action) => { + steps.forEach(step => { + updateReferencesInObject(step.inputs, modifiedIndex, action) + }) +} + const automationActions = store => ({ definitions: async () => { const response = await API.getAutomationDefinitions() @@ -218,10 +250,40 @@ const automationActions = store => ({ if (!automation) { return } + + try { + updateStepReferences(newAutomation.definition.steps, blockIdx, "add") + } catch (e) { + notifications.error("Error adding automation block") + } newAutomation.definition.steps.splice(blockIdx, 0, block) await store.actions.save(newAutomation) }, - deleteAutomationBlock: async block => { + saveAutomationName: async (blockId, name) => { + const automation = get(selectedAutomation) + let newAutomation = cloneDeep(automation) + if (!automation) { + return + } + newAutomation.definition.stepNames = { + ...newAutomation.definition.stepNames, + [blockId]: name.trim(), + } + + await store.actions.save(newAutomation) + }, + deleteAutomationName: async blockId => { + const automation = get(selectedAutomation) + let newAutomation = cloneDeep(automation) + if (!automation) { + return + } + delete newAutomation.definition.stepNames[blockId] + + await store.actions.save(newAutomation) + }, + + deleteAutomationBlock: async (block, blockIdx) => { const automation = get(selectedAutomation) let newAutomation = cloneDeep(automation) @@ -233,7 +295,14 @@ const automationActions = store => ({ newAutomation.definition.steps = newAutomation.definition.steps.filter( step => step.id !== block.id ) + delete newAutomation.definition.stepNames?.[block.id] } + try { + updateStepReferences(newAutomation.definition.steps, blockIdx, "delete") + } catch (e) { + notifications.error("Error deleting automation block") + } + await store.actions.save(newAutomation) }, replace: async (automationId, automation) => { diff --git a/packages/builder/src/builderStore/store/frontend.js b/packages/builder/src/builderStore/store/frontend.js index a567caf87f..a4729b4a8a 100644 --- a/packages/builder/src/builderStore/store/frontend.js +++ b/packages/builder/src/builderStore/store/frontend.js @@ -580,7 +580,7 @@ export const getFrontendStore = () => { let table = validTables.find(table => { return ( table.sourceId !== BUDIBASE_INTERNAL_DB_ID && - table.type === DB_TYPE_INTERNAL + table.sourceType === DB_TYPE_INTERNAL ) }) if (table) { @@ -591,7 +591,7 @@ export const getFrontendStore = () => { table = validTables.find(table => { return ( table.sourceId === BUDIBASE_INTERNAL_DB_ID && - table.type === DB_TYPE_INTERNAL + table.sourceType === DB_TYPE_INTERNAL ) }) if (table) { @@ -599,7 +599,7 @@ export const getFrontendStore = () => { } // Finally try an external table - return validTables.find(table => table.type === DB_TYPE_EXTERNAL) + return validTables.find(table => table.sourceType === DB_TYPE_EXTERNAL) }, enrichEmptySettings: (component, opts) => { if (!component?._component) { diff --git a/packages/builder/src/builderStore/store/screenTemplates/rowListScreen.js b/packages/builder/src/builderStore/store/screenTemplates/rowListScreen.js index b17bd99e10..59bcd0d5e8 100644 --- a/packages/builder/src/builderStore/store/screenTemplates/rowListScreen.js +++ b/packages/builder/src/builderStore/store/screenTemplates/rowListScreen.js @@ -2,14 +2,14 @@ import sanitizeUrl from "./utils/sanitizeUrl" import { Screen } from "./utils/Screen" import { Component } from "./utils/Component" -export default function (datasources) { +export default function (datasources, mode = "table") { if (!Array.isArray(datasources)) { return [] } return datasources.map(datasource => { return { name: `${datasource.label} - List`, - create: () => createScreen(datasource), + create: () => createScreen(datasource, mode), id: ROW_LIST_TEMPLATE, resourceId: datasource.resourceId, } @@ -40,10 +40,24 @@ const generateTableBlock = datasource => { return tableBlock } -const createScreen = datasource => { +const generateGridBlock = datasource => { + const gridBlock = new Component("@budibase/standard-components/gridblock") + gridBlock + .customProps({ + table: datasource, + }) + .instanceName(`${datasource.label} - Grid block`) + return gridBlock +} + +const createScreen = (datasource, mode) => { return new Screen() .route(rowListUrl(datasource)) .instanceName(`${datasource.label} - List`) - .addChild(generateTableBlock(datasource)) + .addChild( + mode === "table" + ? generateTableBlock(datasource) + : generateGridBlock(datasource) + ) .json() } diff --git a/packages/builder/src/components/automation/AutomationBuilder/FlowChart/FlowChart.svelte b/packages/builder/src/components/automation/AutomationBuilder/FlowChart/FlowChart.svelte index 63a3478ef3..54e098c9d5 100644 --- a/packages/builder/src/components/automation/AutomationBuilder/FlowChart/FlowChart.svelte +++ b/packages/builder/src/components/automation/AutomationBuilder/FlowChart/FlowChart.svelte @@ -5,13 +5,7 @@ import TestDataModal from "./TestDataModal.svelte" import { flip } from "svelte/animate" import { fly } from "svelte/transition" - import { - Heading, - Icon, - ActionButton, - notifications, - Modal, - } from "@budibase/bbui" + import { Icon, notifications, Modal } from "@budibase/bbui" import { ActionStepID } from "constants/backend/automations" import UndoRedoControl from "components/common/UndoRedoControl.svelte" import { automationHistoryStore } from "builderStore" @@ -20,9 +14,8 @@ let testDataModal let confirmDeleteDialog - - $: blocks = getBlocks(automation) - + let scrolling = false + $: blocks = getBlocks(automation).filter(x => x.stepId !== ActionStepID.LOOP) const getBlocks = automation => { let blocks = [] if (automation.definition.trigger) { @@ -32,58 +25,71 @@ return blocks } - async function deleteAutomation() { + const deleteAutomation = async () => { try { await automationStore.actions.delete($selectedAutomation) } catch (error) { notifications.error("Error deleting automation") } } + + const handleScroll = e => { + if (e.target.scrollTop >= 30) { + scrolling = true + } else if (e.target.scrollTop) { + // Set scrolling back to false if scrolled back to less than 100px + scrolling = false + } + } -
-
- {automation.name} -
- +
+
+ +
+
+
{ + testDataModal.show() + }} + class="buttons" + > + +
Run test
+
+
-
- { - testDataModal.show() - }} - icon="MultipleCheck" - size="M">Run test - { - $automationStore.showTestPanel = true - }} - size="M">Test Details +
{ + $automationStore.showTestPanel = true + }} + > + Test details
-
- {#each blocks as block, idx (block.id)} -
- {#if block.stepId !== ActionStepID.LOOP} - - {/if} -
- {/each} +
+
+ {#each blocks as block, idx (block.id)} +
+ {#if block.stepId !== ActionStepID.LOOP} + + {/if} +
+ {/each} +
.canvas { padding: var(--spacing-l) var(--spacing-xl); + overflow-y: auto; + max-height: 100%; + } + + .header-left :global(div) { + border-right: none; } /* Fix for firefox not respecting bottom padding in scrolling containers */ .canvas > *:last-child { @@ -117,23 +129,45 @@ } .content { - display: inline-block; - text-align: left; + flex-grow: 1; + padding: 23px 23px 80px; + box-sizing: border-box; + } + + .header.scrolling { + background: var(--background); + border-bottom: var(--border-light); + border-left: var(--border-light); + z-index: 1; } .header { + z-index: 1; display: flex; justify-content: space-between; align-items: center; + padding-left: var(--spacing-l); + transition: background 130ms ease-out; + flex: 0 0 48px; + padding-right: var(--spacing-xl); + } + .controls { + display: flex; + gap: var(--spacing-xl); } - .controls, .buttons { display: flex; justify-content: flex-end; align-items: center; - gap: var(--spacing-xl); - } - .buttons { gap: var(--spacing-s); } + + .buttons:hover { + cursor: pointer; + } + + .disabled { + pointer-events: none; + color: var(--spectrum-global-color-gray-500) !important; + } diff --git a/packages/builder/src/components/automation/AutomationBuilder/FlowChart/FlowItem.svelte b/packages/builder/src/components/automation/AutomationBuilder/FlowChart/FlowItem.svelte index 6c964c84a9..c6d38a4d2e 100644 --- a/packages/builder/src/components/automation/AutomationBuilder/FlowChart/FlowItem.svelte +++ b/packages/builder/src/components/automation/AutomationBuilder/FlowChart/FlowItem.svelte @@ -7,20 +7,16 @@ Detail, Modal, Button, - ActionButton, notifications, Label, + AbsTooltip, } from "@budibase/bbui" import AutomationBlockSetup from "../../SetupPanel/AutomationBlockSetup.svelte" import CreateWebhookModal from "components/automation/Shared/CreateWebhookModal.svelte" import ActionModal from "./ActionModal.svelte" import FlowItemHeader from "./FlowItemHeader.svelte" import RoleSelect from "components/design/settings/controls/RoleSelect.svelte" - import { - ActionStepID, - TriggerStepID, - Features, - } from "constants/backend/automations" + import { ActionStepID, TriggerStepID } from "constants/backend/automations" import { permissions } from "stores/backend" export let block @@ -86,7 +82,7 @@ if (loopBlock) { await automationStore.actions.deleteAutomationBlock(loopBlock) } - await automationStore.actions.deleteAutomationBlock(block) + await automationStore.actions.deleteAutomationBlock(block, blockIdx) } catch (error) { notifications.error("Error saving automation") } @@ -129,6 +125,10 @@
+ + + +
{}}>
@@ -139,9 +139,6 @@ {#if !showLooping}
-
- removeLooping()} icon="DeleteOutline" /> -
(open = !open)} /> {#if open}
- {#if !isTrigger} -
-
- {#if !loopBlock && (block?.features?.[Features.LOOPING] || !block.features)} - addLooping()} icon="Reuse"> - Add Looping - - {/if} - deleteStep()} - icon="DeleteOutline" - /> -
-
- {/if} - {#if isAppAction} - - +
+ + +
{/if} diff --git a/packages/builder/src/components/automation/AutomationBuilder/FlowChart/FlowItemHeader.svelte b/packages/builder/src/components/automation/AutomationBuilder/FlowChart/FlowItemHeader.svelte index c09474a370..3c9e1a13b1 100644 --- a/packages/builder/src/components/automation/AutomationBuilder/FlowChart/FlowItemHeader.svelte +++ b/packages/builder/src/components/automation/AutomationBuilder/FlowChart/FlowItemHeader.svelte @@ -1,8 +1,9 @@ -
-
dispatch("toggle")} class="splitHeader"> +
+
{#if externalActions[block.stepId]} {/if}
- {#if isTrigger} + {#if isHeaderTrigger} Trigger - When this happens: {:else} - Step {idx} - Do this: +
+ Step {idx} +
+ {/if} + + {#if enableNaming} + { + automationName = e.target.value.trim() + }} + on:click={startTyping} + on:blur={async () => { + typing = false + if (automationNameError) { + automationName = stepNames[block.id] || block?.name + } else { + await saveName() + } + }} + /> + {:else} +
+ {automationName} +
{/if} - {block?.name?.toUpperCase() || ""}
{#if showTestStatus && testResult} -
- {status?.message} +
+
+ + {status?.message} + +
+ dispatch("toggle")} + hoverable + name={open ? "ChevronUp" : "ChevronDown"} + />
{/if}
{ onSelect(block) }} > - + {#if !showTestStatus} + {#if !isHeaderTrigger && !loopBlock && (block?.features?.[Features.LOOPING] || !block.features)} + + + + {/if} + {#if !isHeaderTrigger} + + + + {/if} + {/if} + {#if !showTestStatus} + dispatch("toggle")} + hoverable + name={open ? "ChevronUp" : "ChevronDown"} + /> + {/if}
+ {#if automationNameError} +
+ +
+ +
+ +
+ {/if}
diff --git a/packages/builder/src/components/automation/AutomationBuilder/FlowChart/TestDataModal.svelte b/packages/builder/src/components/automation/AutomationBuilder/FlowChart/TestDataModal.svelte index 17d5b35575..5c97d77ae8 100644 --- a/packages/builder/src/components/automation/AutomationBuilder/FlowChart/TestDataModal.svelte +++ b/packages/builder/src/components/automation/AutomationBuilder/FlowChart/TestDataModal.svelte @@ -60,6 +60,7 @@ {#if block.stepId !== ActionStepID.LOOP} (openBlocks[block.id] = !openBlocks[block.id])} isTrigger={idx === 0} diff --git a/packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte b/packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte index ce8c5c344c..9260a197c2 100644 --- a/packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte +++ b/packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte @@ -58,7 +58,6 @@ let fillWidth = true let inputData let codeBindingOpen = false - $: filters = lookForFilters(schemaProperties) || [] $: tempFilters = filters $: stepId = block.stepId @@ -155,7 +154,7 @@ } let blockIdx = allSteps.findIndex(step => step.id === block.id) - // Extract all outputs from all previous steps as available bindins + // Extract all outputs from all previous steps as available bindingsx§x let bindings = [] let loopBlockCount = 0 for (let idx = 0; idx < blockIdx; idx++) { @@ -183,20 +182,19 @@ } } const outputs = Object.entries(schema) - let bindingIcon = "" - let bindindingRank = 0 - + let bindingRank = 0 if (idx === 0) { bindingIcon = automation.trigger.icon } else if (isLoopBlock) { bindingIcon = "Reuse" - bindindingRank = idx + 1 + bindingRank = idx + 1 } else { bindingIcon = allSteps[idx].icon - bindindingRank = idx - loopBlockCount + bindingRank = idx - loopBlockCount } - + let bindingName = + automation.stepNames?.[allSteps[idx - loopBlockCount].id] bindings = bindings.concat( outputs.map(([name, value]) => { let runtimeName = isLoopBlock @@ -205,14 +203,20 @@ ? `steps[${idx - loopBlockCount}].${name}` : `steps.${idx - loopBlockCount}.${name}` const runtime = idx === 0 ? `trigger.${name}` : runtimeName - const categoryName = - idx === 0 - ? "Trigger outputs" - : isLoopBlock - ? "Loop Outputs" - : `Step ${idx - loopBlockCount} outputs` + + let categoryName + if (idx === 0) { + categoryName = "Trigger outputs" + } else if (isLoopBlock) { + categoryName = "Loop Outputs" + } else if (bindingName) { + categoryName = `${bindingName} outputs` + } else { + categoryName = `Step ${idx - loopBlockCount} outputs` + } + return { - readableBinding: runtime, + readableBinding: bindingName ? `${bindingName}.${name}` : runtime, runtimeBinding: runtime, type: value.type, description: value.description, @@ -221,7 +225,7 @@ display: { type: value.type, name: name, - rank: bindindingRank, + rank: bindingRank, }, } }) @@ -277,6 +281,16 @@ return !dependsOn || !!inputData[dependsOn] } + function shouldRenderField(value) { + return ( + value.customType !== "row" && + value.customType !== "code" && + value.customType !== "queryParams" && + value.customType !== "cron" && + value.customType !== "triggerSchema" + ) + } + onMount(async () => { try { await environment.loadVariables() @@ -289,245 +303,248 @@
{#each schemaProperties as [key, value]} {#if canShowField(key, value)} -
- {#if key !== "fields" && value.type !== "boolean"} +
+ {#if key !== "fields" && value.type !== "boolean" && shouldRenderField(value)} {/if} - {#if value.type === "string" && value.enum && canShowField(key, value)} - onChange(e, key)} - /> -
- {:else if value.type === "date"} - onChange(e, key)} - {bindings} - allowJS={true} - updateOnChange={false} - drawerLeft="260px" - > - onChange(e, key)} + placeholder={false} + options={value.enum} + getOptionLabel={(x, idx) => + value.pretty ? value.pretty[idx] : x} /> - - {:else if value.customType === "column"} - onChange(e, key)} - value={inputData[key]} - /> - {:else if value.customType === "email"} - {#if isTestModal} - onChange(e, key)} - {bindings} - fillWidth - updateOnChange={false} - /> - {:else} - + onChange(e, key)} + /> +
+ {:else if value.type === "date"} + onChange(e, key)} {bindings} - allowJS={false} + allowJS={true} updateOnChange={false} drawerLeft="260px" - /> - {/if} - {:else if value.customType === "query"} - onChange(e, key)} - value={inputData[key]} - /> - {:else if value.customType === "cron"} - onChange(e, key)} - value={inputData[key]} - /> - {:else if value.customType === "queryParams"} - onChange(e, key)} - value={inputData[key]} - {bindings} - /> - {:else if value.customType === "table"} - onChange(e, key)} - /> - {:else if value.customType === "row"} - { - if (e.detail?.key) { - onChange(e, e.detail.key) - } else { - onChange(e, key) - } - }} - {bindings} - {isTestModal} - {isUpdateRow} - /> - {:else if value.customType === "webhookUrl"} - onChange(e, key)} - value={inputData[key]} - /> - {:else if value.customType === "fields"} - onChange(e, key)} - {bindings} - {isTestModal} - /> - {:else if value.customType === "triggerSchema"} - onChange(e, key)} - value={inputData[key]} - /> - {:else if value.customType === "code"} - - {#if codeMode == EditorModes.JS} - (codeBindingOpen = !codeBindingOpen)} - quiet - icon={codeBindingOpen ? "ChevronDown" : "ChevronRight"} - > - Bindings - - {#if codeBindingOpen} - 
{JSON.stringify(bindings, null, 2)}
- {/if} - {/if} - { - // need to pass without the value inside - onChange({ detail: e.detail }, key) - inputData[key] = e.detail - }} - completions={stepCompletions} - mode={codeMode} - autocompleteEnabled={codeMode != EditorModes.JS} - height={500} - /> -
- {#if codeMode == EditorModes.Handlebars} - -
-
- Add available bindings by typing - }} - -
-
- {/if} -
- - {:else if value.customType === "loopOption"} - onChange(e, key)} - {bindings} - updateOnChange={false} + value={inputData[key]} + options={Object.keys(table?.schema || {})} /> - {:else} -
+ {:else if value.customType === "filters"} + Define filters + + + (tempFilters = e.detail)} + /> + + {:else if value.customType === "password"} + onChange(e, key)} + value={inputData[key]} + /> + {:else if value.customType === "email"} + {#if isTestModal} + onChange(e, key)} + {bindings} + fillWidth + updateOnChange={false} + /> + {:else} onChange(e, key)} {bindings} + allowJS={false} updateOnChange={false} - placeholder={value.customType === "queryLimit" - ? queryLimit - : ""} drawerLeft="260px" /> -
+ {/if} + {:else if value.customType === "query"} + onChange(e, key)} + value={inputData[key]} + /> + {:else if value.customType === "cron"} + onChange(e, key)} + value={inputData[key]} + /> + {:else if value.customType === "queryParams"} + onChange(e, key)} + value={inputData[key]} + {bindings} + /> + {:else if value.customType === "table"} + onChange(e, key)} + /> + {:else if value.customType === "row"} + { + if (e.detail?.key) { + onChange(e, e.detail.key) + } else { + onChange(e, key) + } + }} + {bindings} + {isTestModal} + {isUpdateRow} + /> + {:else if value.customType === "webhookUrl"} + onChange(e, key)} + value={inputData[key]} + /> + {:else if value.customType === "fields"} + onChange(e, key)} + {bindings} + {isTestModal} + /> + {:else if value.customType === "triggerSchema"} + onChange(e, key)} + value={inputData[key]} + /> + {:else if value.customType === "code"} + + {#if codeMode == EditorModes.JS} + (codeBindingOpen = !codeBindingOpen)} + quiet + icon={codeBindingOpen ? "ChevronDown" : "ChevronRight"} + > + Bindings + + {#if codeBindingOpen} + 
{JSON.stringify(bindings, null, 2)}
+ {/if} + {/if} + { + // need to pass without the value inside + onChange({ detail: e.detail }, key) + inputData[key] = e.detail + }} + completions={stepCompletions} + mode={codeMode} + autocompleteEnabled={codeMode != EditorModes.JS} + height={500} + /> +
+ {#if codeMode == EditorModes.Handlebars} + +
+
+ Add available bindings by typing + }} + +
+
+ {/if} +
+ + {:else if value.customType === "loopOption"} + query._id} - getOptionLabel={query => query.name} - /> +
+ +
+ table.name} - getOptionValue={table => table._id} -/> +
+ +
+ + (column.displayName = e.detail)} - /> - - - removeColumn(column.id)} - disabled={columns.length === 1} - /> -
- {/each} -
- - {:else} -
-
- Add columns to be included in your form below. -
-
- {/if} -
-
- - - {#if columns?.length} - - {/if} -
-
- -
- - - diff --git a/packages/builder/src/components/design/settings/controls/FieldConfiguration/FieldConfiguration.svelte b/packages/builder/src/components/design/settings/controls/FieldConfiguration/FieldConfiguration.svelte index 4169cb7d3d..2b76bc6591 100644 --- a/packages/builder/src/components/design/settings/controls/FieldConfiguration/FieldConfiguration.svelte +++ b/packages/builder/src/components/design/settings/controls/FieldConfiguration/FieldConfiguration.svelte @@ -1,4 +1,5 @@
+
+ + { + let update = fieldList.map(field => ({ + ...field, + active: selectAll, + })) + listUpdated(update) + }} + text="" + bind:value={selectAll} + thin + /> +
{#if fieldList?.length} listUpdated(e.detail)} on:itemChange={processItemUpdate} items={fieldList} listItemKey={"_id"} @@ -171,4 +189,21 @@ .field-configuration :global(.spectrum-ActionButton) { width: 100%; } + .toggle-all { + display: flex; + justify-content: space-between; + } + .toggle-all :global(.spectrum-Switch) { + margin-right: 0px; + padding-right: calc(var(--spacing-s) - 1px); + min-height: unset; + } + .toggle-all :global(.spectrum-Switch .spectrum-Switch-switch) { + margin-top: 0px; + } + .toggle-all span { + color: var(--spectrum-global-color-gray-700); + font-size: 12px; + margin-left: calc(var(--spacing-s) - 1px); + } diff --git a/packages/builder/src/components/design/settings/controls/FieldConfiguration/FieldSetting.svelte b/packages/builder/src/components/design/settings/controls/FieldConfiguration/FieldSetting.svelte index b5cfcb12d9..1d9ce733b8 100644 --- a/packages/builder/src/components/design/settings/controls/FieldConfiguration/FieldSetting.svelte +++ b/packages/builder/src/components/design/settings/controls/FieldConfiguration/FieldSetting.svelte @@ -1,8 +1,11 @@
- -
{item.label || item.field}
+ > +
+ + {item.field} +
+ +
{readableText}
@@ -53,4 +81,20 @@ .list-item-body { justify-content: space-between; } + .type-icon { + display: flex; + gap: var(--spacing-m); + margin: var(--spacing-xl); + margin-bottom: 0px; + height: var(--spectrum-alias-item-height-m); + padding: 0px var(--spectrum-alias-item-padding-m); + border-width: var(--spectrum-actionbutton-border-size); + border-radius: var(--spectrum-alias-border-radius-regular); + border: 1px solid + var( + --spectrum-actionbutton-m-border-color, + var(--spectrum-alias-border-color) + ); + align-items: center; + } diff --git a/packages/builder/src/components/design/settings/controls/SchemaSelect.svelte b/packages/builder/src/components/design/settings/controls/SchemaSelect.svelte index 80e36328f1..dbeeec53ef 100644 --- a/packages/builder/src/components/design/settings/controls/SchemaSelect.svelte +++ b/packages/builder/src/components/design/settings/controls/SchemaSelect.svelte @@ -1,5 +1,5 @@ diff --git a/packages/builder/src/components/integration/QueryViewerSidePanel/PreviewPanel.svelte b/packages/builder/src/components/integration/QueryViewerSidePanel/PreviewPanel.svelte index f379ad18a1..3873669b63 100644 --- a/packages/builder/src/components/integration/QueryViewerSidePanel/PreviewPanel.svelte +++ b/packages/builder/src/components/integration/QueryViewerSidePanel/PreviewPanel.svelte @@ -23,7 +23,7 @@
- +
diff --git a/packages/builder/src/pages/builder/app/[application]/design/[screenId]/[componentId]/_components/Component/ComponentSettingsSection.svelte b/packages/builder/src/pages/builder/app/[application]/design/[screenId]/[componentId]/_components/Component/ComponentSettingsSection.svelte index 639cef332e..65f010e4ec 100644 --- a/packages/builder/src/pages/builder/app/[application]/design/[screenId]/[componentId]/_components/Component/ComponentSettingsSection.svelte +++ b/packages/builder/src/pages/builder/app/[application]/design/[screenId]/[componentId]/_components/Component/ComponentSettingsSection.svelte @@ -1,6 +1,6 @@ @@ -127,28 +151,19 @@ {#if section.visible} {#if section.info}
- {:else if idx === 0 && section.name === "General" && componentDefinition.info} + {:else if idx === 0 && section.name === "General" && componentDefinition?.info && !tag} {/if}
- {#if idx === 0 && !componentInstance._component.endsWith("/layout") && !isScreen && showInstanceName} - updateSetting({ key: "_instanceName" }, val)} - /> - {/if} {#each section.settings as setting (setting.key)} {#if setting.visible} {/if} {/each} -{#if componentDefinition?.block} +{#if componentDefinition?.block && !tag} diff --git a/packages/builder/src/pages/builder/app/[application]/design/[screenId]/[componentId]/_components/Component/DesignSection.svelte b/packages/builder/src/pages/builder/app/[application]/design/[screenId]/[componentId]/_components/Component/DesignSection.svelte index 444ded7e1f..def1fcf24b 100644 --- a/packages/builder/src/pages/builder/app/[application]/design/[screenId]/[componentId]/_components/Component/DesignSection.svelte +++ b/packages/builder/src/pages/builder/app/[application]/design/[screenId]/[componentId]/_components/Component/DesignSection.svelte @@ -1,10 +1,12 @@ + + + {#if styles?.length > 0} {#each styles as style} { - if (component._instanceName) { - return component._instanceName - } - const type = - component._component.replace("@budibase/standard-components/", "") || - "component" - return capitalise(type) - } - const getComponentIcon = component => { const def = store.actions.components.getDefinition(component?._component) return def?.icon diff --git a/packages/builder/src/pages/builder/app/[application]/design/_components/NewScreen/CreateScreenModal.svelte b/packages/builder/src/pages/builder/app/[application]/design/_components/NewScreen/CreateScreenModal.svelte index 9a96242b30..92ed3dcfc7 100644 --- a/packages/builder/src/pages/builder/app/[application]/design/_components/NewScreen/CreateScreenModal.svelte +++ b/packages/builder/src/pages/builder/app/[application]/design/_components/NewScreen/CreateScreenModal.svelte @@ -12,6 +12,7 @@ import { capitalise } from "helpers" import { goto } from "@roxi/routify" + let mode let pendingScreen // Modal refs @@ -100,14 +101,15 @@ } // Handler for NewScreenModal - export const show = mode => { + export const show = newMode => { + mode = newMode selectedTemplates = null blankScreenUrl = null screenMode = mode pendingScreen = null screenAccessRole = Roles.BASIC - if (mode === "table") { + if (mode === "table" || mode === "grid") { datasourceModal.show() } else if (mode === "blank") { let templates = getTemplates($tables.list) @@ -123,6 +125,7 @@ // Handler for DatasourceModal confirmation, move to screen access select const confirmScreenDatasources = async ({ templates }) => { + console.log(templates) selectedTemplates = templates screenAccessRoleModal.show() } @@ -177,6 +180,7 @@ diff --git a/packages/builder/src/pages/builder/app/[application]/design/_components/NewScreen/DatasourceModal.svelte b/packages/builder/src/pages/builder/app/[application]/design/_components/NewScreen/DatasourceModal.svelte index a866cd23d4..731c60a406 100644 --- a/packages/builder/src/pages/builder/app/[application]/design/_components/NewScreen/DatasourceModal.svelte +++ b/packages/builder/src/pages/builder/app/[application]/design/_components/NewScreen/DatasourceModal.svelte @@ -7,6 +7,7 @@ import rowListScreen from "builderStore/store/screenTemplates/rowListScreen" import DatasourceTemplateRow from "./DatasourceTemplateRow.svelte" + export let mode export let onCancel export let onConfirm export let initialScreens = [] @@ -24,7 +25,10 @@ screen => screen.resourceId !== resourceId ) } else { - selectedScreens = [...selectedScreens, rowListScreen([datasource])[0]] + selectedScreens = [ + ...selectedScreens, + rowListScreen([datasource], mode)[0], + ] } } diff --git a/packages/builder/src/pages/builder/app/[application]/design/_components/NewScreen/grid.png b/packages/builder/src/pages/builder/app/[application]/design/_components/NewScreen/grid.png new file mode 100644 index 0000000000..c3efa30a67 Binary files /dev/null and b/packages/builder/src/pages/builder/app/[application]/design/_components/NewScreen/grid.png differ diff --git a/packages/builder/src/pages/builder/app/[application]/design/_components/NewScreen/index.svelte b/packages/builder/src/pages/builder/app/[application]/design/_components/NewScreen/index.svelte index b504940ca7..6b080747b0 100644 --- a/packages/builder/src/pages/builder/app/[application]/design/_components/NewScreen/index.svelte +++ b/packages/builder/src/pages/builder/app/[application]/design/_components/NewScreen/index.svelte @@ -3,6 +3,7 @@ import CreationPage from "components/common/CreationPage.svelte" import blankImage from "./blank.png" import tableImage from "./table.png" + import gridImage from "./grid.png" import CreateScreenModal from "./CreateScreenModal.svelte" import { store } from "builderStore" @@ -43,6 +44,16 @@ View, edit and delete rows on a table
+ +
createScreenModal.show("grid")}> +
+ +
+
+ Grid + View and manipulate rows on a grid +
+
diff --git a/packages/builder/src/pages/builder/app/[application]/settings/automation-history/_components/HistoryDetailsPanel.svelte b/packages/builder/src/pages/builder/app/[application]/settings/automation-history/_components/HistoryDetailsPanel.svelte index 5b9c925130..cde76fa1c0 100644 --- a/packages/builder/src/pages/builder/app/[application]/settings/automation-history/_components/HistoryDetailsPanel.svelte +++ b/packages/builder/src/pages/builder/app/[application]/settings/automation-history/_components/HistoryDetailsPanel.svelte @@ -56,7 +56,7 @@ {/if} {#key history}
- +
{/key} diff --git a/packages/builder/src/pages/builder/portal/apps/_components/PortalSideBar.svelte b/packages/builder/src/pages/builder/portal/apps/_components/PortalSideBar.svelte index 1e21bd7a9a..7989c5f1a8 100644 --- a/packages/builder/src/pages/builder/portal/apps/_components/PortalSideBar.svelte +++ b/packages/builder/src/pages/builder/portal/apps/_components/PortalSideBar.svelte @@ -9,12 +9,18 @@ let searchString let searching = false - $: filteredApps = $apps.filter(app => { - return ( - !searchString || - app.name.toLowerCase().includes(searchString.toLowerCase()) - ) - }) + $: filteredApps = $apps + .filter(app => { + return ( + !searchString || + app.name.toLowerCase().includes(searchString.toLowerCase()) + ) + }) + .sort((a, b) => { + const lowerA = a.name.toLowerCase() + const lowerB = b.name.toLowerCase() + return lowerA > lowerB ? 1 : -1 + }) const startSearching = async () => { searching = true diff --git a/packages/builder/src/pages/builder/portal/apps/onboarding/_components/PanelHeader.svelte b/packages/builder/src/pages/builder/portal/apps/onboarding/_components/PanelHeader.svelte index 34d612dc9e..49b8032726 100644 --- a/packages/builder/src/pages/builder/portal/apps/onboarding/_components/PanelHeader.svelte +++ b/packages/builder/src/pages/builder/portal/apps/onboarding/_components/PanelHeader.svelte @@ -8,11 +8,7 @@
- +
{#if onBack} - +
{:else} {/if} @@ -390,12 +393,15 @@ diff --git a/packages/client/src/components/app/blocks/form/FormBlock.svelte b/packages/client/src/components/app/blocks/form/FormBlock.svelte index 5d57d10ab6..e4d3b55eff 100644 --- a/packages/client/src/components/app/blocks/form/FormBlock.svelte +++ b/packages/client/src/components/app/blocks/form/FormBlock.svelte @@ -10,8 +10,8 @@ export let size export let disabled export let fields - export let labelPosition export let title + export let description export let showDeleteButton export let showSaveButton export let saveButtonLabel @@ -96,8 +96,8 @@ size, disabled, fields: fieldsOrDefault, - labelPosition, title, + description, saveButtonLabel: saveLabel, deleteButtonLabel: deleteLabel, schema, diff --git a/packages/client/src/components/app/blocks/form/InnerFormBlock.svelte b/packages/client/src/components/app/blocks/form/InnerFormBlock.svelte index ec5daa21b1..52ef3ac80c 100644 --- a/packages/client/src/components/app/blocks/form/InnerFormBlock.svelte +++ b/packages/client/src/components/app/blocks/form/InnerFormBlock.svelte @@ -2,6 +2,7 @@ import BlockComponent from "components/BlockComponent.svelte" import Placeholder from "components/app/Placeholder.svelte" import { makePropSafe as safe } from "@budibase/string-templates" + import { getContext } from "svelte" export let dataSource export let actionUrl @@ -9,8 +10,8 @@ export let size export let disabled export let fields - export let labelPosition export let title + export let description export let saveButtonLabel export let deleteButtonLabel export let schema @@ -32,6 +33,7 @@ barcodeqr: "codescanner", bb_reference: "bbreferencefield", } + const context = getContext("context") let formId @@ -135,7 +137,8 @@ actionType: actionType === "Create" ? "Create" : "Update", dataSource, size, - disabled: disabled || actionType === "View", + disabled, + readonly: !disabled && actionType === "View", }} styles={{ normal: { @@ -160,69 +163,85 @@ - {#if renderButtons} + > - {#if renderDeleteButton} - - {/if} - {#if renderSaveButton} - - {/if} - - {/if} + type="heading" + props={{ text: title || "" }} + order={0} + /> + {#if renderButtons} + + {#if renderDeleteButton} + + {/if} + {#if renderSaveButton} + + {/if} + + {/if} + {/if} + {#if description} + + {/if} {#key fields} - - {#each fields as field, idx} - {#if getComponentForField(field) && field.active} - - {/if} - {/each} + +
+ {#each fields as field, idx} + {#if getComponentForField(field) && field.active} + + {/if} + {/each} +
{/key} @@ -232,3 +251,14 @@ text="Choose your table and add some fields to your form to get started" /> {/if} + + diff --git a/packages/client/src/components/app/deprecated/Navigation.svelte b/packages/client/src/components/app/deprecated/Navigation.svelte index d2c9b6986f..7c77424ba4 100644 --- a/packages/client/src/components/app/deprecated/Navigation.svelte +++ b/packages/client/src/components/app/deprecated/Navigation.svelte @@ -4,9 +4,6 @@ const { linkable, styleable } = getContext("sdk") const component = getContext("component") - // BB emblem: https://i.imgur.com/Xhdt1YP.png - // Space logo: https://i.imgur.com/Dn7Xt1G.png - export let logoUrl export let hideLogo diff --git a/packages/client/src/components/app/forms/AttachmentField.svelte b/packages/client/src/components/app/forms/AttachmentField.svelte index e24115ebc0..cc0f7aaac6 100644 --- a/packages/client/src/components/app/forms/AttachmentField.svelte +++ b/packages/client/src/components/app/forms/AttachmentField.svelte @@ -6,11 +6,13 @@ export let field export let label export let disabled = false + export let readonly = false export let compact = false export let validation export let extensions export let onChange export let maximum = undefined + export let span let fieldState let fieldApi @@ -71,33 +73,27 @@ {label} {field} {disabled} + {readonly} {validation} + {span} type="attachment" bind:fieldState bind:fieldApi defaultValue={[]} > -
- {#if fieldState} - - {/if} -
+ {#if fieldState} + + {/if} - - diff --git a/packages/client/src/components/app/forms/BooleanField.svelte b/packages/client/src/components/app/forms/BooleanField.svelte index a65d041c29..1f59ddcfa6 100644 --- a/packages/client/src/components/app/forms/BooleanField.svelte +++ b/packages/client/src/components/app/forms/BooleanField.svelte @@ -6,6 +6,7 @@ export let label export let text export let disabled = false + export let readonly = false export let size export let validation export let defaultValue @@ -39,6 +40,7 @@ {label} {field} {disabled} + {readonly} {validation} defaultValue={isTruthy(defaultValue)} type="boolean" @@ -49,6 +51,7 @@ {/if} diff --git a/packages/client/src/components/app/forms/DateTimeField.svelte b/packages/client/src/components/app/forms/DateTimeField.svelte index 6bcd20d250..7e2598be81 100644 --- a/packages/client/src/components/app/forms/DateTimeField.svelte +++ b/packages/client/src/components/app/forms/DateTimeField.svelte @@ -6,6 +6,7 @@ export let label export let placeholder export let disabled = false + export let readonly = false export let enableTime = true export let timeOnly = false export let time24hr = false @@ -13,6 +14,7 @@ export let validation export let defaultValue export let onChange + export let span let fieldState let fieldApi @@ -29,8 +31,10 @@ {label} {field} {disabled} + {readonly} {validation} {defaultValue} + {span} type="datetime" bind:fieldState bind:fieldApi @@ -40,6 +44,7 @@ value={fieldState.value} on:change={handleChange} disabled={fieldState.disabled} + readonly={fieldState.readonly} error={fieldState.error} id={fieldState.fieldId} appendTo={document.getElementById("flatpickr-root")} diff --git a/packages/client/src/components/app/forms/Field.svelte b/packages/client/src/components/app/forms/Field.svelte index 5d4da5afef..8878b25989 100644 --- a/packages/client/src/components/app/forms/Field.svelte +++ b/packages/client/src/components/app/forms/Field.svelte @@ -1,6 +1,5 @@ - -
- {#key $component.editing} - - {/key} -
- {#if !formContext} - - {:else if !fieldState} - - {:else if schemaType && schemaType !== type && !["options", "longform"].includes(type)} - - {:else} - - {#if fieldState.error} -
{fieldState.error}
- {/if} +
+ {#key $component.editing} + + {/key} +
+ {#if !formContext} + + {:else if !fieldState} + + {:else if schemaType && schemaType !== type && !["options", "longform"].includes(type)} + + {:else} + + {#if fieldState.error} +
{fieldState.error}
{/if} -
+ {/if}
- +
diff --git a/packages/client/src/components/app/forms/Form.svelte b/packages/client/src/components/app/forms/Form.svelte index 87883fe4b6..1a740585f3 100644 --- a/packages/client/src/components/app/forms/Form.svelte +++ b/packages/client/src/components/app/forms/Form.svelte @@ -8,6 +8,7 @@ export let theme export let size export let disabled = false + export let readonly = false export let actionType = "Create" export let initialFormStep = 1 @@ -39,7 +40,7 @@ $: schemaKey = generateSchemaKey(schema) $: initialValues = getInitialValues(actionType, dataSource, $context) $: resetKey = Helpers.hashString( - schemaKey + JSON.stringify(initialValues) + disabled + schemaKey + JSON.stringify(initialValues) + disabled + readonly ) // Returns the closes data context which isn't a built in context @@ -97,6 +98,7 @@ {theme} {size} {disabled} + {readonly} {actionType} {schema} {table} diff --git a/packages/client/src/components/app/forms/InnerForm.svelte b/packages/client/src/components/app/forms/InnerForm.svelte index 4dacf36244..6ebe9de7ec 100644 --- a/packages/client/src/components/app/forms/InnerForm.svelte +++ b/packages/client/src/components/app/forms/InnerForm.svelte @@ -6,6 +6,7 @@ export let dataSource export let disabled = false + export let readonly = false export let initialValues export let size export let schema @@ -148,6 +149,7 @@ type, defaultValue = null, fieldDisabled = false, + fieldReadOnly = false, validationRules, step = 1 ) => { @@ -205,6 +207,7 @@ error: initialError, disabled: disabled || fieldDisabled || (isAutoColumn && !editAutoColumns), + readonly: readonly || fieldReadOnly, defaultValue, validator, lastUpdate: Date.now(), diff --git a/packages/client/src/components/app/forms/JSONField.svelte b/packages/client/src/components/app/forms/JSONField.svelte index c80060d3d6..cf96f54a23 100644 --- a/packages/client/src/components/app/forms/JSONField.svelte +++ b/packages/client/src/components/app/forms/JSONField.svelte @@ -7,6 +7,7 @@ export let label export let placeholder export let disabled = false + export let readonly = false export let defaultValue = "" export let onChange @@ -48,6 +49,7 @@ {label} {field} {disabled} + {readonly} {validation} {defaultValue} type="json" @@ -60,6 +62,7 @@ value={serialiseValue(fieldState.value)} on:change={handleChange} disabled={fieldState.disabled} + readonly={fieldState.readonly} error={fieldState.error} id={fieldState.fieldId} {placeholder} diff --git a/packages/client/src/components/app/forms/LongFormField.svelte b/packages/client/src/components/app/forms/LongFormField.svelte index 8d94f83319..a9087a0a9c 100644 --- a/packages/client/src/components/app/forms/LongFormField.svelte +++ b/packages/client/src/components/app/forms/LongFormField.svelte @@ -8,6 +8,7 @@ export let label export let placeholder export let disabled = false + export let readonly = false export let validation export let defaultValue = "" export let format = "auto" @@ -58,6 +59,7 @@ {label} {field} {disabled} + {readonly} {validation} {defaultValue} type="longform" @@ -71,6 +73,7 @@ value={fieldState.value} on:change={handleChange} disabled={fieldState.disabled} + readonly={fieldState.readonly} error={fieldState.error} id={fieldState.fieldId} {placeholder} @@ -88,6 +91,7 @@ value={fieldState.value} on:change={handleChange} disabled={fieldState.disabled} + readonly={fieldState.readonly} error={fieldState.error} id={fieldState.fieldId} {placeholder} diff --git a/packages/client/src/components/app/forms/MultiFieldSelect.svelte b/packages/client/src/components/app/forms/MultiFieldSelect.svelte index 88e1ec5a8e..519bef4659 100644 --- a/packages/client/src/components/app/forms/MultiFieldSelect.svelte +++ b/packages/client/src/components/app/forms/MultiFieldSelect.svelte @@ -6,6 +6,7 @@ export let label export let placeholder export let disabled = false + export let readonly = false export let validation export let defaultValue export let optionsSource = "schema" @@ -17,6 +18,7 @@ export let onChange export let optionsType = "select" export let direction = "vertical" + export let span let fieldState let fieldApi @@ -55,7 +57,9 @@ {field} {label} {disabled} + {readonly} {validation} + {span} defaultValue={expandedDefaultValue} type="array" bind:fieldState @@ -71,6 +75,7 @@ getOptionValue={flatOptions ? x => x : x => x.value} id={fieldState.fieldId} disabled={fieldState.disabled} + readonly={fieldState.readonly} on:change={handleChange} {placeholder} {options} @@ -81,6 +86,7 @@ value={fieldState.value || []} id={fieldState.fieldId} disabled={fieldState.disabled} + readonly={fieldState.readonly} error={fieldState.error} {options} {direction} diff --git a/packages/client/src/components/app/forms/OptionsField.svelte b/packages/client/src/components/app/forms/OptionsField.svelte index 3c229c0509..f8080419a3 100644 --- a/packages/client/src/components/app/forms/OptionsField.svelte +++ b/packages/client/src/components/app/forms/OptionsField.svelte @@ -6,6 +6,7 @@ export let label export let placeholder export let disabled = false + export let readonly = false export let optionsType = "select" export let validation export let defaultValue @@ -18,6 +19,7 @@ export let direction = "vertical" export let onChange export let sort = true + export let span let fieldState let fieldApi @@ -45,8 +47,10 @@ {field} {label} {disabled} + {readonly} {validation} {defaultValue} + {span} type="options" bind:fieldState bind:fieldApi @@ -58,6 +62,7 @@ value={fieldState.value} id={fieldState.fieldId} disabled={fieldState.disabled} + readonly={fieldState.readonly} error={fieldState.error} {options} {placeholder} @@ -72,6 +77,7 @@ value={fieldState.value} id={fieldState.fieldId} disabled={fieldState.disabled} + readonly={fieldState.readonly} error={fieldState.error} {options} {direction} diff --git a/packages/client/src/components/app/forms/RelationshipField.svelte b/packages/client/src/components/app/forms/RelationshipField.svelte index 544a1a8434..99906fe831 100644 --- a/packages/client/src/components/app/forms/RelationshipField.svelte +++ b/packages/client/src/components/app/forms/RelationshipField.svelte @@ -11,6 +11,7 @@ export let label export let placeholder export let disabled = false + export let readonly = false export let validation export let autocomplete = true export let defaultValue @@ -18,6 +19,7 @@ export let filter export let datasourceType = "table" export let primaryDisplay + export let span let fieldState let fieldApi @@ -137,7 +139,9 @@ typeof value === "object" ? value._id : value ) // Make sure field state is valid - fieldApi.setValue(values) + if (values?.length > 0) { + fieldApi.setValue(values) + } return values } @@ -183,9 +187,11 @@ {label} {field} {disabled} + {readonly} {validation} defaultValue={expandedDefaultValue} {type} + {span} bind:fieldState bind:fieldApi bind:fieldSchema @@ -200,6 +206,7 @@ on:loadMore={loadMore} id={fieldState.fieldId} disabled={fieldState.disabled} + readonly={fieldState.readonly} error={fieldState.error} getOptionLabel={getDisplayName} getOptionValue={option => option._id} diff --git a/packages/client/src/components/app/forms/S3Upload.svelte b/packages/client/src/components/app/forms/S3Upload.svelte index 9985c83bb8..0147cbca6e 100644 --- a/packages/client/src/components/app/forms/S3Upload.svelte +++ b/packages/client/src/components/app/forms/S3Upload.svelte @@ -1,8 +1,7 @@ + + + +
- + {#if searching} + focusedCellId.set(null)} + on:keydown={onInputKeyDown} + data-grid-ignore + /> + {/if} + +
+ +
+
+ +
+
{column.label}
- {#if sortedBy} -
- + + {#if searching} +
+ +
+ {:else} + {#if sortedBy} +
+ +
+ {/if} +
(open = true)}> +
{/if} -
(open = true)}> - -
@@ -235,7 +355,7 @@ disabled={!canBeSortColumn(column.schema.type) || (column.name === $sort.column && $sort.order === "ascending")} > - Sort {ascendingLabel} + Sort {sortingLabels.ascending} - Sort {descendingLabel} + Sort {sortingLabels.descending} Move left @@ -262,6 +382,11 @@ > Hide column + {#if $config.canEditColumns && column.schema.type === "link" && column.schema.tableId === TableNames.USERS} + + Migrate to user column + + {/if} {/if} @@ -283,6 +408,29 @@ background: var(--grid-background-alt); } + /* Icon colors */ + .header-cell :global(.spectrum-Icon) { + color: var(--spectrum-global-color-gray-600); + } + .header-cell :global(.spectrum-Icon.hoverable:hover) { + color: var(--spectrum-global-color-gray-800) !important; + cursor: pointer; + } + + /* Search icon */ + .search-icon { + display: none; + } + .header-cell.searchable:not(.open):hover .search-icon, + .header-cell.searchable.searching .search-icon { + display: block; + } + .header-cell.searchable:not(.open):hover .column-icon, + .header-cell.searchable.searching .column-icon { + display: none; + } + + /* Main center content */ .name { flex: 1 1 auto; width: 0; @@ -290,23 +438,45 @@ text-overflow: ellipsis; overflow: hidden; } + .header-cell.searching .name { + opacity: 0; + pointer-events: none; + } + input { + display: none; + font-family: var(--font-sans); + outline: none; + border: 1px solid transparent; + background: transparent; + color: var(--spectrum-global-color-gray-800); + position: absolute; + top: 0; + left: 0; + width: 100%; + height: 100%; + padding: 0 30px; + border-radius: 2px; + } + input:focus { + border: 1px solid var(--accent-color); + } + input:not(:focus) { + background: var(--spectrum-global-color-gray-200); + } + .header-cell.searching input { + display: block; + } - .more { + /* Right icons */ + .more-icon { display: none; padding: 4px; margin: 0 -4px; } - .header-cell.open .more, - .header-cell:hover .more { + .header-cell.open .more-icon, + .header-cell:hover .more-icon { display: block; } - .more:hover { - cursor: pointer; - } - .more:hover :global(.spectrum-Icon) { - color: var(--spectrum-global-color-gray-800) !important; - } - .header-cell.open .sort-indicator, .header-cell:hover .sort-indicator { display: none; diff --git a/packages/frontend-core/src/components/grid/cells/RelationshipCell.svelte b/packages/frontend-core/src/components/grid/cells/RelationshipCell.svelte index e6d83e0bea..0db022777f 100644 --- a/packages/frontend-core/src/components/grid/cells/RelationshipCell.svelte +++ b/packages/frontend-core/src/components/grid/cells/RelationshipCell.svelte @@ -1,27 +1,10 @@ - - + + + This operation will kick off a migration of the column "{column.schema.name}" + to a new column, with the name provided - this operation may take a moment to + complete. + + + + diff --git a/packages/frontend-core/src/components/grid/layout/GridBody.svelte b/packages/frontend-core/src/components/grid/layout/GridBody.svelte index 762985a4db..0bb2a51fb4 100644 --- a/packages/frontend-core/src/components/grid/layout/GridBody.svelte +++ b/packages/frontend-core/src/components/grid/layout/GridBody.svelte @@ -7,7 +7,7 @@ const { bounds, renderedRows, - renderedColumns, + visibleColumns, rowVerticalInversionIndex, hoveredRowId, dispatch, @@ -17,7 +17,7 @@ let body - $: renderColumnsWidth = $renderedColumns.reduce( + $: columnsWidth = $visibleColumns.reduce( (total, col) => (total += col.width), 0 ) @@ -47,7 +47,7 @@
($hoveredRowId = BlankRowID)} on:mouseleave={$isDragging ? null : () => ($hoveredRowId = null)} on:click={() => dispatch("add-row-inline")} diff --git a/packages/frontend-core/src/components/grid/layout/GridRow.svelte b/packages/frontend-core/src/components/grid/layout/GridRow.svelte index 4754d493bf..4a0db40ee8 100644 --- a/packages/frontend-core/src/components/grid/layout/GridRow.svelte +++ b/packages/frontend-core/src/components/grid/layout/GridRow.svelte @@ -10,7 +10,7 @@ focusedCellId, reorder, selectedRows, - renderedColumns, + visibleColumns, hoveredRowId, selectedCellMap, focusedRow, @@ -19,6 +19,7 @@ isDragging, dispatch, rows, + columnRenderMap, } = getContext("grid") $: rowSelected = !!$selectedRows[row._id] @@ -34,7 +35,7 @@ on:mouseleave={$isDragging ? null : () => ($hoveredRowId = null)} on:click={() => dispatch("rowclick", rows.actions.cleanRow(row))} > - {#each $renderedColumns as column, columnIdx (column.name)} + {#each $visibleColumns as column, columnIdx} {@const cellId = `${row._id}-${column.name}`}
diff --git a/packages/frontend-core/src/components/grid/layout/GridScrollWrapper.svelte b/packages/frontend-core/src/components/grid/layout/GridScrollWrapper.svelte index 05bd261721..2a131809a9 100644 --- a/packages/frontend-core/src/components/grid/layout/GridScrollWrapper.svelte +++ b/packages/frontend-core/src/components/grid/layout/GridScrollWrapper.svelte @@ -11,7 +11,6 @@ maxScrollLeft, bounds, hoveredRowId, - hiddenColumnsWidth, menu, } = getContext("grid") @@ -23,10 +22,10 @@ let initialTouchX let initialTouchY - $: style = generateStyle($scroll, $rowHeight, $hiddenColumnsWidth) + $: style = generateStyle($scroll, $rowHeight) - const generateStyle = (scroll, rowHeight, hiddenWidths) => { - const offsetX = scrollHorizontally ? -1 * scroll.left + hiddenWidths : 0 + const generateStyle = (scroll, rowHeight) => { + const offsetX = scrollHorizontally ? -1 * scroll.left : 0 const offsetY = scrollVertically ? -1 * (scroll.top % rowHeight) : 0 return `transform: translate3d(${offsetX}px, ${offsetY}px, 0);` } diff --git a/packages/frontend-core/src/components/grid/layout/HeaderRow.svelte b/packages/frontend-core/src/components/grid/layout/HeaderRow.svelte index 97b7d054f3..b8655b98b3 100644 --- a/packages/frontend-core/src/components/grid/layout/HeaderRow.svelte +++ b/packages/frontend-core/src/components/grid/layout/HeaderRow.svelte @@ -5,14 +5,14 @@ import HeaderCell from "../cells/HeaderCell.svelte" import { TempTooltip, TooltipType } from "@budibase/bbui" - const { renderedColumns, config, hasNonAutoColumn, datasource, loading } = + const { visibleColumns, config, hasNonAutoColumn, datasource, loading } = getContext("grid")
- {#each $renderedColumns as column, idx} + {#each $visibleColumns as column, idx} diff --git a/packages/frontend-core/src/components/grid/layout/NewColumnButton.svelte b/packages/frontend-core/src/components/grid/layout/NewColumnButton.svelte index d131df26e5..46e9b40fb6 100644 --- a/packages/frontend-core/src/components/grid/layout/NewColumnButton.svelte +++ b/packages/frontend-core/src/components/grid/layout/NewColumnButton.svelte @@ -2,17 +2,16 @@ import { getContext, onMount } from "svelte" import { Icon, Popover, clickOutside } from "@budibase/bbui" - const { renderedColumns, scroll, hiddenColumnsWidth, width, subscribe } = - getContext("grid") + const { visibleColumns, scroll, width, subscribe } = getContext("grid") let anchor let open = false - $: columnsWidth = $renderedColumns.reduce( + $: columnsWidth = $visibleColumns.reduce( (total, col) => (total += col.width), 0 ) - $: end = $hiddenColumnsWidth + columnsWidth - 1 - $scroll.left + $: end = columnsWidth - 1 - $scroll.left $: left = Math.min($width - 40, end) const close = () => { @@ -34,7 +33,7 @@ {#if !visible && !selectedRowCount && $config.canAddRows} @@ -209,29 +212,28 @@
- {#each $renderedColumns as column, columnIdx} + {#each $visibleColumns as column, columnIdx} {@const cellId = `new-${column.name}`} - {#key cellId} - = $columnHorizontalInversionIndex} - {invertY} - > - {#if column?.schema?.autocolumn} -
Can't edit auto column
- {/if} - {#if isAdding} -
- {/if} - - {/key} + = $columnHorizontalInversionIndex} + {invertY} + hidden={!$columnRenderMap[column.name]} + > + {#if column?.schema?.autocolumn} +
Can't edit auto column
+ {/if} + {#if isAdding} +
+ {/if} + {/each}
diff --git a/packages/frontend-core/src/components/grid/overlays/KeyboardManager.svelte b/packages/frontend-core/src/components/grid/overlays/KeyboardManager.svelte index cd23f154b5..8b0a0f0942 100644 --- a/packages/frontend-core/src/components/grid/overlays/KeyboardManager.svelte +++ b/packages/frontend-core/src/components/grid/overlays/KeyboardManager.svelte @@ -21,6 +21,7 @@ const ignoredOriginSelectors = [ ".spectrum-Modal", "#builder-side-panel-container", + "[data-grid-ignore]", ] // Global key listener which intercepts all key events diff --git a/packages/frontend-core/src/components/grid/overlays/ResizeOverlay.svelte b/packages/frontend-core/src/components/grid/overlays/ResizeOverlay.svelte index 13e158b300..9e584ab610 100644 --- a/packages/frontend-core/src/components/grid/overlays/ResizeOverlay.svelte +++ b/packages/frontend-core/src/components/grid/overlays/ResizeOverlay.svelte @@ -2,7 +2,7 @@ import { getContext } from "svelte" import { GutterWidth } from "../lib/constants" - const { resize, renderedColumns, stickyColumn, isReordering, scrollLeft } = + const { resize, visibleColumns, stickyColumn, isReordering, scrollLeft } = getContext("grid") $: offset = GutterWidth + ($stickyColumn?.width || 0) @@ -26,7 +26,7 @@
{/if} - {#each $renderedColumns as column} + {#each $visibleColumns as column}
{ + const { API } = context + + // Cache for the primary display columns of different tables. + // If we ever need to cache table definitions for other purposes then we can + // expand this to be a more generic cache. + let primaryDisplayCache = {} + + const resetPrimaryDisplayCache = () => { + primaryDisplayCache = {} + } + + const getPrimaryDisplayForTableId = async tableId => { + // If we've never encountered this tableId before then store a promise that + // resolves to the primary display so that subsequent invocations before the + // promise completes can reuse this promise + if (!primaryDisplayCache[tableId]) { + primaryDisplayCache[tableId] = new Promise(resolve => { + API.fetchTableDefinition(tableId).then(def => { + const display = def?.primaryDisplay || def?.schema?.[0]?.name + primaryDisplayCache[tableId] = display + resolve(display) + }) + }) + } + + // We await the result so that we account for both promises and primitives + return await primaryDisplayCache[tableId] + } + + return { + cache: { + actions: { + getPrimaryDisplayForTableId, + resetPrimaryDisplayCache, + }, + }, + } +} + +export const initialise = context => { + const { datasource, cache } = context + + // Wipe the caches whenever the datasource changes to ensure we aren't + // storing any stale information + datasource.subscribe(cache.actions.resetPrimaryDisplayCache) +} diff --git a/packages/frontend-core/src/components/grid/stores/datasource.js b/packages/frontend-core/src/components/grid/stores/datasource.js index 0b62194f73..7ee3a19b8a 100644 --- a/packages/frontend-core/src/components/grid/stores/datasource.js +++ b/packages/frontend-core/src/components/grid/stores/datasource.js @@ -1,8 +1,9 @@ -import { derived, get, writable } from "svelte/store" -import { getDatasourceDefinition } from "../../../fetch" +import { derived, get } from "svelte/store" +import { getDatasourceDefinition, getDatasourceSchema } from "../../../fetch" +import { memo } from "../../../utils" export const createStores = () => { - const definition = writable(null) + const definition = memo(null) return { definition, @@ -10,10 +11,15 @@ export const createStores = () => { } export const deriveStores = context => { - const { definition, schemaOverrides, columnWhitelist, datasource } = context + const { API, definition, schemaOverrides, columnWhitelist, datasource } = + context const schema = derived(definition, $definition => { - let schema = $definition?.schema + let schema = getDatasourceSchema({ + API, + datasource: get(datasource), + definition: $definition, + }) if (!schema) { return null } @@ -154,11 +160,6 @@ export const createActions = context => { return getAPI()?.actions.canUseColumn(name) } - // Gets the default number of rows for a single page - const getFeatures = () => { - return getAPI()?.actions.getFeatures() - } - return { datasource: { ...datasource, @@ -171,7 +172,6 @@ export const createActions = context => { getRow, isDatasourceValid, canUseColumn, - getFeatures, }, }, } diff --git a/packages/frontend-core/src/components/grid/stores/datasources/nonPlus.js b/packages/frontend-core/src/components/grid/stores/datasources/nonPlus.js index a05e1f7d37..acdf509278 100644 --- a/packages/frontend-core/src/components/grid/stores/datasources/nonPlus.js +++ b/packages/frontend-core/src/components/grid/stores/datasources/nonPlus.js @@ -35,11 +35,6 @@ export const createActions = context => { return $columns.some(col => col.name === name) || $sticky?.name === name } - const getFeatures = () => { - // We don't support any features - return {} - } - return { nonPlus: { actions: { @@ -50,7 +45,6 @@ export const createActions = context => { getRow, isDatasourceValid, canUseColumn, - getFeatures, }, }, } @@ -66,6 +60,8 @@ export const initialise = context => { datasource, sort, filter, + inlineFilters, + allFilters, nonPlus, initialFilter, initialSortColumn, @@ -87,6 +83,7 @@ export const initialise = context => { // Wipe state filter.set(get(initialFilter)) + inlineFilters.set([]) sort.set({ column: get(initialSortColumn), order: get(initialSortOrder) || "ascending", @@ -94,14 +91,14 @@ export const initialise = context => { // Update fetch when filter changes unsubscribers.push( - filter.subscribe($filter => { + allFilters.subscribe($allFilters => { // Ensure we're updating the correct fetch const $fetch = get(fetch) if (!isSameDatasource($fetch?.options?.datasource, $datasource)) { return } $fetch.update({ - filter: $filter, + filter: $allFilters, }) }) ) diff --git a/packages/frontend-core/src/components/grid/stores/datasources/table.js b/packages/frontend-core/src/components/grid/stores/datasources/table.js index 9ced1530ba..847dfd2c6b 100644 --- a/packages/frontend-core/src/components/grid/stores/datasources/table.js +++ b/packages/frontend-core/src/components/grid/stores/datasources/table.js @@ -1,5 +1,4 @@ import { get } from "svelte/store" -import TableFetch from "../../../../fetch/TableFetch" const SuppressErrors = true @@ -46,10 +45,6 @@ export const createActions = context => { return $columns.some(col => col.name === name) || $sticky?.name === name } - const getFeatures = () => { - return new TableFetch({ API }).determineFeatureFlags() - } - return { table: { actions: { @@ -60,7 +55,6 @@ export const createActions = context => { getRow, isDatasourceValid, canUseColumn, - getFeatures, }, }, } @@ -71,6 +65,8 @@ export const initialise = context => { datasource, fetch, filter, + inlineFilters, + allFilters, sort, table, initialFilter, @@ -93,6 +89,7 @@ export const initialise = context => { // Wipe state filter.set(get(initialFilter)) + inlineFilters.set([]) sort.set({ column: get(initialSortColumn), order: get(initialSortOrder) || "ascending", @@ -100,14 +97,14 @@ export const initialise = context => { // Update fetch when filter changes unsubscribers.push( - filter.subscribe($filter => { + allFilters.subscribe($allFilters => { // Ensure we're updating the correct fetch const $fetch = get(fetch) if ($fetch?.options?.datasource?.tableId !== $datasource.tableId) { return } $fetch.update({ - filter: $filter, + filter: $allFilters, }) }) ) diff --git a/packages/frontend-core/src/components/grid/stores/datasources/viewV2.js b/packages/frontend-core/src/components/grid/stores/datasources/viewV2.js index f0572003c2..ed31d0ae44 100644 --- a/packages/frontend-core/src/components/grid/stores/datasources/viewV2.js +++ b/packages/frontend-core/src/components/grid/stores/datasources/viewV2.js @@ -1,5 +1,4 @@ import { get } from "svelte/store" -import ViewV2Fetch from "../../../../fetch/ViewV2Fetch" const SuppressErrors = true @@ -46,10 +45,6 @@ export const createActions = context => { ) } - const getFeatures = () => { - return new ViewV2Fetch({ API }).determineFeatureFlags() - } - return { viewV2: { actions: { @@ -60,7 +55,6 @@ export const createActions = context => { getRow, isDatasourceValid, canUseColumn, - getFeatures, }, }, } @@ -73,6 +67,8 @@ export const initialise = context => { sort, rows, filter, + inlineFilters, + allFilters, subscribe, viewV2, initialFilter, @@ -97,6 +93,7 @@ export const initialise = context => { // Reset state for new view filter.set(get(initialFilter)) + inlineFilters.set([]) sort.set({ column: get(initialSortColumn), order: get(initialSortOrder) || "ascending", @@ -143,21 +140,19 @@ export const initialise = context => { order: $sort.order || "ascending", }, }) - await rows.actions.refreshData() } } - // Otherwise just update the fetch - else { - // Ensure we're updating the correct fetch - const $fetch = get(fetch) - if ($fetch?.options?.datasource?.tableId !== $datasource.tableId) { - return - } - $fetch.update({ - sortOrder: $sort.order || "ascending", - sortColumn: $sort.column, - }) + + // Also update the fetch to ensure the new sort is respected. + // Ensure we're updating the correct fetch. + const $fetch = get(fetch) + if ($fetch?.options?.datasource?.tableId !== $datasource.tableId) { + return } + $fetch.update({ + sortOrder: $sort.order, + sortColumn: $sort.column, + }) }) ) @@ -176,20 +171,25 @@ export const initialise = context => { ...$view, query: $filter, }) - await rows.actions.refreshData() } } - // Otherwise just update the fetch - else { - // Ensure we're updating the correct fetch - const $fetch = get(fetch) - if ($fetch?.options?.datasource?.tableId !== $datasource.tableId) { - return - } - $fetch.update({ - filter: $filter, - }) + }) + ) + + // Keep fetch up to date with filters. + // If we're able to save filters against the view then we only need to apply + // inline filters to the fetch, as saved filters are applied server side. + // If we can't save filters, then all filters must be applied to the fetch. + unsubscribers.push( + allFilters.subscribe($allFilters => { + // Ensure we're updating the correct fetch + const $fetch = get(fetch) + if ($fetch?.options?.datasource?.tableId !== $datasource.tableId) { + return } + $fetch.update({ + filter: $allFilters, + }) }) ) diff --git a/packages/frontend-core/src/components/grid/stores/filter.js b/packages/frontend-core/src/components/grid/stores/filter.js index a59c98ccdd..a16b101bbb 100644 --- a/packages/frontend-core/src/components/grid/stores/filter.js +++ b/packages/frontend-core/src/components/grid/stores/filter.js @@ -1,13 +1,79 @@ -import { writable, get } from "svelte/store" +import { writable, get, derived } from "svelte/store" +import { FieldType } from "@budibase/types" export const createStores = context => { const { props } = context // Initialise to default props const filter = writable(get(props).initialFilter) + const inlineFilters = writable([]) return { filter, + inlineFilters, + } +} + +export const deriveStores = context => { + const { filter, inlineFilters } = context + + const allFilters = derived( + [filter, inlineFilters], + ([$filter, $inlineFilters]) => { + return [...($filter || []), ...$inlineFilters] + } + ) + + return { + allFilters, + } +} + +export const createActions = context => { + const { filter, inlineFilters } = context + + const addInlineFilter = (column, value) => { + const filterId = `inline-${column.name}` + const type = column.schema.type + let inlineFilter = { + field: column.name, + id: filterId, + operator: "string", + valueType: "value", + type, + value, + } + + // Add overrides specific so the certain column type + if (type === FieldType.NUMBER) { + inlineFilter.value = parseFloat(value) + inlineFilter.operator = "equal" + } else if (type === FieldType.BIGINT) { + inlineFilter.operator = "equal" + } else if (type === FieldType.ARRAY) { + inlineFilter.operator = "contains" + } + + // Add this filter + inlineFilters.update($inlineFilters => { + // Remove any existing inline filter for this column + $inlineFilters = $inlineFilters?.filter(x => x.id !== filterId) + + // Add new one if a value exists + if (value) { + $inlineFilters.push(inlineFilter) + } + return $inlineFilters + }) + } + + return { + filter: { + ...filter, + actions: { + addInlineFilter, + }, + }, } } diff --git a/packages/frontend-core/src/components/grid/stores/index.js b/packages/frontend-core/src/components/grid/stores/index.js index 10fe932aab..6dfff6531b 100644 --- a/packages/frontend-core/src/components/grid/stores/index.js +++ b/packages/frontend-core/src/components/grid/stores/index.js @@ -19,6 +19,7 @@ import * as Datasource from "./datasource" import * as Table from "./datasources/table" import * as ViewV2 from "./datasources/viewV2" import * as NonPlus from "./datasources/nonPlus" +import * as Cache from "./cache" const DependencyOrderedStores = [ Sort, @@ -42,6 +43,7 @@ const DependencyOrderedStores = [ Clipboard, Config, Notifications, + Cache, ] export const attachStores = context => { diff --git a/packages/frontend-core/src/components/grid/stores/rows.js b/packages/frontend-core/src/components/grid/stores/rows.js index 49adb62936..82185d6b91 100644 --- a/packages/frontend-core/src/components/grid/stores/rows.js +++ b/packages/frontend-core/src/components/grid/stores/rows.js @@ -8,6 +8,7 @@ export const createStores = () => { const rows = writable([]) const loading = writable(false) const loaded = writable(false) + const refreshing = writable(false) const rowChangeCache = writable({}) const inProgressChanges = writable({}) const hasNextPage = writable(false) @@ -53,6 +54,7 @@ export const createStores = () => { fetch, rowLookupMap, loaded, + refreshing, loading, rowChangeCache, inProgressChanges, @@ -66,7 +68,7 @@ export const createActions = context => { rows, rowLookupMap, definition, - filter, + allFilters, loading, sort, datasource, @@ -82,6 +84,7 @@ export const createActions = context => { notifications, fetch, isDatasourcePlus, + refreshing, } = context const instanceLoaded = writable(false) @@ -108,23 +111,23 @@ export const createActions = context => { // Tick to allow other reactive logic to update stores when datasource changes // before proceeding. This allows us to wipe filters etc if needed. await tick() - const $filter = get(filter) + const $allFilters = get(allFilters) const $sort = get(sort) - // Determine how many rows to fetch per page - const features = datasource.actions.getFeatures() - const limit = features?.supportsPagination ? RowPageSize : null - // Create new fetch model const newFetch = fetchData({ API, datasource: $datasource, options: { - filter: $filter, + filter: $allFilters, sortColumn: $sort.column, sortOrder: $sort.order, - limit, + limit: RowPageSize, paginate: true, + + // Disable client side limiting, so that for queries and custom data + // sources we don't impose fake row limits. We want all the data. + clientSideLimiting: false, }, }) @@ -176,6 +179,9 @@ export const createActions = context => { // Notify that we're loaded loading.set(false) } + + // Update refreshing state + refreshing.set($fetch.loading) }) fetch.set(newFetch) diff --git a/packages/frontend-core/src/components/grid/stores/viewport.js b/packages/frontend-core/src/components/grid/stores/viewport.js index 6c0c4708b9..8df8acd0f4 100644 --- a/packages/frontend-core/src/components/grid/stores/viewport.js +++ b/packages/frontend-core/src/components/grid/stores/viewport.js @@ -1,4 +1,4 @@ -import { derived, get } from "svelte/store" +import { derived } from "svelte/store" import { MaxCellRenderHeight, MaxCellRenderWidthOverflow, @@ -50,12 +50,11 @@ export const deriveStores = context => { const interval = MinColumnWidth return Math.round($scrollLeft / interval) * interval }) - const renderedColumns = derived( + const columnRenderMap = derived( [visibleColumns, scrollLeftRounded, width], - ([$visibleColumns, $scrollLeft, $width], set) => { + ([$visibleColumns, $scrollLeft, $width]) => { if (!$visibleColumns.length) { - set([]) - return + return {} } let startColIdx = 0 let rightEdge = $visibleColumns[0].width @@ -75,34 +74,16 @@ export const deriveStores = context => { leftEdge += $visibleColumns[endColIdx].width endColIdx++ } - // Render an additional column on either side to account for - // debounce column updates based on scroll position - const next = $visibleColumns.slice( - Math.max(0, startColIdx - 1), - endColIdx + 1 - ) - const current = get(renderedColumns) - if (JSON.stringify(next) !== JSON.stringify(current)) { - set(next) - } - } - ) - const hiddenColumnsWidth = derived( - [renderedColumns, visibleColumns], - ([$renderedColumns, $visibleColumns]) => { - const idx = $visibleColumns.findIndex( - col => col.name === $renderedColumns[0]?.name - ) - let width = 0 - if (idx > 0) { - for (let i = 0; i < idx; i++) { - width += $visibleColumns[i].width - } - } - return width - }, - 0 + // Only update the store if different + let next = {} + $visibleColumns + .slice(Math.max(0, startColIdx), endColIdx) + .forEach(col => { + next[col.name] = true + }) + return next + } ) // Determine the row index at which we should start vertically inverting cell @@ -130,12 +111,12 @@ export const deriveStores = context => { // Determine the column index at which we should start horizontally inverting // cell dropdowns const columnHorizontalInversionIndex = derived( - [renderedColumns, scrollLeft, width], - ([$renderedColumns, $scrollLeft, $width]) => { + [visibleColumns, scrollLeft, width], + ([$visibleColumns, $scrollLeft, $width]) => { const cutoff = $width + $scrollLeft - ScrollBarSize * 3 - let inversionIdx = $renderedColumns.length - for (let i = $renderedColumns.length - 1; i >= 0; i--, inversionIdx--) { - const rightEdge = $renderedColumns[i].left + $renderedColumns[i].width + let inversionIdx = $visibleColumns.length + for (let i = $visibleColumns.length - 1; i >= 0; i--, inversionIdx--) { + const rightEdge = $visibleColumns[i].left + $visibleColumns[i].width if (rightEdge + MaxCellRenderWidthOverflow <= cutoff) { break } @@ -148,8 +129,7 @@ export const deriveStores = context => { scrolledRowCount, visualRowCapacity, renderedRows, - renderedColumns, - hiddenColumnsWidth, + columnRenderMap, rowVerticalInversionIndex, columnHorizontalInversionIndex, } diff --git a/packages/frontend-core/src/fetch/DataFetch.js b/packages/frontend-core/src/fetch/DataFetch.js index 857072601e..92115efef0 100644 --- a/packages/frontend-core/src/fetch/DataFetch.js +++ b/packages/frontend-core/src/fetch/DataFetch.js @@ -43,6 +43,11 @@ export default class DataFetch { // Pagination config paginate: true, + + // Client side feature customisation + clientSideSearching: true, + clientSideSorting: true, + clientSideLimiting: true, } // State of the fetch @@ -208,24 +213,32 @@ export default class DataFetch { * Fetches some filtered, sorted and paginated data */ async getPage() { - const { sortColumn, sortOrder, sortType, limit } = this.options + const { + sortColumn, + sortOrder, + sortType, + limit, + clientSideSearching, + clientSideSorting, + clientSideLimiting, + } = this.options const { query } = get(this.store) // Get the actual data let { rows, info, hasNextPage, cursor, error } = await this.getData() // If we don't support searching, do a client search - if (!this.features.supportsSearch) { + if (!this.features.supportsSearch && clientSideSearching) { rows = runLuceneQuery(rows, query) } // If we don't support sorting, do a client-side sort - if (!this.features.supportsSort) { + if (!this.features.supportsSort && clientSideSorting) { rows = luceneSort(rows, sortColumn, sortOrder, sortType) } // If we don't support pagination, do a client-side limit - if (!this.features.supportsPagination) { + if (!this.features.supportsPagination && clientSideLimiting) { rows = luceneLimit(rows, limit) } diff --git a/packages/frontend-core/src/fetch/UserFetch.js b/packages/frontend-core/src/fetch/UserFetch.js index b1478c3a6d..65bfe36058 100644 --- a/packages/frontend-core/src/fetch/UserFetch.js +++ b/packages/frontend-core/src/fetch/UserFetch.js @@ -33,7 +33,7 @@ export default class UserFetch extends DataFetch { let finalQuery // convert old format to new one - we now allow use of the lucene format const { appId, paginated, ...rest } = query - if (!LuceneUtils.hasFilters(query) && rest.email) { + if (!LuceneUtils.hasFilters(query) && rest.email != null) { finalQuery = { string: { email: rest.email } } } else { finalQuery = rest diff --git a/packages/frontend-core/src/fetch/ViewV2Fetch.js b/packages/frontend-core/src/fetch/ViewV2Fetch.js index b9eaf4bdf7..9d2f8c103a 100644 --- a/packages/frontend-core/src/fetch/ViewV2Fetch.js +++ b/packages/frontend-core/src/fetch/ViewV2Fetch.js @@ -35,9 +35,28 @@ export default class ViewV2Fetch extends DataFetch { } async getData() { - const { datasource, limit, sortColumn, sortOrder, sortType, paginate } = - this.options - const { cursor, query } = get(this.store) + const { + datasource, + limit, + sortColumn, + sortOrder, + sortType, + paginate, + filter, + } = this.options + const { cursor, query, definition } = get(this.store) + + // If sort/filter params are not defined, update options to store the + // params built in to this view. This ensures that we can accurately + // compare old and new params and skip a redundant API call. + if (!sortColumn && definition.sort?.field) { + this.options.sortColumn = definition.sort.field + this.options.sortOrder = definition.sort.order + } + if (!filter?.length && definition.query?.length) { + this.options.filter = definition.query + } + try { const res = await this.API.viewV2.fetch({ viewId: datasource.id, diff --git a/packages/frontend-core/src/fetch/index.js b/packages/frontend-core/src/fetch/index.js index d133942bb7..a41a859351 100644 --- a/packages/frontend-core/src/fetch/index.js +++ b/packages/frontend-core/src/fetch/index.js @@ -32,12 +32,24 @@ export const fetchData = ({ API, datasource, options }) => { return new Fetch({ API, datasource, ...options }) } -// Fetches the definition of any type of datasource -export const getDatasourceDefinition = async ({ API, datasource }) => { +// Creates an empty fetch instance with no datasource configured, so no data +// will initially be loaded +const createEmptyFetchInstance = ({ API, datasource }) => { const handler = DataFetchMap[datasource?.type] if (!handler) { return null } - const instance = new handler({ API }) - return await instance.getDefinition(datasource) + return new handler({ API }) +} + +// Fetches the definition of any type of datasource +export const getDatasourceDefinition = async ({ API, datasource }) => { + const instance = createEmptyFetchInstance({ API, datasource }) + return await instance?.getDefinition(datasource) +} + +// Fetches the schema of any type of datasource +export const getDatasourceSchema = ({ API, datasource, definition }) => { + const instance = createEmptyFetchInstance({ API, datasource }) + return instance?.getSchema(datasource, definition) } diff --git a/packages/pro b/packages/pro index 570d14aa44..2cf6f28380 160000 --- a/packages/pro +++ b/packages/pro @@ -1 +1 @@ -Subproject commit 570d14aa44aa88f4d053856322210f0008ba5c76 +Subproject commit 2cf6f28380d3ab22128b8a889d622fd5adfa31fc diff --git a/packages/server/Dockerfile b/packages/server/Dockerfile index e1b3b208c7..ea4c5b217a 100644 --- a/packages/server/Dockerfile +++ b/packages/server/Dockerfile @@ -38,7 +38,7 @@ RUN apt update && apt upgrade -y \ COPY package.json . COPY dist/yarn.lock . -RUN yarn install --production=true \ +RUN yarn install --production=true --network-timeout 1000000 \ # Remove unneeded data from file system to reduce image size && yarn cache clean && apt-get remove -y --purge --auto-remove g++ make python \ && rm -rf /tmp/* /root/.node-gyp /usr/local/lib/node_modules/npm/node_modules/node-gyp diff --git a/packages/server/Dockerfile.v2 b/packages/server/Dockerfile.v2 index d5a86b037d..f737570fcd 100644 --- a/packages/server/Dockerfile.v2 +++ b/packages/server/Dockerfile.v2 @@ -44,7 +44,7 @@ RUN chmod +x ./scripts/removeWorkspaceDependencies.sh WORKDIR /string-templates COPY packages/string-templates/package.json package.json RUN ../scripts/removeWorkspaceDependencies.sh package.json -RUN --mount=type=cache,target=/root/.yarn YARN_CACHE_FOLDER=/root/.yarn yarn install --production=true +RUN --mount=type=cache,target=/root/.yarn YARN_CACHE_FOLDER=/root/.yarn yarn install --production=true --network-timeout 1000000 COPY packages/string-templates . @@ -57,7 +57,7 @@ COPY scripts/removeWorkspaceDependencies.sh scripts/removeWorkspaceDependencies. RUN chmod +x ./scripts/removeWorkspaceDependencies.sh RUN ./scripts/removeWorkspaceDependencies.sh package.json -RUN --mount=type=cache,target=/root/.yarn YARN_CACHE_FOLDER=/root/.yarn yarn install --production=true \ +RUN --mount=type=cache,target=/root/.yarn YARN_CACHE_FOLDER=/root/.yarn yarn install --production=true --network-timeout 1000000 \ # Remove unneeded data from file system to reduce image size && yarn cache clean && apt-get remove -y --purge --auto-remove g++ make python jq \ && rm -rf /tmp/* /root/.node-gyp /usr/local/lib/node_modules/npm/node_modules/node-gyp @@ -67,6 +67,11 @@ COPY packages/server/docker_run.sh . COPY packages/server/builder/ builder/ COPY packages/server/client/ client/ +ARG BUDIBASE_VERSION +# Ensuring the version argument is sent +RUN test -n "$BUDIBASE_VERSION" +ENV BUDIBASE_VERSION=$BUDIBASE_VERSION + EXPOSE 4001 # have to add node environment production after install diff --git a/packages/server/__mocks__/aws-sdk.ts b/packages/server/__mocks__/aws-sdk.ts index 8a66f0e213..fa6d099f56 100644 --- a/packages/server/__mocks__/aws-sdk.ts +++ b/packages/server/__mocks__/aws-sdk.ts @@ -70,6 +70,13 @@ module AwsMock { Contents: {}, }) ) + + // @ts-ignore + this.getObject = jest.fn( + response({ + Body: "", + }) + ) } aws.DynamoDB = { DocumentClient } diff --git a/packages/server/package.json b/packages/server/package.json index 18644e29eb..cc093acf52 100644 --- a/packages/server/package.json +++ b/packages/server/package.json @@ -18,7 +18,6 @@ "test": "bash scripts/test.sh", "test:memory": "jest --maxWorkers=2 --logHeapUsage --forceExit", "test:watch": "jest --watch", - "build:docker": "yarn build && docker build . -t app-service --label version=$BUDIBASE_RELEASE_VERSION", "run:docker": "node dist/index.js", "run:docker:cluster": "pm2-runtime start pm2.config.js", "dev:stack:up": "node scripts/dev/manage.js up", diff --git a/packages/server/scripts/dev/manage.js b/packages/server/scripts/dev/manage.js index 5db45040bf..13639b6bfd 100644 --- a/packages/server/scripts/dev/manage.js +++ b/packages/server/scripts/dev/manage.js @@ -47,6 +47,7 @@ async function init() { TENANT_FEATURE_FLAGS: "*:LICENSING,*:USER_GROUPS,*:ONBOARDING_TOUR", HTTP_MIGRATIONS: "0", HTTP_LOGGING: "0", + VERSION: "0.0.0+local", } let envFile = "" Object.keys(envFileJson).forEach(key => { diff --git a/packages/server/scripts/integrations/postgres/docker-compose.yml b/packages/server/scripts/integrations/postgres/docker-compose.yml index 88efd0301d..0e8e30ecdb 100644 --- a/packages/server/scripts/integrations/postgres/docker-compose.yml +++ b/packages/server/scripts/integrations/postgres/docker-compose.yml @@ -2,7 +2,7 @@ version: "3.8" services: db: container_name: postgres - image: postgres:15 + image: postgres:15-bullseye restart: unless-stopped environment: POSTGRES_USER: root diff --git a/packages/server/src/api/controllers/application.ts b/packages/server/src/api/controllers/application.ts index 4afd7b23f9..4e4c66858e 100644 --- a/packages/server/src/api/controllers/application.ts +++ b/packages/server/src/api/controllers/application.ts @@ -32,11 +32,8 @@ import { tenancy, users, } from "@budibase/backend-core" -import { USERS_TABLE_SCHEMA } from "../../constants" -import { - buildDefaultDocs, - DEFAULT_BB_DATASOURCE_ID, -} from "../../db/defaultData/datasource_bb_default" +import { USERS_TABLE_SCHEMA, DEFAULT_BB_DATASOURCE_ID } from "../../constants" +import { buildDefaultDocs } from "../../db/defaultData/datasource_bb_default" import { removeAppFromUserRoles } from "../../utilities/workerRequests" import { stringToReadStream } from "../../utilities" import { doesUserHaveLock } from "../../utilities/redis" diff --git a/packages/server/src/api/controllers/auth.ts b/packages/server/src/api/controllers/auth.ts index eabfe10bab..9b1b78ed9e 100644 --- a/packages/server/src/api/controllers/auth.ts +++ b/packages/server/src/api/controllers/auth.ts @@ -26,7 +26,7 @@ export async function fetchSelf(ctx: UserCtx) { } const appId = context.getAppId() - let user: ContextUser = await getFullUser(ctx, userId) + let user: ContextUser = await getFullUser(userId) // this shouldn't be returned by the app self delete user.roles // forward the csrf token from the session diff --git a/packages/server/src/api/controllers/datasource.ts b/packages/server/src/api/controllers/datasource.ts index b50c2464f0..39bc612b32 100644 --- a/packages/server/src/api/controllers/datasource.ts +++ b/packages/server/src/api/controllers/datasource.ts @@ -12,7 +12,6 @@ import { CreateDatasourceResponse, Datasource, DatasourcePlus, - ExternalTable, FetchDatasourceInfoRequest, FetchDatasourceInfoResponse, IntegrationBase, @@ -59,7 +58,7 @@ async function buildSchemaHelper(datasource: Datasource): Promise { const connector = (await getConnector(datasource)) as DatasourcePlus return await connector.buildSchema( datasource._id!, - datasource.entities! as Record + datasource.entities! as Record ) } @@ -338,7 +337,7 @@ export async function destroy(ctx: UserCtx) { if (datasource.type === dbCore.BUDIBASE_DATASOURCE_TYPE) { await destroyInternalTablesBySourceId(datasourceId) } else { - const queries = await db.allDocs(getQueryParams(datasourceId, null)) + const queries = await db.allDocs(getQueryParams(datasourceId)) await db.bulkDocs( queries.rows.map((row: any) => ({ _id: row.id, diff --git a/packages/server/src/api/controllers/deploy/index.ts b/packages/server/src/api/controllers/deploy/index.ts index 66439d3411..2cf3da3dda 100644 --- a/packages/server/src/api/controllers/deploy/index.ts +++ b/packages/server/src/api/controllers/deploy/index.ts @@ -106,7 +106,6 @@ export async function fetchDeployments(ctx: any) { } ctx.body = Object.values(deployments.history).reverse() } catch (err) { - console.error(err) ctx.body = [] } } diff --git a/packages/server/src/api/controllers/layout.ts b/packages/server/src/api/controllers/layout.ts index c00252d643..69e4ad91ed 100644 --- a/packages/server/src/api/controllers/layout.ts +++ b/packages/server/src/api/controllers/layout.ts @@ -1,7 +1,7 @@ import { EMPTY_LAYOUT } from "../../constants/layouts" import { generateLayoutID, getScreenParams } from "../../db/utils" import { events, context } from "@budibase/backend-core" -import { BBContext } from "@budibase/types" +import { BBContext, Layout } from "@budibase/types" export async function save(ctx: BBContext) { const db = context.getAppDB() @@ -30,12 +30,12 @@ export async function destroy(ctx: BBContext) { layoutRev = ctx.params.layoutRev const layoutsUsedByScreens = ( - await db.allDocs( + await db.allDocs( getScreenParams(null, { include_docs: true, }) ) - ).rows.map(element => element.doc.layoutId) + ).rows.map(element => element.doc!.layoutId) if (layoutsUsedByScreens.includes(layoutId)) { ctx.throw(400, "Cannot delete a layout that's being used by a screen") } diff --git a/packages/server/src/api/controllers/permission.ts b/packages/server/src/api/controllers/permission.ts index a9cd686674..e2bd6c40e5 100644 --- a/packages/server/src/api/controllers/permission.ts +++ b/packages/server/src/api/controllers/permission.ts @@ -25,12 +25,12 @@ const SUPPORTED_LEVELS = CURRENTLY_SUPPORTED_LEVELS // utility function to stop this repetition - permissions always stored under roles async function getAllDBRoles(db: Database) { - const body = await db.allDocs( + const body = await db.allDocs( getRoleParams(null, { include_docs: true, }) ) - return body.rows.map(row => row.doc) + return body.rows.map(row => row.doc!) } async function updatePermissionOnRole( @@ -79,7 +79,7 @@ async function updatePermissionOnRole( ) { rolePermissions[resourceId] = typeof rolePermissions[resourceId] === "string" - ? [rolePermissions[resourceId]] + ? [rolePermissions[resourceId] as unknown as string] : [] } // handle the removal/updating the role which has this permission first diff --git a/packages/server/src/api/controllers/public/utils.ts b/packages/server/src/api/controllers/public/utils.ts index 1272fcb36a..1d67b49e0d 100644 --- a/packages/server/src/api/controllers/public/utils.ts +++ b/packages/server/src/api/controllers/public/utils.ts @@ -1,12 +1,12 @@ import { context } from "@budibase/backend-core" -import { isExternalTable } from "../../../integrations/utils" +import { isExternalTableID } from "../../../integrations/utils" import { APP_PREFIX, DocumentType } from "../../../db/utils" export async function addRev( body: { _id?: string; _rev?: string }, tableId?: string ) { - if (!body._id || (tableId && isExternalTable(tableId))) { + if (!body._id || (tableId && isExternalTableID(tableId))) { return body } let id = body._id diff --git a/packages/server/src/api/controllers/role.ts b/packages/server/src/api/controllers/role.ts index ed23009706..ae6b89e6d4 100644 --- a/packages/server/src/api/controllers/role.ts +++ b/packages/server/src/api/controllers/role.ts @@ -1,6 +1,18 @@ -import { context, db as dbCore, events, roles } from "@budibase/backend-core" +import { + context, + db as dbCore, + events, + roles, + Header, +} from "@budibase/backend-core" import { getUserMetadataParams, InternalTables } from "../../db/utils" -import { Database, Role, UserCtx, UserRoles } from "@budibase/types" +import { + Database, + Role, + UserCtx, + UserMetadata, + UserRoles, +} from "@budibase/types" import { sdk as sharedSdk } from "@budibase/shared-core" import sdk from "../../sdk" @@ -109,12 +121,12 @@ export async function destroy(ctx: UserCtx) { const role = await db.get(roleId) // first check no users actively attached to role const users = ( - await db.allDocs( + await db.allDocs( getUserMetadataParams(undefined, { include_docs: true, }) ) - ).rows.map(row => row.doc) + ).rows.map(row => row.doc!) const usersWithRole = users.filter(user => user.roleId === roleId) if (usersWithRole.length !== 0) { ctx.throw(400, "Cannot delete role when it is in use.") @@ -143,4 +155,20 @@ export async function accessible(ctx: UserCtx) { } else { ctx.body = await roles.getUserRoleIdHierarchy(roleId!) } + + // If a custom role is provided in the header, filter out higher level roles + const roleHeader = ctx.header?.[Header.PREVIEW_ROLE] as string + if (roleHeader && !Object.keys(roles.BUILTIN_ROLE_IDS).includes(roleHeader)) { + const inherits = (await roles.getRole(roleHeader))?.inherits + const orderedRoles = ctx.body.reverse() + let filteredRoles = [roleHeader] + for (let role of orderedRoles) { + filteredRoles = [role, ...filteredRoles] + if (role === inherits) { + break + } + } + filteredRoles.pop() + ctx.body = [roleHeader, ...filteredRoles] + } } diff --git a/packages/server/src/api/controllers/row/ExternalRequest.ts b/packages/server/src/api/controllers/row/ExternalRequest.ts index d14bd622ec..485a653ae9 100644 --- a/packages/server/src/api/controllers/row/ExternalRequest.ts +++ b/packages/server/src/api/controllers/row/ExternalRequest.ts @@ -29,7 +29,6 @@ import { buildSqlFieldList, generateIdForRow, sqlOutputProcessing, - squashRelationshipColumns, updateRelationshipColumns, fixArrayTypes, isManyToMany, @@ -217,7 +216,7 @@ function basicProcessing({ thisRow._id = generateIdForRow(row, table, isLinked) thisRow.tableId = table._id thisRow._rev = "rev" - return processFormulas(table, thisRow) + return thisRow } function isOneSide( @@ -349,6 +348,33 @@ export class ExternalRequest { return { row: newRow, manyRelationships } } + processRelationshipFields( + table: Table, + row: Row, + relationships: RelationshipsJson[] + ): Row { + for (let relationship of relationships) { + const linkedTable = this.tables[relationship.tableName] + if (!linkedTable || !row[relationship.column]) { + continue + } + for (let key of Object.keys(row[relationship.column])) { + let relatedRow: Row = row[relationship.column][key] + // add this row as context for the relationship + for (let col of Object.values(linkedTable.schema)) { + if (col.type === FieldType.LINK && col.tableId === table._id) { + relatedRow[col.name] = [row] + } + } + // process additional types + relatedRow = processDates(table, relatedRow) + relatedRow = processFormulas(linkedTable, relatedRow) + row[relationship.column][key] = relatedRow + } + } + return row + } + outputProcessing( rows: Row[] = [], table: Table, @@ -391,13 +417,14 @@ export class ExternalRequest { ) } - // Process some additional data types - let finalRowArray = Object.values(finalRows) - finalRowArray = processDates(table, finalRowArray) - finalRowArray = processFormulas(table, finalRowArray) as Row[] - return finalRowArray.map((row: Row) => - squashRelationshipColumns(table, tableMap, row, relationships) + // make sure all related rows are correct + let finalRowArray = Object.values(finalRows).map(row => + this.processRelationshipFields(table, row, relationships) ) + + // process some additional types + finalRowArray = processDates(table, finalRowArray) + return finalRowArray } /** @@ -487,7 +514,7 @@ export class ExternalRequest { linkPrimary, linkSecondary, }: { - row: Record + row: Row linkPrimary: string linkSecondary?: string }) { diff --git a/packages/server/src/api/controllers/row/external.ts b/packages/server/src/api/controllers/row/external.ts index 0515b6b97e..287b2ae6aa 100644 --- a/packages/server/src/api/controllers/row/external.ts +++ b/packages/server/src/api/controllers/row/external.ts @@ -76,6 +76,7 @@ export async function patch(ctx: UserCtx) { relationships: true, }) const enrichedRow = await outputProcessing(table, row, { + squash: true, preserveLinks: true, }) return { @@ -119,7 +120,10 @@ export async function save(ctx: UserCtx) { }) return { ...response, - row: await outputProcessing(table, row, { preserveLinks: true }), + row: await outputProcessing(table, row, { + preserveLinks: true, + squash: true, + }), } } else { return response @@ -140,7 +144,7 @@ export async function find(ctx: UserCtx): Promise { const table = await sdk.tables.getTable(tableId) // Preserving links, as the outputProcessing does not support external rows yet and we don't need it in this use case return await outputProcessing(table, row, { - squash: false, + squash: true, preserveLinks: true, }) } @@ -207,7 +211,7 @@ export async function fetchEnrichedRow(ctx: UserCtx) { // don't support composite keys right now const linkedIds = links.map((link: Row) => breakRowIdField(link._id!)[0]) const primaryLink = linkedTable.primary?.[0] as string - row[fieldName] = await handleRequest(Operation.READ, linkedTableId!, { + const relatedRows = await handleRequest(Operation.READ, linkedTableId!, { tables, filters: { oneOf: { @@ -216,6 +220,10 @@ export async function fetchEnrichedRow(ctx: UserCtx) { }, includeSqlRelationships: IncludeRelationship.INCLUDE, }) + row[fieldName] = await outputProcessing(linkedTable, relatedRows, { + squash: true, + preserveLinks: true, + }) } return row } diff --git a/packages/server/src/api/controllers/row/index.ts b/packages/server/src/api/controllers/row/index.ts index 0ccbf5cacf..018283c8c5 100644 --- a/packages/server/src/api/controllers/row/index.ts +++ b/packages/server/src/api/controllers/row/index.ts @@ -1,7 +1,7 @@ import { quotas } from "@budibase/pro" import * as internal from "./internal" import * as external from "./external" -import { isExternalTable } from "../../../integrations/utils" +import { isExternalTableID } from "../../../integrations/utils" import { Ctx, UserCtx, @@ -30,7 +30,7 @@ import { Format } from "../view/exporters" export * as views from "./views" function pickApi(tableId: any) { - if (isExternalTable(tableId)) { + if (isExternalTableID(tableId)) { return external } return internal @@ -227,7 +227,7 @@ export async function search(ctx: Ctx) { export async function validate(ctx: Ctx) { const tableId = utils.getTableId(ctx) // external tables are hard to validate currently - if (isExternalTable(tableId)) { + if (isExternalTableID(tableId)) { ctx.body = { valid: true, errors: {} } } else { ctx.body = await sdk.rows.utils.validate({ @@ -254,7 +254,7 @@ export const exportRows = async ( const format = ctx.query.format - const { rows, columns, query } = ctx.request.body + const { rows, columns, query, sort, sortOrder } = ctx.request.body if (typeof format !== "string" || !exporters.isFormat(format)) { ctx.throw( 400, @@ -272,6 +272,8 @@ export const exportRows = async ( rowIds: rows, columns, query, + sort, + sortOrder, }) ctx.attachment(fileName) return apiFileReturn(content) diff --git a/packages/server/src/api/controllers/row/internal.ts b/packages/server/src/api/controllers/row/internal.ts index a972a0a53a..46f828626d 100644 --- a/packages/server/src/api/controllers/row/internal.ts +++ b/packages/server/src/api/controllers/row/internal.ts @@ -1,10 +1,5 @@ import * as linkRows from "../../../db/linkedRows" -import { - generateRowID, - getMultiIDParams, - getTableIDFromRowID, - InternalTables, -} from "../../../db/utils" +import { generateRowID, InternalTables } from "../../../db/utils" import * as userController from "../user" import { cleanupAttachments, @@ -100,7 +95,7 @@ export async function patch(ctx: UserCtx) { if (isUserTable) { // the row has been updated, need to put it into the ctx ctx.request.body = row as any - await userController.updateMetadata(ctx) + await userController.updateMetadata(ctx as any) return { row: ctx.body as Row, table } } @@ -340,17 +335,18 @@ export async function fetchEnrichedRow(ctx: UserCtx) { const tableId = utils.getTableId(ctx) const rowId = ctx.params.rowId as string // need table to work out where links go in row, as well as the link docs - let response = await Promise.all([ + const [table, row, links] = await Promise.all([ sdk.tables.getTable(tableId), utils.findRow(ctx, tableId, rowId), linkRows.getLinkDocuments({ tableId, rowId, fieldName }), ]) - const table = response[0] as Table - const row = response[1] as Row - const linkVals = response[2] as LinkDocumentValue[] + const linkVals = links as LinkDocumentValue[] + // look up the actual rows based on the ids - const params = getMultiIDParams(linkVals.map(linkVal => linkVal.id)) - let linkedRows = (await db.allDocs(params)).rows.map(row => row.doc) + let linkedRows = await db.getMultiple( + linkVals.map(linkVal => linkVal.id), + { allowMissing: true } + ) // get the linked tables const linkTableIds = getLinkedTableIDs(table as Table) diff --git a/packages/server/src/api/controllers/row/staticFormula.ts b/packages/server/src/api/controllers/row/staticFormula.ts index 6f426c6fa0..8d52b6a05c 100644 --- a/packages/server/src/api/controllers/row/staticFormula.ts +++ b/packages/server/src/api/controllers/row/staticFormula.ts @@ -86,12 +86,12 @@ export async function updateAllFormulasInTable(table: Table) { const db = context.getAppDB() // start by getting the raw rows (which will be written back to DB after update) let rows = ( - await db.allDocs( + await db.allDocs( getRowParams(table._id, null, { include_docs: true, }) ) - ).rows.map(row => row.doc) + ).rows.map(row => row.doc!) // now enrich the rows, note the clone so that we have the base state of the // rows so that we don't write any of the enriched information back let enrichedRows = await outputProcessing(table, cloneDeep(rows), { @@ -101,12 +101,12 @@ export async function updateAllFormulasInTable(table: Table) { for (let row of rows) { // find the enriched row, if found process the formulas const enrichedRow = enrichedRows.find( - (enriched: any) => enriched._id === row._id + (enriched: Row) => enriched._id === row._id ) if (enrichedRow) { const processed = processFormulas(table, cloneDeep(row), { dynamic: false, - contextRows: enrichedRow, + contextRows: [enrichedRow], }) // values have changed, need to add to bulk docs to update if (!isEqual(processed, row)) { @@ -139,7 +139,7 @@ export async function finaliseRow( // use enriched row to generate formulas for saving, specifically only use as context row = processFormulas(table, row, { dynamic: false, - contextRows: enrichedRow, + contextRows: [enrichedRow], }) // don't worry about rev, tables handle rev/lastID updates // if another row has been written since processing this will @@ -163,7 +163,9 @@ export async function finaliseRow( const response = await db.put(row) // for response, calculate the formulas for the enriched row enrichedRow._rev = response.rev - enrichedRow = await processFormulas(table, enrichedRow, { dynamic: false }) + enrichedRow = processFormulas(table, enrichedRow, { + dynamic: false, + }) // this updates the related formulas in other rows based on the relations to this row if (updateFormula) { await updateRelatedFormula(table, enrichedRow) diff --git a/packages/server/src/api/controllers/static/index.ts b/packages/server/src/api/controllers/static/index.ts index 889b2c8738..6f5cde102d 100644 --- a/packages/server/src/api/controllers/static/index.ts +++ b/packages/server/src/api/controllers/static/index.ts @@ -1,3 +1,5 @@ +import { ValidFileExtensions } from "@budibase/shared-core" + require("svelte/register") import { join } from "../../../utilities/centralPath" @@ -10,35 +12,28 @@ import { TOP_LEVEL_PATH, } from "../../../utilities/fileSystem" import env from "../../../environment" -import { context, objectStore, utils, configs } from "@budibase/backend-core" +import { + context, + objectStore, + utils, + configs, + BadRequestError, +} from "@budibase/backend-core" import AWS from "aws-sdk" import fs from "fs" import sdk from "../../../sdk" import * as pro from "@budibase/pro" -import { App, DocumentType } from "@budibase/types" +import { + App, + Ctx, + DocumentType, + ProcessAttachmentResponse, + Upload, +} from "@budibase/types" const send = require("koa-send") -async function prepareUpload({ s3Key, bucket, metadata, file }: any) { - const response = await objectStore.upload({ - bucket, - metadata, - filename: s3Key, - path: file.path, - type: file.type, - }) - - // don't store a URL, work this out on the way out as the URL could change - return { - size: file.size, - name: file.name, - url: objectStore.getAppFileUrl(s3Key), - extension: [...file.name.split(".")].pop(), - key: response.Key, - } -} - -export const toggleBetaUiFeature = async function (ctx: any) { +export const toggleBetaUiFeature = async function (ctx: Ctx) { const cookieName = `beta:${ctx.params.feature}` if (ctx.cookies.get(cookieName)) { @@ -66,40 +61,73 @@ export const toggleBetaUiFeature = async function (ctx: any) { } } -export const serveBuilder = async function (ctx: any) { +export const serveBuilder = async function (ctx: Ctx) { const builderPath = join(TOP_LEVEL_PATH, "builder") await send(ctx, ctx.file, { root: builderPath }) } -export const uploadFile = async function (ctx: any) { - let files = - ctx.request.files.file.length > 1 - ? Array.from(ctx.request.files.file) - : [ctx.request.files.file] +export const uploadFile = async function ( + ctx: Ctx<{}, ProcessAttachmentResponse> +) { + const file = ctx.request?.files?.file + if (!file) { + throw new BadRequestError("No file provided") + } - const uploads = files.map(async (file: any) => { - const fileExtension = [...file.name.split(".")].pop() - // filenames converted to UUIDs so they are unique - const processedFileName = `${uuid.v4()}.${fileExtension}` + let files = file && Array.isArray(file) ? Array.from(file) : [file] - return prepareUpload({ - file, - s3Key: `${context.getProdAppId()}/attachments/${processedFileName}`, - bucket: ObjectStoreBuckets.APPS, + ctx.body = await Promise.all( + files.map(async file => { + if (!file.name) { + throw new BadRequestError( + "Attempted to upload a file without a filename" + ) + } + + const extension = [...file.name.split(".")].pop() + if (!extension) { + throw new BadRequestError( + `File "${file.name}" has no extension, an extension is required to upload a file` + ) + } + + if (!env.SELF_HOSTED && !ValidFileExtensions.includes(extension)) { + throw new BadRequestError( + `File "${file.name}" has an invalid extension: "${extension}"` + ) + } + + // filenames converted to UUIDs so they are unique + const processedFileName = `${uuid.v4()}.${extension}` + + const s3Key = `${context.getProdAppId()}/attachments/${processedFileName}` + + const response = await objectStore.upload({ + bucket: ObjectStoreBuckets.APPS, + filename: s3Key, + path: file.path, + type: file.type, + }) + + return { + size: file.size, + name: file.name, + url: objectStore.getAppFileUrl(s3Key), + extension, + key: response.Key, + } }) - }) - - ctx.body = await Promise.all(uploads) + ) } -export const deleteObjects = async function (ctx: any) { +export const deleteObjects = async function (ctx: Ctx) { ctx.body = await objectStore.deleteFiles( ObjectStoreBuckets.APPS, ctx.request.body.keys ) } -export const serveApp = async function (ctx: any) { +export const serveApp = async function (ctx: Ctx) { const bbHeaderEmbed = ctx.request.get("x-budibase-embed")?.toLowerCase() === "true" @@ -124,7 +152,7 @@ export const serveApp = async function (ctx: any) { const { head, html, css } = App.render({ metaImage: branding?.metaImageUrl || - "https://res.cloudinary.com/daog6scxm/image/upload/v1666109324/meta-images/budibase-meta-image_uukc1m.png", + "https://res.cloudinary.com/daog6scxm/image/upload/v1698759482/meta-images/plain-branded-meta-image-coral_ocxmgu.png", metaDescription: branding?.metaDescription || "", metaTitle: branding?.metaTitle || `${appInfo.name} - built with Budibase`, @@ -162,7 +190,7 @@ export const serveApp = async function (ctx: any) { metaTitle: branding?.metaTitle, metaImage: branding?.metaImageUrl || - "https://res.cloudinary.com/daog6scxm/image/upload/v1666109324/meta-images/budibase-meta-image_uukc1m.png", + "https://res.cloudinary.com/daog6scxm/image/upload/v1698759482/meta-images/plain-branded-meta-image-coral_ocxmgu.png", metaDescription: branding?.metaDescription || "", favicon: branding.faviconUrl !== "" @@ -180,7 +208,7 @@ export const serveApp = async function (ctx: any) { } } -export const serveBuilderPreview = async function (ctx: any) { +export const serveBuilderPreview = async function (ctx: Ctx) { const db = context.getAppDB({ skip_setup: true }) const appInfo = await db.get(DocumentType.APP_METADATA) @@ -196,18 +224,30 @@ export const serveBuilderPreview = async function (ctx: any) { } } -export const serveClientLibrary = async function (ctx: any) { +export const serveClientLibrary = async function (ctx: Ctx) { + const appId = context.getAppId() || (ctx.request.query.appId as string) let rootPath = join(NODE_MODULES_PATH, "@budibase", "client", "dist") - // incase running from TS directly - if (env.isDev() && !fs.existsSync(rootPath)) { - rootPath = join(require.resolve("@budibase/client"), "..") + if (!appId) { + ctx.throw(400, "No app ID provided - cannot fetch client library.") + } + if (env.isProd()) { + ctx.body = await objectStore.getReadStream( + ObjectStoreBuckets.APPS, + objectStore.clientLibraryPath(appId!) + ) + ctx.set("Content-Type", "application/javascript") + } else if (env.isDev()) { + // incase running from TS directly + const tsPath = join(require.resolve("@budibase/client"), "..") + return send(ctx, "budibase-client.js", { + root: !fs.existsSync(rootPath) ? tsPath : rootPath, + }) + } else { + ctx.throw(500, "Unable to retrieve client library.") } - return send(ctx, "budibase-client.js", { - root: rootPath, - }) } -export const getSignedUploadURL = async function (ctx: any) { +export const getSignedUploadURL = async function (ctx: Ctx) { // Ensure datasource is valid let datasource try { @@ -246,7 +286,7 @@ export const getSignedUploadURL = async function (ctx: any) { const params = { Bucket: bucket, Key: key } signedUrl = s3.getSignedUrl("putObject", params) publicUrl = `https://${bucket}.s3.${awsRegion}.amazonaws.com/${key}` - } catch (error) { + } catch (error: any) { ctx.throw(400, error) } } diff --git a/packages/server/src/api/controllers/table/index.ts b/packages/server/src/api/controllers/table/index.ts index afb2a9d12d..db2bd672d0 100644 --- a/packages/server/src/api/controllers/table/index.ts +++ b/packages/server/src/api/controllers/table/index.ts @@ -5,18 +5,27 @@ import { isSchema, validate as validateSchema, } from "../../../utilities/schema" -import { isExternalTable, isSQL } from "../../../integrations/utils" +import { + isExternalTable, + isExternalTableID, + isSQL, +} from "../../../integrations/utils" import { events } from "@budibase/backend-core" import { BulkImportRequest, BulkImportResponse, + DocumentType, FetchTablesResponse, + MigrateRequest, + MigrateResponse, + Row, SaveTableRequest, SaveTableResponse, Table, TableResponse, + TableSourceType, UserCtx, - Row, + SEPARATOR, } from "@budibase/types" import sdk from "../../../sdk" import { jsonFromCsvString } from "../../../utilities/csv" @@ -24,12 +33,10 @@ import { builderSocket } from "../../../websockets" import { cloneDeep, isEqual } from "lodash" function pickApi({ tableId, table }: { tableId?: string; table?: Table }) { - if (table && !tableId) { - tableId = table._id - } - if (table && table.type === "external") { + if (table && isExternalTable(table)) { return external - } else if (tableId && isExternalTable(tableId)) { + } + if (tableId && isExternalTableID(tableId)) { return external } return internal @@ -46,8 +53,8 @@ export async function fetch(ctx: UserCtx) { if (entities) { return Object.values(entities).map
((entity: Table) => ({ ...entity, - type: "external", - sourceId: datasource._id, + sourceType: TableSourceType.EXTERNAL, + sourceId: datasource._id!, sql: isSQL(datasource), })) } else { @@ -158,3 +165,19 @@ export async function validateExistingTableImport(ctx: UserCtx) { ctx.status = 422 } } + +export async function migrate(ctx: UserCtx) { + const { oldColumn, newColumn } = ctx.request.body + let tableId = ctx.params.tableId as string + const table = await sdk.tables.getTable(tableId) + let result = await sdk.tables.migrate(table, oldColumn, newColumn) + + for (let table of result.tablesUpdated) { + builderSocket?.emitTableUpdate(ctx, table, { + includeOriginator: true, + }) + } + + ctx.status = 200 + ctx.body = { message: `Column ${oldColumn.name} migrated.` } +} diff --git a/packages/server/src/api/controllers/table/internal.ts b/packages/server/src/api/controllers/table/internal.ts index 822ff8a75d..bb94f2bc01 100644 --- a/packages/server/src/api/controllers/table/internal.ts +++ b/packages/server/src/api/controllers/table/internal.ts @@ -7,6 +7,7 @@ import { SaveTableRequest, SaveTableResponse, Table, + TableSourceType, UserCtx, } from "@budibase/types" import sdk from "../../../sdk" @@ -16,10 +17,11 @@ export async function save(ctx: UserCtx) { let tableToSave: Table & { _rename?: RenameColumn } = { - type: "table", _id: generateTableID(), - views: {}, ...rest, + type: "table", + sourceType: TableSourceType.INTERNAL, + views: {}, } const renaming = tableToSave._rename delete tableToSave._rename diff --git a/packages/server/src/api/controllers/table/utils.ts b/packages/server/src/api/controllers/table/utils.ts index 01aed1a2bb..0cb41d41fb 100644 --- a/packages/server/src/api/controllers/table/utils.ts +++ b/packages/server/src/api/controllers/table/utils.ts @@ -335,29 +335,33 @@ export async function checkForViewUpdates( columnRename?: RenameColumn ) { const views = await getViews() - const tableViews = views.filter(view => view.meta.tableId === table._id) + const tableViews = views.filter(view => view.meta?.tableId === table._id) // Check each table view to see if impacted by this table action for (let view of tableViews) { let needsUpdated = false + const viewMetadata = view.meta as any + if (!viewMetadata) { + continue + } // First check for renames, otherwise check for deletions if (columnRename) { // Update calculation field if required - if (view.meta.field === columnRename.old) { - view.meta.field = columnRename.updated + if (viewMetadata.field === columnRename.old) { + viewMetadata.field = columnRename.updated needsUpdated = true } // Update group by field if required - if (view.meta.groupBy === columnRename.old) { - view.meta.groupBy = columnRename.updated + if (viewMetadata.groupBy === columnRename.old) { + viewMetadata.groupBy = columnRename.updated needsUpdated = true } // Update filters if required - if (view.meta.filters) { - view.meta.filters.forEach((filter: any) => { + if (viewMetadata.filters) { + viewMetadata.filters.forEach((filter: any) => { if (filter.key === columnRename.old) { filter.key = columnRename.updated needsUpdated = true @@ -367,26 +371,26 @@ export async function checkForViewUpdates( } else if (deletedColumns) { deletedColumns.forEach((column: string) => { // Remove calculation statement if required - if (view.meta.field === column) { - delete view.meta.field - delete view.meta.calculation - delete view.meta.groupBy + if (viewMetadata.field === column) { + delete viewMetadata.field + delete viewMetadata.calculation + delete viewMetadata.groupBy needsUpdated = true } // Remove group by field if required - if (view.meta.groupBy === column) { - delete view.meta.groupBy + if (viewMetadata.groupBy === column) { + delete viewMetadata.groupBy needsUpdated = true } // Remove filters referencing deleted field if required - if (view.meta.filters && view.meta.filters.length) { - const initialLength = view.meta.filters.length - view.meta.filters = view.meta.filters.filter((filter: any) => { + if (viewMetadata.filters && viewMetadata.filters.length) { + const initialLength = viewMetadata.filters.length + viewMetadata.filters = viewMetadata.filters.filter((filter: any) => { return filter.key !== column }) - if (initialLength !== view.meta.filters.length) { + if (initialLength !== viewMetadata.filters.length) { needsUpdated = true } } @@ -399,15 +403,16 @@ export async function checkForViewUpdates( (field: any) => field.name == view.groupBy ) const newViewTemplate = viewTemplate( - view.meta, + viewMetadata, groupByField?.type === FieldTypes.ARRAY ) - await saveView(null, view.name, newViewTemplate) - if (!newViewTemplate.meta.schema) { - newViewTemplate.meta.schema = table.schema + const viewName = view.name! + await saveView(null, viewName, newViewTemplate) + if (!newViewTemplate.meta?.schema) { + newViewTemplate.meta!.schema = table.schema } - if (table.views?.[view.name]) { - table.views[view.name] = newViewTemplate.meta as View + if (table.views?.[viewName]) { + table.views[viewName] = newViewTemplate.meta as View } } } diff --git a/packages/server/src/api/controllers/user.ts b/packages/server/src/api/controllers/user.ts index b6c3e7c6bd..108e29fd3d 100644 --- a/packages/server/src/api/controllers/user.ts +++ b/packages/server/src/api/controllers/user.ts @@ -1,14 +1,26 @@ import { generateUserFlagID, InternalTables } from "../../db/utils" import { getFullUser } from "../../utilities/users" import { context } from "@budibase/backend-core" -import { Ctx, UserCtx } from "@budibase/types" +import { + ContextUserMetadata, + Ctx, + FetchUserMetadataResponse, + FindUserMetadataResponse, + Flags, + SetFlagRequest, + UserCtx, + UserMetadata, +} from "@budibase/types" import sdk from "../../sdk" +import { DocumentInsertResponse } from "@budibase/nano" -export async function fetchMetadata(ctx: Ctx) { +export async function fetchMetadata(ctx: Ctx) { ctx.body = await sdk.users.fetchMetadata() } -export async function updateSelfMetadata(ctx: UserCtx) { +export async function updateSelfMetadata( + ctx: UserCtx +) { // overwrite the ID with current users ctx.request.body._id = ctx.user?._id // make sure no stale rev @@ -18,19 +30,21 @@ export async function updateSelfMetadata(ctx: UserCtx) { await updateMetadata(ctx) } -export async function updateMetadata(ctx: UserCtx) { +export async function updateMetadata( + ctx: UserCtx +) { const db = context.getAppDB() const user = ctx.request.body - // this isn't applicable to the user - delete user.roles - const metadata = { + const metadata: ContextUserMetadata = { tableId: InternalTables.USER_METADATA, ...user, } + // this isn't applicable to the user + delete metadata.roles ctx.body = await db.put(metadata) } -export async function destroyMetadata(ctx: UserCtx) { +export async function destroyMetadata(ctx: UserCtx) { const db = context.getAppDB() try { const dbUser = await sdk.users.get(ctx.params.id) @@ -43,11 +57,15 @@ export async function destroyMetadata(ctx: UserCtx) { } } -export async function findMetadata(ctx: UserCtx) { - ctx.body = await getFullUser(ctx, ctx.params.id) +export async function findMetadata( + ctx: UserCtx +) { + ctx.body = await getFullUser(ctx.params.id) } -export async function setFlag(ctx: UserCtx) { +export async function setFlag( + ctx: UserCtx +) { const userId = ctx.user?._id const { flag, value } = ctx.request.body if (!flag) { @@ -55,9 +73,9 @@ export async function setFlag(ctx: UserCtx) { } const flagDocId = generateUserFlagID(userId!) const db = context.getAppDB() - let doc + let doc: Flags try { - doc = await db.get(flagDocId) + doc = await db.get(flagDocId) } catch (err) { doc = { _id: flagDocId } } @@ -66,13 +84,13 @@ export async function setFlag(ctx: UserCtx) { ctx.body = { message: "Flag set successfully" } } -export async function getFlags(ctx: UserCtx) { +export async function getFlags(ctx: UserCtx) { const userId = ctx.user?._id const docId = generateUserFlagID(userId!) const db = context.getAppDB() - let doc + let doc: Flags try { - doc = await db.get(docId) + doc = await db.get(docId) } catch (err) { doc = { _id: docId } } diff --git a/packages/server/src/api/controllers/view/utils.ts b/packages/server/src/api/controllers/view/utils.ts index 1ae065684e..1229ff0e0f 100644 --- a/packages/server/src/api/controllers/view/utils.ts +++ b/packages/server/src/api/controllers/view/utils.ts @@ -7,13 +7,19 @@ import { import env from "../../../environment" import { context } from "@budibase/backend-core" import viewBuilder from "./viewBuilder" -import { Database, DocumentType } from "@budibase/types" +import { + Database, + DBView, + DocumentType, + DesignDocument, + InMemoryView, +} from "@budibase/types" export async function getView(viewName: string) { const db = context.getAppDB() if (env.SELF_HOSTED) { - const designDoc = await db.get("_design/database") - return designDoc.views[viewName] + const designDoc = await db.get("_design/database") + return designDoc.views?.[viewName] } else { // This is a table view, don't read the view from the DB if (viewName.startsWith(DocumentType.TABLE + SEPARATOR)) { @@ -21,7 +27,7 @@ export async function getView(viewName: string) { } try { - const viewDoc = await db.get(generateMemoryViewID(viewName)) + const viewDoc = await db.get(generateMemoryViewID(viewName)) return viewDoc.view } catch (err: any) { // Return null when PouchDB doesn't found the view @@ -34,30 +40,33 @@ export async function getView(viewName: string) { } } -export async function getViews() { +export async function getViews(): Promise { const db = context.getAppDB() - const response = [] + const response: DBView[] = [] if (env.SELF_HOSTED) { - const designDoc = await db.get("_design/database") - for (let name of Object.keys(designDoc.views)) { + const designDoc = await db.get("_design/database") + for (let name of Object.keys(designDoc.views || {})) { // Only return custom views, not built ins const viewNames = Object.values(ViewName) as string[] if (viewNames.indexOf(name) !== -1) { continue } - response.push({ - name, - ...designDoc.views[name], - }) + const view = designDoc.views?.[name] + if (view) { + response.push({ + name, + ...view, + }) + } } } else { const views = ( - await db.allDocs( + await db.allDocs( getMemoryViewParams({ include_docs: true, }) ) - ).rows.map(row => row.doc) + ).rows.map(row => row.doc!) for (let viewDoc of views) { response.push({ name: viewDoc.name, @@ -71,11 +80,11 @@ export async function getViews() { export async function saveView( originalName: string | null, viewName: string, - viewTemplate: any + viewTemplate: DBView ) { const db = context.getAppDB() if (env.SELF_HOSTED) { - const designDoc = await db.get("_design/database") + const designDoc = await db.get("_design/database") designDoc.views = { ...designDoc.views, [viewName]: viewTemplate, @@ -88,17 +97,17 @@ export async function saveView( } else { const id = generateMemoryViewID(viewName) const originalId = originalName ? generateMemoryViewID(originalName) : null - const viewDoc: any = { + const viewDoc: InMemoryView = { _id: id, view: viewTemplate, name: viewName, - tableId: viewTemplate.meta.tableId, + tableId: viewTemplate.meta!.tableId, } try { - const old = await db.get(id) + const old = await db.get(id) if (originalId) { - const originalDoc = await db.get(originalId) - await db.remove(originalDoc._id, originalDoc._rev) + const originalDoc = await db.get(originalId) + await db.remove(originalDoc._id!, originalDoc._rev) } if (old && old._rev) { viewDoc._rev = old._rev @@ -113,52 +122,65 @@ export async function saveView( export async function deleteView(viewName: string) { const db = context.getAppDB() if (env.SELF_HOSTED) { - const designDoc = await db.get("_design/database") - const view = designDoc.views[viewName] - delete designDoc.views[viewName] + const designDoc = await db.get("_design/database") + const view = designDoc.views?.[viewName] + delete designDoc.views?.[viewName] await db.put(designDoc) return view } else { const id = generateMemoryViewID(viewName) - const viewDoc = await db.get(id) - await db.remove(viewDoc._id, viewDoc._rev) + const viewDoc = await db.get(id) + await db.remove(viewDoc._id!, viewDoc._rev) return viewDoc.view } } export async function migrateToInMemoryView(db: Database, viewName: string) { // delete the view initially - const designDoc = await db.get("_design/database") + const designDoc = await db.get("_design/database") + const meta = designDoc.views?.[viewName].meta + if (!meta) { + throw new Error("Unable to migrate view - no metadata") + } // run the view back through the view builder to update it - const view = viewBuilder(designDoc.views[viewName].meta) - delete designDoc.views[viewName] + const view = viewBuilder(meta) + delete designDoc.views?.[viewName] await db.put(designDoc) - await exports.saveView(db, null, viewName, view) + await saveView(null, viewName, view) } export async function migrateToDesignView(db: Database, viewName: string) { - let view = await db.get(generateMemoryViewID(viewName)) - const designDoc = await db.get("_design/database") - designDoc.views[viewName] = viewBuilder(view.view.meta) + let view = await db.get(generateMemoryViewID(viewName)) + const designDoc = await db.get("_design/database") + const meta = view.view.meta + if (!meta) { + throw new Error("Unable to migrate view - no metadata") + } + if (!designDoc.views) { + designDoc.views = {} + } + designDoc.views[viewName] = viewBuilder(meta) await db.put(designDoc) - await db.remove(view._id, view._rev) + await db.remove(view._id!, view._rev) } export async function getFromDesignDoc(db: Database, viewName: string) { - const designDoc = await db.get("_design/database") - let view = designDoc.views[viewName] + const designDoc = await db.get("_design/database") + let view = designDoc.views?.[viewName] if (view == null) { throw { status: 404, message: "Unable to get view" } } return view } -export async function getFromMemoryDoc(db: Database, viewName: string) { - let view = await db.get(generateMemoryViewID(viewName)) +export async function getFromMemoryDoc( + db: Database, + viewName: string +): Promise { + let view = await db.get(generateMemoryViewID(viewName)) if (view) { - view = view.view + return view.view } else { throw { status: 404, message: "Unable to get view" } } - return view } diff --git a/packages/server/src/api/controllers/view/viewBuilder.ts b/packages/server/src/api/controllers/view/viewBuilder.ts index cbe7e72d04..3df9df6657 100644 --- a/packages/server/src/api/controllers/view/viewBuilder.ts +++ b/packages/server/src/api/controllers/view/viewBuilder.ts @@ -1,13 +1,4 @@ -import { ViewFilter } from "@budibase/types" - -type ViewTemplateOpts = { - field: string - tableId: string - groupBy: string - filters: ViewFilter[] - calculation: string - groupByMulti: boolean -} +import { ViewFilter, ViewTemplateOpts, DBView } from "@budibase/types" const TOKEN_MAP: Record = { EQUALS: "===", @@ -146,7 +137,7 @@ function parseEmitExpression(field: string, groupBy: string) { export default function ( { field, tableId, groupBy, filters = [], calculation }: ViewTemplateOpts, groupByMulti?: boolean -) { +): DBView { // first filter can't have a conjunction if (filters && filters.length > 0 && filters[0].conjunction) { delete filters[0].conjunction diff --git a/packages/server/src/api/controllers/view/views.ts b/packages/server/src/api/controllers/view/views.ts index 039c03dcc7..a0a08fc06d 100644 --- a/packages/server/src/api/controllers/view/views.ts +++ b/packages/server/src/api/controllers/view/views.ts @@ -47,8 +47,11 @@ export async function save(ctx: Ctx) { // add views to table document if (!table.views) table.views = {} - if (!view.meta.schema) { - view.meta.schema = table.schema + if (!view.meta?.schema) { + view.meta = { + ...view.meta!, + schema: table.schema, + } } table.views[viewName] = { ...view.meta, name: viewName } if (originalName) { @@ -125,10 +128,13 @@ export async function destroy(ctx: Ctx) { const db = context.getAppDB() const viewName = decodeURIComponent(ctx.params.viewName) const view = await deleteView(viewName) + if (!view || !view.meta) { + ctx.throw(400, "Unable to delete view - no metadata/view not found.") + } const table = await sdk.tables.getTable(view.meta.tableId) delete table.views![viewName] await db.put(table) - await events.view.deleted(view) + await events.view.deleted(view as View) ctx.body = view builderSocket?.emitTableUpdate(ctx, table) @@ -147,7 +153,7 @@ export async function exportView(ctx: Ctx) { ) } - if (view) { + if (view && view.meta) { ctx.params.viewName = viewName // Fetch view rows ctx.query = { diff --git a/packages/server/src/api/routes/public/index.ts b/packages/server/src/api/routes/public/index.ts index 4cc1eff8a4..f27f3f8857 100644 --- a/packages/server/src/api/routes/public/index.ts +++ b/packages/server/src/api/routes/public/index.ts @@ -15,64 +15,71 @@ import env from "../../../environment" const Router = require("@koa/router") const { RateLimit, Stores } = require("koa2-ratelimit") import { middleware, redis } from "@budibase/backend-core" +import { SelectableDatabase } from "@budibase/backend-core/src/redis/utils" -const PREFIX = "/api/public/v1" -// allow a lot more requests when in test -const DEFAULT_API_REQ_LIMIT_PER_SEC = env.isTest() ? 100 : 10 - -function getApiLimitPerSecond(): number { - if (!env.API_REQ_LIMIT_PER_SEC) { - return DEFAULT_API_REQ_LIMIT_PER_SEC +interface KoaRateLimitOptions { + socket: { + host: string + port: number } - return parseInt(env.API_REQ_LIMIT_PER_SEC) + password?: string + database?: number } -let rateLimitStore: any = null -if (!env.isTest()) { - const REDIS_OPTS = redis.utils.getRedisOptions() - let options - if (REDIS_OPTS.redisProtocolUrl) { - // fully qualified redis URL - options = { - url: REDIS_OPTS.redisProtocolUrl, +const PREFIX = "/api/public/v1" + +// type can't be known - untyped libraries +let limiter: any, rateLimitStore: any +if (!env.DISABLE_RATE_LIMITING) { + // allow a lot more requests when in test + const DEFAULT_API_REQ_LIMIT_PER_SEC = env.isTest() ? 100 : 10 + + function getApiLimitPerSecond(): number { + if (!env.API_REQ_LIMIT_PER_SEC) { + return DEFAULT_API_REQ_LIMIT_PER_SEC } - } else { - options = { + return parseInt(env.API_REQ_LIMIT_PER_SEC) + } + + if (!env.isTest()) { + const { password, host, port } = redis.utils.getRedisConnectionDetails() + let options: KoaRateLimitOptions = { socket: { - host: REDIS_OPTS.host, - port: REDIS_OPTS.port, + host: host, + port: port, }, } - if (REDIS_OPTS.opts?.password || REDIS_OPTS.opts.redisOptions?.password) { - // @ts-ignore - options.password = - REDIS_OPTS.opts.password || REDIS_OPTS.opts.redisOptions.password + if (password) { + options.password = password } if (!env.REDIS_CLUSTERED) { - // @ts-ignore // Can't set direct redis db in clustered env - options.database = 1 + options.database = SelectableDatabase.RATE_LIMITING } + rateLimitStore = new Stores.Redis(options) + RateLimit.defaultOptions({ + store: rateLimitStore, + }) } - rateLimitStore = new Stores.Redis(options) - RateLimit.defaultOptions({ - store: rateLimitStore, + // rate limiting, allows for 2 requests per second + limiter = RateLimit.middleware({ + interval: { sec: 1 }, + // per ip, per interval + max: getApiLimitPerSecond(), }) +} else { + console.log("**** PUBLIC API RATE LIMITING DISABLED ****") } -// rate limiting, allows for 2 requests per second -const limiter = RateLimit.middleware({ - interval: { sec: 1 }, - // per ip, per interval - max: getApiLimitPerSecond(), -}) const publicRouter = new Router({ prefix: PREFIX, }) -publicRouter.use(limiter) +if (limiter) { + publicRouter.use(limiter) +} function addMiddleware( endpoints: any, diff --git a/packages/server/src/api/routes/row.ts b/packages/server/src/api/routes/row.ts index c29cb65eac..516bfd20c6 100644 --- a/packages/server/src/api/routes/row.ts +++ b/packages/server/src/api/routes/row.ts @@ -11,128 +11,24 @@ const { PermissionType, PermissionLevel } = permissions const router: Router = new Router() router - /** - * @api {get} /api/:sourceId/:rowId/enrich Get an enriched row - * @apiName Get an enriched row - * @apiGroup rows - * @apiPermission table read access - * @apiDescription This API is only useful when dealing with rows that have relationships. - * Normally when a row is a returned from the API relationships will only have the structure - * `{ primaryDisplay: "name", _id: ... }` but this call will return the full related rows - * for each relationship instead. - * - * @apiParam {string} rowId The ID of the row which is to be retrieved and enriched. - * - * @apiSuccess {object} row The response body will be the enriched row. - */ .get( "/api/:sourceId/:rowId/enrich", paramSubResource("sourceId", "rowId"), authorized(PermissionType.TABLE, PermissionLevel.READ), rowController.fetchEnrichedRow ) - /** - * @api {get} /api/:sourceId/rows Get all rows in a table - * @apiName Get all rows in a table - * @apiGroup rows - * @apiPermission table read access - * @apiDescription This is a deprecated endpoint that should not be used anymore, instead use the search endpoint. - * This endpoint gets all of the rows within the specified table - it is not heavily used - * due to its lack of support for pagination. With SQL tables this will retrieve up to a limit and then - * will simply stop. - * - * @apiParam {string} sourceId The ID of the table to retrieve all rows within. - * - * @apiSuccess {object[]} rows The response body will be an array of all rows found. - */ .get( "/api/:sourceId/rows", paramResource("sourceId"), authorized(PermissionType.TABLE, PermissionLevel.READ), rowController.fetch ) - /** - * @api {get} /api/:sourceId/rows/:rowId Retrieve a single row - * @apiName Retrieve a single row - * @apiGroup rows - * @apiPermission table read access - * @apiDescription This endpoint retrieves only the specified row. If you wish to retrieve - * a row by anything other than its _id field, use the search endpoint. - * - * @apiParam {string} sourceId The ID of the table to retrieve a row from. - * @apiParam {string} rowId The ID of the row to retrieve. - * - * @apiSuccess {object} body The response body will be the row that was found. - */ .get( "/api/:sourceId/rows/:rowId", paramSubResource("sourceId", "rowId"), authorized(PermissionType.TABLE, PermissionLevel.READ), rowController.find ) - /** - * @api {post} /api/:sourceId/search Search for rows in a table - * @apiName Search for rows in a table - * @apiGroup rows - * @apiPermission table read access - * @apiDescription This is the primary method of accessing rows in Budibase, the data provider - * and data UI in the builder are built atop this. All filtering, sorting and pagination is - * handled through this, for internal and external (datasource plus, e.g. SQL) tables. - * - * @apiParam {string} sourceId The ID of the table to retrieve rows from. - * - * @apiParam (Body) {boolean} [paginate] If pagination is required then this should be set to true, - * defaults to false. - * @apiParam (Body) {object} [query] This contains a set of filters which should be applied, if none - * specified then the request will be unfiltered. An example with all of the possible query - * options has been supplied below. - * @apiParam (Body) {number} [limit] This sets a limit for the number of rows that will be returned, - * this will be implemented at the database level if supported for performance reasons. This - * is useful when paginating to set exactly how many rows per page. - * @apiParam (Body) {string} [bookmark] If pagination is enabled then a bookmark will be returned - * with each successful search request, this should be supplied back to get the next page. - * @apiParam (Body) {object} [sort] If sort is desired this should contain the name of the column to - * sort on. - * @apiParam (Body) {string} [sortOrder] If sort is enabled then this can be either "descending" or - * "ascending" as required. - * @apiParam (Body) {string} [sortType] If sort is enabled then you must specify the type of search - * being used, either "string" or "number". This is only used for internal tables. - * - * @apiParamExample {json} Example: - * { - * "tableId": "ta_70260ff0b85c467ca74364aefc46f26d", - * "query": { - * "string": {}, - * "fuzzy": {}, - * "range": { - * "columnName": { - * "high": 20, - * "low": 10, - * } - * }, - * "equal": { - * "columnName": "someValue" - * }, - * "notEqual": {}, - * "empty": {}, - * "notEmpty": {}, - * "oneOf": { - * "columnName": ["value"] - * } - * }, - * "limit": 10, - * "sort": "name", - * "sortOrder": "descending", - * "sortType": "string", - * "paginate": true - * } - * - * @apiSuccess {object[]} rows An array of rows that was found based on the supplied parameters. - * @apiSuccess {boolean} hasNextPage If pagination was enabled then this specifies whether or - * not there is another page after this request. - * @apiSuccess {string} bookmark The bookmark to be sent with the next request to get the next - * page. - */ .post( "/api/:sourceId/search", internalSearchValidator(), @@ -148,30 +44,6 @@ router authorized(PermissionType.TABLE, PermissionLevel.READ), rowController.search ) - /** - * @api {post} /api/:sourceId/rows Creates a new row - * @apiName Creates a new row - * @apiGroup rows - * @apiPermission table write access - * @apiDescription This API will create a new row based on the supplied body. If the - * body includes an "_id" field then it will update an existing row if the field - * links to one. Please note that "_id", "_rev" and "tableId" are fields that are - * already used by Budibase tables and cannot be used for columns. - * - * @apiParam {string} sourceId The ID of the table to save a row to. - * - * @apiParam (Body) {string} [_id] If the row exists already then an ID for the row must be provided. - * @apiParam (Body) {string} [_rev] If working with an existing row for an internal table its revision - * must also be provided. - * @apiParam (Body) {string} tableId The ID of the table should also be specified in the row body itself. - * @apiParam (Body) {any} [any] Any field supplied in the body will be assessed to see if it matches - * a column in the specified table. All other fields will be dropped and not stored. - * - * @apiSuccess {string} _id The ID of the row that was just saved, if it was just created this - * is the rows new ID. - * @apiSuccess {string} [_rev] If saving to an internal table a revision will also be returned. - * @apiSuccess {object} body The contents of the row that was saved will be returned as well. - */ .post( "/api/:sourceId/rows", paramResource("sourceId"), @@ -179,14 +51,6 @@ router trimViewRowInfo, rowController.save ) - /** - * @api {patch} /api/:sourceId/rows Updates a row - * @apiName Update a row - * @apiGroup rows - * @apiPermission table write access - * @apiDescription This endpoint is identical to the row creation endpoint but instead it will - * error if an _id isn't provided, it will only function for existing rows. - */ .patch( "/api/:sourceId/rows", paramResource("sourceId"), @@ -194,52 +58,12 @@ router trimViewRowInfo, rowController.patch ) - /** - * @api {post} /api/:sourceId/rows/validate Validate inputs for a row - * @apiName Validate inputs for a row - * @apiGroup rows - * @apiPermission table write access - * @apiDescription When attempting to save a row you may want to check if the row is valid - * given the table schema, this will iterate through all the constraints on the table and - * check if the request body is valid. - * - * @apiParam {string} sourceId The ID of the table the row is to be validated for. - * - * @apiParam (Body) {any} [any] Any fields provided in the request body will be tested - * against the table schema and constraints. - * - * @apiSuccess {boolean} valid If inputs provided are acceptable within the table schema this - * will be true, if it is not then then errors property will be populated. - * @apiSuccess {object} [errors] A key value map of information about fields on the input - * which do not match the table schema. The key name will be the column names that have breached - * the schema. - */ .post( "/api/:sourceId/rows/validate", paramResource("sourceId"), authorized(PermissionType.TABLE, PermissionLevel.WRITE), rowController.validate ) - /** - * @api {delete} /api/:sourceId/rows Delete rows - * @apiName Delete rows - * @apiGroup rows - * @apiPermission table write access - * @apiDescription This endpoint can delete a single row, or delete them in a bulk - * fashion. - * - * @apiParam {string} sourceId The ID of the table the row is to be deleted from. - * - * @apiParam (Body) {object[]} [rows] If bulk deletion is desired then provide the rows in this - * key of the request body that are to be deleted. - * @apiParam (Body) {string} [_id] If deleting a single row then provide its ID in this field. - * @apiParam (Body) {string} [_rev] If deleting a single row from an internal table then provide its - * revision here. - * - * @apiSuccess {object[]|object} body If deleting bulk then the response body will be an array - * of the deleted rows, if deleting a single row then the body will contain a "row" property which - * is the deleted row. - */ .delete( "/api/:sourceId/rows", paramResource("sourceId"), @@ -247,20 +71,6 @@ router trimViewRowInfo, rowController.destroy ) - - /** - * @api {post} /api/:sourceId/rows/exportRows Export Rows - * @apiName Export rows - * @apiGroup rows - * @apiPermission table write access - * @apiDescription This API can export a number of provided rows - * - * @apiParam {string} sourceId The ID of the table the row is to be deleted from. - * - * @apiParam (Body) {object[]} [rows] The row IDs which are to be exported - * - * @apiSuccess {object[]|object} - */ .post( "/api/:sourceId/rows/exportRows", paramResource("sourceId"), diff --git a/packages/server/src/api/routes/static.ts b/packages/server/src/api/routes/static.ts index 0012764b40..bd3f1aba2e 100644 --- a/packages/server/src/api/routes/static.ts +++ b/packages/server/src/api/routes/static.ts @@ -27,15 +27,9 @@ router.param("file", async (file: any, ctx: any, next: any) => { return next() }) -// only used in development for retrieving the client library, -// in production the client lib is always stored in the object store. -if (env.isDev()) { - router.get("/api/assets/client", controller.serveClientLibrary) -} - router - // TODO: for now this builder endpoint is not authorized/secured, will need to be .get("/builder/:file*", controller.serveBuilder) + .get("/api/assets/client", controller.serveClientLibrary) .post("/api/attachments/process", authorized(BUILDER), controller.uploadFile) .post( "/api/attachments/delete", diff --git a/packages/server/src/api/routes/table.ts b/packages/server/src/api/routes/table.ts index 7ffa5acb3e..b947fa5e0b 100644 --- a/packages/server/src/api/routes/table.ts +++ b/packages/server/src/api/routes/table.ts @@ -9,99 +9,13 @@ const { BUILDER, PermissionLevel, PermissionType } = permissions const router: Router = new Router() router - /** - * @api {get} /api/tables Fetch all tables - * @apiName Fetch all tables - * @apiGroup tables - * @apiPermission table read access - * @apiDescription This endpoint retrieves all of the tables which have been created in - * an app. This includes all of the external and internal tables; to tell the difference - * between these look for the "type" property on each table, either being "internal" or "external". - * - * @apiSuccess {object[]} body The response body will be the list of tables that was found - as - * this does not take any parameters the only error scenario is no access. - */ .get("/api/tables", authorized(BUILDER), tableController.fetch) - /** - * @api {get} /api/tables/:id Fetch a single table - * @apiName Fetch a single table - * @apiGroup tables - * @apiPermission table read access - * @apiDescription Retrieves a single table this could be be internal or external based on - * the provided table ID. - * - * @apiParam {string} id The ID of the table which is to be retrieved. - * - * @apiSuccess {object[]} body The response body will be the table that was found. - */ .get( "/api/tables/:tableId", paramResource("tableId"), authorized(PermissionType.TABLE, PermissionLevel.READ, { schema: true }), tableController.find ) - /** - * @api {post} /api/tables Save a table - * @apiName Save a table - * @apiGroup tables - * @apiPermission builder - * @apiDescription Create or update a table with this endpoint, this will function for both internal - * external tables. - * - * @apiParam (Body) {string} [_id] If updating an existing table then the ID of the table must be specified. - * @apiParam (Body) {string} [_rev] If updating an existing internal table then the revision must also be specified. - * @apiParam (Body) {string} type] This should either be "internal" or "external" depending on the table type - - * this will default to internal. - * @apiParam (Body) {string} [sourceId] If creating an external table then this should be set to the datasource ID. If - * building an internal table this does not need to be set, although it will be returned as "bb_internal". - * @apiParam (Body) {string} name The name of the table, this will be used in the UI. To rename the table simply - * supply the table structure to this endpoint with the name changed. - * @apiParam (Body) {object} schema A key value object which has all of the columns in the table as the keys in this - * object. For each column a "type" and "constraints" must be specified, with some types requiring further information. - * More information about the schema structure can be found in the Typescript definitions. - * @apiParam (Body) {string} [primaryDisplay] The name of the column which should be used when displaying rows - * from this table as relationships. - * @apiParam (Body) {object[]} [indexes] Specifies the search indexes - this is deprecated behaviour with the introduction - * of lucene indexes. This functionality is only available for internal tables. - * @apiParam (Body) {object} [_rename] If a column is to be renamed then the "old" column name should be set in this - * structure, and the "updated", new column name should also be supplied. The schema should also be updated, this field - * lets the server know that a field hasn't just been deleted, that the data has moved to a new name, this will fix - * the rows in the table. This functionality is only available for internal tables. - * @apiParam (Body) {object[]} [rows] When creating a table using a compatible data source, an array of objects to be imported into the new table can be provided. - * - * @apiParamExample {json} Example: - * { - * "_id": "ta_05541307fa0f4044abee071ca2a82119", - * "_rev": "10-0fbe4e78f69b255d79f1017e2eeef807", - * "type": "internal", - * "views": {}, - * "name": "tableName", - * "schema": { - * "column": { - * "type": "string", - * "constraints": { - * "type": "string", - * "length": { - * "maximum": null - * }, - * "presence": false - * }, - * "name": "column" - * }, - * }, - * "primaryDisplay": "column", - * "indexes": [], - * "sourceId": "bb_internal", - * "_rename": { - * "old": "columnName", - * "updated": "newColumnName", - * }, - * "rows": [] - * } - * - * @apiSuccess {object} table The response body will contain the table structure after being cleaned up and - * saved to the database. - */ .post( "/api/tables", // allows control over updating a table @@ -125,41 +39,12 @@ router authorized(BUILDER), tableController.validateExistingTableImport ) - /** - * @api {post} /api/tables/:tableId/:revId Delete a table - * @apiName Delete a table - * @apiGroup tables - * @apiPermission builder - * @apiDescription This endpoint will delete a table and all of its associated data, for this reason it is - * quite dangerous - it will work for internal and external tables. - * - * @apiParam {string} tableId The ID of the table which is to be deleted. - * @apiParam {string} [revId] If deleting an internal table then the revision must also be supplied (_rev), for - * external tables this can simply be set to anything, e.g. "external". - * - * @apiSuccess {string} message A message stating that the table was deleted successfully. - */ .delete( "/api/tables/:tableId/:revId", paramResource("tableId"), authorized(BUILDER), tableController.destroy ) - /** - * @api {post} /api/tables/:tableId/:revId Import CSV to existing table - * @apiName Import CSV to existing table - * @apiGroup tables - * @apiPermission builder - * @apiDescription This endpoint will import data to existing tables, internal or external. It is used in combination - * with the CSV validation endpoint. Take the output of the CSV validation endpoint and pass it to this endpoint to - * import the data; please note this will only import fields that already exist on the table/match the type. - * - * @apiParam {string} tableId The ID of the table which the data should be imported to. - * - * @apiParam (Body) {object[]} rows An array of objects representing the rows to be imported, key-value pairs not matching the table schema will be ignored. - * - * @apiSuccess {string} message A message stating that the data was imported successfully. - */ .post( "/api/tables/:tableId/import", paramResource("tableId"), @@ -167,4 +52,11 @@ router tableController.bulkImport ) + .post( + "/api/tables/:tableId/migrate", + paramResource("tableId"), + authorized(BUILDER), + tableController.migrate + ) + export default router diff --git a/packages/server/src/api/routes/tests/__snapshots__/datasource.spec.ts.snap b/packages/server/src/api/routes/tests/__snapshots__/datasource.spec.ts.snap index 2894f597ab..8dc472173c 100644 --- a/packages/server/src/api/routes/tests/__snapshots__/datasource.spec.ts.snap +++ b/packages/server/src/api/routes/tests/__snapshots__/datasource.spec.ts.snap @@ -7,7 +7,7 @@ exports[`/datasources fetch returns all the datasources from the server 1`] = ` "entities": [ { "_id": "ta_users", - "_rev": "1-2375e1bc58aeec664dc1b1f04ad43e44", + "_rev": "1-73b7912e6cbdd3d696febc60f3715844", "createdAt": "2020-01-01T00:00:00.000Z", "name": "Users", "primaryDisplay": "email", @@ -21,7 +21,6 @@ exports[`/datasources fetch returns all the datasources from the server 1`] = ` "presence": true, "type": "string", }, - "fieldName": "email", "name": "email", "type": "string", }, @@ -30,7 +29,6 @@ exports[`/datasources fetch returns all the datasources from the server 1`] = ` "presence": false, "type": "string", }, - "fieldName": "firstName", "name": "firstName", "type": "string", }, @@ -39,7 +37,6 @@ exports[`/datasources fetch returns all the datasources from the server 1`] = ` "presence": false, "type": "string", }, - "fieldName": "lastName", "name": "lastName", "type": "string", }, @@ -54,7 +51,6 @@ exports[`/datasources fetch returns all the datasources from the server 1`] = ` "presence": false, "type": "string", }, - "fieldName": "roleId", "name": "roleId", "type": "options", }, @@ -67,11 +63,12 @@ exports[`/datasources fetch returns all the datasources from the server 1`] = ` "presence": false, "type": "string", }, - "fieldName": "status", "name": "status", "type": "options", }, }, + "sourceId": "bb_internal", + "sourceType": "internal", "type": "table", "updatedAt": "2020-01-01T00:00:00.000Z", "views": {}, diff --git a/packages/server/src/api/routes/tests/attachment.spec.ts b/packages/server/src/api/routes/tests/attachment.spec.ts new file mode 100644 index 0000000000..14d2e845f6 --- /dev/null +++ b/packages/server/src/api/routes/tests/attachment.spec.ts @@ -0,0 +1,49 @@ +import * as setup from "./utilities" +import { APIError } from "@budibase/types" + +describe("/api/applications/:appId/sync", () => { + let config = setup.getConfig() + + afterAll(setup.afterAll) + beforeAll(async () => { + await config.init() + }) + + describe("/api/attachments/process", () => { + it("should accept an image file upload", async () => { + let resp = await config.api.attachment.process( + "1px.jpg", + Buffer.from([0]) + ) + expect(resp.length).toBe(1) + + let upload = resp[0] + expect(upload.url.endsWith(".jpg")).toBe(true) + expect(upload.extension).toBe("jpg") + expect(upload.size).toBe(1) + expect(upload.name).toBe("1px.jpg") + }) + + it("should reject an upload with a malicious file extension", async () => { + await config.withEnv({ SELF_HOSTED: undefined }, async () => { + let resp = (await config.api.attachment.process( + "ohno.exe", + Buffer.from([0]), + { expectStatus: 400 } + )) as unknown as APIError + expect(resp.message).toContain("invalid extension") + }) + }) + + it("should reject an upload with no file", async () => { + let resp = (await config.api.attachment.process( + undefined as any, + undefined as any, + { + expectStatus: 400, + } + )) as unknown as APIError + expect(resp.message).toContain("No file provided") + }) + }) +}) diff --git a/packages/server/src/api/routes/tests/backup.spec.ts b/packages/server/src/api/routes/tests/backup.spec.ts index 92e0176060..d12b5e1507 100644 --- a/packages/server/src/api/routes/tests/backup.spec.ts +++ b/packages/server/src/api/routes/tests/backup.spec.ts @@ -5,6 +5,8 @@ import sdk from "../../../sdk" import { checkBuilderEndpoint } from "./utilities/TestFunctions" import { mocks } from "@budibase/backend-core/tests" +mocks.licenses.useBackups() + describe("/backups", () => { let request = setup.getRequest() let config = setup.getConfig() @@ -12,16 +14,17 @@ describe("/backups", () => { afterAll(setup.afterAll) beforeEach(async () => { + tk.reset() await config.init() }) - describe("exportAppDump", () => { + describe("/api/backups/export", () => { it("should be able to export app", async () => { - const res = await request - .post(`/api/backups/export?appId=${config.getAppId()}`) - .set(config.defaultHeaders()) - .expect(200) - expect(res.headers["content-type"]).toEqual("application/gzip") + const { body, headers } = await config.api.backup.exportBasicBackup( + config.getAppId()! + ) + expect(body instanceof Buffer).toBe(true) + expect(headers["content-type"]).toEqual("application/gzip") expect(events.app.exported).toBeCalledTimes(1) }) @@ -36,11 +39,11 @@ describe("/backups", () => { it("should infer the app name from the app", async () => { tk.freeze(mocks.date.MOCK_DATE) - const res = await request - .post(`/api/backups/export?appId=${config.getAppId()}`) - .set(config.defaultHeaders()) + const { headers } = await config.api.backup.exportBasicBackup( + config.getAppId()! + ) - expect(res.headers["content-disposition"]).toEqual( + expect(headers["content-disposition"]).toEqual( `attachment; filename="${ config.getApp()!.name }-export-${mocks.date.MOCK_DATE.getTime()}.tar.gz"` @@ -48,6 +51,21 @@ describe("/backups", () => { }) }) + describe("/api/backups/import", () => { + it("should be able to import an app", async () => { + const appId = config.getAppId()! + const automation = await config.createAutomation() + await config.createAutomationLog(automation, appId) + await config.createScreen() + const exportRes = await config.api.backup.createBackup(appId) + expect(exportRes.backupId).toBeDefined() + const importRes = await config.api.backup.importBackup( + appId, + exportRes.backupId + ) + }) + }) + describe("calculateBackupStats", () => { it("should be able to calculate the backup statistics", async () => { await config.createAutomation() diff --git a/packages/server/src/api/routes/tests/debug.spec.ts b/packages/server/src/api/routes/tests/debug.spec.ts index 23ee43fc73..26e98d93f9 100644 --- a/packages/server/src/api/routes/tests/debug.spec.ts +++ b/packages/server/src/api/routes/tests/debug.spec.ts @@ -41,7 +41,7 @@ describe("/component", () => { .expect("Content-Type", /json/) .expect(200) expect(res.body).toEqual({ - budibaseVersion: "0.0.0", + budibaseVersion: "0.0.0+jest", cpuArch: "arm64", cpuCores: 1, cpuInfo: "test", diff --git a/packages/server/src/api/routes/tests/dev.spec.js b/packages/server/src/api/routes/tests/dev.spec.js index 111f3dbd5b..af1dc82a9d 100644 --- a/packages/server/src/api/routes/tests/dev.spec.js +++ b/packages/server/src/api/routes/tests/dev.spec.js @@ -1,6 +1,6 @@ const setup = require("./utilities") const { events } = require("@budibase/backend-core") -const version = require("../../../../package.json").version + describe("/dev", () => { let request = setup.getRequest() @@ -32,9 +32,9 @@ describe("/dev", () => { .expect("Content-Type", /json/) .expect(200) - expect(res.body.version).toBe(version) + expect(res.body.version).toBe('0.0.0+jest') expect(events.installation.versionChecked).toBeCalledTimes(1) - expect(events.installation.versionChecked).toBeCalledWith(version) + expect(events.installation.versionChecked).toBeCalledWith('0.0.0+jest') }) }) }) \ No newline at end of file diff --git a/packages/server/src/api/routes/tests/role.spec.js b/packages/server/src/api/routes/tests/role.spec.js index c8e383d5ed..d133a69d64 100644 --- a/packages/server/src/api/routes/tests/role.spec.js +++ b/packages/server/src/api/routes/tests/role.spec.js @@ -158,5 +158,25 @@ describe("/roles", () => { expect(res.body.length).toBe(1) expect(res.body[0]).toBe("PUBLIC") }) + + it("should not fetch higher level accessible roles when a custom role header is provided", async () => { + await createRole({ + name: `CUSTOM_ROLE`, + inherits: roles.BUILTIN_ROLE_IDS.BASIC, + permissionId: permissions.BuiltinPermissionID.READ_ONLY, + version: "name", + }) + const res = await request + .get("/api/roles/accessible") + .set({ + ...config.defaultHeaders(), + "x-budibase-role": "CUSTOM_ROLE" + }) + .expect(200) + expect(res.body.length).toBe(3) + expect(res.body[0]).toBe("CUSTOM_ROLE") + expect(res.body[1]).toBe("BASIC") + expect(res.body[2]).toBe("PUBLIC") + }) }) }) diff --git a/packages/server/src/api/routes/tests/routing.spec.js b/packages/server/src/api/routes/tests/routing.spec.js index ff6d7aba1d..4076f4879c 100644 --- a/packages/server/src/api/routes/tests/routing.spec.js +++ b/packages/server/src/api/routes/tests/routing.spec.js @@ -1,5 +1,5 @@ const setup = require("./utilities") -const { basicScreen } = setup.structures +const { basicScreen, powerScreen } = setup.structures const { checkBuilderEndpoint, runInProd } = require("./utilities/TestFunctions") const { roles } = require("@budibase/backend-core") const { BUILTIN_ROLE_IDS } = roles @@ -12,19 +12,14 @@ const route = "/test" describe("/routing", () => { let request = setup.getRequest() let config = setup.getConfig() - let screen, screen2 + let basic, power afterAll(setup.afterAll) beforeAll(async () => { await config.init() - screen = basicScreen() - screen.routing.route = route - screen = await config.createScreen(screen) - screen2 = basicScreen() - screen2.routing.roleId = BUILTIN_ROLE_IDS.POWER - screen2.routing.route = route - screen2 = await config.createScreen(screen2) + basic = await config.createScreen(basicScreen(route)) + power = await config.createScreen(powerScreen(route)) await config.publish() }) @@ -61,8 +56,8 @@ describe("/routing", () => { expect(res.body.routes[route]).toEqual({ subpaths: { [route]: { - screenId: screen._id, - roleId: screen.routing.roleId + screenId: basic._id, + roleId: basic.routing.roleId } } }) @@ -80,8 +75,8 @@ describe("/routing", () => { expect(res.body.routes[route]).toEqual({ subpaths: { [route]: { - screenId: screen2._id, - roleId: screen2.routing.roleId + screenId: power._id, + roleId: power.routing.roleId } } }) @@ -101,8 +96,8 @@ describe("/routing", () => { expect(res.body.routes).toBeDefined() expect(res.body.routes[route].subpaths[route]).toBeDefined() const subpath = res.body.routes[route].subpaths[route] - expect(subpath.screens[screen2.routing.roleId]).toEqual(screen2._id) - expect(subpath.screens[screen.routing.roleId]).toEqual(screen._id) + expect(subpath.screens[power.routing.roleId]).toEqual(power._id) + expect(subpath.screens[basic.routing.roleId]).toEqual(basic._id) }) it("make sure it is a builder only endpoint", async () => { diff --git a/packages/server/src/api/routes/tests/row.spec.ts b/packages/server/src/api/routes/tests/row.spec.ts index 4c2e7a7494..060f6e46c1 100644 --- a/packages/server/src/api/routes/tests/row.spec.ts +++ b/packages/server/src/api/routes/tests/row.spec.ts @@ -10,6 +10,8 @@ import { FieldSchema, FieldType, FieldTypeSubtypes, + FormulaTypes, + INTERNAL_TABLE_SOURCE_ID, MonthlyQuotaName, PermissionLevel, QuotaUsageType, @@ -21,6 +23,7 @@ import { SortType, StaticQuotaName, Table, + TableSourceType, } from "@budibase/types" import { expectAnyExternalColsAttributes, @@ -30,6 +33,7 @@ import { structures, } from "@budibase/backend-core/tests" import _ from "lodash" +import * as uuid from "uuid" const timestamp = new Date("2023-01-26T11:48:57.597Z").toISOString() tk.freeze(timestamp) @@ -47,7 +51,12 @@ describe.each([ let table: Table let tableId: string - afterAll(setup.afterAll) + afterAll(async () => { + if (dsProvider) { + await dsProvider.stopContainer() + } + setup.afterAll() + }) beforeAll(async () => { await config.init() @@ -61,10 +70,12 @@ describe.each([ const generateTableConfig: () => SaveTableRequest = () => { return { - name: generator.word(), + name: uuid.v4(), type: "table", primary: ["id"], primaryDisplay: "name", + sourceType: TableSourceType.INTERNAL, + sourceId: INTERNAL_TABLE_SOURCE_ID, schema: { id: { type: FieldType.AUTO, @@ -134,9 +145,22 @@ describe.each([ } : undefined + async function createTable( + cfg: Omit, + opts?: { skipReassigning: boolean } + ) { + let table + if (dsProvider) { + table = await config.createExternalTable(cfg, opts) + } else { + table = await config.createTable(cfg, opts) + } + return table + } + beforeAll(async () => { const tableConfig = generateTableConfig() - const table = await config.createTable(tableConfig) + let table = await createTable(tableConfig) tableId = table._id! }) @@ -165,7 +189,7 @@ describe.each([ const queryUsage = await getQueryUsage() const tableConfig = generateTableConfig() - const newTable = await config.createTable( + const newTable = await createTable( { ...tableConfig, name: "TestTableAuto", @@ -242,7 +266,7 @@ describe.each([ }) it("should list all rows for given tableId", async () => { - const table = await config.createTable(generateTableConfig(), { + const table = await createTable(generateTableConfig(), { skipReassigning: true, }) const tableId = table._id! @@ -323,7 +347,7 @@ describe.each([ inclusion: ["Alpha", "Beta", "Gamma"], }, } - const table = await config.createTable({ + const table = await createTable({ name: "TestTable2", type: "table", schema: { @@ -438,7 +462,8 @@ describe.each([ describe("view save", () => { it("views have extra data trimmed", async () => { - const table = await config.createTable({ + const table = await createTable({ + type: "table", name: "orders", primary: ["OrderID"], schema: { @@ -458,7 +483,7 @@ describe.each([ }) const createViewResponse = await config.createView({ - name: generator.word(), + name: uuid.v4(), schema: { Country: { visible: true, @@ -494,7 +519,7 @@ describe.each([ describe("patch", () => { beforeAll(async () => { const tableConfig = generateTableConfig() - table = await config.createTable(tableConfig) + table = await createTable(tableConfig) }) it("should update only the fields that are supplied", async () => { @@ -503,20 +528,17 @@ describe.each([ const rowUsage = await getRowUsage() const queryUsage = await getQueryUsage() - const res = await config.api.row.patch(table._id!, { + const row = await config.api.row.patch(table._id!, { _id: existing._id!, _rev: existing._rev!, tableId: table._id!, name: "Updated Name", }) - expect((res as any).res.statusMessage).toEqual( - `${table.name} updated successfully.` - ) - expect(res.body.name).toEqual("Updated Name") - expect(res.body.description).toEqual(existing.description) + expect(row.name).toEqual("Updated Name") + expect(row.description).toEqual(existing.description) - const savedRow = await loadRow(res.body._id, table._id!) + const savedRow = await loadRow(row._id!, table._id!) expect(savedRow.body.description).toEqual(existing.description) expect(savedRow.body.name).toEqual("Updated Name") @@ -543,12 +565,62 @@ describe.each([ await assertRowUsage(rowUsage) await assertQueryUsage(queryUsage) }) + + it("should not overwrite links if those links are not set", async () => { + let linkField: FieldSchema = { + type: FieldType.LINK, + name: "", + fieldName: "", + constraints: { + type: "array", + presence: false, + }, + relationshipType: RelationshipType.ONE_TO_MANY, + tableId: InternalTable.USER_METADATA, + } + + let table = await config.api.table.create({ + name: "TestTable", + type: "table", + sourceType: TableSourceType.INTERNAL, + sourceId: INTERNAL_TABLE_SOURCE_ID, + schema: { + user1: { ...linkField, name: "user1", fieldName: "user1" }, + user2: { ...linkField, name: "user2", fieldName: "user2" }, + }, + }) + + let user1 = await config.createUser() + let user2 = await config.createUser() + + let row = await config.api.row.save(table._id!, { + user1: [{ _id: user1._id }], + user2: [{ _id: user2._id }], + }) + + let getResp = await config.api.row.get(table._id!, row._id!) + expect(getResp.body.user1[0]._id).toEqual(user1._id) + expect(getResp.body.user2[0]._id).toEqual(user2._id) + + let patchResp = await config.api.row.patch(table._id!, { + _id: row._id!, + _rev: row._rev!, + tableId: table._id!, + user1: [{ _id: user2._id }], + }) + expect(patchResp.user1[0]._id).toEqual(user2._id) + expect(patchResp.user2[0]._id).toEqual(user2._id) + + getResp = await config.api.row.get(table._id!, row._id!) + expect(getResp.body.user1[0]._id).toEqual(user2._id) + expect(getResp.body.user2[0]._id).toEqual(user2._id) + }) }) describe("destroy", () => { beforeAll(async () => { const tableConfig = generateTableConfig() - table = await config.createTable(tableConfig) + table = await createTable(tableConfig) }) it("should be able to delete a row", async () => { @@ -566,7 +638,7 @@ describe.each([ describe("validate", () => { beforeAll(async () => { const tableConfig = generateTableConfig() - table = await config.createTable(tableConfig) + table = await createTable(tableConfig) }) it("should return no errors on valid row", async () => { @@ -603,7 +675,7 @@ describe.each([ describe("bulkDelete", () => { beforeAll(async () => { const tableConfig = generateTableConfig() - table = await config.createTable(tableConfig) + table = await createTable(tableConfig) }) it("should be able to delete a bulk set of rows", async () => { @@ -687,7 +759,7 @@ describe.each([ describe("fetchView", () => { beforeEach(async () => { const tableConfig = generateTableConfig() - table = await config.createTable(tableConfig) + table = await createTable(tableConfig) }) it("should be able to fetch tables contents via 'view'", async () => { @@ -735,7 +807,7 @@ describe.each([ describe("fetchEnrichedRows", () => { beforeAll(async () => { const tableConfig = generateTableConfig() - table = await config.createTable(tableConfig) + table = await createTable(tableConfig) }) it("should allow enriching some linked rows", async () => { @@ -746,7 +818,8 @@ describe.each([ RelationshipType.ONE_TO_MANY, ["link"], { - name: generator.word(), + // Making sure that the combined table name + column name is within postgres limits + name: uuid.v4().replace(/-/g, "").substring(0, 16), type: "table", primary: ["id"], primaryDisplay: "id", @@ -808,7 +881,7 @@ describe.each([ describe("attachments", () => { beforeAll(async () => { const tableConfig = generateTableConfig() - table = await config.createTable(tableConfig) + table = await createTable(tableConfig) }) it("should allow enriching attachment rows", async () => { @@ -839,7 +912,7 @@ describe.each([ describe("exportData", () => { beforeAll(async () => { const tableConfig = generateTableConfig() - table = await config.createTable(tableConfig) + table = await createTable(tableConfig) }) it("should allow exporting all columns", async () => { @@ -879,7 +952,9 @@ describe.each([ describe("view 2.0", () => { async function userTable(): Promise
{ return { - name: `users_${generator.word()}`, + name: `users_${uuid.v4()}`, + sourceId: INTERNAL_TABLE_SOURCE_ID, + sourceType: TableSourceType.INTERNAL, type: "table", primary: ["id"], schema: { @@ -925,7 +1000,7 @@ describe.each([ describe("create", () => { it("should persist a new row with only the provided view fields", async () => { - const table = await config.createTable(await userTable()) + const table = await createTable(await userTable()) const view = await config.createView({ schema: { name: { visible: true }, @@ -960,7 +1035,7 @@ describe.each([ describe("patch", () => { it("should update only the view fields for a row", async () => { - const table = await config.createTable(await userTable()) + const table = await createTable(await userTable()) const tableId = table._id! const view = await config.createView({ schema: { @@ -1001,7 +1076,7 @@ describe.each([ describe("destroy", () => { it("should be able to delete a row", async () => { - const table = await config.createTable(await userTable()) + const table = await createTable(await userTable()) const tableId = table._id! const view = await config.createView({ schema: { @@ -1025,7 +1100,7 @@ describe.each([ }) it("should be able to delete multiple rows", async () => { - const table = await config.createTable(await userTable()) + const table = await createTable(await userTable()) const tableId = table._id! const view = await config.createView({ schema: { @@ -1061,7 +1136,9 @@ describe.each([ const viewSchema = { age: { visible: true }, name: { visible: true } } async function userTable(): Promise
{ return { - name: `users_${generator.word()}`, + name: `users_${uuid.v4()}`, + sourceId: INTERNAL_TABLE_SOURCE_ID, + sourceType: TableSourceType.INTERNAL, type: "table", primary: ["id"], schema: { @@ -1088,7 +1165,7 @@ describe.each([ } it("returns empty rows from view when no schema is passed", async () => { - const table = await config.createTable(await userTable()) + const table = await createTable(await userTable()) const rows = await Promise.all( Array.from({ length: 10 }, () => config.api.row.save(table._id!, { tableId: table._id }) @@ -1119,7 +1196,7 @@ describe.each([ }) it("searching respects the view filters", async () => { - const table = await config.createTable(await userTable()) + const table = await createTable(await userTable()) await Promise.all( Array.from({ length: 10 }, () => @@ -1243,7 +1320,7 @@ describe.each([ describe("sorting", () => { beforeAll(async () => { - const table = await config.createTable(await userTable()) + const table = await createTable(await userTable()) const users = [ { name: "Alice", age: 25 }, { name: "Bob", age: 30 }, @@ -1310,7 +1387,7 @@ describe.each([ }) it("when schema is defined, defined columns and row attributes are returned", async () => { - const table = await config.createTable(await userTable()) + const table = await createTable(await userTable()) const rows = await Promise.all( Array.from({ length: 10 }, () => config.api.row.save(table._id!, { @@ -1341,7 +1418,7 @@ describe.each([ }) it("views without data can be returned", async () => { - const table = await config.createTable(await userTable()) + const table = await createTable(await userTable()) const createViewResponse = await config.createView() const response = await config.api.viewV2.search(createViewResponse.id) @@ -1350,7 +1427,7 @@ describe.each([ }) it("respects the limit parameter", async () => { - await config.createTable(await userTable()) + await createTable(await userTable()) await Promise.all(Array.from({ length: 10 }, () => config.createRow())) const limit = generator.integer({ min: 1, max: 8 }) @@ -1365,7 +1442,7 @@ describe.each([ }) it("can handle pagination", async () => { - await config.createTable(await userTable()) + await createTable(await userTable()) await Promise.all(Array.from({ length: 10 }, () => config.createRow())) const createViewResponse = await config.createView() @@ -1443,7 +1520,7 @@ describe.each([ let tableId: string beforeAll(async () => { - await config.createTable(await userTable()) + await createTable(await userTable()) await Promise.all( Array.from({ length: 10 }, () => config.createRow()) ) @@ -1521,13 +1598,13 @@ describe.each([ let o2mTable: Table let m2mTable: Table beforeAll(async () => { - o2mTable = await config.createTable( + o2mTable = await createTable( { ...generateTableConfig(), name: "o2m" }, { skipReassigning: true, } ) - m2mTable = await config.createTable( + m2mTable = await createTable( { ...generateTableConfig(), name: "m2m" }, { skipReassigning: true, @@ -1556,7 +1633,7 @@ describe.each([ }), (tableId: string) => config.api.row.save(tableId, { - name: generator.word(), + name: uuid.v4(), description: generator.paragraph(), tableId, }), @@ -1597,9 +1674,9 @@ describe.each([ const tableConfig = generateTableConfig() if (config.datasource) { - tableConfig.sourceId = config.datasource._id + tableConfig.sourceId = config.datasource._id! if (config.datasource.plus) { - tableConfig.type = "external" + tableConfig.sourceType = TableSourceType.EXTERNAL } } const table = await config.api.table.create({ @@ -1924,4 +2001,52 @@ describe.each([ }) }) }) + + describe("Formula fields", () => { + let relationshipTable: Table, tableId: string, relatedRow: Row + + beforeAll(async () => { + const otherTableId = config.table!._id! + const cfg = generateTableConfig() + relationshipTable = await config.createLinkedTable( + RelationshipType.ONE_TO_MANY, + ["links"], + { + ...cfg, + // needs to be a short name + name: "b", + schema: { + ...cfg.schema, + formula: { + name: "formula", + type: FieldType.FORMULA, + formula: "{{ links.0.name }}", + formulaType: FormulaTypes.DYNAMIC, + }, + }, + } + ) + + tableId = relationshipTable._id! + + relatedRow = await config.api.row.save(otherTableId, { + name: generator.word(), + description: generator.paragraph(), + }) + await config.api.row.save(tableId, { + name: generator.word(), + description: generator.paragraph(), + tableId, + links: [relatedRow._id], + }) + }) + + it("should be able to search for rows containing formulas", async () => { + const { rows } = await config.api.row.search(tableId) + expect(rows.length).toBe(1) + expect(rows[0].links.length).toBe(1) + const row = rows[0] + expect(row.formula).toBe(relatedRow.name) + }) + }) }) diff --git a/packages/server/src/api/routes/tests/static.spec.js b/packages/server/src/api/routes/tests/static.spec.js index 13d963d057..a28d9ecd79 100644 --- a/packages/server/src/api/routes/tests/static.spec.js +++ b/packages/server/src/api/routes/tests/static.spec.js @@ -5,11 +5,15 @@ describe("/static", () => { let request = setup.getRequest() let config = setup.getConfig() let app + let cleanupEnv - afterAll(setup.afterAll) + afterAll(() => { + setup.afterAll() + cleanupEnv() + }) beforeAll(async () => { - config.modeSelf() + cleanupEnv = config.setEnv({ SELF_HOSTED: "true" }) app = await config.init() }) diff --git a/packages/server/src/api/routes/tests/table.spec.ts b/packages/server/src/api/routes/tests/table.spec.ts index ded54729b9..4743bca814 100644 --- a/packages/server/src/api/routes/tests/table.spec.ts +++ b/packages/server/src/api/routes/tests/table.spec.ts @@ -1,16 +1,24 @@ -import { events, context } from "@budibase/backend-core" +import { context, events } from "@budibase/backend-core" import { - FieldType, - SaveTableRequest, - RelationshipType, - Table, - ViewCalculation, AutoFieldSubTypes, + FieldSubtype, + FieldType, + INTERNAL_TABLE_SOURCE_ID, + InternalTable, + RelationshipType, + Row, + SaveTableRequest, + Table, + TableSourceType, + User, + ViewCalculation, } from "@budibase/types" import { checkBuilderEndpoint } from "./utilities/TestFunctions" import * as setup from "./utilities" -const { basicTable } = setup.structures import sdk from "../../../sdk" +import uuid from "uuid" + +const { basicTable } = setup.structures describe("/tables", () => { let request = setup.getRequest() @@ -239,7 +247,8 @@ describe("/tables", () => { .expect(200) const fetchedTable = res.body[0] expect(fetchedTable.name).toEqual(testTable.name) - expect(fetchedTable.type).toEqual("internal") + expect(fetchedTable.type).toEqual("table") + expect(fetchedTable.sourceType).toEqual("internal") }) it("should apply authorization to endpoint", async () => { @@ -417,4 +426,342 @@ describe("/tables", () => { }) }) }) + + describe("migrate", () => { + let users: User[] + beforeAll(async () => { + users = await Promise.all([ + config.createUser({ email: `${uuid.v4()}@example.com` }), + config.createUser({ email: `${uuid.v4()}@example.com` }), + config.createUser({ email: `${uuid.v4()}@example.com` }), + ]) + }) + + it("should successfully migrate a one-to-many user relationship to a user column", async () => { + const table = await config.api.table.create({ + name: "table", + type: "table", + sourceId: INTERNAL_TABLE_SOURCE_ID, + sourceType: TableSourceType.INTERNAL, + schema: { + "user relationship": { + type: FieldType.LINK, + fieldName: "test", + name: "user relationship", + constraints: { + type: "array", + presence: false, + }, + relationshipType: RelationshipType.ONE_TO_MANY, + tableId: InternalTable.USER_METADATA, + }, + }, + }) + + const rows = await Promise.all( + users.map(u => + config.api.row.save(table._id!, { "user relationship": [u] }) + ) + ) + + await config.api.table.migrate(table._id!, { + oldColumn: table.schema["user relationship"], + newColumn: { + name: "user column", + type: FieldType.BB_REFERENCE, + subtype: FieldSubtype.USER, + }, + }) + + const migratedTable = await config.api.table.get(table._id!) + expect(migratedTable.schema["user column"]).toBeDefined() + expect(migratedTable.schema["user relationship"]).not.toBeDefined() + + const migratedRows = await config.api.row.fetch(table._id!) + + rows.sort((a, b) => a._id!.localeCompare(b._id!)) + migratedRows.sort((a, b) => a._id!.localeCompare(b._id!)) + + for (const [i, row] of rows.entries()) { + const migratedRow = migratedRows[i] + expect(migratedRow["user column"]).toBeDefined() + expect(migratedRow["user relationship"]).not.toBeDefined() + expect(row["user relationship"][0]._id).toEqual( + migratedRow["user column"][0]._id + ) + } + }) + + it("should succeed when the row is created from the other side of the relationship", async () => { + // We found a bug just after releasing this feature where if the row was created from the + // users table, not the table linking to it, the migration would succeed but lose the data. + // This happened because the order of the documents in the link was reversed. + const table = await config.api.table.create({ + name: "table", + type: "table", + sourceId: INTERNAL_TABLE_SOURCE_ID, + sourceType: TableSourceType.INTERNAL, + schema: { + "user relationship": { + type: FieldType.LINK, + fieldName: "test", + name: "user relationship", + constraints: { + type: "array", + presence: false, + }, + relationshipType: RelationshipType.MANY_TO_ONE, + tableId: InternalTable.USER_METADATA, + }, + }, + }) + + let testRow = await config.api.row.save(table._id!, {}) + + await Promise.all( + users.map(u => + config.api.row.patch(InternalTable.USER_METADATA, { + tableId: InternalTable.USER_METADATA, + _rev: u._rev!, + _id: u._id!, + test: [testRow], + }) + ) + ) + + await config.api.table.migrate(table._id!, { + oldColumn: table.schema["user relationship"], + newColumn: { + name: "user column", + type: FieldType.BB_REFERENCE, + subtype: FieldSubtype.USERS, + }, + }) + + const migratedTable = await config.api.table.get(table._id!) + expect(migratedTable.schema["user column"]).toBeDefined() + expect(migratedTable.schema["user relationship"]).not.toBeDefined() + + const resp = await config.api.row.get(table._id!, testRow._id!) + const migratedRow = resp.body as Row + + expect(migratedRow["user column"]).toBeDefined() + expect(migratedRow["user relationship"]).not.toBeDefined() + expect(migratedRow["user column"]).toHaveLength(3) + expect(migratedRow["user column"].map((u: Row) => u._id)).toEqual( + expect.arrayContaining(users.map(u => u._id)) + ) + }) + + it("should successfully migrate a many-to-many user relationship to a users column", async () => { + const table = await config.api.table.create({ + name: "table", + type: "table", + sourceId: INTERNAL_TABLE_SOURCE_ID, + sourceType: TableSourceType.INTERNAL, + schema: { + "user relationship": { + type: FieldType.LINK, + fieldName: "test", + name: "user relationship", + constraints: { + type: "array", + presence: false, + }, + relationshipType: RelationshipType.MANY_TO_MANY, + tableId: InternalTable.USER_METADATA, + }, + }, + }) + + const row1 = await config.api.row.save(table._id!, { + "user relationship": [users[0], users[1]], + }) + + const row2 = await config.api.row.save(table._id!, { + "user relationship": [users[1], users[2]], + }) + + await config.api.table.migrate(table._id!, { + oldColumn: table.schema["user relationship"], + newColumn: { + name: "user column", + type: FieldType.BB_REFERENCE, + subtype: FieldSubtype.USERS, + }, + }) + + const migratedTable = await config.api.table.get(table._id!) + expect(migratedTable.schema["user column"]).toBeDefined() + expect(migratedTable.schema["user relationship"]).not.toBeDefined() + + const row1Migrated = (await config.api.row.get(table._id!, row1._id!)) + .body as Row + expect(row1Migrated["user relationship"]).not.toBeDefined() + expect(row1Migrated["user column"].map((r: Row) => r._id)).toEqual( + expect.arrayContaining([users[0]._id, users[1]._id]) + ) + + const row2Migrated = (await config.api.row.get(table._id!, row2._id!)) + .body as Row + expect(row2Migrated["user relationship"]).not.toBeDefined() + expect(row2Migrated["user column"].map((r: Row) => r._id)).toEqual( + expect.arrayContaining([users[1]._id, users[2]._id]) + ) + }) + + it("should successfully migrate a many-to-one user relationship to a users column", async () => { + const table = await config.api.table.create({ + name: "table", + type: "table", + sourceId: INTERNAL_TABLE_SOURCE_ID, + sourceType: TableSourceType.INTERNAL, + schema: { + "user relationship": { + type: FieldType.LINK, + fieldName: "test", + name: "user relationship", + constraints: { + type: "array", + presence: false, + }, + relationshipType: RelationshipType.MANY_TO_ONE, + tableId: InternalTable.USER_METADATA, + }, + }, + }) + + const row1 = await config.api.row.save(table._id!, { + "user relationship": [users[0], users[1]], + }) + + const row2 = await config.api.row.save(table._id!, { + "user relationship": [users[2]], + }) + + await config.api.table.migrate(table._id!, { + oldColumn: table.schema["user relationship"], + newColumn: { + name: "user column", + type: FieldType.BB_REFERENCE, + subtype: FieldSubtype.USERS, + }, + }) + + const migratedTable = await config.api.table.get(table._id!) + expect(migratedTable.schema["user column"]).toBeDefined() + expect(migratedTable.schema["user relationship"]).not.toBeDefined() + + const row1Migrated = (await config.api.row.get(table._id!, row1._id!)) + .body as Row + expect(row1Migrated["user relationship"]).not.toBeDefined() + expect(row1Migrated["user column"].map((r: Row) => r._id)).toEqual( + expect.arrayContaining([users[0]._id, users[1]._id]) + ) + + const row2Migrated = (await config.api.row.get(table._id!, row2._id!)) + .body as Row + expect(row2Migrated["user relationship"]).not.toBeDefined() + expect(row2Migrated["user column"].map((r: Row) => r._id)).toEqual([ + users[2]._id, + ]) + }) + + describe("unhappy paths", () => { + let table: Table + beforeAll(async () => { + table = await config.api.table.create({ + name: "table", + type: "table", + sourceId: INTERNAL_TABLE_SOURCE_ID, + sourceType: TableSourceType.INTERNAL, + schema: { + "user relationship": { + type: FieldType.LINK, + fieldName: "test", + name: "user relationship", + constraints: { + type: "array", + presence: false, + }, + relationshipType: RelationshipType.MANY_TO_ONE, + tableId: InternalTable.USER_METADATA, + }, + num: { + type: FieldType.NUMBER, + name: "num", + constraints: { + type: "number", + presence: false, + }, + }, + }, + }) + }) + + it("should fail if the new column name is blank", async () => { + await config.api.table.migrate( + table._id!, + { + oldColumn: table.schema["user relationship"], + newColumn: { + name: "", + type: FieldType.BB_REFERENCE, + subtype: FieldSubtype.USERS, + }, + }, + { expectStatus: 400 } + ) + }) + + it("should fail if the new column name is a reserved name", async () => { + await config.api.table.migrate( + table._id!, + { + oldColumn: table.schema["user relationship"], + newColumn: { + name: "_id", + type: FieldType.BB_REFERENCE, + subtype: FieldSubtype.USERS, + }, + }, + { expectStatus: 400 } + ) + }) + + it("should fail if the new column name is the same as an existing column", async () => { + await config.api.table.migrate( + table._id!, + { + oldColumn: table.schema["user relationship"], + newColumn: { + name: "num", + type: FieldType.BB_REFERENCE, + subtype: FieldSubtype.USERS, + }, + }, + { expectStatus: 400 } + ) + }) + + it("should fail if the old column name isn't a column in the table", async () => { + await config.api.table.migrate( + table._id!, + { + oldColumn: { + name: "not a column", + type: FieldType.BB_REFERENCE, + subtype: FieldSubtype.USERS, + }, + newColumn: { + name: "new column", + type: FieldType.BB_REFERENCE, + subtype: FieldSubtype.USERS, + }, + }, + { expectStatus: 400 } + ) + }) + }) + }) }) diff --git a/packages/server/src/api/routes/tests/user.spec.js b/packages/server/src/api/routes/tests/user.spec.js deleted file mode 100644 index e8ffd8df2b..0000000000 --- a/packages/server/src/api/routes/tests/user.spec.js +++ /dev/null @@ -1,208 +0,0 @@ -const { roles, utils } = require("@budibase/backend-core") -const { checkPermissionsEndpoint } = require("./utilities/TestFunctions") -const setup = require("./utilities") -const { BUILTIN_ROLE_IDS } = roles - -jest.setTimeout(30000) - -jest.mock("../../../utilities/workerRequests", () => ({ - getGlobalUsers: jest.fn(() => { - return {} - }), - getGlobalSelf: jest.fn(() => { - return {} - }), - deleteGlobalUser: jest.fn(), -})) - -describe("/users", () => { - let request = setup.getRequest() - let config = setup.getConfig() - - afterAll(setup.afterAll) - - beforeAll(async () => { - await config.init() - }) - - describe("fetch", () => { - it("returns a list of users from an instance db", async () => { - await config.createUser({ id: "uuidx" }) - await config.createUser({ id: "uuidy" }) - const res = await request - .get(`/api/users/metadata`) - .set(config.defaultHeaders()) - .expect("Content-Type", /json/) - .expect(200) - - expect(res.body.length).toBe(3) - expect(res.body.find(u => u._id === `ro_ta_users_us_uuidx`)).toBeDefined() - expect(res.body.find(u => u._id === `ro_ta_users_us_uuidy`)).toBeDefined() - }) - - it("should apply authorization to endpoint", async () => { - await config.createUser() - await checkPermissionsEndpoint({ - config, - request, - method: "GET", - url: `/api/users/metadata`, - passRole: BUILTIN_ROLE_IDS.ADMIN, - failRole: BUILTIN_ROLE_IDS.PUBLIC, - }) - }) - }) - - describe("update", () => { - it("should be able to update the user", async () => { - const user = await config.createUser({ id: `us_update${utils.newid()}` }) - user.roleId = BUILTIN_ROLE_IDS.BASIC - delete user._rev - const res = await request - .put(`/api/users/metadata`) - .set(config.defaultHeaders()) - .send(user) - .expect(200) - .expect("Content-Type", /json/) - expect(res.body.ok).toEqual(true) - }) - - it("should be able to update the user multiple times", async () => { - const user = await config.createUser() - delete user._rev - - const res1 = await request - .put(`/api/users/metadata`) - .set(config.defaultHeaders()) - .send({ ...user, roleId: BUILTIN_ROLE_IDS.BASIC }) - .expect(200) - .expect("Content-Type", /json/) - - const res = await request - .put(`/api/users/metadata`) - .set(config.defaultHeaders()) - .send({ ...user, _rev: res1.body.rev, roleId: BUILTIN_ROLE_IDS.POWER }) - .expect(200) - .expect("Content-Type", /json/) - - expect(res.body.ok).toEqual(true) - }) - - it("should require the _rev field for multiple updates", async () => { - const user = await config.createUser() - delete user._rev - - await request - .put(`/api/users/metadata`) - .set(config.defaultHeaders()) - .send({ ...user, roleId: BUILTIN_ROLE_IDS.BASIC }) - .expect(200) - .expect("Content-Type", /json/) - - await request - .put(`/api/users/metadata`) - .set(config.defaultHeaders()) - .send({ ...user, roleId: BUILTIN_ROLE_IDS.POWER }) - .expect(409) - .expect("Content-Type", /json/) - }) - }) - - describe("destroy", () => { - it("should be able to delete the user", async () => { - const user = await config.createUser() - const res = await request - .delete(`/api/users/metadata/${user._id}`) - .set(config.defaultHeaders()) - .expect(200) - .expect("Content-Type", /json/) - expect(res.body.message).toBeDefined() - }) - }) - - describe("find", () => { - it("should be able to find the user", async () => { - const user = await config.createUser() - const res = await request - .get(`/api/users/metadata/${user._id}`) - .set(config.defaultHeaders()) - .expect(200) - .expect("Content-Type", /json/) - expect(res.body._id).toEqual(user._id) - expect(res.body.roleId).toEqual(BUILTIN_ROLE_IDS.ADMIN) - expect(res.body.tableId).toBeDefined() - }) - }) - - describe("setFlag", () => { - it("should throw an error if a flag is not provided", async () => { - await config.createUser() - const res = await request - .post(`/api/users/flags`) - .set(config.defaultHeaders()) - .send({ value: "test" }) - .expect(400) - .expect("Content-Type", /json/) - expect(res.body.message).toEqual( - "Must supply a 'flag' field in request body." - ) - }) - - it("should be able to set a flag on the user", async () => { - await config.createUser() - const res = await request - .post(`/api/users/flags`) - .set(config.defaultHeaders()) - .send({ value: "test", flag: "test" }) - .expect(200) - .expect("Content-Type", /json/) - expect(res.body.message).toEqual("Flag set successfully") - }) - }) - - describe("getFlags", () => { - it("should get flags for a specific user", async () => { - let flagData = { value: "test", flag: "test" } - await config.createUser() - await request - .post(`/api/users/flags`) - .set(config.defaultHeaders()) - .send(flagData) - .expect(200) - .expect("Content-Type", /json/) - - const res = await request - .get(`/api/users/flags`) - .set(config.defaultHeaders()) - .expect(200) - .expect("Content-Type", /json/) - expect(res.body[flagData.value]).toEqual(flagData.flag) - }) - }) - - describe("setFlag", () => { - it("should throw an error if a flag is not provided", async () => { - await config.createUser() - const res = await request - .post(`/api/users/flags`) - .set(config.defaultHeaders()) - .send({ value: "test" }) - .expect(400) - .expect("Content-Type", /json/) - expect(res.body.message).toEqual( - "Must supply a 'flag' field in request body." - ) - }) - - it("should be able to set a flag on the user", async () => { - await config.createUser() - const res = await request - .post(`/api/users/flags`) - .set(config.defaultHeaders()) - .send({ value: "test", flag: "test" }) - .expect(200) - .expect("Content-Type", /json/) - expect(res.body.message).toEqual("Flag set successfully") - }) - }) -}) diff --git a/packages/server/src/api/routes/tests/user.spec.ts b/packages/server/src/api/routes/tests/user.spec.ts new file mode 100644 index 0000000000..e6349099d7 --- /dev/null +++ b/packages/server/src/api/routes/tests/user.spec.ts @@ -0,0 +1,144 @@ +import { roles, utils } from "@budibase/backend-core" +import { checkPermissionsEndpoint } from "./utilities/TestFunctions" +import * as setup from "./utilities" +import { UserMetadata } from "@budibase/types" + +jest.setTimeout(30000) + +jest.mock("../../../utilities/workerRequests", () => ({ + getGlobalUsers: jest.fn(() => { + return {} + }), + getGlobalSelf: jest.fn(() => { + return {} + }), + deleteGlobalUser: jest.fn(), +})) + +describe("/users", () => { + let request = setup.getRequest() + let config = setup.getConfig() + + afterAll(setup.afterAll) + + beforeAll(async () => { + await config.init() + }) + + describe("fetch", () => { + it("returns a list of users from an instance db", async () => { + await config.createUser({ id: "uuidx" }) + await config.createUser({ id: "uuidy" }) + + const res = await config.api.user.fetch() + expect(res.length).toBe(3) + + const ids = res.map(u => u._id) + expect(ids).toContain(`ro_ta_users_us_uuidx`) + expect(ids).toContain(`ro_ta_users_us_uuidy`) + }) + + it("should apply authorization to endpoint", async () => { + await config.createUser() + await checkPermissionsEndpoint({ + config, + request, + method: "GET", + url: `/api/users/metadata`, + passRole: roles.BUILTIN_ROLE_IDS.ADMIN, + failRole: roles.BUILTIN_ROLE_IDS.PUBLIC, + }) + }) + }) + + describe("update", () => { + it("should be able to update the user", async () => { + const user: UserMetadata = await config.createUser({ + id: `us_update${utils.newid()}`, + }) + user.roleId = roles.BUILTIN_ROLE_IDS.BASIC + delete user._rev + const res = await config.api.user.update(user) + expect(res.ok).toEqual(true) + }) + + it("should be able to update the user multiple times", async () => { + const user = await config.createUser() + delete user._rev + + const res1 = await config.api.user.update({ + ...user, + roleId: roles.BUILTIN_ROLE_IDS.BASIC, + }) + const res2 = await config.api.user.update({ + ...user, + _rev: res1.rev, + roleId: roles.BUILTIN_ROLE_IDS.POWER, + }) + expect(res2.ok).toEqual(true) + }) + + it("should require the _rev field for multiple updates", async () => { + const user = await config.createUser() + delete user._rev + + await config.api.user.update({ + ...user, + roleId: roles.BUILTIN_ROLE_IDS.BASIC, + }) + await config.api.user.update( + { ...user, roleId: roles.BUILTIN_ROLE_IDS.POWER }, + { expectStatus: 409 } + ) + }) + }) + + describe("destroy", () => { + it("should be able to delete the user", async () => { + const user = await config.createUser() + const res = await config.api.user.destroy(user._id!) + expect(res.message).toBeDefined() + }) + }) + + describe("find", () => { + it("should be able to find the user", async () => { + const user = await config.createUser() + const res = await config.api.user.find(user._id!) + expect(res._id).toEqual(user._id) + expect(res.roleId).toEqual(roles.BUILTIN_ROLE_IDS.ADMIN) + expect(res.tableId).toBeDefined() + }) + }) + + describe("setFlag", () => { + it("should throw an error if a flag is not provided", async () => { + await config.createUser() + const res = await request + .post(`/api/users/flags`) + .set(config.defaultHeaders()) + .send({ value: "test" }) + .expect(400) + .expect("Content-Type", /json/) + expect(res.body.message).toEqual( + "Must supply a 'flag' field in request body." + ) + }) + + it("should be able to set a flag on the user", async () => { + await config.createUser() + const res = await config.api.user.setFlag("test", true) + expect(res.message).toEqual("Flag set successfully") + }) + }) + + describe("getFlags", () => { + it("should get flags for a specific user", async () => { + await config.createUser() + await config.api.user.setFlag("test", "test") + + const res = await config.api.user.getFlags() + expect(res.test).toEqual("test") + }) + }) +}) diff --git a/packages/server/src/api/routes/tests/viewV2.spec.ts b/packages/server/src/api/routes/tests/viewV2.spec.ts index 40060aef48..b03a73ddda 100644 --- a/packages/server/src/api/routes/tests/viewV2.spec.ts +++ b/packages/server/src/api/routes/tests/viewV2.spec.ts @@ -3,10 +3,12 @@ import { CreateViewRequest, FieldSchema, FieldType, + INTERNAL_TABLE_SOURCE_ID, SearchQueryOperators, SortOrder, SortType, Table, + TableSourceType, UIFieldMetadata, UpdateViewRequest, ViewV2, @@ -18,6 +20,8 @@ function priceTable(): Table { return { name: "table", type: "table", + sourceId: INTERNAL_TABLE_SOURCE_ID, + sourceType: TableSourceType.INTERNAL, schema: { Price: { type: FieldType.NUMBER, @@ -54,10 +58,10 @@ describe.each([ }, }) - return config.createTable({ + return config.createExternalTable({ ...priceTable(), sourceId: datasource._id, - type: "external", + sourceType: TableSourceType.EXTERNAL, }) }, ], diff --git a/packages/server/src/api/routes/tests/webhook.spec.ts b/packages/server/src/api/routes/tests/webhook.spec.ts index e7046d07c8..118bfca95f 100644 --- a/packages/server/src/api/routes/tests/webhook.spec.ts +++ b/packages/server/src/api/routes/tests/webhook.spec.ts @@ -8,11 +8,15 @@ describe("/webhooks", () => { let request = setup.getRequest() let config = setup.getConfig() let webhook: Webhook + let cleanupEnv: () => void - afterAll(setup.afterAll) + afterAll(() => { + setup.afterAll() + cleanupEnv() + }) const setupTest = async () => { - config.modeSelf() + cleanupEnv = config.setEnv({ SELF_HOSTED: "true" }) await config.init() const autoConfig = basicAutomation() autoConfig.definition.trigger.schema = { diff --git a/packages/server/src/automations/tests/automation.spec.ts b/packages/server/src/automations/tests/automation.spec.ts index 67ff6d40ec..c37c9cc7ce 100644 --- a/packages/server/src/automations/tests/automation.spec.ts +++ b/packages/server/src/automations/tests/automation.spec.ts @@ -36,7 +36,7 @@ describe("Run through some parts of the automations system", () => { it("should be able to init in builder", async () => { const automation: Automation = { ...basicAutomation(), - appId: config.appId, + appId: config.appId!, } const fields: any = { a: 1, appId: config.appId } await triggers.externalTrigger(automation, fields) diff --git a/packages/server/src/automations/tests/updateRow.spec.js b/packages/server/src/automations/tests/updateRow.spec.js deleted file mode 100644 index 77383d80e9..0000000000 --- a/packages/server/src/automations/tests/updateRow.spec.js +++ /dev/null @@ -1,44 +0,0 @@ -const setup = require("./utilities") - -describe("test the update row action", () => { - let table, row, inputs - let config = setup.getConfig() - - beforeAll(async () => { - await config.init() - table = await config.createTable() - row = await config.createRow() - inputs = { - rowId: row._id, - row: { - ...row, - name: "Updated name", - // put a falsy option in to be removed - description: "", - } - } - }) - - afterAll(setup.afterAll) - - it("should be able to run the action", async () => { - const res = await setup.runStep(setup.actions.UPDATE_ROW.stepId, inputs) - expect(res.success).toEqual(true) - const updatedRow = await config.getRow(table._id, res.id) - expect(updatedRow.name).toEqual("Updated name") - expect(updatedRow.description).not.toEqual("") - }) - - it("should check invalid inputs return an error", async () => { - const res = await setup.runStep(setup.actions.UPDATE_ROW.stepId, {}) - expect(res.success).toEqual(false) - }) - - it("should return an error when table doesn't exist", async () => { - const res = await setup.runStep(setup.actions.UPDATE_ROW.stepId, { - row: { _id: "invalid" }, - rowId: "invalid", - }) - expect(res.success).toEqual(false) - }) -}) diff --git a/packages/server/src/automations/tests/updateRow.spec.ts b/packages/server/src/automations/tests/updateRow.spec.ts new file mode 100644 index 0000000000..7e369f1ecb --- /dev/null +++ b/packages/server/src/automations/tests/updateRow.spec.ts @@ -0,0 +1,169 @@ +import { + FieldSchema, + FieldType, + INTERNAL_TABLE_SOURCE_ID, + InternalTable, + RelationshipType, + Row, + Table, + TableSourceType, +} from "@budibase/types" + +import * as setup from "./utilities" +import * as uuid from "uuid" + +describe("test the update row action", () => { + let table: Table, row: Row, inputs: any + let config = setup.getConfig() + + beforeAll(async () => { + await config.init() + table = await config.createTable() + row = await config.createRow() + inputs = { + rowId: row._id, + row: { + ...row, + name: "Updated name", + // put a falsy option in to be removed + description: "", + }, + } + }) + + afterAll(setup.afterAll) + + it("should be able to run the action", async () => { + const res = await setup.runStep(setup.actions.UPDATE_ROW.stepId, inputs) + expect(res.success).toEqual(true) + const updatedRow = await config.getRow(table._id!, res.id) + expect(updatedRow.name).toEqual("Updated name") + expect(updatedRow.description).not.toEqual("") + }) + + it("should check invalid inputs return an error", async () => { + const res = await setup.runStep(setup.actions.UPDATE_ROW.stepId, {}) + expect(res.success).toEqual(false) + }) + + it("should return an error when table doesn't exist", async () => { + const res = await setup.runStep(setup.actions.UPDATE_ROW.stepId, { + row: { _id: "invalid" }, + rowId: "invalid", + }) + expect(res.success).toEqual(false) + }) + + it("should not overwrite links if those links are not set", async () => { + let linkField: FieldSchema = { + type: FieldType.LINK, + name: "", + fieldName: "", + constraints: { + type: "array", + presence: false, + }, + relationshipType: RelationshipType.ONE_TO_MANY, + tableId: InternalTable.USER_METADATA, + } + + let table = await config.api.table.create({ + name: uuid.v4(), + type: "table", + sourceType: TableSourceType.INTERNAL, + sourceId: INTERNAL_TABLE_SOURCE_ID, + schema: { + user1: { ...linkField, name: "user1", fieldName: uuid.v4() }, + user2: { ...linkField, name: "user2", fieldName: uuid.v4() }, + }, + }) + + let user1 = await config.createUser() + let user2 = await config.createUser() + + let row = await config.api.row.save(table._id!, { + user1: [{ _id: user1._id }], + user2: [{ _id: user2._id }], + }) + + let getResp = await config.api.row.get(table._id!, row._id!) + expect(getResp.body.user1[0]._id).toEqual(user1._id) + expect(getResp.body.user2[0]._id).toEqual(user2._id) + + let stepResp = await setup.runStep(setup.actions.UPDATE_ROW.stepId, { + rowId: row._id, + row: { + _id: row._id, + _rev: row._rev, + tableId: row.tableId, + user1: [user2._id], + user2: "", + }, + }) + expect(stepResp.success).toEqual(true) + + getResp = await config.api.row.get(table._id!, row._id!) + expect(getResp.body.user1[0]._id).toEqual(user2._id) + expect(getResp.body.user2[0]._id).toEqual(user2._id) + }) + + it("should overwrite links if those links are not set and we ask it do", async () => { + let linkField: FieldSchema = { + type: FieldType.LINK, + name: "", + fieldName: "", + constraints: { + type: "array", + presence: false, + }, + relationshipType: RelationshipType.ONE_TO_MANY, + tableId: InternalTable.USER_METADATA, + } + + let table = await config.api.table.create({ + name: uuid.v4(), + type: "table", + sourceType: TableSourceType.INTERNAL, + sourceId: INTERNAL_TABLE_SOURCE_ID, + schema: { + user1: { ...linkField, name: "user1", fieldName: uuid.v4() }, + user2: { ...linkField, name: "user2", fieldName: uuid.v4() }, + }, + }) + + let user1 = await config.createUser() + let user2 = await config.createUser() + + let row = await config.api.row.save(table._id!, { + user1: [{ _id: user1._id }], + user2: [{ _id: user2._id }], + }) + + let getResp = await config.api.row.get(table._id!, row._id!) + expect(getResp.body.user1[0]._id).toEqual(user1._id) + expect(getResp.body.user2[0]._id).toEqual(user2._id) + + let stepResp = await setup.runStep(setup.actions.UPDATE_ROW.stepId, { + rowId: row._id, + row: { + _id: row._id, + _rev: row._rev, + tableId: row.tableId, + user1: [user2._id], + user2: "", + }, + meta: { + fields: { + user2: { + clearRelationships: true, + }, + }, + }, + }) + expect(stepResp.success).toEqual(true) + + getResp = await config.api.row.get(table._id!, row._id!) + expect(getResp.body.user1[0]._id).toEqual(user2._id) + expect(getResp.body.user2).toBeUndefined() + }) +}) diff --git a/packages/server/src/automations/tests/utilities/index.ts b/packages/server/src/automations/tests/utilities/index.ts index 9ba4f950f3..cd3ea289ca 100644 --- a/packages/server/src/automations/tests/utilities/index.ts +++ b/packages/server/src/automations/tests/utilities/index.ts @@ -4,11 +4,11 @@ import { BUILTIN_ACTION_DEFINITIONS, getAction } from "../../actions" import emitter from "../../../events/index" import env from "../../../environment" -let config: any +let config: TestConfig -export function getConfig() { +export function getConfig(): TestConfig { if (!config) { - config = new TestConfig(false) + config = new TestConfig(true) } return config } diff --git a/packages/server/src/automations/triggers.ts b/packages/server/src/automations/triggers.ts index 922bc10343..ac977bbefb 100644 --- a/packages/server/src/automations/triggers.ts +++ b/packages/server/src/automations/triggers.ts @@ -20,10 +20,10 @@ const JOB_OPTS = { async function getAllAutomations() { const db = context.getAppDB() - let automations = await db.allDocs( + let automations = await db.allDocs( getAutomationParams(null, { include_docs: true }) ) - return automations.rows.map(row => row.doc) + return automations.rows.map(row => row.doc!) } async function queueRelevantRowAutomations( @@ -45,19 +45,19 @@ async function queueRelevantRowAutomations( for (let automation of automations) { let automationDef = automation.definition - let automationTrigger = automationDef ? automationDef.trigger : {} + let automationTrigger = automationDef?.trigger // don't queue events which are for dev apps, only way to test automations is // running tests on them, in production the test flag will never // be checked due to lazy evaluation (first always false) if ( !env.ALLOW_DEV_AUTOMATIONS && isDevAppID(event.appId) && - !(await checkTestFlag(automation._id)) + !(await checkTestFlag(automation._id!)) ) { continue } if ( - automationTrigger.inputs && + automationTrigger?.inputs && automationTrigger.inputs.tableId === event.row.tableId ) { await automationQueue.add({ automation, event }, JOB_OPTS) @@ -94,7 +94,7 @@ export async function externalTrigger( automation: Automation, params: { fields: Record; timeout?: number }, { getResponses }: { getResponses?: boolean } = {} -) { +): Promise { if ( automation.definition != null && automation.definition.trigger != null && diff --git a/packages/server/src/constants/index.ts b/packages/server/src/constants/index.ts index b37a4b36c1..fb5c42e7b8 100644 --- a/packages/server/src/constants/index.ts +++ b/packages/server/src/constants/index.ts @@ -1,5 +1,11 @@ -import { objectStore, roles, constants } from "@budibase/backend-core" -import { FieldType as FieldTypes } from "@budibase/types" +import { constants, objectStore, roles } from "@budibase/backend-core" +import { + FieldType as FieldTypes, + INTERNAL_TABLE_SOURCE_ID, + Table, + TableSourceType, +} from "@budibase/types" + export { FieldType as FieldTypes, RelationshipType, @@ -70,9 +76,11 @@ export enum SortDirection { DESCENDING = "DESCENDING", } -export const USERS_TABLE_SCHEMA = { +export const USERS_TABLE_SCHEMA: Table = { _id: "ta_users", type: "table", + sourceId: INTERNAL_TABLE_SOURCE_ID, + sourceType: TableSourceType.INTERNAL, views: {}, name: "Users", // TODO: ADMIN PANEL - when implemented this doesn't need to be carried out @@ -87,12 +95,10 @@ export const USERS_TABLE_SCHEMA = { }, presence: true, }, - fieldName: "email", name: "email", }, firstName: { name: "firstName", - fieldName: "firstName", type: FieldTypes.STRING, constraints: { type: FieldTypes.STRING, @@ -101,7 +107,6 @@ export const USERS_TABLE_SCHEMA = { }, lastName: { name: "lastName", - fieldName: "lastName", type: FieldTypes.STRING, constraints: { type: FieldTypes.STRING, @@ -109,7 +114,6 @@ export const USERS_TABLE_SCHEMA = { }, }, roleId: { - fieldName: "roleId", name: "roleId", type: FieldTypes.OPTIONS, constraints: { @@ -119,7 +123,6 @@ export const USERS_TABLE_SCHEMA = { }, }, status: { - fieldName: "status", name: "status", type: FieldTypes.OPTIONS, constraints: { @@ -169,3 +172,8 @@ export enum AutomationErrors { export const ObjectStoreBuckets = objectStore.ObjectStoreBuckets export const MAX_AUTOMATION_RECURRING_ERRORS = 5 export const GOOGLE_SHEETS_PRIMARY_KEY = "rowNumber" +export const DEFAULT_JOBS_TABLE_ID = "ta_bb_jobs" +export const DEFAULT_INVENTORY_TABLE_ID = "ta_bb_inventory" +export const DEFAULT_EXPENSES_TABLE_ID = "ta_bb_expenses" +export const DEFAULT_EMPLOYEE_TABLE_ID = "ta_bb_employee" +export const DEFAULT_BB_DATASOURCE_ID = "datasource_internal_bb_default" diff --git a/packages/server/src/constants/screens.ts b/packages/server/src/constants/screens.ts index 23e36a65b8..6c88b0f957 100644 --- a/packages/server/src/constants/screens.ts +++ b/packages/server/src/constants/screens.ts @@ -1,7 +1,15 @@ import { roles } from "@budibase/backend-core" import { BASE_LAYOUT_PROP_IDS } from "./layouts" -export function createHomeScreen() { +export function createHomeScreen( + config: { + roleId: string + route: string + } = { + roleId: roles.BUILTIN_ROLE_IDS.BASIC, + route: "/", + } +) { return { description: "", url: "", @@ -40,8 +48,8 @@ export function createHomeScreen() { gap: "M", }, routing: { - route: "/", - roleId: roles.BUILTIN_ROLE_IDS.BASIC, + route: config.route, + roleId: config.roleId, }, name: "home-screen", } diff --git a/packages/server/src/db/defaultData/datasource_bb_default.ts b/packages/server/src/db/defaultData/datasource_bb_default.ts index 48d4876de1..b430f9ffb6 100644 --- a/packages/server/src/db/defaultData/datasource_bb_default.ts +++ b/packages/server/src/db/defaultData/datasource_bb_default.ts @@ -1,4 +1,12 @@ -import { FieldTypes, AutoFieldSubTypes } from "../../constants" +import { + AutoFieldSubTypes, + FieldTypes, + DEFAULT_BB_DATASOURCE_ID, + DEFAULT_INVENTORY_TABLE_ID, + DEFAULT_EMPLOYEE_TABLE_ID, + DEFAULT_EXPENSES_TABLE_ID, + DEFAULT_JOBS_TABLE_ID, +} from "../../constants" import { importToRows } from "../../api/controllers/table/utils" import { cloneDeep } from "lodash/fp" import LinkDocument from "../linkedRows/LinkDocument" @@ -8,19 +16,14 @@ import { jobsImport } from "./jobsImport" import { expensesImport } from "./expensesImport" import { db as dbCore } from "@budibase/backend-core" import { - Table, - Row, - RelationshipType, FieldType, + RelationshipType, + Row, + Table, TableSchema, + TableSourceType, } from "@budibase/types" -export const DEFAULT_JOBS_TABLE_ID = "ta_bb_jobs" -export const DEFAULT_INVENTORY_TABLE_ID = "ta_bb_inventory" -export const DEFAULT_EXPENSES_TABLE_ID = "ta_bb_expenses" -export const DEFAULT_EMPLOYEE_TABLE_ID = "ta_bb_employee" -export const DEFAULT_BB_DATASOURCE_ID = "datasource_internal_bb_default" - const defaultDatasource = { _id: DEFAULT_BB_DATASOURCE_ID, type: dbCore.BUDIBASE_DATASOURCE_TYPE, @@ -89,9 +92,10 @@ const AUTO_COLUMNS: TableSchema = { export const DEFAULT_INVENTORY_TABLE_SCHEMA: Table = { _id: DEFAULT_INVENTORY_TABLE_ID, - type: "internal", + type: "table", views: {}, sourceId: DEFAULT_BB_DATASOURCE_ID, + sourceType: TableSourceType.INTERNAL, primaryDisplay: "Item Name", name: "Inventory", schema: { @@ -198,10 +202,11 @@ export const DEFAULT_INVENTORY_TABLE_SCHEMA: Table = { export const DEFAULT_EMPLOYEE_TABLE_SCHEMA: Table = { _id: DEFAULT_EMPLOYEE_TABLE_ID, - type: "internal", + type: "table", views: {}, name: "Employees", sourceId: DEFAULT_BB_DATASOURCE_ID, + sourceType: TableSourceType.INTERNAL, primaryDisplay: "First Name", schema: { "First Name": { @@ -346,9 +351,10 @@ export const DEFAULT_EMPLOYEE_TABLE_SCHEMA: Table = { export const DEFAULT_JOBS_TABLE_SCHEMA: Table = { _id: DEFAULT_JOBS_TABLE_ID, - type: "internal", + type: "table", name: "Jobs", sourceId: DEFAULT_BB_DATASOURCE_ID, + sourceType: TableSourceType.INTERNAL, primaryDisplay: "Job ID", schema: { "Job ID": { @@ -503,10 +509,11 @@ export const DEFAULT_JOBS_TABLE_SCHEMA: Table = { export const DEFAULT_EXPENSES_TABLE_SCHEMA: Table = { _id: DEFAULT_EXPENSES_TABLE_ID, - type: "internal", + type: "table", views: {}, name: "Expenses", sourceId: DEFAULT_BB_DATASOURCE_ID, + sourceType: TableSourceType.INTERNAL, primaryDisplay: "Expense ID", schema: { "Expense ID": { diff --git a/packages/server/src/db/inMemoryView.ts b/packages/server/src/db/inMemoryView.ts index 4e9301f4ee..724bc725ce 100644 --- a/packages/server/src/db/inMemoryView.ts +++ b/packages/server/src/db/inMemoryView.ts @@ -1,5 +1,5 @@ import newid from "./newid" -import { Row, View, Document } from "@budibase/types" +import { Row, Document, DBView } from "@budibase/types" // bypass the main application db config // use in memory pouchdb directly @@ -7,7 +7,7 @@ import { db as dbCore } from "@budibase/backend-core" const Pouch = dbCore.getPouch({ inMemory: true }) export async function runView( - view: View, + view: DBView, calculation: string, group: boolean, data: Row[] diff --git a/packages/server/src/db/linkedRows/index.ts b/packages/server/src/db/linkedRows/index.ts index 7a7a06551e..7af3f9392f 100644 --- a/packages/server/src/db/linkedRows/index.ts +++ b/packages/server/src/db/linkedRows/index.ts @@ -2,19 +2,25 @@ import LinkController from "./LinkController" import { IncludeDocs, getLinkDocuments, - createLinkView, getUniqueByProp, getRelatedTableForField, getLinkedTableIDs, getLinkedTable, } from "./linkUtils" import flatten from "lodash/flatten" -import { getMultiIDParams, USER_METDATA_PREFIX } from "../utils" +import { USER_METDATA_PREFIX } from "../utils" import partition from "lodash/partition" import { getGlobalUsersFromMetadata } from "../../utilities/global" import { processFormulas } from "../../utilities/rowProcessor" import { context } from "@budibase/backend-core" -import { Table, Row, LinkDocumentValue, FieldType } from "@budibase/types" +import { + Table, + Row, + LinkDocumentValue, + FieldType, + LinkDocument, + ContextUser, +} from "@budibase/types" import sdk from "../../sdk" export { IncludeDocs, getLinkDocuments, createLinkView } from "./linkUtils" @@ -73,18 +79,16 @@ async function getFullLinkedDocs(links: LinkDocumentValue[]) { const db = context.getAppDB() const linkedRowIds = links.map(link => link.id) const uniqueRowIds = [...new Set(linkedRowIds)] - let dbRows = (await db.allDocs(getMultiIDParams(uniqueRowIds))).rows.map( - row => row.doc - ) + let dbRows = await db.getMultiple(uniqueRowIds, { allowMissing: true }) // convert the unique db rows back to a full list of linked rows const linked = linkedRowIds .map(id => dbRows.find(row => row && row._id === id)) - .filter(row => row != null) + .filter(row => row != null) as Row[] // need to handle users as specific cases let [users, other] = partition(linked, linkRow => - linkRow._id.startsWith(USER_METDATA_PREFIX) + linkRow._id!.startsWith(USER_METDATA_PREFIX) ) - users = await getGlobalUsersFromMetadata(users) + users = await getGlobalUsersFromMetadata(users as ContextUser[]) return [...other, ...users] } @@ -176,7 +180,7 @@ export async function attachFullLinkedDocs( // clear any existing links that could be dupe'd rows = clearRelationshipFields(table, rows) // now get the docs and combine into the rows - let linked = [] + let linked: Row[] = [] if (linksWithoutFromRow.length > 0) { linked = await getFullLinkedDocs(linksWithoutFromRow) } @@ -189,7 +193,7 @@ export async function attachFullLinkedDocs( if (opts?.fromRow && opts?.fromRow?._id === link.id) { linkedRow = opts.fromRow! } else { - linkedRow = linked.find(row => row._id === link.id) + linkedRow = linked.find(row => row._id === link.id)! } if (linkedRow) { const linkedTableId = diff --git a/packages/server/src/db/linkedRows/linkUtils.ts b/packages/server/src/db/linkedRows/linkUtils.ts index c74674a865..5942e7e5a1 100644 --- a/packages/server/src/db/linkedRows/linkUtils.ts +++ b/packages/server/src/db/linkedRows/linkUtils.ts @@ -2,7 +2,13 @@ import { ViewName, getQueryIndex, isRelationshipColumn } from "../utils" import { FieldTypes } from "../../constants" import { createLinkView } from "../views/staticViews" import { context, logging } from "@budibase/backend-core" -import { LinkDocument, LinkDocumentValue, Table } from "@budibase/types" +import { + DatabaseQueryOpts, + LinkDocument, + LinkDocumentValue, + Table, +} from "@budibase/types" +import sdk from "../../sdk" export { createLinkView } from "../views/staticViews" @@ -36,13 +42,13 @@ export async function getLinkDocuments(args: { }): Promise { const { tableId, rowId, fieldName, includeDocs } = args const db = context.getAppDB() - let params: any + let params: DatabaseQueryOpts if (rowId) { params = { key: [tableId, rowId] } } // only table is known else { - params = { startKey: [tableId], endKey: [tableId, {}] } + params = { startkey: [tableId], endkey: [tableId, {}] } } if (includeDocs) { params.include_docs = true @@ -105,12 +111,11 @@ export function getLinkedTableIDs(table: Table): string[] { } export async function getLinkedTable(id: string, tables: Table[]) { - const db = context.getAppDB() let linkedTable = tables.find(table => table._id === id) if (linkedTable) { return linkedTable } - linkedTable = await db.get(id) + linkedTable = await sdk.tables.getTable(id) if (linkedTable) { tables.push(linkedTable) } diff --git a/packages/server/src/db/utils.ts b/packages/server/src/db/utils.ts index 69424539c0..c703ced098 100644 --- a/packages/server/src/db/utils.ts +++ b/packages/server/src/db/utils.ts @@ -5,6 +5,8 @@ import { FieldSchema, RelationshipFieldMetadata, VirtualDocumentType, + INTERNAL_TABLE_SOURCE_ID, + DatabaseQueryOpts, } from "@budibase/types" import { FieldTypes } from "../constants" export { DocumentType, VirtualDocumentType } from "@budibase/types" @@ -18,7 +20,7 @@ export const enum AppStatus { } export const BudibaseInternalDB = { - _id: "bb_internal", + _id: INTERNAL_TABLE_SOURCE_ID, type: dbCore.BUDIBASE_DATASOURCE_TYPE, name: "Budibase DB", source: "BUDIBASE", @@ -236,7 +238,10 @@ export function getAutomationMetadataParams(otherProps: any = {}) { /** * Gets parameters for retrieving a query, this is a utility function for the getDocParams function. */ -export function getQueryParams(datasourceId?: Optional, otherProps: any = {}) { +export function getQueryParams( + datasourceId?: Optional, + otherProps: Partial = {} +) { if (datasourceId == null) { return getDocParams(DocumentType.QUERY, null, otherProps) } @@ -263,7 +268,7 @@ export function generateMetadataID(type: string, entityId: string) { export function getMetadataParams( type: string, entityId?: Optional, - otherProps: any = {} + otherProps: Partial = {} ) { let docId = `${type}${SEPARATOR}` if (entityId != null) { @@ -276,7 +281,9 @@ export function generateMemoryViewID(viewName: string) { return `${DocumentType.MEM_VIEW}${SEPARATOR}${viewName}` } -export function getMemoryViewParams(otherProps: any = {}) { +export function getMemoryViewParams( + otherProps: Partial = {} +) { return getDocParams(DocumentType.MEM_VIEW, null, otherProps) } @@ -290,16 +297,6 @@ export function generateJunctionTableID(tableId1: string, tableId2: string) { return `${first}${SEPARATOR}${second}` } -/** - * This can be used with the db.allDocs to get a list of IDs - */ -export function getMultiIDParams(ids: string[]) { - return { - keys: ids, - include_docs: true, - } -} - /** * Generates a new view ID. * @returns The new view ID which the view doc can be stored under. diff --git a/packages/server/src/environment.ts b/packages/server/src/environment.ts index a1701535ce..c126a61c22 100644 --- a/packages/server/src/environment.ts +++ b/packages/server/src/environment.ts @@ -61,6 +61,7 @@ const environment = { ALLOW_DEV_AUTOMATIONS: process.env.ALLOW_DEV_AUTOMATIONS, DISABLE_THREADING: process.env.DISABLE_THREADING, DISABLE_AUTOMATION_LOGS: process.env.DISABLE_AUTOMATION_LOGS, + DISABLE_RATE_LIMITING: process.env.DISABLE_RATE_LIMITING, MULTI_TENANCY: process.env.MULTI_TENANCY, ENABLE_ANALYTICS: process.env.ENABLE_ANALYTICS, SELF_HOSTED: process.env.SELF_HOSTED, @@ -75,7 +76,6 @@ const environment = { }, isTest: coreEnv.isTest, isJest: coreEnv.isJest, - isDev: coreEnv.isDev, isProd: () => { return !coreEnv.isDev() diff --git a/packages/server/src/integration-test/postgres.spec.ts b/packages/server/src/integration-test/postgres.spec.ts index 90f0fc9f2c..8dc49a9489 100644 --- a/packages/server/src/integration-test/postgres.spec.ts +++ b/packages/server/src/integration-test/postgres.spec.ts @@ -1,6 +1,4 @@ import fetch from "node-fetch" -// @ts-ignore -fetch.mockSearch() import { generateMakeRequest, MakeRequestResponse, @@ -13,12 +11,15 @@ import { RelationshipType, Row, Table, + TableSourceType, } from "@budibase/types" import _ from "lodash" import { generator } from "@budibase/backend-core/tests" import { utils } from "@budibase/backend-core" import { databaseTestProviders } from "../integrations/tests/utils" import { Client } from "pg" +// @ts-ignore +fetch.mockSearch() const config = setup.getConfig()! @@ -52,7 +53,7 @@ describe("postgres integrations", () => { async function createAuxTable(prefix: string) { return await config.createTable({ name: `${prefix}_${generator.word({ length: 6 })}`, - type: "external", + type: "table", primary: ["id"], primaryDisplay: "title", schema: { @@ -67,6 +68,7 @@ describe("postgres integrations", () => { }, }, sourceId: postgresDatasource._id, + sourceType: TableSourceType.EXTERNAL, }) } @@ -88,7 +90,7 @@ describe("postgres integrations", () => { primaryPostgresTable = await config.createTable({ name: `p_${generator.word({ length: 6 })}`, - type: "external", + type: "table", primary: ["id"], schema: { id: { @@ -143,6 +145,7 @@ describe("postgres integrations", () => { }, }, sourceId: postgresDatasource._id, + sourceType: TableSourceType.EXTERNAL, }) }) @@ -249,7 +252,7 @@ describe("postgres integrations", () => { async function createDefaultPgTable() { return await config.createTable({ name: generator.word({ length: 10 }), - type: "external", + type: "table", primary: ["id"], schema: { id: { @@ -259,6 +262,7 @@ describe("postgres integrations", () => { }, }, sourceId: postgresDatasource._id, + sourceType: TableSourceType.EXTERNAL, }) } @@ -919,7 +923,6 @@ describe("postgres integrations", () => { [m2mFieldName]: [ { _id: row._id, - primaryDisplay: "Invalid display column", }, ], }) @@ -928,7 +931,6 @@ describe("postgres integrations", () => { [m2mFieldName]: [ { _id: row._id, - primaryDisplay: "Invalid display column", }, ], }) diff --git a/packages/server/src/integrations/googlesheets.ts b/packages/server/src/integrations/googlesheets.ts index 57b6682cc8..58c867ea0b 100644 --- a/packages/server/src/integrations/googlesheets.ts +++ b/packages/server/src/integrations/googlesheets.ts @@ -10,11 +10,12 @@ import { QueryJson, QueryType, Row, + Schema, SearchFilters, SortJson, - ExternalTable, + Table, TableRequest, - Schema, + TableSourceType, } from "@budibase/types" import { OAuth2Client } from "google-auth-library" import { @@ -262,11 +263,13 @@ class GoogleSheetsIntegration implements DatasourcePlus { id?: string ) { // base table - const table: ExternalTable = { + const table: Table = { + type: "table", name: title, primary: [GOOGLE_SHEETS_PRIMARY_KEY], schema: {}, sourceId: datasourceId, + sourceType: TableSourceType.EXTERNAL, } if (id) { table._id = id @@ -283,7 +286,7 @@ class GoogleSheetsIntegration implements DatasourcePlus { async buildSchema( datasourceId: string, - entities: Record + entities: Record ): Promise { // not fully configured yet if (!this.config.auth) { @@ -291,7 +294,7 @@ class GoogleSheetsIntegration implements DatasourcePlus { } await this.connect() const sheets = this.client.sheetsByIndex - const tables: Record = {} + const tables: Record = {} let errors: Record = {} await utils.parallelForeach( sheets, diff --git a/packages/server/src/integrations/microsoftSqlServer.ts b/packages/server/src/integrations/microsoftSqlServer.ts index ff68026369..c615e5ba48 100644 --- a/packages/server/src/integrations/microsoftSqlServer.ts +++ b/packages/server/src/integrations/microsoftSqlServer.ts @@ -2,7 +2,7 @@ import { DatasourceFieldType, Integration, Operation, - ExternalTable, + Table, TableSchema, QueryJson, QueryType, @@ -12,6 +12,7 @@ import { ConnectionInfo, SourceName, Schema, + TableSourceType, } from "@budibase/types" import { getSqlQuery, @@ -380,7 +381,7 @@ class SqlServerIntegration extends Sql implements DatasourcePlus { */ async buildSchema( datasourceId: string, - entities: Record + entities: Record ): Promise { await this.connect() let tableInfo: MSSQLTablesResponse[] = await this.runSQL(this.TABLES_SQL) @@ -394,7 +395,7 @@ class SqlServerIntegration extends Sql implements DatasourcePlus { .map((record: any) => record.TABLE_NAME) .filter((name: string) => this.MASTER_TABLES.indexOf(name) === -1) - const tables: Record = {} + const tables: Record = {} for (let tableName of tableNames) { // get the column definition (type) const definition = await this.runSQL( @@ -439,7 +440,9 @@ class SqlServerIntegration extends Sql implements DatasourcePlus { } tables[tableName] = { _id: buildExternalTableId(datasourceId, tableName), + type: "table", sourceId: datasourceId, + sourceType: TableSourceType.EXTERNAL, primary: primaryKeys, name: tableName, schema, diff --git a/packages/server/src/integrations/mysql.ts b/packages/server/src/integrations/mysql.ts index 3a954da9bd..e89393d251 100644 --- a/packages/server/src/integrations/mysql.ts +++ b/packages/server/src/integrations/mysql.ts @@ -4,13 +4,14 @@ import { QueryType, QueryJson, SqlQuery, - ExternalTable, + Table, TableSchema, DatasourcePlus, DatasourceFeature, ConnectionInfo, SourceName, Schema, + TableSourceType, } from "@budibase/types" import { getSqlQuery, @@ -278,9 +279,9 @@ class MySQLIntegration extends Sql implements DatasourcePlus { async buildSchema( datasourceId: string, - entities: Record + entities: Record ): Promise { - const tables: { [key: string]: ExternalTable } = {} + const tables: { [key: string]: Table } = {} await this.connect() try { @@ -317,8 +318,10 @@ class MySQLIntegration extends Sql implements DatasourcePlus { } if (!tables[tableName]) { tables[tableName] = { + type: "table", _id: buildExternalTableId(datasourceId, tableName), sourceId: datasourceId, + sourceType: TableSourceType.EXTERNAL, primary: primaryKeys, name: tableName, schema, diff --git a/packages/server/src/integrations/oracle.ts b/packages/server/src/integrations/oracle.ts index b3936320ac..c6a871e41f 100644 --- a/packages/server/src/integrations/oracle.ts +++ b/packages/server/src/integrations/oracle.ts @@ -5,11 +5,12 @@ import { QueryJson, QueryType, SqlQuery, - ExternalTable, + Table, DatasourcePlus, DatasourceFeature, ConnectionInfo, Schema, + TableSourceType, } from "@budibase/types" import { buildExternalTableId, @@ -263,25 +264,27 @@ class OracleIntegration extends Sql implements DatasourcePlus { */ async buildSchema( datasourceId: string, - entities: Record + entities: Record ): Promise { const columnsResponse = await this.internalQuery({ sql: this.COLUMNS_SQL, }) const oracleTables = this.mapColumns(columnsResponse) - const tables: { [key: string]: ExternalTable } = {} + const tables: { [key: string]: Table } = {} // iterate each table Object.values(oracleTables).forEach(oracleTable => { let table = tables[oracleTable.name] if (!table) { table = { + type: "table", _id: buildExternalTableId(datasourceId, oracleTable.name), primary: [], name: oracleTable.name, schema: {}, sourceId: datasourceId, + sourceType: TableSourceType.EXTERNAL, } tables[oracleTable.name] = table } diff --git a/packages/server/src/integrations/postgres.ts b/packages/server/src/integrations/postgres.ts index 8479cd05d8..4d7dc33d75 100644 --- a/packages/server/src/integrations/postgres.ts +++ b/packages/server/src/integrations/postgres.ts @@ -5,12 +5,13 @@ import { QueryType, QueryJson, SqlQuery, - ExternalTable, + Table, DatasourcePlus, DatasourceFeature, ConnectionInfo, SourceName, Schema, + TableSourceType, } from "@budibase/types" import { getSqlQuery, @@ -273,7 +274,7 @@ class PostgresIntegration extends Sql implements DatasourcePlus { */ async buildSchema( datasourceId: string, - entities: Record + entities: Record ): Promise { let tableKeys: { [key: string]: string[] } = {} await this.openConnection() @@ -300,7 +301,7 @@ class PostgresIntegration extends Sql implements DatasourcePlus { const columnsResponse: { rows: PostgresColumn[] } = await this.client.query(this.COLUMNS_SQL) - const tables: { [key: string]: ExternalTable } = {} + const tables: { [key: string]: Table } = {} for (let column of columnsResponse.rows) { const tableName: string = column.table_name @@ -309,11 +310,13 @@ class PostgresIntegration extends Sql implements DatasourcePlus { // table key doesn't exist yet if (!tables[tableName] || !tables[tableName].schema) { tables[tableName] = { + type: "table", _id: buildExternalTableId(datasourceId, tableName), primary: tableKeys[tableName] || [], name: tableName, schema: {}, sourceId: datasourceId, + sourceType: TableSourceType.EXTERNAL, } } diff --git a/packages/server/src/integrations/redis.ts b/packages/server/src/integrations/redis.ts index 879a790550..6a6331ccd4 100644 --- a/packages/server/src/integrations/redis.ts +++ b/packages/server/src/integrations/redis.ts @@ -165,10 +165,22 @@ class RedisIntegration { // commands split line by line const commands = query.json.trim().split("\n") let pipelineCommands = [] + let tokenised // process each command separately for (let command of commands) { - const tokenised = command.trim().split(" ") + const valueToken = command.trim().match(/".*"/) + if (valueToken?.[0]) { + tokenised = [ + ...command + .substring(0, command.indexOf(valueToken[0]) - 1) + .trim() + .split(" "), + valueToken?.[0], + ] + } else { + tokenised = command.trim().split(" ") + } // Pipeline only accepts lower case commands tokenised[0] = tokenised[0].toLowerCase() pipelineCommands.push(tokenised) diff --git a/packages/server/src/integrations/tests/googlesheets.spec.ts b/packages/server/src/integrations/tests/googlesheets.spec.ts index 748baddc39..10ec7815d6 100644 --- a/packages/server/src/integrations/tests/googlesheets.spec.ts +++ b/packages/server/src/integrations/tests/googlesheets.spec.ts @@ -30,18 +30,24 @@ GoogleSpreadsheet.mockImplementation(() => mockGoogleIntegration) import { structures } from "@budibase/backend-core/tests" import TestConfiguration from "../../tests/utilities/TestConfiguration" import GoogleSheetsIntegration from "../googlesheets" -import { FieldType, Table, TableSchema } from "@budibase/types" +import { FieldType, Table, TableSchema, TableSourceType } from "@budibase/types" +import { generateDatasourceID } from "../../db/utils" describe("Google Sheets Integration", () => { let integration: any, config = new TestConfiguration() + let cleanupEnv: () => void beforeAll(() => { - config.setGoogleAuth("test") + cleanupEnv = config.setEnv({ + GOOGLE_CLIENT_ID: "test", + GOOGLE_CLIENT_SECRET: "test", + }) }) afterAll(async () => { - await config.end() + cleanupEnv() + config.end() }) beforeEach(async () => { @@ -60,7 +66,10 @@ describe("Google Sheets Integration", () => { function createBasicTable(name: string, columns: string[]): Table { return { + type: "table", name, + sourceId: generateDatasourceID(), + sourceType: TableSourceType.EXTERNAL, schema: { ...columns.reduce((p, c) => { p[c] = { diff --git a/packages/server/src/integrations/tests/redis.spec.ts b/packages/server/src/integrations/tests/redis.spec.ts index 9521d58a51..942da99530 100644 --- a/packages/server/src/integrations/tests/redis.spec.ts +++ b/packages/server/src/integrations/tests/redis.spec.ts @@ -85,4 +85,21 @@ describe("Redis Integration", () => { ["get", "foo"], ]) }) + + it("calls the pipeline method with double quoted phrase values", async () => { + const body = { + json: 'SET foo "What a wonderful world!"\nGET foo', + } + + // ioredis-mock doesn't support pipelines + config.integration.client.pipeline = jest.fn(() => ({ + exec: jest.fn(() => [[]]), + })) + + await config.integration.command(body) + expect(config.integration.client.pipeline).toHaveBeenCalledWith([ + ["set", "foo", '"What a wonderful world!"'], + ["get", "foo"], + ]) + }) }) diff --git a/packages/server/src/integrations/tests/utils/postgres.ts b/packages/server/src/integrations/tests/utils/postgres.ts index b749551721..f65d33e3e0 100644 --- a/packages/server/src/integrations/tests/utils/postgres.ts +++ b/packages/server/src/integrations/tests/utils/postgres.ts @@ -1,39 +1,47 @@ import { Datasource, SourceName } from "@budibase/types" import { GenericContainer, Wait, StartedTestContainer } from "testcontainers" +import env from "../../../environment" let container: StartedTestContainer | undefined +const isMac = process.platform === "darwin" + export async function getDsConfig(): Promise { - if (!container) { - container = await new GenericContainer("postgres") - .withExposedPorts(5432) - .withEnv("POSTGRES_PASSWORD", "password") - .withWaitStrategy( - Wait.forLogMessage( - "PostgreSQL init process complete; ready for start up." + try { + if (!container) { + // postgres 15-bullseye safer bet on Linux + const version = isMac ? undefined : "15-bullseye" + container = await new GenericContainer("postgres", version) + .withExposedPorts(5432) + .withEnv("POSTGRES_PASSWORD", "password") + .withWaitStrategy( + Wait.forLogMessage( + "PostgreSQL init process complete; ready for start up." + ) ) - ) - .start() - } + .start() + } + const host = container.getContainerIpAddress() + const port = container.getMappedPort(5432) - const host = container.getContainerIpAddress() - const port = container.getMappedPort(5432) - - return { - type: "datasource_plus", - source: SourceName.POSTGRES, - plus: true, - config: { - host, - port, - database: "postgres", - user: "postgres", - password: "password", - schema: "public", - ssl: false, - rejectUnauthorized: false, - ca: false, - }, + return { + type: "datasource_plus", + source: SourceName.POSTGRES, + plus: true, + config: { + host, + port, + database: "postgres", + user: "postgres", + password: "password", + schema: "public", + ssl: false, + rejectUnauthorized: false, + ca: false, + }, + } + } catch (err) { + throw new Error("**UNABLE TO CREATE TO POSTGRES CONTAINER**") } } diff --git a/packages/server/src/integrations/utils.ts b/packages/server/src/integrations/utils.ts index 59ba26ae81..6f63bc260e 100644 --- a/packages/server/src/integrations/utils.ts +++ b/packages/server/src/integrations/utils.ts @@ -1,12 +1,12 @@ -import { InvalidColumns } from "../constants" import { SqlQuery, Table, Datasource, FieldType, - ExternalTable, + TableSourceType, } from "@budibase/types" import { DocumentType, SEPARATOR } from "../db/utils" +import { InvalidColumns, DEFAULT_BB_DATASOURCE_ID } from "../constants" import { helpers } from "@budibase/shared-core" const DOUBLE_SEPARATOR = `${SEPARATOR}${SEPARATOR}` @@ -83,10 +83,29 @@ export enum SqlClient { SQL_LITE = "sqlite3", } -export function isExternalTable(tableId: string) { +export function isExternalTableID(tableId: string) { return tableId.includes(DocumentType.DATASOURCE) } +export function isInternalTableID(tableId: string) { + return !isExternalTableID(tableId) +} + +export function isExternalTable(table: Table) { + if ( + table?.sourceId && + table.sourceId.includes(DocumentType.DATASOURCE + SEPARATOR) && + table?.sourceId !== DEFAULT_BB_DATASOURCE_ID + ) { + return true + } else if (table?.sourceType === TableSourceType.EXTERNAL) { + return true + } else if (table?._id && isExternalTableID(table._id)) { + return true + } + return false +} + export function buildExternalTableId(datasourceId: string, tableName: string) { // encode spaces if (tableName.includes(" ")) { @@ -297,9 +316,9 @@ function copyExistingPropsOver( * @param entities The old list of tables, if there was any to look for definitions in. */ export function finaliseExternalTables( - tables: Record, - entities: Record -): Record { + tables: Record, + entities: Record +): Record { let finalTables: Record = {} const tableIds = Object.values(tables).map(table => table._id!) for (let [name, table] of Object.entries(tables)) { @@ -312,7 +331,7 @@ export function finaliseExternalTables( } export function checkExternalTables( - tables: Record + tables: Record ): Record { const invalidColumns = Object.values(InvalidColumns) as string[] const errors: Record = {} diff --git a/packages/server/src/middleware/tests/currentapp.spec.js b/packages/server/src/middleware/tests/currentapp.spec.js index b80800fd96..22e47b0a6e 100644 --- a/packages/server/src/middleware/tests/currentapp.spec.js +++ b/packages/server/src/middleware/tests/currentapp.spec.js @@ -9,11 +9,11 @@ function mockWorker() { return { _id: "us_uuid1", roles: { - "app_test": "BASIC", + app_test: "BASIC", }, roleId: "BASIC", } - } + }, })) } @@ -109,7 +109,7 @@ class TestConfiguration { path: "", cookies: { set: jest.fn(), - } + }, } } diff --git a/packages/server/src/middleware/tests/trimViewRowInfo.spec.ts b/packages/server/src/middleware/tests/trimViewRowInfo.spec.ts index bf717d5828..40ff88c1e5 100644 --- a/packages/server/src/middleware/tests/trimViewRowInfo.spec.ts +++ b/packages/server/src/middleware/tests/trimViewRowInfo.spec.ts @@ -1,5 +1,12 @@ import { generator } from "@budibase/backend-core/tests" -import { BBRequest, FieldType, Row, Table } from "@budibase/types" +import { + BBRequest, + FieldType, + Row, + Table, + INTERNAL_TABLE_SOURCE_ID, + TableSourceType, +} from "@budibase/types" import * as utils from "../../db/utils" import trimViewRowInfoMiddleware from "../trimViewRowInfo" @@ -73,6 +80,8 @@ describe("trimViewRowInfo middleware", () => { const table: Table = { _id: tableId, name: generator.word(), + sourceId: INTERNAL_TABLE_SOURCE_ID, + sourceType: TableSourceType.INTERNAL, type: "table", schema: { name: { diff --git a/packages/server/src/middleware/trimViewRowInfo.ts b/packages/server/src/middleware/trimViewRowInfo.ts index 6a7448262b..95b085a08f 100644 --- a/packages/server/src/middleware/trimViewRowInfo.ts +++ b/packages/server/src/middleware/trimViewRowInfo.ts @@ -1,7 +1,6 @@ import { Ctx, Row } from "@budibase/types" import * as utils from "../db/utils" import sdk from "../sdk" -import { db } from "@budibase/backend-core" import { Next } from "koa" import { getTableId } from "../api/controllers/row/utils" diff --git a/packages/server/src/migrations/functions/backfill/global/configs.ts b/packages/server/src/migrations/functions/backfill/global/configs.ts index 1b76727bbe..04eb9caff2 100644 --- a/packages/server/src/migrations/functions/backfill/global/configs.ts +++ b/packages/server/src/migrations/functions/backfill/global/configs.ts @@ -11,10 +11,11 @@ import { isOIDCConfig, isSettingsConfig, ConfigType, + DatabaseQueryOpts, } from "@budibase/types" import env from "./../../../../environment" -export const getConfigParams = () => { +export function getConfigParams(): DatabaseQueryOpts { return { include_docs: true, startkey: `${DocumentType.CONFIG}${SEPARATOR}`, diff --git a/packages/server/src/sdk/app/applications/import.ts b/packages/server/src/sdk/app/applications/import.ts index 158e4772b2..c3415bdb36 100644 --- a/packages/server/src/sdk/app/applications/import.ts +++ b/packages/server/src/sdk/app/applications/import.ts @@ -91,7 +91,7 @@ async function getImportableDocuments(db: Database) { // map the responses to the document itself let documents: Document[] = [] for (let response of await Promise.all(docPromises)) { - documents = documents.concat(response.rows.map(row => row.doc)) + documents = documents.concat(response.rows.map(row => row.doc!)) } // remove the _rev, stops it being written documents.forEach(doc => { diff --git a/packages/server/src/sdk/app/applications/sync.ts b/packages/server/src/sdk/app/applications/sync.ts index 6e1e6747e1..e73b3396d9 100644 --- a/packages/server/src/sdk/app/applications/sync.ts +++ b/packages/server/src/sdk/app/applications/sync.ts @@ -3,7 +3,11 @@ import { db as dbCore, context, logging, roles } from "@budibase/backend-core" import { User, ContextUser, UserGroup } from "@budibase/types" import { sdk as proSdk } from "@budibase/pro" import sdk from "../../" -import { getGlobalUsers, processUser } from "../../../utilities/global" +import { + getGlobalUsers, + getRawGlobalUsers, + processUser, +} from "../../../utilities/global" import { generateUserMetadataID, InternalTables } from "../../../db/utils" type DeletedUser = { _id: string; deleted: boolean } @@ -77,9 +81,7 @@ async function syncUsersToApp( export async function syncUsersToAllApps(userIds: string[]) { // list of users, if one has been deleted it will be undefined in array - const users = (await getGlobalUsers(userIds, { - noProcessing: true, - })) as User[] + const users = await getRawGlobalUsers(userIds) const groups = await proSdk.groups.fetch() const finalUsers: (User | DeletedUser)[] = [] for (let userId of userIds) { diff --git a/packages/server/src/sdk/app/backups/exports.ts b/packages/server/src/sdk/app/backups/exports.ts index d5ea31cdf5..c349dcb927 100644 --- a/packages/server/src/sdk/app/backups/exports.ts +++ b/packages/server/src/sdk/app/backups/exports.ts @@ -26,7 +26,6 @@ export interface DBDumpOpts { export interface ExportOpts extends DBDumpOpts { tar?: boolean excludeRows?: boolean - excludeLogs?: boolean encryptPassword?: string } @@ -83,14 +82,15 @@ export async function exportDB( }) } -function defineFilter(excludeRows?: boolean, excludeLogs?: boolean) { - const ids = [USER_METDATA_PREFIX, LINK_USER_METADATA_PREFIX] +function defineFilter(excludeRows?: boolean) { + const ids = [ + USER_METDATA_PREFIX, + LINK_USER_METADATA_PREFIX, + AUTOMATION_LOG_PREFIX, + ] if (excludeRows) { ids.push(TABLE_ROW_PREFIX) } - if (excludeLogs) { - ids.push(AUTOMATION_LOG_PREFIX) - } return (doc: any) => !ids.map(key => doc._id.includes(key)).reduce((prev, curr) => prev || curr) } @@ -118,7 +118,7 @@ export async function exportApp(appId: string, config?: ExportOpts) { fs.writeFileSync(join(tmpPath, path), contents) } } - // get all of the files + // get all the files else { tmpPath = await objectStore.retrieveDirectory( ObjectStoreBuckets.APPS, @@ -141,7 +141,7 @@ export async function exportApp(appId: string, config?: ExportOpts) { // enforce an export of app DB to the tmp path const dbPath = join(tmpPath, DB_EXPORT_FILE) await exportDB(appId, { - filter: defineFilter(config?.excludeRows, config?.excludeLogs), + filter: defineFilter(config?.excludeRows), exportPath: dbPath, }) @@ -191,7 +191,6 @@ export async function streamExportApp({ }) { const tmpPath = await exportApp(appId, { excludeRows, - excludeLogs: true, tar: true, encryptPassword, }) diff --git a/packages/server/src/sdk/app/datasources/datasources.ts b/packages/server/src/sdk/app/datasources/datasources.ts index fb5d04b03e..51cceeab94 100644 --- a/packages/server/src/sdk/app/datasources/datasources.ts +++ b/packages/server/src/sdk/app/datasources/datasources.ts @@ -51,12 +51,12 @@ export async function fetch(opts?: { // Get external datasources const datasources = ( - await db.allDocs( + await db.allDocs( getDatasourceParams(null, { include_docs: true, }) ) - ).rows.map(row => row.doc) + ).rows.map(row => row.doc!) const allDatasources: Datasource[] = await sdk.datasources.removeSecrets([ bbInternalDb, @@ -271,5 +271,5 @@ export async function getExternalDatasources(): Promise { }) ) - return externalDatasources.rows.map(r => r.doc) + return externalDatasources.rows.map(r => r.doc!) } diff --git a/packages/server/src/sdk/app/links/index.ts b/packages/server/src/sdk/app/links/index.ts new file mode 100644 index 0000000000..6655a76656 --- /dev/null +++ b/packages/server/src/sdk/app/links/index.ts @@ -0,0 +1,5 @@ +import * as links from "./links" + +export default { + ...links, +} diff --git a/packages/server/src/sdk/app/links/links.ts b/packages/server/src/sdk/app/links/links.ts new file mode 100644 index 0000000000..5d3420341f --- /dev/null +++ b/packages/server/src/sdk/app/links/links.ts @@ -0,0 +1,39 @@ +import { context, docIds } from "@budibase/backend-core" + +import { + DatabaseQueryOpts, + LinkDocument, + LinkDocumentValue, +} from "@budibase/types" +import { ViewName, getQueryIndex } from "../../../../src/db/utils" + +export async function fetch(tableId: string): Promise { + if (!docIds.isTableId(tableId)) { + throw new Error(`Invalid tableId: ${tableId}`) + } + + const db = context.getAppDB() + const params: DatabaseQueryOpts = { + startkey: [tableId], + endkey: [tableId, {}], + } + const linkRows = (await db.query(getQueryIndex(ViewName.LINK), params)).rows + return linkRows.map(row => row.value as LinkDocumentValue) +} + +export async function fetchWithDocument( + tableId: string +): Promise { + if (!docIds.isTableId(tableId)) { + throw new Error(`Invalid tableId: ${tableId}`) + } + + const db = context.getAppDB() + const params: DatabaseQueryOpts = { + startkey: [tableId], + endkey: [tableId, {}], + include_docs: true, + } + const linkRows = (await db.query(getQueryIndex(ViewName.LINK), params)).rows + return linkRows.map(row => row.doc as LinkDocument) +} diff --git a/packages/server/src/sdk/app/rows/external.ts b/packages/server/src/sdk/app/rows/external.ts index 8bcf89a3f5..beae02e134 100644 --- a/packages/server/src/sdk/app/rows/external.ts +++ b/packages/server/src/sdk/app/rows/external.ts @@ -1,4 +1,4 @@ -import { IncludeRelationship, Operation, Row } from "@budibase/types" +import { IncludeRelationship, Operation } from "@budibase/types" import { handleRequest } from "../../../api/controllers/row/external" import { breakRowIdField } from "../../../integrations/utils" diff --git a/packages/server/src/sdk/app/rows/search.ts b/packages/server/src/sdk/app/rows/search.ts index 24ed3f680b..f4fb258213 100644 --- a/packages/server/src/sdk/app/rows/search.ts +++ b/packages/server/src/sdk/app/rows/search.ts @@ -1,5 +1,11 @@ -import { SearchFilters, SearchParams, Row } from "@budibase/types" -import { isExternalTable } from "../../../integrations/utils" +import { + Row, + SearchFilters, + SearchParams, + SortOrder, + SortType, +} from "@budibase/types" +import { isExternalTableID } from "../../../integrations/utils" import * as internal from "./search/internal" import * as external from "./search/external" import { Format } from "../../../api/controllers/view/exporters" @@ -13,7 +19,7 @@ export interface ViewParams { } function pickApi(tableId: any) { - if (isExternalTable(tableId)) { + if (isExternalTableID(tableId)) { return external } return internal @@ -62,6 +68,8 @@ export interface ExportRowsParams { rowIds?: string[] columns?: string[] query?: SearchFilters + sort?: string + sortOrder?: SortOrder } export interface ExportRowsResult { @@ -79,6 +87,10 @@ export async function fetch(tableId: string): Promise { return pickApi(tableId).fetch(tableId) } +export async function fetchRaw(tableId: string): Promise { + return pickApi(tableId).fetchRaw(tableId) +} + export async function fetchView( tableId: string, viewName: string, diff --git a/packages/server/src/sdk/app/rows/search/external.ts b/packages/server/src/sdk/app/rows/search/external.ts index b673d45938..a7886ef54f 100644 --- a/packages/server/src/sdk/app/rows/search/external.ts +++ b/packages/server/src/sdk/app/rows/search/external.ts @@ -80,7 +80,10 @@ export async function search(options: SearchParams) { rows = rows.map((r: any) => pick(r, fields)) } - rows = await outputProcessing(table, rows, { preserveLinks: true }) + rows = await outputProcessing(table, rows, { + preserveLinks: true, + squash: true, + }) // need wrapper object for bookmarks etc when paginating return { rows, hasNextPage, bookmark: bookmark && bookmark + 1 } @@ -98,12 +101,12 @@ export async function search(options: SearchParams) { export async function exportRows( options: ExportRowsParams ): Promise { - const { tableId, format, columns, rowIds } = options + const { tableId, format, columns, rowIds, query, sort, sortOrder } = options const { datasourceId, tableName } = breakExternalTableId(tableId) - let query: SearchFilters = {} + let requestQuery: SearchFilters = {} if (rowIds?.length) { - query = { + requestQuery = { oneOf: { _id: rowIds.map((row: string) => { const ids = JSON.parse( @@ -119,6 +122,8 @@ export async function exportRows( }), }, } + } else { + requestQuery = query || {} } const datasource = await sdk.datasources.get(datasourceId!) @@ -126,7 +131,7 @@ export async function exportRows( throw new HTTPError("Datasource has not been configured for plus API.", 400) } - let result = await search({ tableId, query }) + let result = await search({ tableId, query: requestQuery, sort, sortOrder }) let rows: Row[] = [] // Filter data to only specified columns if required @@ -183,6 +188,13 @@ export async function fetch(tableId: string): Promise { const table = await sdk.tables.getTable(tableId) return await outputProcessing(table, response, { preserveLinks: true, + squash: true, + }) +} + +export async function fetchRaw(tableId: string): Promise { + return await handleRequest(Operation.READ, tableId, { + includeSqlRelationships: IncludeRelationship.INCLUDE, }) } diff --git a/packages/server/src/sdk/app/rows/search/internal.ts b/packages/server/src/sdk/app/rows/search/internal.ts index acc6a8d4da..4cbe84557c 100644 --- a/packages/server/src/sdk/app/rows/search/internal.ts +++ b/packages/server/src/sdk/app/rows/search/internal.ts @@ -85,7 +85,7 @@ export async function search(options: SearchParams) { export async function exportRows( options: ExportRowsParams ): Promise { - const { tableId, format, rowIds, columns, query } = options + const { tableId, format, rowIds, columns, query, sort, sortOrder } = options const db = context.getAppDB() const table = await sdk.tables.getTable(tableId) @@ -100,7 +100,12 @@ export async function exportRows( result = await outputProcessing(table, response) } else if (query) { - let searchResponse = await search({ tableId, query }) + let searchResponse = await search({ + tableId, + query, + sort, + sortOrder, + }) result = searchResponse.rows } @@ -141,14 +146,13 @@ export async function exportRows( } export async function fetch(tableId: string): Promise { - const db = context.getAppDB() - const table = await sdk.tables.getTable(tableId) - const rows = await getRawTableData(db, tableId) + const rows = await fetchRaw(tableId) return await outputProcessing(table, rows) } -async function getRawTableData(db: Database, tableId: string) { +export async function fetchRaw(tableId: string): Promise { + const db = context.getAppDB() let rows if (tableId === InternalTables.USER_METADATA) { rows = await sdk.users.fetchMetadata() @@ -182,8 +186,8 @@ export async function fetchView( group: !!group, }) } else { - const tableId = viewInfo.meta.tableId - const data = await getRawTableData(db, tableId) + const tableId = viewInfo.meta!.tableId + const data = await fetchRaw(tableId!) response = await inMemoryViews.runView( viewInfo, calculation as string, @@ -197,13 +201,9 @@ export async function fetchView( response.rows = response.rows.map(row => row.doc) let table: Table try { - table = await sdk.tables.getTable(viewInfo.meta.tableId) + table = await sdk.tables.getTable(viewInfo.meta!.tableId) } catch (err) { - /* istanbul ignore next */ - table = { - name: "", - schema: {}, - } + throw new Error("Unable to retrieve view table.") } rows = await outputProcessing(table, response.rows) } diff --git a/packages/server/src/sdk/app/rows/search/tests/external.spec.ts b/packages/server/src/sdk/app/rows/search/tests/external.spec.ts index b3bddfbc97..c92155230a 100644 --- a/packages/server/src/sdk/app/rows/search/tests/external.spec.ts +++ b/packages/server/src/sdk/app/rows/search/tests/external.spec.ts @@ -7,6 +7,7 @@ import { SourceName, Table, SearchParams, + TableSourceType, } from "@budibase/types" import TestConfiguration from "../../../../../tests/utilities/TestConfiguration" @@ -15,6 +16,7 @@ import { expectAnyExternalColsAttributes, generator, } from "@budibase/backend-core/tests" +import datasource from "../../../../../api/routes/datasource" jest.unmock("mysql2/promise") @@ -23,36 +25,7 @@ jest.setTimeout(30000) describe.skip("external", () => { const config = new TestConfiguration() - let externalDatasource: Datasource - - const tableData: Table = { - name: generator.word(), - type: "external", - primary: ["id"], - schema: { - id: { - name: "id", - type: FieldType.AUTO, - autocolumn: true, - }, - name: { - name: "name", - type: FieldType.STRING, - }, - surname: { - name: "surname", - type: FieldType.STRING, - }, - age: { - name: "age", - type: FieldType.NUMBER, - }, - address: { - name: "address", - type: FieldType.STRING, - }, - }, - } + let externalDatasource: Datasource, tableData: Table beforeAll(async () => { const container = await new GenericContainer("mysql") @@ -84,12 +57,43 @@ describe.skip("external", () => { }, }, }) + + tableData = { + name: generator.word(), + type: "table", + primary: ["id"], + sourceId: externalDatasource._id!, + sourceType: TableSourceType.EXTERNAL, + schema: { + id: { + name: "id", + type: FieldType.AUTO, + autocolumn: true, + }, + name: { + name: "name", + type: FieldType.STRING, + }, + surname: { + name: "surname", + type: FieldType.STRING, + }, + age: { + name: "age", + type: FieldType.NUMBER, + }, + address: { + name: "address", + type: FieldType.STRING, + }, + }, + } }) describe("search", () => { const rows: Row[] = [] beforeAll(async () => { - const table = await config.createTable({ + const table = await config.createExternalTable({ ...tableData, sourceId: externalDatasource._id, }) diff --git a/packages/server/src/sdk/app/rows/search/tests/internal.spec.ts b/packages/server/src/sdk/app/rows/search/tests/internal.spec.ts index b3e98a1149..d82af66e3d 100644 --- a/packages/server/src/sdk/app/rows/search/tests/internal.spec.ts +++ b/packages/server/src/sdk/app/rows/search/tests/internal.spec.ts @@ -1,4 +1,11 @@ -import { FieldType, Row, Table, SearchParams } from "@budibase/types" +import { + FieldType, + Row, + Table, + SearchParams, + INTERNAL_TABLE_SOURCE_ID, + TableSourceType, +} from "@budibase/types" import TestConfiguration from "../../../../../tests/utilities/TestConfiguration" import { search } from "../internal" import { @@ -12,6 +19,8 @@ describe("internal", () => { const tableData: Table = { name: generator.word(), type: "table", + sourceId: INTERNAL_TABLE_SOURCE_ID, + sourceType: TableSourceType.INTERNAL, schema: { name: { name: "name", diff --git a/packages/server/src/sdk/app/rows/search/tests/utils.spec.ts b/packages/server/src/sdk/app/rows/search/tests/utils.spec.ts index d946eea432..055628c41c 100644 --- a/packages/server/src/sdk/app/rows/search/tests/utils.spec.ts +++ b/packages/server/src/sdk/app/rows/search/tests/utils.spec.ts @@ -3,14 +3,19 @@ import { db as dbCore } from "@budibase/backend-core" import { FieldType, FieldTypeSubtypes, - Table, + INTERNAL_TABLE_SOURCE_ID, SearchParams, + Table, + TableSourceType, } from "@budibase/types" const tableId = "ta_a" const tableWithUserCol: Table = { + type: "table", _id: tableId, name: "table", + sourceId: INTERNAL_TABLE_SOURCE_ID, + sourceType: TableSourceType.INTERNAL, schema: { user: { name: "user", @@ -21,8 +26,11 @@ const tableWithUserCol: Table = { } const tableWithUsersCol: Table = { + type: "table", _id: tableId, name: "table", + sourceId: INTERNAL_TABLE_SOURCE_ID, + sourceType: TableSourceType.INTERNAL, schema: { user: { name: "user", diff --git a/packages/server/src/sdk/app/tables/external/index.ts b/packages/server/src/sdk/app/tables/external/index.ts index 402baada78..f445fcaf08 100644 --- a/packages/server/src/sdk/app/tables/external/index.ts +++ b/packages/server/src/sdk/app/tables/external/index.ts @@ -35,10 +35,10 @@ export async function save( opts?: { tableId?: string; renaming?: RenameColumn } ) { let tableToSave: TableRequest = { + ...update, type: "table", _id: buildExternalTableId(datasourceId, update.name), sourceId: datasourceId, - ...update, } const tableId = opts?.tableId || update._id diff --git a/packages/server/src/sdk/app/tables/external/utils.ts b/packages/server/src/sdk/app/tables/external/utils.ts index 10c755a7d6..bde812dd3d 100644 --- a/packages/server/src/sdk/app/tables/external/utils.ts +++ b/packages/server/src/sdk/app/tables/external/utils.ts @@ -6,6 +6,7 @@ import { RelationshipFieldMetadata, RelationshipType, Table, + TableSourceType, } from "@budibase/types" import { FieldTypes } from "../../../../constants" import { @@ -76,12 +77,16 @@ export function generateManyLinkSchema( const primary = table.name + table.primary[0] const relatedPrimary = relatedTable.name + relatedTable.primary[0] const jcTblName = generateJunctionTableName(column, table, relatedTable) + const datasourceId = datasource._id! // first create the new table - const junctionTable = { - _id: buildExternalTableId(datasource._id!, jcTblName), + const junctionTable: Table = { + type: "table", + _id: buildExternalTableId(datasourceId, jcTblName), name: jcTblName, primary: [primary, relatedPrimary], constrained: [primary, relatedPrimary], + sourceId: datasourceId, + sourceType: TableSourceType.EXTERNAL, schema: { [primary]: foreignKeyStructure(primary, { toTable: table.name, diff --git a/packages/server/src/sdk/app/tables/getters.ts b/packages/server/src/sdk/app/tables/getters.ts index 02cef748c5..72a6ab61f1 100644 --- a/packages/server/src/sdk/app/tables/getters.ts +++ b/packages/server/src/sdk/app/tables/getters.ts @@ -1,42 +1,62 @@ import { context } from "@budibase/backend-core" -import { - BudibaseInternalDB, - getMultiIDParams, - getTableParams, -} from "../../../db/utils" +import { getTableParams } from "../../../db/utils" import { breakExternalTableId, - isExternalTable, + isExternalTableID, isSQL, } from "../../../integrations/utils" import { - AllDocsResponse, Database, + INTERNAL_TABLE_SOURCE_ID, Table, TableResponse, + TableSourceType, TableViewsResponse, } from "@budibase/types" import datasources from "../datasources" import sdk from "../../../sdk" -function processInternalTables(docs: AllDocsResponse): Table[] { - return docs.rows.map((tableDoc: any) => ({ - ...tableDoc.doc, - type: "internal", - sourceId: tableDoc.doc.sourceId || BudibaseInternalDB._id, - })) +export function processTable(table: Table): Table { + if (!table) { + return table + } + if (table._id && isExternalTableID(table._id)) { + return { + ...table, + type: "table", + sourceType: TableSourceType.EXTERNAL, + } + } else { + return { + ...table, + type: "table", + sourceId: table.sourceId || INTERNAL_TABLE_SOURCE_ID, + sourceType: TableSourceType.INTERNAL, + } + } +} + +export function processTables(tables: Table[]): Table[] { + return tables.map(table => processTable(table)) +} + +function processEntities(tables: Record) { + for (let key of Object.keys(tables)) { + tables[key] = processTable(tables[key]) + } + return tables } export async function getAllInternalTables(db?: Database): Promise { if (!db) { db = context.getAppDB() } - const internalTables = await db.allDocs( + const internalTables = await db.allDocs
( getTableParams(null, { include_docs: true, }) ) - return processInternalTables(internalTables) + return processTables(internalTables.rows.map(row => row.doc!)) } async function getAllExternalTables(): Promise { @@ -48,7 +68,7 @@ async function getAllExternalTables(): Promise { final = final.concat(Object.values(entities)) } } - return final + return processTables(final) } export async function getExternalTable( @@ -56,19 +76,24 @@ export async function getExternalTable( tableName: string ): Promise
{ const entities = await getExternalTablesInDatasource(datasourceId) - return entities[tableName] + if (!entities[tableName]) { + throw new Error(`Unable to find table named "${tableName}"`) + } + return processTable(entities[tableName]) } export async function getTable(tableId: string): Promise
{ const db = context.getAppDB() - if (isExternalTable(tableId)) { + let output: Table + if (isExternalTableID(tableId)) { let { datasourceId, tableName } = breakExternalTableId(tableId) const datasource = await datasources.get(datasourceId!) const table = await getExternalTable(datasourceId!, tableName!) - return { ...table, sql: isSQL(datasource) } + output = { ...table, sql: isSQL(datasource) } } else { - return db.get(tableId) + output = await db.get
(tableId) } + return processTable(output) } export async function getAllTables() { @@ -76,7 +101,7 @@ export async function getAllTables() { getAllInternalTables(), getAllExternalTables(), ]) - return [...internal, ...external] + return processTables([...internal, ...external]) } export async function getExternalTablesInDatasource( @@ -86,12 +111,14 @@ export async function getExternalTablesInDatasource( if (!datasource || !datasource.entities) { throw new Error("Datasource is not configured fully.") } - return datasource.entities + return processEntities(datasource.entities) } export async function getTables(tableIds: string[]): Promise { - const externalTableIds = tableIds.filter(tableId => isExternalTable(tableId)), - internalTableIds = tableIds.filter(tableId => !isExternalTable(tableId)) + const externalTableIds = tableIds.filter(tableId => + isExternalTableID(tableId) + ), + internalTableIds = tableIds.filter(tableId => !isExternalTableID(tableId)) let tables: Table[] = [] if (externalTableIds.length) { const externalTables = await getAllExternalTables() @@ -103,12 +130,12 @@ export async function getTables(tableIds: string[]): Promise { } if (internalTableIds.length) { const db = context.getAppDB() - const internalTableDocs = await db.allDocs( - getMultiIDParams(internalTableIds) - ) - tables = tables.concat(processInternalTables(internalTableDocs)) + const internalTables = await db.getMultiple
(internalTableIds, { + allowMissing: true, + }) + tables = tables.concat(internalTables) } - return tables + return processTables(tables) } export function enrichViewSchemas(table: Table): TableResponse { diff --git a/packages/server/src/sdk/app/tables/index.ts b/packages/server/src/sdk/app/tables/index.ts index 8542250517..ed71498d44 100644 --- a/packages/server/src/sdk/app/tables/index.ts +++ b/packages/server/src/sdk/app/tables/index.ts @@ -2,10 +2,12 @@ import { populateExternalTableSchemas } from "./validation" import * as getters from "./getters" import * as updates from "./update" import * as utils from "./utils" +import { migrate } from "./migration" export default { populateExternalTableSchemas, ...updates, ...getters, ...utils, + migrate, } diff --git a/packages/server/src/sdk/app/tables/migration.ts b/packages/server/src/sdk/app/tables/migration.ts new file mode 100644 index 0000000000..e282251bfb --- /dev/null +++ b/packages/server/src/sdk/app/tables/migration.ts @@ -0,0 +1,212 @@ +import { BadRequestError, context, db as dbCore } from "@budibase/backend-core" +import { + BBReferenceFieldMetadata, + FieldSchema, + FieldSubtype, + InternalTable, + isBBReferenceField, + isRelationshipField, + LinkDocument, + LinkInfo, + RelationshipFieldMetadata, + RelationshipType, + Row, + Table, +} from "@budibase/types" +import sdk from "../../../sdk" +import { isExternalTableID } from "../../../integrations/utils" +import { EventType, updateLinks } from "../../../db/linkedRows" +import { cloneDeep } from "lodash" + +export interface MigrationResult { + tablesUpdated: Table[] +} + +export async function migrate( + table: Table, + oldColumn: FieldSchema, + newColumn: FieldSchema +): Promise { + if (newColumn.name in table.schema) { + throw new BadRequestError(`Column "${newColumn.name}" already exists`) + } + + if (newColumn.name === "") { + throw new BadRequestError(`Column name cannot be empty`) + } + + if (dbCore.isInternalColumnName(newColumn.name)) { + throw new BadRequestError(`Column name cannot be a reserved column name`) + } + + table.schema[newColumn.name] = newColumn + table = await sdk.tables.saveTable(table) + + let migrator = getColumnMigrator(table, oldColumn, newColumn) + try { + return await migrator.doMigration() + } catch (e) { + // If the migration fails then we need to roll back the table schema + // change. + delete table.schema[newColumn.name] + await sdk.tables.saveTable(table) + throw e + } +} + +interface ColumnMigrator { + doMigration(): Promise +} + +function getColumnMigrator( + table: Table, + oldColumn: FieldSchema, + newColumn: FieldSchema +): ColumnMigrator { + // For now, we're only supporting migrations of user relationships to user + // columns in internal tables. In the future, we may want to support other + // migrations but for now return an error if we aren't migrating a user + // relationship. + if (isExternalTableID(table._id!)) { + throw new BadRequestError("External tables cannot be migrated") + } + + if (!(oldColumn.name in table.schema)) { + throw new BadRequestError(`Column "${oldColumn.name}" does not exist`) + } + + if (!isBBReferenceField(newColumn)) { + throw new BadRequestError(`Column "${newColumn.name}" is not a user column`) + } + + if (newColumn.subtype !== "user" && newColumn.subtype !== "users") { + throw new BadRequestError(`Column "${newColumn.name}" is not a user column`) + } + + if (!isRelationshipField(oldColumn)) { + throw new BadRequestError( + `Column "${oldColumn.name}" is not a user relationship` + ) + } + + if (oldColumn.tableId !== InternalTable.USER_METADATA) { + throw new BadRequestError( + `Column "${oldColumn.name}" is not a user relationship` + ) + } + + if (oldColumn.relationshipType === RelationshipType.ONE_TO_MANY) { + if (newColumn.subtype !== FieldSubtype.USER) { + throw new BadRequestError( + `Column "${oldColumn.name}" is a one-to-many column but "${newColumn.name}" is not a single user column` + ) + } + return new SingleUserColumnMigrator(table, oldColumn, newColumn) + } + if ( + oldColumn.relationshipType === RelationshipType.MANY_TO_MANY || + oldColumn.relationshipType === RelationshipType.MANY_TO_ONE + ) { + if (newColumn.subtype !== FieldSubtype.USERS) { + throw new BadRequestError( + `Column "${oldColumn.name}" is a ${oldColumn.relationshipType} column but "${newColumn.name}" is not a multi user column` + ) + } + return new MultiUserColumnMigrator(table, oldColumn, newColumn) + } + + throw new BadRequestError(`Unknown migration type`) +} + +abstract class UserColumnMigrator implements ColumnMigrator { + constructor( + protected table: Table, + protected oldColumn: RelationshipFieldMetadata, + protected newColumn: BBReferenceFieldMetadata + ) {} + + abstract updateRow(row: Row, linkInfo: LinkInfo): void + + pickUserTableLinkSide(link: LinkDocument): LinkInfo { + if (link.doc1.tableId === InternalTable.USER_METADATA) { + return link.doc1 + } else { + return link.doc2 + } + } + + pickOtherTableLinkSide(link: LinkDocument): LinkInfo { + if (link.doc1.tableId === InternalTable.USER_METADATA) { + return link.doc2 + } else { + return link.doc1 + } + } + + async doMigration(): Promise { + let oldTable = cloneDeep(this.table) + let rows = await sdk.rows.fetchRaw(this.table._id!) + let rowsById = rows.reduce((acc, row) => { + acc[row._id!] = row + return acc + }, {} as Record) + + let links = await sdk.links.fetchWithDocument(this.table._id!) + for (let link of links) { + const userSide = this.pickUserTableLinkSide(link) + const otherSide = this.pickOtherTableLinkSide(link) + if ( + otherSide.tableId !== this.table._id || + otherSide.fieldName !== this.oldColumn.name || + userSide.tableId !== InternalTable.USER_METADATA + ) { + continue + } + + let row = rowsById[otherSide.rowId] + if (!row) { + // This can happen if the row has been deleted but the link hasn't, + // which was a state that was found during the initial testing of this + // feature. Not sure exactly what can cause it, but best to be safe. + continue + } + + this.updateRow(row, userSide) + } + + let db = context.getAppDB() + await db.bulkDocs(rows) + + delete this.table.schema[this.oldColumn.name] + this.table = await sdk.tables.saveTable(this.table) + await updateLinks({ + eventType: EventType.TABLE_UPDATED, + table: this.table, + oldTable, + }) + + let otherTable = await sdk.tables.getTable(this.oldColumn.tableId) + return { + tablesUpdated: [this.table, otherTable], + } + } +} + +class SingleUserColumnMigrator extends UserColumnMigrator { + updateRow(row: Row, linkInfo: LinkInfo): void { + row[this.newColumn.name] = dbCore.getGlobalIDFromUserMetadataID( + linkInfo.rowId + ) + } +} + +class MultiUserColumnMigrator extends UserColumnMigrator { + updateRow(row: Row, linkInfo: LinkInfo): void { + if (!row[this.newColumn.name]) { + row[this.newColumn.name] = [] + } + row[this.newColumn.name].push( + dbCore.getGlobalIDFromUserMetadataID(linkInfo.rowId) + ) + } +} diff --git a/packages/server/src/sdk/app/tables/tests/tables.spec.ts b/packages/server/src/sdk/app/tables/tests/tables.spec.ts index 78ebe59f01..457988c476 100644 --- a/packages/server/src/sdk/app/tables/tests/tables.spec.ts +++ b/packages/server/src/sdk/app/tables/tests/tables.spec.ts @@ -1,4 +1,10 @@ -import { FieldType, Table, ViewV2 } from "@budibase/types" +import { + FieldType, + INTERNAL_TABLE_SOURCE_ID, + Table, + TableSourceType, + ViewV2, +} from "@budibase/types" import { generator } from "@budibase/backend-core/tests" import sdk from "../../.." @@ -13,6 +19,8 @@ describe("table sdk", () => { _id: generator.guid(), name: "TestTable", type: "table", + sourceId: INTERNAL_TABLE_SOURCE_ID, + sourceType: TableSourceType.INTERNAL, schema: { name: { type: FieldType.STRING, diff --git a/packages/server/src/sdk/app/tables/tests/validation.spec.ts b/packages/server/src/sdk/app/tables/tests/validation.spec.ts index 5347eede90..66b4222005 100644 --- a/packages/server/src/sdk/app/tables/tests/validation.spec.ts +++ b/packages/server/src/sdk/app/tables/tests/validation.spec.ts @@ -1,73 +1,92 @@ import { populateExternalTableSchemas } from "../validation" import { cloneDeep } from "lodash/fp" -import { AutoReason, Datasource, Table } from "@budibase/types" +import { + AutoReason, + Datasource, + FieldType, + RelationshipType, + SourceName, + Table, + TableSourceType, +} from "@budibase/types" import { isEqual } from "lodash" +import { generateDatasourceID } from "../../../../db/utils" -const SCHEMA = { +const datasourceId = generateDatasourceID() + +const SCHEMA: Datasource = { + source: SourceName.POSTGRES, + type: "datasource", + _id: datasourceId, entities: { client: { + type: "table", _id: "tableA", name: "client", primary: ["idC"], primaryDisplay: "Name", + sourceId: datasourceId, + sourceType: TableSourceType.EXTERNAL, schema: { idC: { autocolumn: true, externalType: "int unsigned", name: "idC", - type: "number", + type: FieldType.NUMBER, }, Name: { autocolumn: false, externalType: "varchar(255)", name: "Name", - type: "string", + type: FieldType.STRING, }, project: { fieldName: "idC", foreignKey: "idC", main: true, name: "project", - relationshipType: "many-to-one", + relationshipType: RelationshipType.MANY_TO_ONE, tableId: "tableB", - type: "link", + type: FieldType.LINK, }, }, }, project: { + type: "table", _id: "tableB", name: "project", primary: ["idP"], primaryDisplay: "Name", + sourceId: datasourceId, + sourceType: TableSourceType.EXTERNAL, schema: { idC: { externalType: "int unsigned", name: "idC", - type: "number", + type: FieldType.NUMBER, }, idP: { autocolumn: true, externalType: "int unsigned", name: "idProject", - type: "number", + type: FieldType.NUMBER, }, Name: { autocolumn: false, externalType: "varchar(255)", name: "Name", - type: "string", + type: FieldType.STRING, }, client: { fieldName: "idC", foreignKey: "idC", name: "client", - relationshipType: "one-to-many", + relationshipType: RelationshipType.ONE_TO_MANY, tableId: "tableA", - type: "link", + type: FieldType.LINK, }, }, sql: true, - type: "table", }, }, } @@ -95,12 +114,12 @@ describe("validation and update of external table schemas", () => { function noOtherTableChanges(response: any) { checkOtherColumns( response.entities!.client!, - SCHEMA.entities.client as Table, + SCHEMA.entities!.client, OTHER_CLIENT_COLS ) checkOtherColumns( response.entities!.project!, - SCHEMA.entities.project as Table, + SCHEMA.entities!.project, OTHER_PROJECT_COLS ) } diff --git a/packages/server/src/sdk/app/tables/update.ts b/packages/server/src/sdk/app/tables/update.ts index 9bba4a967e..5c762e628b 100644 --- a/packages/server/src/sdk/app/tables/update.ts +++ b/packages/server/src/sdk/app/tables/update.ts @@ -1,23 +1,30 @@ import { Table, RenameColumn } from "@budibase/types" -import { isExternalTable } from "../../../integrations/utils" +import { isExternalTableID } from "../../../integrations/utils" import sdk from "../../index" import { context } from "@budibase/backend-core" import { isExternal } from "./utils" +import { DocumentInsertResponse } from "@budibase/nano" import * as external from "./external" import * as internal from "./internal" +import { cloneDeep } from "lodash" export * as external from "./external" export * as internal from "./internal" -export async function saveTable(table: Table) { +export async function saveTable(table: Table): Promise
{ const db = context.getAppDB() - if (isExternalTable(table._id!)) { + let resp: DocumentInsertResponse + if (isExternalTableID(table._id!)) { const datasource = await sdk.datasources.get(table.sourceId!) datasource.entities![table.name] = table - await db.put(datasource) + resp = await db.put(datasource) } else { - await db.put(table) + resp = await db.put(table) } + + let tableClone = cloneDeep(table) + tableClone._rev = resp.rev + return tableClone } export async function update(table: Table, renaming?: RenameColumn) { diff --git a/packages/server/src/sdk/app/tables/utils.ts b/packages/server/src/sdk/app/tables/utils.ts index 88543e7c4c..b8e3d888af 100644 --- a/packages/server/src/sdk/app/tables/utils.ts +++ b/packages/server/src/sdk/app/tables/utils.ts @@ -1,10 +1,10 @@ -import { Table } from "@budibase/types" -import { isExternalTable } from "../../../integrations/utils" +import { Table, TableSourceType } from "@budibase/types" +import { isExternalTableID } from "../../../integrations/utils" export function isExternal(opts: { table?: Table; tableId?: string }): boolean { - if (opts.table && opts.table.type === "external") { + if (opts.table && opts.table.sourceType === TableSourceType.EXTERNAL) { return true - } else if (opts.tableId && isExternalTable(opts.tableId)) { + } else if (opts.tableId && isExternalTableID(opts.tableId)) { return true } return false diff --git a/packages/server/src/sdk/app/views/index.ts b/packages/server/src/sdk/app/views/index.ts index 927f82cc68..67e7158f21 100644 --- a/packages/server/src/sdk/app/views/index.ts +++ b/packages/server/src/sdk/app/views/index.ts @@ -4,13 +4,13 @@ import { cloneDeep } from "lodash" import sdk from "../../../sdk" import * as utils from "../../../db/utils" -import { isExternalTable } from "../../../integrations/utils" +import { isExternalTableID } from "../../../integrations/utils" import * as internal from "./internal" import * as external from "./external" function pickApi(tableId: any) { - if (isExternalTable(tableId)) { + if (isExternalTableID(tableId)) { return external } return internal diff --git a/packages/server/src/sdk/app/views/tests/views.spec.ts b/packages/server/src/sdk/app/views/tests/views.spec.ts index 8fcc6405ef..508285651a 100644 --- a/packages/server/src/sdk/app/views/tests/views.spec.ts +++ b/packages/server/src/sdk/app/views/tests/views.spec.ts @@ -2,8 +2,10 @@ import _ from "lodash" import { FieldSchema, FieldType, + INTERNAL_TABLE_SOURCE_ID, Table, TableSchema, + TableSourceType, ViewV2, } from "@budibase/types" import { generator } from "@budibase/backend-core/tests" @@ -14,6 +16,8 @@ describe("table sdk", () => { _id: generator.guid(), name: "TestTable", type: "table", + sourceId: INTERNAL_TABLE_SOURCE_ID, + sourceType: TableSourceType.INTERNAL, schema: { name: { type: FieldType.STRING, diff --git a/packages/server/src/sdk/index.ts b/packages/server/src/sdk/index.ts index 24eb1ebf3c..c3057e3d4f 100644 --- a/packages/server/src/sdk/index.ts +++ b/packages/server/src/sdk/index.ts @@ -5,6 +5,7 @@ import { default as applications } from "./app/applications" import { default as datasources } from "./app/datasources" import { default as queries } from "./app/queries" import { default as rows } from "./app/rows" +import { default as links } from "./app/links" import { default as users } from "./users" import { default as plugins } from "./plugins" import * as views from "./app/views" @@ -22,6 +23,7 @@ const sdk = { plugins, views, permissions, + links, } // default export for TS diff --git a/packages/server/src/sdk/tests/tables.spec.ts b/packages/server/src/sdk/tests/tables.spec.ts new file mode 100644 index 0000000000..0e3cd73cfd --- /dev/null +++ b/packages/server/src/sdk/tests/tables.spec.ts @@ -0,0 +1,39 @@ +import TestConfig from "../../tests/utilities/TestConfiguration" +import { basicTable } from "../../tests/utilities/structures" +import { Table } from "@budibase/types" +import sdk from "../" + +describe("tables", () => { + const config = new TestConfig() + let table: Table + + beforeAll(async () => { + await config.init() + table = await config.api.table.create(basicTable()) + }) + + describe("getTables", () => { + it("should be able to retrieve tables", async () => { + await config.doInContext(config.appId, async () => { + const tables = await sdk.tables.getTables([table._id!]) + expect(tables.length).toBe(1) + expect(tables[0]._id).toBe(table._id) + expect(tables[0].name).toBe(table.name) + }) + }) + + it("shouldn't fail when retrieving tables that don't exist", async () => { + await config.doInContext(config.appId, async () => { + const tables = await sdk.tables.getTables(["unknown"]) + expect(tables.length).toBe(0) + }) + }) + + it("should de-duplicate the IDs", async () => { + await config.doInContext(config.appId, async () => { + const tables = await sdk.tables.getTables([table._id!, table._id!]) + expect(tables.length).toBe(1) + }) + }) + }) +}) diff --git a/packages/server/src/sdk/users/tests/utils.spec.ts b/packages/server/src/sdk/users/tests/utils.spec.ts index 5c6777df59..f7c9413ebd 100644 --- a/packages/server/src/sdk/users/tests/utils.spec.ts +++ b/packages/server/src/sdk/users/tests/utils.spec.ts @@ -39,12 +39,12 @@ describe("syncGlobalUsers", () => { expect(metadata).toHaveLength(3) expect(metadata).toContainEqual( expect.objectContaining({ - _id: db.generateUserMetadataID(user1._id), + _id: db.generateUserMetadataID(user1._id!), }) ) expect(metadata).toContainEqual( expect.objectContaining({ - _id: db.generateUserMetadataID(user2._id), + _id: db.generateUserMetadataID(user2._id!), }) ) }) @@ -59,7 +59,7 @@ describe("syncGlobalUsers", () => { expect(metadata).toHaveLength(1) expect(metadata).not.toContainEqual( expect.objectContaining({ - _id: db.generateUserMetadataID(user._id), + _id: db.generateUserMetadataID(user._id!), }) ) }) @@ -70,7 +70,7 @@ describe("syncGlobalUsers", () => { const group = await proSdk.groups.save(structures.userGroups.userGroup()) const user1 = await config.createUser({ admin: false, builder: false }) const user2 = await config.createUser({ admin: false, builder: false }) - await proSdk.groups.addUsers(group.id, [user1._id, user2._id]) + await proSdk.groups.addUsers(group.id, [user1._id!, user2._id!]) await config.doInContext(config.appId, async () => { await syncGlobalUsers() @@ -87,12 +87,12 @@ describe("syncGlobalUsers", () => { expect(metadata).toHaveLength(3) expect(metadata).toContainEqual( expect.objectContaining({ - _id: db.generateUserMetadataID(user1._id), + _id: db.generateUserMetadataID(user1._id!), }) ) expect(metadata).toContainEqual( expect.objectContaining({ - _id: db.generateUserMetadataID(user2._id), + _id: db.generateUserMetadataID(user2._id!), }) ) }) @@ -109,7 +109,7 @@ describe("syncGlobalUsers", () => { { appId: config.prodAppId!, roleId: roles.BUILTIN_ROLE_IDS.BASIC }, ], }) - await proSdk.groups.addUsers(group.id, [user1._id, user2._id]) + await proSdk.groups.addUsers(group.id, [user1._id!, user2._id!]) await config.doInContext(config.appId, async () => { await syncGlobalUsers() diff --git a/packages/server/src/sdk/users/utils.ts b/packages/server/src/sdk/users/utils.ts index 03ddc954e9..9632ac29d8 100644 --- a/packages/server/src/sdk/users/utils.ts +++ b/packages/server/src/sdk/users/utils.ts @@ -7,12 +7,17 @@ import { InternalTables, } from "../../db/utils" import isEqual from "lodash/isEqual" -import { ContextUser, UserMetadata, User, Database } from "@budibase/types" +import { + ContextUser, + UserMetadata, + Database, + ContextUserMetadata, +} from "@budibase/types" export function combineMetadataAndUser( user: ContextUser, metadata: UserMetadata | UserMetadata[] -) { +): ContextUserMetadata | null { const metadataId = generateUserMetadataID(user._id!) const found = Array.isArray(metadata) ? metadata.find(doc => doc._id === metadataId) @@ -51,33 +56,33 @@ export function combineMetadataAndUser( return null } -export async function rawUserMetadata(db?: Database) { +export async function rawUserMetadata(db?: Database): Promise { if (!db) { db = context.getAppDB() } return ( - await db.allDocs( + await db.allDocs( getUserMetadataParams(null, { include_docs: true, }) ) - ).rows.map(row => row.doc) + ).rows.map(row => row.doc!) } -export async function fetchMetadata() { +export async function fetchMetadata(): Promise { const global = await getGlobalUsers() const metadata = await rawUserMetadata() - const users = [] + const users: ContextUserMetadata[] = [] for (let user of global) { // find the metadata that matches up to the global ID - const info = metadata.find(meta => meta._id.includes(user._id)) + const info = metadata.find(meta => meta._id!.includes(user._id!)) // remove these props, not for the correct DB users.push({ ...user, ...info, tableId: InternalTables.USER_METADATA, // make sure the ID is always a local ID, not a global one - _id: generateUserMetadataID(user._id), + _id: generateUserMetadataID(user._id!), }) } return users @@ -90,9 +95,10 @@ export async function syncGlobalUsers() { if (!(await db.exists())) { continue } - const resp = await Promise.all([getGlobalUsers(), rawUserMetadata(db)]) - const users = resp[0] as User[] - const metadata = resp[1] as UserMetadata[] + const [users, metadata] = await Promise.all([ + getGlobalUsers(), + rawUserMetadata(db), + ]) const toWrite = [] for (let user of users) { const combined = combineMetadataAndUser(user, metadata) diff --git a/packages/server/src/tests/jestEnv.ts b/packages/server/src/tests/jestEnv.ts index 34c51009aa..4763208c54 100644 --- a/packages/server/src/tests/jestEnv.ts +++ b/packages/server/src/tests/jestEnv.ts @@ -9,3 +9,4 @@ process.env.LOG_LEVEL = process.env.LOG_LEVEL || "error" process.env.MOCK_REDIS = "1" process.env.PLATFORM_URL = "http://localhost:10000" process.env.REDIS_PASSWORD = "budibase" +process.env.BUDIBASE_VERSION = "0.0.0+jest" diff --git a/packages/server/src/tests/utilities/TestConfiguration.ts b/packages/server/src/tests/utilities/TestConfiguration.ts index cec8c8aa12..3a14a87d2a 100644 --- a/packages/server/src/tests/utilities/TestConfiguration.ts +++ b/packages/server/src/tests/utilities/TestConfiguration.ts @@ -2,37 +2,31 @@ import { generator, mocks, structures } from "@budibase/backend-core/tests" // init the licensing mock import * as pro from "@budibase/pro" -mocks.licenses.init(pro) - -// use unlimited license by default -mocks.licenses.useUnlimited() - import { init as dbInit } from "../../db" -dbInit() import env from "../../environment" import { - basicTable, - basicRow, - basicRole, basicAutomation, - basicDatasource, - basicQuery, - basicScreen, - basicLayout, - basicWebhook, basicAutomationResults, + basicDatasource, + basicLayout, + basicQuery, + basicRole, + basicRow, + basicScreen, + basicTable, + basicWebhook, } from "./structures" import { - constants, - tenancy, - sessions, + auth, cache, + constants, context, db as dbCore, encryption, - auth, - roles, env as coreEnv, + roles, + sessions, + tenancy, } from "@budibase/backend-core" import * as controllers from "./controllers" import { cleanup } from "../../utilities/fileSystem" @@ -43,21 +37,32 @@ import supertest from "supertest" import { App, AuthToken, + Automation, + CreateViewRequest, Datasource, + FieldType, + INTERNAL_TABLE_SOURCE_ID, + RelationshipFieldMetadata, + RelationshipType, Row, + SearchFilters, SourceName, Table, - SearchFilters, + TableSourceType, + User, UserRoles, - Automation, View, - FieldType, - RelationshipType, - CreateViewRequest, - RelationshipFieldMetadata, } from "@budibase/types" import API from "./api" +import { cloneDeep } from "lodash" + +mocks.licenses.init(pro) + +// use unlimited license by default +mocks.licenses.useUnlimited() + +dbInit() type DefaultUserValues = { globalUserId: string @@ -67,6 +72,11 @@ type DefaultUserValues = { csrfToken: string } +interface TableToBuild extends Omit { + sourceId?: string + sourceType?: TableSourceType +} + class TestConfiguration { server: any request: supertest.SuperTest | undefined @@ -188,30 +198,38 @@ class TestConfiguration { } } - // MODES - setMultiTenancy = (value: boolean) => { - env._set("MULTI_TENANCY", value) - coreEnv._set("MULTI_TENANCY", value) + async withEnv(newEnvVars: Partial, f: () => Promise) { + let cleanup = this.setEnv(newEnvVars) + try { + await f() + } finally { + cleanup() + } } - setSelfHosted = (value: boolean) => { - env._set("SELF_HOSTED", value) - coreEnv._set("SELF_HOSTED", value) - } + /* + * Sets the environment variables to the given values and returns a function + * that can be called to reset the environment variables to their original values. + */ + setEnv(newEnvVars: Partial): () => void { + const oldEnv = cloneDeep(env) + const oldCoreEnv = cloneDeep(coreEnv) - setGoogleAuth = (value: string) => { - env._set("GOOGLE_CLIENT_ID", value) - env._set("GOOGLE_CLIENT_SECRET", value) - coreEnv._set("GOOGLE_CLIENT_ID", value) - coreEnv._set("GOOGLE_CLIENT_SECRET", value) - } + let key: keyof typeof newEnvVars + for (key in newEnvVars) { + env._set(key, newEnvVars[key]) + coreEnv._set(key, newEnvVars[key]) + } - modeCloud = () => { - this.setSelfHosted(false) - } + return () => { + for (const [key, value] of Object.entries(oldEnv)) { + env._set(key, value) + } - modeSelf = () => { - this.setSelfHosted(true) + for (const [key, value] of Object.entries(oldCoreEnv)) { + coreEnv._set(key, value) + } + } } // UTILS @@ -246,7 +264,7 @@ class TestConfiguration { admin = false, email = this.defaultUserValues.email, roles, - }: any = {}) { + }: any = {}): Promise { const db = tenancy.getTenantDB(this.getTenantId()) let existing try { @@ -254,7 +272,7 @@ class TestConfiguration { } catch (err) { existing = { email } } - const user = { + const user: User = { _id: id, ...existing, roles: roles || {}, @@ -294,7 +312,7 @@ class TestConfiguration { admin?: boolean roles?: UserRoles } = {} - ) { + ): Promise { let { id, firstName, lastName, email, builder, admin, roles } = user firstName = firstName || this.defaultUserValues.firstName lastName = lastName || this.defaultUserValues.lastName @@ -314,10 +332,7 @@ class TestConfiguration { roles, }) await cache.user.invalidateUser(globalId) - return { - ...resp, - globalId, - } + return resp } async createGroup(roleId: string = roles.BUILTIN_ROLE_IDS.BASIC) { @@ -495,13 +510,14 @@ class TestConfiguration { // create dev app // clear any old app this.appId = null - await context.doInAppContext(null, async () => { - this.app = await this._req( + this.app = await context.doInAppContext(null, async () => { + const app = await this._req( { name: appName }, null, controllers.app.create ) - this.appId = this.app?.appId! + this.appId = app.appId! + return app }) return await context.doInAppContext(this.appId, async () => { // create production app @@ -510,7 +526,7 @@ class TestConfiguration { this.allApps.push(this.prodApp) this.allApps.push(this.app) - return this.app + return this.app! }) } @@ -522,7 +538,7 @@ class TestConfiguration { return context.doInAppContext(prodAppId, async () => { const db = context.getProdAppDB() - return await db.get(dbCore.DocumentType.APP_METADATA) + return await db.get(dbCore.DocumentType.APP_METADATA) }) } @@ -540,10 +556,12 @@ class TestConfiguration { // TABLE async updateTable( - config?: Table, + config?: TableToBuild, { skipReassigning } = { skipReassigning: false } ): Promise
{ config = config || basicTable() + config.sourceType = config.sourceType || TableSourceType.INTERNAL + config.sourceId = config.sourceId || INTERNAL_TABLE_SOURCE_ID const response = await this._req(config, null, controllers.table.save) if (!skipReassigning) { this.table = response @@ -551,18 +569,32 @@ class TestConfiguration { return response } - async createTable(config?: Table, options = { skipReassigning: false }) { + async createTable( + config?: TableToBuild, + options = { skipReassigning: false } + ) { if (config != null && config._id) { delete config._id } config = config || basicTable() - if (this.datasource && !config.sourceId) { - config.sourceId = this.datasource._id - if (this.datasource.plus) { - config.type = "external" - } + if (!config.sourceId) { + config.sourceId = INTERNAL_TABLE_SOURCE_ID } + return this.updateTable(config, options) + } + async createExternalTable( + config?: TableToBuild, + options = { skipReassigning: false } + ) { + if (config != null && config._id) { + delete config._id + } + config = config || basicTable() + if (this.datasource?._id) { + config.sourceId = this.datasource._id + config.sourceType = TableSourceType.EXTERNAL + } return this.updateTable(config, options) } @@ -574,12 +606,15 @@ class TestConfiguration { async createLinkedTable( relationshipType = RelationshipType.ONE_TO_MANY, links: any = ["link"], - config?: Table + config?: TableToBuild ) { if (!this.table) { throw "Must have created a table first." } const tableConfig = config || basicTable() + if (!tableConfig.sourceId) { + tableConfig.sourceId = INTERNAL_TABLE_SOURCE_ID + } tableConfig.primaryDisplay = "name" for (let link of links) { tableConfig.schema[link] = { @@ -591,15 +626,12 @@ class TestConfiguration { } as RelationshipFieldMetadata } - if (this.datasource && !tableConfig.sourceId) { + if (this.datasource?._id) { tableConfig.sourceId = this.datasource._id - if (this.datasource.plus) { - tableConfig.type = "external" - } + tableConfig.sourceType = TableSourceType.EXTERNAL } - const linkedTable = await this.createTable(tableConfig) - return linkedTable + return await this.createTable(tableConfig) } async createAttachmentTable() { @@ -774,8 +806,9 @@ class TestConfiguration { // AUTOMATION LOG - async createAutomationLog(automation: Automation) { - return await context.doInAppContext(this.getProdAppId(), async () => { + async createAutomationLog(automation: Automation, appId?: string) { + appId = appId || this.getProdAppId() + return await context.doInAppContext(appId!, async () => { return await pro.sdk.automations.logs.storeLog( automation, basicAutomationResults(automation._id!) diff --git a/packages/server/src/tests/utilities/api/attachment.ts b/packages/server/src/tests/utilities/api/attachment.ts new file mode 100644 index 0000000000..a466f1a67e --- /dev/null +++ b/packages/server/src/tests/utilities/api/attachment.ts @@ -0,0 +1,35 @@ +import { + APIError, + Datasource, + ProcessAttachmentResponse, +} from "@budibase/types" +import TestConfiguration from "../TestConfiguration" +import { TestAPI } from "./base" +import fs from "fs" + +export class AttachmentAPI extends TestAPI { + constructor(config: TestConfiguration) { + super(config) + } + + process = async ( + name: string, + file: Buffer | fs.ReadStream | string, + { expectStatus } = { expectStatus: 200 } + ): Promise => { + const result = await this.request + .post(`/api/attachments/process`) + .attach("file", file, name) + .set(this.config.defaultHeaders()) + + if (result.statusCode !== expectStatus) { + throw new Error( + `Expected status ${expectStatus} but got ${ + result.statusCode + }, body: ${JSON.stringify(result.body)}` + ) + } + + return result.body + } +} diff --git a/packages/server/src/tests/utilities/api/backup.ts b/packages/server/src/tests/utilities/api/backup.ts new file mode 100644 index 0000000000..f9cbc7086e --- /dev/null +++ b/packages/server/src/tests/utilities/api/backup.ts @@ -0,0 +1,45 @@ +import { + CreateAppBackupResponse, + ImportAppBackupResponse, +} from "@budibase/types" +import TestConfiguration from "../TestConfiguration" +import { TestAPI } from "./base" + +export class BackupAPI extends TestAPI { + constructor(config: TestConfiguration) { + super(config) + } + + exportBasicBackup = async (appId: string) => { + const result = await this.request + .post(`/api/backups/export?appId=${appId}`) + .set(this.config.defaultHeaders()) + .expect("Content-Type", /application\/gzip/) + .expect(200) + return { + body: result.body as Buffer, + headers: result.headers, + } + } + + createBackup = async (appId: string) => { + const result = await this.request + .post(`/api/apps/${appId}/backups`) + .set(this.config.defaultHeaders()) + .expect("Content-Type", /json/) + .expect(200) + return result.body as CreateAppBackupResponse + } + + importBackup = async ( + appId: string, + backupId: string + ): Promise => { + const result = await this.request + .post(`/api/apps/${appId}/backups/${backupId}/import`) + .set(this.config.defaultHeaders()) + .expect("Content-Type", /json/) + .expect(200) + return result.body as ImportAppBackupResponse + } +} diff --git a/packages/server/src/tests/utilities/api/index.ts b/packages/server/src/tests/utilities/api/index.ts index fce8237760..20b96f7a99 100644 --- a/packages/server/src/tests/utilities/api/index.ts +++ b/packages/server/src/tests/utilities/api/index.ts @@ -7,6 +7,9 @@ import { DatasourceAPI } from "./datasource" import { LegacyViewAPI } from "./legacyView" import { ScreenAPI } from "./screen" import { ApplicationAPI } from "./application" +import { BackupAPI } from "./backup" +import { AttachmentAPI } from "./attachment" +import { UserAPI } from "./user" export default class API { table: TableAPI @@ -17,6 +20,9 @@ export default class API { datasource: DatasourceAPI screen: ScreenAPI application: ApplicationAPI + backup: BackupAPI + attachment: AttachmentAPI + user: UserAPI constructor(config: TestConfiguration) { this.table = new TableAPI(config) @@ -27,5 +33,8 @@ export default class API { this.datasource = new DatasourceAPI(config) this.screen = new ScreenAPI(config) this.application = new ApplicationAPI(config) + this.backup = new BackupAPI(config) + this.attachment = new AttachmentAPI(config) + this.user = new UserAPI(config) } } diff --git a/packages/server/src/tests/utilities/api/row.ts b/packages/server/src/tests/utilities/api/row.ts index bb880bb7da..3d4cf6c82c 100644 --- a/packages/server/src/tests/utilities/api/row.ts +++ b/packages/server/src/tests/utilities/api/row.ts @@ -6,6 +6,7 @@ import { ExportRowsRequest, BulkImportRequest, BulkImportResponse, + SearchRowResponse, } from "@budibase/types" import TestConfiguration from "../TestConfiguration" import { TestAPI } from "./base" @@ -55,7 +56,13 @@ export class RowAPI extends TestAPI { .send(row) .set(this.config.defaultHeaders()) .expect("Content-Type", /json/) - .expect(expectStatus) + if (resp.status !== expectStatus) { + throw new Error( + `Expected status ${expectStatus} but got ${ + resp.status + }, body: ${JSON.stringify(resp.body)}` + ) + } return resp.body as Row } @@ -77,13 +84,20 @@ export class RowAPI extends TestAPI { sourceId: string, row: PatchRowRequest, { expectStatus } = { expectStatus: 200 } - ) => { - return this.request + ): Promise => { + let resp = await this.request .patch(`/api/${sourceId}/rows`) .send(row) .set(this.config.defaultHeaders()) .expect("Content-Type", /json/) - .expect(expectStatus) + if (resp.status !== expectStatus) { + throw new Error( + `Expected status ${expectStatus} but got ${ + resp.status + }, body: ${JSON.stringify(resp.body)}` + ) + } + return resp.body as Row } delete = async ( @@ -141,7 +155,7 @@ export class RowAPI extends TestAPI { search = async ( sourceId: string, { expectStatus } = { expectStatus: 200 } - ): Promise => { + ): Promise => { const request = this.request .post(`/api/${sourceId}/search`) .set(this.config.defaultHeaders()) diff --git a/packages/server/src/tests/utilities/api/table.ts b/packages/server/src/tests/utilities/api/table.ts index 04432a788a..ffd9e19ee8 100644 --- a/packages/server/src/tests/utilities/api/table.ts +++ b/packages/server/src/tests/utilities/api/table.ts @@ -1,4 +1,10 @@ -import { SaveTableRequest, SaveTableResponse, Table } from "@budibase/types" +import { + MigrateRequest, + MigrateResponse, + SaveTableRequest, + SaveTableResponse, + Table, +} from "@budibase/types" import TestConfiguration from "../TestConfiguration" import { TestAPI } from "./base" @@ -16,7 +22,15 @@ export class TableAPI extends TestAPI { .send(data) .set(this.config.defaultHeaders()) .expect("Content-Type", /json/) - .expect(expectStatus) + + if (res.status !== expectStatus) { + throw new Error( + `Expected status ${expectStatus} but got ${ + res.status + } with body ${JSON.stringify(res.body)}` + ) + } + return res.body } @@ -42,4 +56,23 @@ export class TableAPI extends TestAPI { .expect(expectStatus) return res.body } + + migrate = async ( + tableId: string, + data: MigrateRequest, + { expectStatus } = { expectStatus: 200 } + ): Promise => { + const res = await this.request + .post(`/api/tables/${tableId}/migrate`) + .send(data) + .set(this.config.defaultHeaders()) + if (res.status !== expectStatus) { + throw new Error( + `Expected status ${expectStatus} but got ${ + res.status + } with body ${JSON.stringify(res.body)}` + ) + } + return res.body + } } diff --git a/packages/server/src/tests/utilities/api/user.ts b/packages/server/src/tests/utilities/api/user.ts new file mode 100644 index 0000000000..2ed23c0461 --- /dev/null +++ b/packages/server/src/tests/utilities/api/user.ts @@ -0,0 +1,157 @@ +import { + FetchUserMetadataResponse, + FindUserMetadataResponse, + Flags, + UserMetadata, +} from "@budibase/types" +import TestConfiguration from "../TestConfiguration" +import { TestAPI } from "./base" +import { DocumentInsertResponse } from "@budibase/nano" + +export class UserAPI extends TestAPI { + constructor(config: TestConfiguration) { + super(config) + } + + fetch = async ( + { expectStatus } = { expectStatus: 200 } + ): Promise => { + const res = await this.request + .get(`/api/users/metadata`) + .set(this.config.defaultHeaders()) + .expect("Content-Type", /json/) + + if (res.status !== expectStatus) { + throw new Error( + `Expected status ${expectStatus} but got ${ + res.status + } with body ${JSON.stringify(res.body)}` + ) + } + + return res.body + } + + find = async ( + id: string, + { expectStatus } = { expectStatus: 200 } + ): Promise => { + const res = await this.request + .get(`/api/users/metadata/${id}`) + .set(this.config.defaultHeaders()) + .expect("Content-Type", /json/) + + if (res.status !== expectStatus) { + throw new Error( + `Expected status ${expectStatus} but got ${ + res.status + } with body ${JSON.stringify(res.body)}` + ) + } + + return res.body + } + + update = async ( + user: UserMetadata, + { expectStatus } = { expectStatus: 200 } + ): Promise => { + const res = await this.request + .put(`/api/users/metadata`) + .set(this.config.defaultHeaders()) + .send(user) + .expect("Content-Type", /json/) + + if (res.status !== expectStatus) { + throw new Error( + `Expected status ${expectStatus} but got ${ + res.status + } with body ${JSON.stringify(res.body)}` + ) + } + + return res.body as DocumentInsertResponse + } + + updateSelf = async ( + user: UserMetadata, + { expectStatus } = { expectStatus: 200 } + ): Promise => { + const res = await this.request + .post(`/api/users/metadata/self`) + .set(this.config.defaultHeaders()) + .send(user) + .expect("Content-Type", /json/) + + if (res.status !== expectStatus) { + throw new Error( + `Expected status ${expectStatus} but got ${ + res.status + } with body ${JSON.stringify(res.body)}` + ) + } + + return res.body as DocumentInsertResponse + } + + destroy = async ( + id: string, + { expectStatus } = { expectStatus: 200 } + ): Promise<{ message: string }> => { + const res = await this.request + .delete(`/api/users/metadata/${id}`) + .set(this.config.defaultHeaders()) + .expect("Content-Type", /json/) + + if (res.status !== expectStatus) { + throw new Error( + `Expected status ${expectStatus} but got ${ + res.status + } with body ${JSON.stringify(res.body)}` + ) + } + + return res.body as { message: string } + } + + setFlag = async ( + flag: string, + value: any, + { expectStatus } = { expectStatus: 200 } + ): Promise<{ message: string }> => { + const res = await this.request + .post(`/api/users/flags`) + .set(this.config.defaultHeaders()) + .send({ flag, value }) + .expect("Content-Type", /json/) + + if (res.status !== expectStatus) { + throw new Error( + `Expected status ${expectStatus} but got ${ + res.status + } with body ${JSON.stringify(res.body)}` + ) + } + + return res.body as { message: string } + } + + getFlags = async ( + { expectStatus } = { expectStatus: 200 } + ): Promise => { + const res = await this.request + .get(`/api/users/flags`) + .set(this.config.defaultHeaders()) + .expect("Content-Type", /json/) + + if (res.status !== expectStatus) { + throw new Error( + `Expected status ${expectStatus} but got ${ + res.status + } with body ${JSON.stringify(res.body)}` + ) + } + + return res.body as Flags + } +} diff --git a/packages/server/src/tests/utilities/structures.ts b/packages/server/src/tests/utilities/structures.ts index d3e92ea34d..b680c6ff19 100644 --- a/packages/server/src/tests/utilities/structures.ts +++ b/packages/server/src/tests/utilities/structures.ts @@ -19,12 +19,17 @@ import { FieldType, SourceName, Table, + INTERNAL_TABLE_SOURCE_ID, + TableSourceType, } from "@budibase/types" +const { BUILTIN_ROLE_IDS } = roles export function basicTable(): Table { return { name: "TestTable", type: "table", + sourceId: INTERNAL_TABLE_SOURCE_ID, + sourceType: TableSourceType.INTERNAL, schema: { name: { type: FieldType.STRING, @@ -322,8 +327,22 @@ export function basicUser(role: string) { } } -export function basicScreen() { - return createHomeScreen() +export function basicScreen(route: string = "/") { + return createHomeScreen({ + roleId: BUILTIN_ROLE_IDS.BASIC, + route, + }) +} + +export function powerScreen(route: string = "/") { + return createHomeScreen({ + roleId: BUILTIN_ROLE_IDS.POWER, + route, + }) +} + +export function customScreen(config: { roleId: string; route: string }) { + return createHomeScreen(config) } export function basicLayout() { diff --git a/packages/server/src/threads/automation.ts b/packages/server/src/threads/automation.ts index 9241289e86..d1fcc2be72 100644 --- a/packages/server/src/threads/automation.ts +++ b/packages/server/src/threads/automation.ts @@ -241,7 +241,7 @@ class Orchestrator { }) } - async execute() { + async execute(): Promise { // this will retrieve from context created at start of thread this._context.env = await sdkUtils.getEnvironmentVariables() let automation = this._automation diff --git a/packages/server/src/utilities/global.ts b/packages/server/src/utilities/global.ts index 5aa201990c..bbb84c1882 100644 --- a/packages/server/src/utilities/global.ts +++ b/packages/server/src/utilities/global.ts @@ -1,4 +1,4 @@ -import { getMultiIDParams, getGlobalIDFromUserMetadataID } from "../db/utils" +import { getGlobalIDFromUserMetadataID } from "../db/utils" import { roles, db as dbCore, @@ -71,69 +71,65 @@ export async function processUser( return user } -export async function getCachedSelf(ctx: UserCtx, appId: string) { +export async function getCachedSelf( + ctx: UserCtx, + appId: string +): Promise { // this has to be tenant aware, can't depend on the context to find it out // running some middlewares before the tenancy causes context to break const user = await cache.user.getUser(ctx.user?._id!) return processUser(user, { appId }) } -export async function getRawGlobalUser(userId: string) { +export async function getRawGlobalUser(userId: string): Promise { const db = tenancy.getGlobalDB() return db.get(getGlobalIDFromUserMetadataID(userId)) } -export async function getGlobalUser(userId: string) { +export async function getGlobalUser(userId: string): Promise { const appId = context.getAppId() let user = await getRawGlobalUser(userId) return processUser(user, { appId }) } -export async function getGlobalUsers( - userIds?: string[], - opts?: { noProcessing?: boolean } -) { - const appId = context.getAppId() +export async function getRawGlobalUsers(userIds?: string[]): Promise { const db = tenancy.getGlobalDB() - let globalUsers + let globalUsers: User[] if (userIds) { - globalUsers = (await db.allDocs(getMultiIDParams(userIds))).rows.map( - row => row.doc - ) + globalUsers = await db.getMultiple(userIds, { allowMissing: true }) } else { globalUsers = ( - await db.allDocs( + await db.allDocs( dbCore.getGlobalUserParams(null, { include_docs: true, }) ) - ).rows.map(row => row.doc) + ).rows.map(row => row.doc!) } - globalUsers = globalUsers + return globalUsers .filter(user => user != null) .map(user => { delete user.password delete user.forceResetPassword return user }) +} - if (opts?.noProcessing || !appId) { - return globalUsers - } else { - // pass in the groups, meaning we don't actually need to retrieve them for - // each user individually - const allGroups = await groups.fetch() - return Promise.all( - globalUsers.map(user => processUser(user, { groups: allGroups })) - ) - } +export async function getGlobalUsers( + userIds?: string[] +): Promise { + const users = await getRawGlobalUsers(userIds) + const allGroups = await groups.fetch() + return Promise.all( + users.map(user => processUser(user, { groups: allGroups })) + ) } export async function getGlobalUsersFromMetadata(users: ContextUser[]) { const globalUsers = await getGlobalUsers(users.map(user => user._id!)) return users.map(user => { const globalUser = globalUsers.find( - globalUser => globalUser && user._id?.includes(globalUser._id) + globalUser => globalUser && user._id?.includes(globalUser._id!) ) return { ...globalUser, diff --git a/packages/server/src/utilities/routing/index.ts b/packages/server/src/utilities/routing/index.ts index de966a946b..82d45743ce 100644 --- a/packages/server/src/utilities/routing/index.ts +++ b/packages/server/src/utilities/routing/index.ts @@ -1,9 +1,9 @@ import { createRoutingView } from "../../db/views/staticViews" import { ViewName, getQueryIndex, UNICODE_MAX } from "../../db/utils" import { context } from "@budibase/backend-core" -import { ScreenRouting } from "@budibase/types" +import { ScreenRouting, Document } from "@budibase/types" -type ScreenRoutesView = { +interface ScreenRoutesView extends Document { id: string routing: ScreenRouting } diff --git a/packages/server/src/utilities/rowProcessor/index.ts b/packages/server/src/utilities/rowProcessor/index.ts index 9efb3c72aa..0e53422a4f 100644 --- a/packages/server/src/utilities/rowProcessor/index.ts +++ b/packages/server/src/utilities/rowProcessor/index.ts @@ -2,7 +2,12 @@ import * as linkRows from "../../db/linkedRows" import { FieldTypes, AutoFieldSubTypes } from "../../constants" import { processFormulas, fixAutoColumnSubType } from "./utils" import { ObjectStoreBuckets } from "../../constants" -import { context, db as dbCore, objectStore } from "@budibase/backend-core" +import { + context, + db as dbCore, + objectStore, + utils, +} from "@budibase/backend-core" import { InternalTables } from "../../db/utils" import { TYPE_TRANSFORM_MAP } from "./map" import { FieldSubtype, Row, RowAttachment, Table } from "@budibase/types" @@ -11,7 +16,7 @@ import { processInputBBReferences, processOutputBBReferences, } from "./bbReferenceProcessor" -import { isExternalTable } from "../../integrations/utils" +import { isExternalTableID } from "../../integrations/utils" export * from "./utils" type AutoColumnProcessingOpts = { @@ -45,7 +50,7 @@ function getRemovedAttachmentKeys( /** * This will update any auto columns that are found on the row/table with the correct information based on * time now and the current logged in user making the request. - * @param user The user to be used for an appId as well as the createdBy and createdAt fields. + * @param userId The user to be used for an appId as well as the createdBy and createdAt fields. * @param table The table which is to be used for the schema, as well as handling auto IDs incrementing. * @param row The row which is to be updated with information for the auto columns. * @param opts specific options for function to carry out optional features. @@ -227,6 +232,11 @@ export async function outputProcessing( }) : safeRows + // make sure squash is enabled if needed + if (!opts.squash && utils.hasCircularStructure(rows)) { + opts.squash = true + } + // process complex types: attachements, bb references... for (let [property, column] of Object.entries(table.schema)) { if (column.type === FieldTypes.ATTACHMENT) { @@ -235,7 +245,7 @@ export async function outputProcessing( continue } row[property].forEach((attachment: RowAttachment) => { - attachment.url = objectStore.getAppFileUrl(attachment.key) + attachment.url ??= objectStore.getAppFileUrl(attachment.key) }) } } else if ( @@ -252,7 +262,7 @@ export async function outputProcessing( } // process formulas after the complex types had been processed - enriched = processFormulas(table, enriched, { dynamic: true }) as Row[] + enriched = processFormulas(table, enriched, { dynamic: true }) if (opts.squash) { enriched = (await linkRows.squashLinksToPrimaryDisplay( @@ -261,7 +271,7 @@ export async function outputProcessing( )) as Row[] } // remove null properties to match internal API - if (isExternalTable(table._id!)) { + if (isExternalTableID(table._id!)) { for (let row of enriched) { for (let key of Object.keys(row)) { if (row[key] === null) { diff --git a/packages/server/src/utilities/rowProcessor/tests/inputProcessing.spec.ts b/packages/server/src/utilities/rowProcessor/tests/inputProcessing.spec.ts index 18d5128986..b6c1db9159 100644 --- a/packages/server/src/utilities/rowProcessor/tests/inputProcessing.spec.ts +++ b/packages/server/src/utilities/rowProcessor/tests/inputProcessing.spec.ts @@ -1,6 +1,12 @@ import { inputProcessing } from ".." import { generator, structures } from "@budibase/backend-core/tests" -import { FieldType, FieldTypeSubtypes, Table } from "@budibase/types" +import { + FieldType, + FieldTypeSubtypes, + INTERNAL_TABLE_SOURCE_ID, + Table, + TableSourceType, +} from "@budibase/types" import * as bbReferenceProcessor from "../bbReferenceProcessor" jest.mock("../bbReferenceProcessor", (): typeof bbReferenceProcessor => ({ @@ -20,6 +26,8 @@ describe("rowProcessor - inputProcessing", () => { _id: generator.guid(), name: "TestTable", type: "table", + sourceId: INTERNAL_TABLE_SOURCE_ID, + sourceType: TableSourceType.INTERNAL, schema: { name: { type: FieldType.STRING, @@ -70,6 +78,8 @@ describe("rowProcessor - inputProcessing", () => { _id: generator.guid(), name: "TestTable", type: "table", + sourceId: INTERNAL_TABLE_SOURCE_ID, + sourceType: TableSourceType.INTERNAL, schema: { name: { type: FieldType.STRING, @@ -110,6 +120,8 @@ describe("rowProcessor - inputProcessing", () => { _id: generator.guid(), name: "TestTable", type: "table", + sourceId: INTERNAL_TABLE_SOURCE_ID, + sourceType: TableSourceType.INTERNAL, schema: { name: { type: FieldType.STRING, @@ -150,6 +162,8 @@ describe("rowProcessor - inputProcessing", () => { _id: generator.guid(), name: "TestTable", type: "table", + sourceId: INTERNAL_TABLE_SOURCE_ID, + sourceType: TableSourceType.INTERNAL, schema: { name: { type: FieldType.STRING, diff --git a/packages/server/src/utilities/rowProcessor/tests/outputProcessing.spec.ts b/packages/server/src/utilities/rowProcessor/tests/outputProcessing.spec.ts index ecb8856c88..03584ef53b 100644 --- a/packages/server/src/utilities/rowProcessor/tests/outputProcessing.spec.ts +++ b/packages/server/src/utilities/rowProcessor/tests/outputProcessing.spec.ts @@ -2,7 +2,9 @@ import { FieldSubtype, FieldType, FieldTypeSubtypes, + INTERNAL_TABLE_SOURCE_ID, Table, + TableSourceType, } from "@budibase/types" import { outputProcessing } from ".." import { generator, structures } from "@budibase/backend-core/tests" @@ -26,6 +28,8 @@ describe("rowProcessor - outputProcessing", () => { _id: generator.guid(), name: "TestTable", type: "table", + sourceId: INTERNAL_TABLE_SOURCE_ID, + sourceType: TableSourceType.INTERNAL, schema: { name: { type: FieldType.STRING, @@ -71,6 +75,8 @@ describe("rowProcessor - outputProcessing", () => { _id: generator.guid(), name: "TestTable", type: "table", + sourceId: INTERNAL_TABLE_SOURCE_ID, + sourceType: TableSourceType.INTERNAL, schema: { name: { type: FieldType.STRING, @@ -108,6 +114,8 @@ describe("rowProcessor - outputProcessing", () => { _id: generator.guid(), name: "TestTable", type: "table", + sourceId: INTERNAL_TABLE_SOURCE_ID, + sourceType: TableSourceType.INTERNAL, schema: { name: { type: FieldType.STRING, diff --git a/packages/server/src/utilities/rowProcessor/utils.ts b/packages/server/src/utilities/rowProcessor/utils.ts index 48697af6a9..9eb725dd7c 100644 --- a/packages/server/src/utilities/rowProcessor/utils.ts +++ b/packages/server/src/utilities/rowProcessor/utils.ts @@ -12,6 +12,11 @@ import { Table, } from "@budibase/types" +interface FormulaOpts { + dynamic?: boolean + contextRows?: Row[] +} + /** * If the subtype has been lost for any reason this works out what * subtype the auto column should be. @@ -40,52 +45,50 @@ export function fixAutoColumnSubType( /** * Looks through the rows provided and finds formulas - which it then processes. */ -export function processFormulas( +export function processFormulas( table: Table, - rows: Row[] | Row, - { dynamic, contextRows }: any = { dynamic: true } -) { - const single = !Array.isArray(rows) - let rowArray: Row[] - if (single) { - rowArray = [rows] - contextRows = contextRows ? [contextRows] : contextRows - } else { - rowArray = rows - } - for (let [column, schema] of Object.entries(table.schema)) { - if (schema.type !== FieldTypes.FORMULA) { - continue - } + inputRows: T, + { dynamic, contextRows }: FormulaOpts = { dynamic: true } +): T { + const rows = Array.isArray(inputRows) ? inputRows : [inputRows] + if (rows) + for (let [column, schema] of Object.entries(table.schema)) { + if (schema.type !== FieldTypes.FORMULA) { + continue + } - const isStatic = schema.formulaType === FormulaTypes.STATIC + const isStatic = schema.formulaType === FormulaTypes.STATIC - if ( - schema.formula == null || - (dynamic && isStatic) || - (!dynamic && !isStatic) - ) { - continue - } - // iterate through rows and process formula - for (let i = 0; i < rowArray.length; i++) { - let row = rowArray[i] - let context = contextRows ? contextRows[i] : row - rowArray[i] = { - ...row, - [column]: processStringSync(schema.formula, context), + if ( + schema.formula == null || + (dynamic && isStatic) || + (!dynamic && !isStatic) + ) { + continue + } + // iterate through rows and process formula + for (let i = 0; i < rows.length; i++) { + let row = rows[i] + let context = contextRows ? contextRows[i] : row + rows[i] = { + ...row, + [column]: processStringSync(schema.formula, context), + } } } - } - return single ? rowArray[0] : rowArray + return Array.isArray(inputRows) ? rows : rows[0] } /** * Processes any date columns and ensures that those without the ignoreTimezones * flag set are parsed as UTC rather than local time. */ -export function processDates(table: Table, rows: Row[]) { - let datesWithTZ = [] +export function processDates( + table: Table, + inputRows: T +): T { + let rows = Array.isArray(inputRows) ? inputRows : [inputRows] + let datesWithTZ: string[] = [] for (let [column, schema] of Object.entries(table.schema)) { if (schema.type !== FieldTypes.DATETIME) { continue @@ -102,5 +105,6 @@ export function processDates(table: Table, rows: Row[]) { } } } - return rows + + return Array.isArray(inputRows) ? rows : rows[0] } diff --git a/packages/server/src/utilities/users.ts b/packages/server/src/utilities/users.ts index bbc1370355..73b2f48b15 100644 --- a/packages/server/src/utilities/users.ts +++ b/packages/server/src/utilities/users.ts @@ -1,11 +1,13 @@ import { InternalTables } from "../db/utils" import { getGlobalUser } from "./global" import { context, roles } from "@budibase/backend-core" -import { UserCtx } from "@budibase/types" +import { ContextUserMetadata, UserCtx, UserMetadata } from "@budibase/types" -export async function getFullUser(ctx: UserCtx, userId: string) { +export async function getFullUser( + userId: string +): Promise { const global = await getGlobalUser(userId) - let metadata: any = {} + let metadata: UserMetadata | undefined = undefined // always prefer the user metadata _id and _rev delete global._id @@ -14,11 +16,11 @@ export async function getFullUser(ctx: UserCtx, userId: string) { try { // this will throw an error if the db doesn't exist, or there is no appId const db = context.getAppDB() - metadata = await db.get(userId) + metadata = await db.get(userId) + delete metadata.csrfToken } catch (err) { // it is fine if there is no user metadata yet } - delete metadata.csrfToken return { ...metadata, ...global, diff --git a/packages/server/src/websockets/builder.ts b/packages/server/src/websockets/builder.ts index d2fdbca20c..a47d3048d3 100644 --- a/packages/server/src/websockets/builder.ts +++ b/packages/server/src/websockets/builder.ts @@ -1,5 +1,5 @@ import authorized from "../middleware/authorized" -import { BaseSocket } from "./websocket" +import { BaseSocket, EmitOptions } from "./websocket" import { permissions, events, context } from "@budibase/backend-core" import http from "http" import Koa from "koa" @@ -16,6 +16,7 @@ import { gridSocket } from "./index" import { clearLock, updateLock } from "../utilities/redis" import { Socket } from "socket.io" import { BuilderSocketEvent } from "@budibase/shared-core" +import { processTable } from "../sdk/app/tables/getters" export default class BuilderSocket extends BaseSocket { constructor(app: Koa, server: http.Server) { @@ -100,11 +101,22 @@ export default class BuilderSocket extends BaseSocket { }) } - emitTableUpdate(ctx: any, table: Table) { - this.emitToRoom(ctx, ctx.appId, BuilderSocketEvent.TableChange, { - id: table._id, - table, - }) + emitTableUpdate(ctx: any, table: Table, options?: EmitOptions) { + // This was added to make sure that sourceId is always present when + // sending this message to clients. Without this, tables without a + // sourceId (e.g. ta_users) won't get correctly updated client-side. + table = processTable(table) + + this.emitToRoom( + ctx, + ctx.appId, + BuilderSocketEvent.TableChange, + { + id: table._id, + table, + }, + options + ) gridSocket?.emitTableUpdate(ctx, table) } diff --git a/packages/server/src/websockets/websocket.ts b/packages/server/src/websockets/websocket.ts index ffaf9e2763..1dba108d24 100644 --- a/packages/server/src/websockets/websocket.ts +++ b/packages/server/src/websockets/websocket.ts @@ -11,6 +11,14 @@ import { SocketSession } from "@budibase/types" import { v4 as uuid } from "uuid" import { createContext, runMiddlewares } from "./middleware" +export interface EmitOptions { + // Whether to include the originator of the request from the broadcast, + // defaults to false because it is assumed that the user who triggered + // an action will already have the changes of that action reflected in their + // own UI, so there is no need to send them again. + includeOriginator?: boolean +} + const anonUser = () => ({ _id: uuid(), email: "user@mail.com", @@ -270,10 +278,17 @@ export class BaseSocket { // Emit an event to everyone in a room, including metadata of whom // the originator of the request was - emitToRoom(ctx: any, room: string | string[], event: string, payload: any) { - this.io.in(room).emit(event, { - ...payload, - apiSessionId: ctx.headers?.[Header.SESSION_ID], - }) + emitToRoom( + ctx: any, + room: string | string[], + event: string, + payload: any, + options?: EmitOptions + ) { + let emitPayload = { ...payload } + if (!options?.includeOriginator) { + emitPayload.apiSessionId = ctx.headers?.[Header.SESSION_ID] + } + this.io.in(room).emit(event, emitPayload) } } diff --git a/packages/shared-core/src/constants.ts b/packages/shared-core/src/constants.ts index 725c246e2f..e7c6feb20a 100644 --- a/packages/shared-core/src/constants.ts +++ b/packages/shared-core/src/constants.ts @@ -96,3 +96,45 @@ export enum BuilderSocketEvent { export const SocketSessionTTL = 60 export const ValidQueryNameRegex = /^[^()]*$/ export const ValidColumnNameRegex = /^[_a-zA-Z0-9\s]*$/g +export const ValidFileExtensions = [ + "avif", + "css", + "csv", + "docx", + "drawio", + "editorconfig", + "edl", + "enc", + "export", + "geojson", + "gif", + "htm", + "html", + "ics", + "iqy", + "jfif", + "jpeg", + "jpg", + "json", + "log", + "md", + "mid", + "odt", + "pdf", + "png", + "ris", + "rtf", + "svg", + "tex", + "toml", + "twig", + "txt", + "url", + "wav", + "webp", + "xls", + "xlsx", + "xml", + "yaml", + "yml", +] diff --git a/packages/string-templates/.eslintrc b/packages/string-templates/.eslintrc deleted file mode 100644 index 3431bf04fb..0000000000 --- a/packages/string-templates/.eslintrc +++ /dev/null @@ -1,12 +0,0 @@ -{ - "globals": { - "emit": true, - "key": true - }, - "env": { - "node": true - }, - "extends": ["eslint:recommended"], - "rules": { - } -} \ No newline at end of file diff --git a/packages/types/package.json b/packages/types/package.json index 1db667e669..1b602097c7 100644 --- a/packages/types/package.json +++ b/packages/types/package.json @@ -15,7 +15,7 @@ }, "jest": {}, "devDependencies": { - "@budibase/nano": "10.1.2", + "@budibase/nano": "10.1.3", "@types/koa": "2.13.4", "@types/node": "18.17.0", "@types/pouchdb": "6.4.0", diff --git a/packages/types/src/api/web/app/attachment.ts b/packages/types/src/api/web/app/attachment.ts new file mode 100644 index 0000000000..792bdf3885 --- /dev/null +++ b/packages/types/src/api/web/app/attachment.ts @@ -0,0 +1,9 @@ +export interface Upload { + size: number + name: string + url: string + extension: string + key: string +} + +export type ProcessAttachmentResponse = Upload[] diff --git a/packages/types/src/api/web/app/backup.ts b/packages/types/src/api/web/app/backup.ts index c9a8d07f5e..f77707e9c6 100644 --- a/packages/types/src/api/web/app/backup.ts +++ b/packages/types/src/api/web/app/backup.ts @@ -20,3 +20,8 @@ export interface CreateAppBackupResponse { export interface UpdateAppBackupRequest { name: string } + +export interface ImportAppBackupResponse { + restoreId: string + message: string +} diff --git a/packages/types/src/api/web/app/index.ts b/packages/types/src/api/web/app/index.ts index 276d7fa7c1..cb1cea2b08 100644 --- a/packages/types/src/api/web/app/index.ts +++ b/packages/types/src/api/web/app/index.ts @@ -5,3 +5,5 @@ export * from "./view" export * from "./rows" export * from "./table" export * from "./permission" +export * from "./attachment" +export * from "./user" diff --git a/packages/types/src/api/web/app/rows.ts b/packages/types/src/api/web/app/rows.ts index 62ea90a6a4..dad3286754 100644 --- a/packages/types/src/api/web/app/rows.ts +++ b/packages/types/src/api/web/app/rows.ts @@ -1,5 +1,6 @@ import { SearchFilters, SearchParams } from "../../../sdk" import { Row } from "../../../documents" +import { SortOrder } from "../../../api" import { ReadStream } from "fs" export interface SaveRowRequest extends Row {} @@ -34,6 +35,8 @@ export interface ExportRowsRequest { rows: string[] columns?: string[] query?: SearchFilters + sort?: string + sortOrder?: SortOrder } export type ExportRowsResponse = ReadStream diff --git a/packages/types/src/api/web/app/table.ts b/packages/types/src/api/web/app/table.ts index cb5faaa9ea..f4d6720516 100644 --- a/packages/types/src/api/web/app/table.ts +++ b/packages/types/src/api/web/app/table.ts @@ -1,4 +1,5 @@ import { + FieldSchema, Row, Table, TableRequest, @@ -33,3 +34,12 @@ export interface BulkImportRequest { export interface BulkImportResponse { message: string } + +export interface MigrateRequest { + oldColumn: FieldSchema + newColumn: FieldSchema +} + +export interface MigrateResponse { + message: string +} diff --git a/packages/types/src/api/web/app/user.ts b/packages/types/src/api/web/app/user.ts new file mode 100644 index 0000000000..7faec83e9c --- /dev/null +++ b/packages/types/src/api/web/app/user.ts @@ -0,0 +1,9 @@ +import { ContextUserMetadata } from "../../../" + +export type FetchUserMetadataResponse = ContextUserMetadata[] +export type FindUserMetadataResponse = ContextUserMetadata + +export interface SetFlagRequest { + flag: string + value: any +} diff --git a/packages/types/src/api/web/user.ts b/packages/types/src/api/web/user.ts index a1e039cfd7..0de42622e6 100644 --- a/packages/types/src/api/web/user.ts +++ b/packages/types/src/api/web/user.ts @@ -10,6 +10,7 @@ export interface SaveUserResponse { export interface UserDetails { _id: string email: string + password?: string } export interface BulkUserRequest { @@ -49,12 +50,14 @@ export type InviteUsersRequest = InviteUserRequest[] export interface InviteUsersResponse { successful: { email: string }[] unsuccessful: { email: string; reason: string }[] + created?: boolean } export interface SearchUsersRequest { bookmark?: string query?: SearchQuery appId?: string + limit?: number paginate?: boolean } diff --git a/packages/types/src/documents/account/flag.ts b/packages/types/src/documents/account/flag.ts new file mode 100644 index 0000000000..a214348fe7 --- /dev/null +++ b/packages/types/src/documents/account/flag.ts @@ -0,0 +1,5 @@ +import { Document } from "../../" + +export interface Flags extends Document { + [key: string]: any +} diff --git a/packages/types/src/documents/account/index.ts b/packages/types/src/documents/account/index.ts index 663fb91b58..1e0c800f39 100644 --- a/packages/types/src/documents/account/index.ts +++ b/packages/types/src/documents/account/index.ts @@ -1,2 +1,3 @@ export * from "./account" export * from "./user" +export * from "./flag" diff --git a/packages/types/src/documents/app/layout.ts b/packages/types/src/documents/app/layout.ts index db046e3d92..06542f680d 100644 --- a/packages/types/src/documents/app/layout.ts +++ b/packages/types/src/documents/app/layout.ts @@ -2,4 +2,5 @@ import { Document } from "../document" export interface Layout extends Document { props: any + layoutId?: string } diff --git a/packages/types/src/documents/app/links.ts b/packages/types/src/documents/app/links.ts index 3f3a83740a..2a9595d99f 100644 --- a/packages/types/src/documents/app/links.ts +++ b/packages/types/src/documents/app/links.ts @@ -1,18 +1,16 @@ import { Document } from "../document" +export interface LinkInfo { + rowId: string + fieldName: string + tableId: string +} + export interface LinkDocument extends Document { type: string tableId: string - doc1: { - rowId: string - fieldName: string - tableId: string - } - doc2: { - rowId: string - fieldName: string - tableId: string - } + doc1: LinkInfo + doc2: LinkInfo } export interface LinkDocumentValue { diff --git a/packages/types/src/documents/app/table/schema.ts b/packages/types/src/documents/app/table/schema.ts index e529a8e8b7..19a7303072 100644 --- a/packages/types/src/documents/app/table/schema.ts +++ b/packages/types/src/documents/app/table/schema.ts @@ -102,6 +102,7 @@ export interface BBReferenceFieldMetadata extends Omit { type: FieldType.BB_REFERENCE subtype: FieldSubtype.USER | FieldSubtype.USERS + relationshipType?: RelationshipType } export interface FieldConstraints { @@ -164,3 +165,33 @@ export type FieldSchema = export interface TableSchema { [key: string]: FieldSchema } + +export function isRelationshipField( + field: FieldSchema +): field is RelationshipFieldMetadata { + return field.type === FieldType.LINK +} + +export function isManyToMany( + field: RelationshipFieldMetadata +): field is ManyToManyRelationshipFieldMetadata { + return field.relationshipType === RelationshipType.MANY_TO_MANY +} + +export function isOneToMany( + field: RelationshipFieldMetadata +): field is OneToManyRelationshipFieldMetadata { + return field.relationshipType === RelationshipType.ONE_TO_MANY +} + +export function isManyToOne( + field: RelationshipFieldMetadata +): field is ManyToOneRelationshipFieldMetadata { + return field.relationshipType === RelationshipType.MANY_TO_ONE +} + +export function isBBReferenceField( + field: FieldSchema +): field is BBReferenceFieldMetadata { + return field.type === FieldType.BB_REFERENCE +} diff --git a/packages/types/src/documents/app/table/table.ts b/packages/types/src/documents/app/table/table.ts index bbaaddf1d6..f0e6079aef 100644 --- a/packages/types/src/documents/app/table/table.ts +++ b/packages/types/src/documents/app/table/table.ts @@ -3,15 +3,23 @@ import { View, ViewV2 } from "../view" import { RenameColumn } from "../../../sdk" import { TableSchema } from "./schema" +export const INTERNAL_TABLE_SOURCE_ID = "bb_internal" + +export enum TableSourceType { + EXTERNAL = "external", + INTERNAL = "internal", +} + export interface Table extends Document { - type?: string + type: "table" + sourceType: TableSourceType views?: { [key: string]: View | ViewV2 } name: string originalName?: string + sourceId: string primary?: string[] schema: TableSchema primaryDisplay?: string - sourceId?: string relatedFormula?: string[] constrained?: string[] sql?: boolean @@ -20,10 +28,6 @@ export interface Table extends Document { rowHeight?: number } -export interface ExternalTable extends Table { - sourceId: string -} - export interface TableRequest extends Table { _rename?: RenameColumn created?: boolean diff --git a/packages/types/src/documents/app/user.ts b/packages/types/src/documents/app/user.ts index 4defd4a414..207997245e 100644 --- a/packages/types/src/documents/app/user.ts +++ b/packages/types/src/documents/app/user.ts @@ -1,6 +1,6 @@ -import { Document } from "../document" +import { User } from "../global" +import { Row } from "./row" +import { ContextUser } from "../../sdk" -export interface UserMetadata extends Document { - roleId: string - email?: string -} +export type UserMetadata = User & Row +export type ContextUserMetadata = ContextUser & Row diff --git a/packages/types/src/documents/app/view.ts b/packages/types/src/documents/app/view.ts index 0d79e2c505..b5a22ec592 100644 --- a/packages/types/src/documents/app/view.ts +++ b/packages/types/src/documents/app/view.ts @@ -1,5 +1,24 @@ import { SearchFilter, SortOrder, SortType } from "../../api" import { UIFieldMetadata } from "./table" +import { Document } from "../document" +import { DBView } from "../../sdk" + +export type ViewTemplateOpts = { + field: string + tableId: string + groupBy: string + filters: ViewFilter[] + schema: any + calculation: string + groupByMulti?: boolean +} + +export interface InMemoryView extends Document { + view: DBView + name: string + tableId: string + groupBy?: string +} export interface View { name?: string @@ -10,7 +29,7 @@ export interface View { calculation?: ViewCalculation map?: string reduce?: any - meta?: Record + meta?: ViewTemplateOpts } export interface ViewV2 { diff --git a/packages/types/src/documents/pouch.ts b/packages/types/src/documents/pouch.ts index d484f4700d..11efc502be 100644 --- a/packages/types/src/documents/pouch.ts +++ b/packages/types/src/documents/pouch.ts @@ -1,17 +1,19 @@ +import { Document } from "../" + export interface RowValue { rev: string deleted: boolean } -export interface RowResponse { +export interface RowResponse { id: string key: string error: string value: T | RowValue - doc?: T | any + doc?: T } -export interface AllDocsResponse { +export interface AllDocsResponse { offset: number total_rows: number rows: RowResponse[] diff --git a/packages/types/src/sdk/datasources.ts b/packages/types/src/sdk/datasources.ts index 39a10961de..7a335eb3b9 100644 --- a/packages/types/src/sdk/datasources.ts +++ b/packages/types/src/sdk/datasources.ts @@ -1,4 +1,4 @@ -import { ExternalTable, Table } from "../documents" +import { Table } from "../documents" export const PASSWORD_REPLACEMENT = "--secret-value--" @@ -176,7 +176,7 @@ export interface IntegrationBase { } export interface Schema { - tables: Record + tables: Record errors: Record } @@ -187,7 +187,7 @@ export interface DatasourcePlus extends IntegrationBase { getStringConcat(parts: string[]): string buildSchema( datasourceId: string, - entities: Record + entities: Record ): Promise getTableNames(): Promise } diff --git a/packages/types/src/sdk/db.ts b/packages/types/src/sdk/db.ts index 1ba408b1d4..b8d5c0068d 100644 --- a/packages/types/src/sdk/db.ts +++ b/packages/types/src/sdk/db.ts @@ -1,5 +1,5 @@ import Nano from "@budibase/nano" -import { AllDocsResponse, AnyDocument, Document } from "../" +import { AllDocsResponse, AnyDocument, Document, ViewTemplateOpts } from "../" import { Writable } from "stream" export enum SearchIndex { @@ -20,6 +20,37 @@ export enum SortOption { DESCENDING = "desc", } +export type IndexAnalyzer = { + name: string + default?: string + fields?: Record +} + +export type DBView = { + name?: string + map: string + reduce?: string + meta?: ViewTemplateOpts + groupBy?: string +} + +export interface DesignDocument extends Document { + // we use this static reference for all design documents + _id: "_design/database" + language?: string + // CouchDB views + views?: { + [viewName: string]: DBView + } + // Lucene indexes + indexes?: { + [indexName: string]: { + index: string + analyzer?: string | IndexAnalyzer + } + } +} + export type CouchFindOptions = { selector: PouchDB.Find.Selector fields?: string[] @@ -54,15 +85,18 @@ export type DatabaseDeleteIndexOpts = { type?: string | undefined } +type DBPrimitiveKey = string | number | {} +export type DatabaseKey = DBPrimitiveKey | DBPrimitiveKey[] + export type DatabaseQueryOpts = { include_docs?: boolean - startkey?: string - endkey?: string + startkey?: DatabaseKey + endkey?: DatabaseKey limit?: number skip?: number descending?: boolean - key?: string - keys?: string[] + key?: DatabaseKey + keys?: DatabaseKey[] group?: boolean startkey_docid?: string } @@ -88,7 +122,11 @@ export interface Database { exists(): Promise checkSetup(): Promise> - get(id?: string): Promise + get(id?: string): Promise + getMultiple( + ids: string[], + opts?: { allowMissing?: boolean } + ): Promise remove( id: string | Document, rev?: string @@ -98,9 +136,11 @@ export interface Database { opts?: DatabasePutOpts ): Promise bulkDocs(documents: AnyDocument[]): Promise - allDocs(params: DatabaseQueryOpts): Promise> sql(sql: string): Promise - query( + allDocs( + params: DatabaseQueryOpts + ): Promise> + query( viewName: string, params: DatabaseQueryOpts ): Promise> diff --git a/packages/types/src/sdk/featureFlag.ts b/packages/types/src/sdk/featureFlag.ts index e3935bc7ee..ca0046696a 100644 --- a/packages/types/src/sdk/featureFlag.ts +++ b/packages/types/src/sdk/featureFlag.ts @@ -1,8 +1,7 @@ export enum FeatureFlag { LICENSING = "LICENSING", - // Feature IDs in Posthog - PER_CREATOR_PER_USER_PRICE = "18873", - PER_CREATOR_PER_USER_PRICE_ALERT = "18530", + PER_CREATOR_PER_USER_PRICE = "PER_CREATOR_PER_USER_PRICE", + PER_CREATOR_PER_USER_PRICE_ALERT = "PER_CREATOR_PER_USER_PRICE_ALERT", } export interface TenantFeatureFlags { diff --git a/packages/types/src/sdk/licensing/plan.ts b/packages/types/src/sdk/licensing/plan.ts index 1604dfb8af..5ac8b1c9f6 100644 --- a/packages/types/src/sdk/licensing/plan.ts +++ b/packages/types/src/sdk/licensing/plan.ts @@ -7,7 +7,9 @@ export enum PlanType { /** @deprecated */ PREMIUM = "premium", PREMIUM_PLUS = "premium_plus", + /** @deprecated */ BUSINESS = "business", + ENTERPRISE_BASIC = "enterprise_basic", ENTERPRISE = "enterprise", } diff --git a/packages/types/src/sdk/migrations.ts b/packages/types/src/sdk/migrations.ts index 4667ed0c8f..0692b27f8e 100644 --- a/packages/types/src/sdk/migrations.ts +++ b/packages/types/src/sdk/migrations.ts @@ -46,7 +46,7 @@ export enum MigrationName { GLOBAL_INFO_SYNC_USERS = "global_info_sync_users", TABLE_SETTINGS_LINKS_TO_ACTIONS = "table_settings_links_to_actions", // increment this number to re-activate this migration - SYNC_QUOTAS = "sync_quotas_1", + SYNC_QUOTAS = "sync_quotas_2", } export interface MigrationDefinition { diff --git a/packages/worker/Dockerfile b/packages/worker/Dockerfile index 4230ee86f8..50f1bb78b9 100644 --- a/packages/worker/Dockerfile +++ b/packages/worker/Dockerfile @@ -14,7 +14,7 @@ RUN yarn global add pm2 COPY package.json . COPY dist/yarn.lock . -RUN yarn install --production=true +RUN yarn install --production=true --network-timeout 1000000 # Remove unneeded data from file system to reduce image size RUN apk del .gyp \ && yarn cache clean diff --git a/packages/worker/Dockerfile.v2 b/packages/worker/Dockerfile.v2 index 0d60db6fc5..4706ca155a 100644 --- a/packages/worker/Dockerfile.v2 +++ b/packages/worker/Dockerfile.v2 @@ -19,7 +19,7 @@ RUN chmod +x ./scripts/removeWorkspaceDependencies.sh WORKDIR /string-templates COPY packages/string-templates/package.json package.json RUN ../scripts/removeWorkspaceDependencies.sh package.json -RUN --mount=type=cache,target=/root/.yarn YARN_CACHE_FOLDER=/root/.yarn yarn install --production=true +RUN --mount=type=cache,target=/root/.yarn YARN_CACHE_FOLDER=/root/.yarn yarn install --production=true --network-timeout 1000000 COPY packages/string-templates . @@ -30,7 +30,7 @@ RUN cd ../string-templates && yarn link && cd - && yarn link @budibase/string-te RUN ../scripts/removeWorkspaceDependencies.sh package.json -RUN --mount=type=cache,target=/root/.yarn YARN_CACHE_FOLDER=/root/.yarn yarn install --production=true +RUN --mount=type=cache,target=/root/.yarn YARN_CACHE_FOLDER=/root/.yarn yarn install --production=true --network-timeout 1000000 # Remove unneeded data from file system to reduce image size RUN apk del .gyp \ && yarn cache clean @@ -50,4 +50,9 @@ ENV POSTHOG_TOKEN=phc_bIjZL7oh2GEUd2vqvTBH8WvrX0fWTFQMs6H5KQxiUxU ENV TENANT_FEATURE_FLAGS=*:LICENSING,*:USER_GROUPS,*:ONBOARDING_TOUR ENV ACCOUNT_PORTAL_URL=https://account.budibase.app +ARG BUDIBASE_VERSION +# Ensuring the version argument is sent +RUN test -n "$BUDIBASE_VERSION" +ENV BUDIBASE_VERSION=$BUDIBASE_VERSION + CMD ["./docker_run.sh"] diff --git a/packages/worker/package.json b/packages/worker/package.json index 1eee3f020f..ec86575395 100644 --- a/packages/worker/package.json +++ b/packages/worker/package.json @@ -20,7 +20,6 @@ "run:docker": "node dist/index.js", "debug": "yarn build && node --expose-gc --inspect=9223 dist/index.js", "run:docker:cluster": "pm2-runtime start pm2.config.js", - "build:docker": "yarn build && docker build . -t worker-service --label version=$BUDIBASE_RELEASE_VERSION", "dev:stack:init": "node ./scripts/dev/manage.js init", "dev:builder": "npm run dev:stack:init && nodemon", "dev:built": "yarn run dev:stack:init && yarn run run:docker", diff --git a/packages/worker/scripts/dev/manage.js b/packages/worker/scripts/dev/manage.js index ecf5defd47..9e6a57d4bf 100644 --- a/packages/worker/scripts/dev/manage.js +++ b/packages/worker/scripts/dev/manage.js @@ -31,6 +31,7 @@ async function init() { TENANT_FEATURE_FLAGS: "*:LICENSING,*:USER_GROUPS,*:ONBOARDING_TOUR", ENABLE_EMAIL_TEST_MODE: 1, HTTP_LOGGING: 0, + VERSION: "0.0.0+local", } let envFile = "" Object.keys(envFileJson).forEach(key => { diff --git a/packages/worker/src/api/controllers/global/users.ts b/packages/worker/src/api/controllers/global/users.ts index 8de3a1444e..82a1578c88 100644 --- a/packages/worker/src/api/controllers/global/users.ts +++ b/packages/worker/src/api/controllers/global/users.ts @@ -1,8 +1,3 @@ -import { - checkInviteCode, - getInviteCodes, - updateInviteCode, -} from "../../../utilities/redis" import * as userSdk from "../../../sdk/users" import env from "../../../environment" import { @@ -16,6 +11,7 @@ import { Ctx, InviteUserRequest, InviteUsersRequest, + InviteUsersResponse, MigrationType, SaveUserResponse, SearchUsersRequest, @@ -189,7 +185,10 @@ export const destroy = async (ctx: any) => { export const getAppUsers = async (ctx: Ctx) => { const body = ctx.request.body - const users = await userSdk.db.getUsersByAppAccess(body?.appId) + const users = await userSdk.db.getUsersByAppAccess({ + appId: body.appId, + limit: body.limit, + }) ctx.body = { data: users } } @@ -246,59 +245,35 @@ export const tenantUserLookup = async (ctx: any) => { /* Encapsulate the app user onboarding flows here. */ -export const onboardUsers = async (ctx: Ctx) => { - const request = ctx.request.body - const isBulkCreate = "create" in request - - const emailConfigured = await isEmailConfigured() - - let onboardingResponse - - if (isBulkCreate) { - // @ts-ignore - const { users, groups, roles } = request.create - const assignUsers = users.map((user: User) => (user.roles = roles)) - onboardingResponse = await userSdk.db.bulkCreate(assignUsers, groups) - ctx.body = onboardingResponse - } else if (emailConfigured) { - onboardingResponse = await inviteMultiple(ctx) - } else if (!emailConfigured) { - const inviteRequest = ctx.request.body - - let createdPasswords: any = {} - - const users: User[] = inviteRequest.map(invite => { - let password = Math.random().toString(36).substring(2, 22) - - // Temp password to be passed to the user. - createdPasswords[invite.email] = password - - return { - email: invite.email, - password, - forceResetPassword: true, - roles: invite.userInfo.apps, - admin: invite.userInfo.admin, - builder: invite.userInfo.builder, - tenantId: tenancy.getTenantId(), - } - }) - let bulkCreateReponse = await userSdk.db.bulkCreate(users, []) - - // Apply temporary credentials - ctx.body = { - ...bulkCreateReponse, - successful: bulkCreateReponse?.successful.map(user => { - return { - ...user, - password: createdPasswords[user.email], - } - }), - created: true, - } - } else { - ctx.throw(400, "User onboarding failed") +export const onboardUsers = async ( + ctx: Ctx +) => { + if (await isEmailConfigured()) { + await inviteMultiple(ctx) + return } + + let createdPasswords: Record = {} + const users: User[] = ctx.request.body.map(invite => { + let password = Math.random().toString(36).substring(2, 22) + createdPasswords[invite.email] = password + + return { + email: invite.email, + password, + forceResetPassword: true, + roles: invite.userInfo.apps, + admin: invite.userInfo.admin, + builder: invite.userInfo.builder, + tenantId: tenancy.getTenantId(), + } + }) + + let resp = await userSdk.db.bulkCreate(users) + for (const user of resp.successful) { + user.password = createdPasswords[user.email] + } + ctx.body = { ...resp, created: true } } export const invite = async (ctx: Ctx) => { @@ -325,18 +300,18 @@ export const invite = async (ctx: Ctx) => { } export const inviteMultiple = async (ctx: Ctx) => { - const request = ctx.request.body - ctx.body = await userSdk.invite(request) + ctx.body = await userSdk.invite(ctx.request.body) } export const checkInvite = async (ctx: any) => { const { code } = ctx.params let invite try { - invite = await checkInviteCode(code, false) + invite = await cache.invite.getCode(code) } catch (e) { console.warn("Error getting invite from code", e) ctx.throw(400, "There was a problem with the invite") + return } ctx.body = { email: invite.email, @@ -344,14 +319,12 @@ export const checkInvite = async (ctx: any) => { } export const getUserInvites = async (ctx: any) => { - let invites try { // Restricted to the currently authenticated tenant - invites = await getInviteCodes() + ctx.body = await cache.invite.getInviteCodes() } catch (e) { ctx.throw(400, "There was a problem fetching invites") } - ctx.body = invites } export const updateInvite = async (ctx: any) => { @@ -362,12 +335,10 @@ export const updateInvite = async (ctx: any) => { let invite try { - invite = await checkInviteCode(code, false) - if (!invite) { - throw new Error("The invite could not be retrieved") - } + invite = await cache.invite.getCode(code) } catch (e) { ctx.throw(400, "There was a problem with the invite") + return } let updated = { @@ -392,7 +363,7 @@ export const updateInvite = async (ctx: any) => { } } - await updateInviteCode(code, updated) + await cache.invite.updateCode(code, updated) ctx.body = { ...invite } } @@ -402,7 +373,8 @@ export const inviteAccept = async ( const { inviteCode, password, firstName, lastName } = ctx.request.body try { // info is an extension of the user object that was stored by global - const { email, info }: any = await checkInviteCode(inviteCode) + const { email, info }: any = await cache.invite.getCode(inviteCode) + await cache.invite.deleteCode(inviteCode) const user = await tenancy.doInTenant(info.tenantId, async () => { let request: any = { firstName, diff --git a/packages/worker/src/api/routes/global/tests/groups.spec.ts b/packages/worker/src/api/routes/global/tests/groups.spec.ts index afeaae952c..8f0739a812 100644 --- a/packages/worker/src/api/routes/global/tests/groups.spec.ts +++ b/packages/worker/src/api/routes/global/tests/groups.spec.ts @@ -1,7 +1,7 @@ import { events } from "@budibase/backend-core" import { generator } from "@budibase/backend-core/tests" import { structures, TestConfiguration, mocks } from "../../../../tests" -import { UserGroup } from "@budibase/types" +import { User, UserGroup } from "@budibase/types" mocks.licenses.useGroups() @@ -231,4 +231,39 @@ describe("/api/global/groups", () => { }) }) }) + + describe("with global builder role", () => { + let builder: User + let group: UserGroup + + beforeAll(async () => { + builder = await config.createUser({ + builder: { global: true }, + admin: { global: false }, + }) + await config.createSession(builder) + + let resp = await config.api.groups.saveGroup( + structures.groups.UserGroup() + ) + group = resp.body as UserGroup + }) + + it("find should return 200", async () => { + await config.withUser(builder, async () => { + await config.api.groups.searchUsers(group._id!, { + emailSearch: `user1`, + }) + }) + }) + + it("update should return 200", async () => { + await config.withUser(builder, async () => { + await config.api.groups.updateGroupUsers(group._id!, { + add: [builder._id!], + remove: [], + }) + }) + }) + }) }) diff --git a/packages/worker/src/api/routes/global/tests/users.spec.ts b/packages/worker/src/api/routes/global/tests/users.spec.ts index a446d10ed0..a85933255a 100644 --- a/packages/worker/src/api/routes/global/tests/users.spec.ts +++ b/packages/worker/src/api/routes/global/tests/users.spec.ts @@ -1,11 +1,12 @@ import { InviteUsersResponse, User } from "@budibase/types" -jest.mock("nodemailer") import { TestConfiguration, mocks, structures } from "../../../../tests" -const sendMailMock = mocks.email.mock() import { events, tenancy, accounts as _accounts } from "@budibase/backend-core" import * as userSdk from "../../../../sdk/users" +jest.mock("nodemailer") +const sendMailMock = mocks.email.mock() + const accounts = jest.mocked(_accounts) describe("/api/global/users", () => { @@ -54,6 +55,24 @@ describe("/api/global/users", () => { expect(events.user.invited).toBeCalledTimes(0) }) + it("should not invite the same user twice", async () => { + const email = structures.users.newEmail() + await config.api.users.sendUserInvite(sendMailMock, email) + + jest.clearAllMocks() + + const { code, res } = await config.api.users.sendUserInvite( + sendMailMock, + email, + 400 + ) + + expect(res.body.message).toBe(`Unavailable`) + expect(sendMailMock).toHaveBeenCalledTimes(0) + expect(code).toBeUndefined() + expect(events.user.invited).toBeCalledTimes(0) + }) + it("should be able to create new user from invite", async () => { const email = structures.users.newEmail() const { code } = await config.api.users.sendUserInvite( @@ -101,6 +120,23 @@ describe("/api/global/users", () => { expect(sendMailMock).toHaveBeenCalledTimes(0) expect(events.user.invited).toBeCalledTimes(0) }) + + it("should not be able to generate an invitation for user that has already been invited", async () => { + const email = structures.users.newEmail() + await config.api.users.sendUserInvite(sendMailMock, email) + + jest.clearAllMocks() + + const request = [{ email: email, userInfo: {} }] + const res = await config.api.users.sendMultiUserInvite(request) + + const body = res.body as InviteUsersResponse + expect(body.successful.length).toBe(0) + expect(body.unsuccessful.length).toBe(1) + expect(body.unsuccessful[0].reason).toBe("Unavailable") + expect(sendMailMock).toHaveBeenCalledTimes(0) + expect(events.user.invited).toBeCalledTimes(0) + }) }) describe("POST /api/global/users/bulk", () => { @@ -569,9 +605,13 @@ describe("/api/global/users", () => { { query: { equal: { firstName: user.firstName } }, }, - 501 + { status: 501 } ) }) + + it("should throw an error if public query performed", async () => { + await config.api.users.searchUsers({}, { status: 403, noHeaders: true }) + }) }) describe("DELETE /api/global/users/:userId", () => { @@ -629,4 +669,25 @@ describe("/api/global/users", () => { expect(response.body.message).toBe("Unable to delete self.") }) }) + + describe("POST /api/global/users/onboard", () => { + it("should successfully onboard a user", async () => { + const response = await config.api.users.onboardUser([ + { email: structures.users.newEmail(), userInfo: {} }, + ]) + expect(response.successful.length).toBe(1) + expect(response.unsuccessful.length).toBe(0) + }) + + it("should not onboard a user who has been invited", async () => { + const email = structures.users.newEmail() + await config.api.users.sendUserInvite(sendMailMock, email) + + const response = await config.api.users.onboardUser([ + { email, userInfo: {} }, + ]) + expect(response.successful.length).toBe(0) + expect(response.unsuccessful.length).toBe(1) + }) + }) }) diff --git a/packages/worker/src/api/routes/global/users.ts b/packages/worker/src/api/routes/global/users.ts index a57f7834ac..3c9cfd2f41 100644 --- a/packages/worker/src/api/routes/global/users.ts +++ b/packages/worker/src/api/routes/global/users.ts @@ -72,7 +72,8 @@ router ) .get("/api/global/users", auth.builderOrAdmin, controller.fetch) - .post("/api/global/users/search", auth.builderOrAdmin, controller.search) + // search can be used by any user now, to retrieve users for user column + .post("/api/global/users/search", controller.search) .delete("/api/global/users/:id", auth.adminOnly, controller.destroy) .get( "/api/global/users/count/:appId", diff --git a/packages/worker/src/constants/templates/base.hbs b/packages/worker/src/constants/templates/base.hbs index 438197b5d2..9a7d906aa9 100644 --- a/packages/worker/src/constants/templates/base.hbs +++ b/packages/worker/src/constants/templates/base.hbs @@ -19,7 +19,7 @@ } a { - color: #3869D4 !important; + color: #6E56FF !important; } a img { @@ -109,11 +109,11 @@ /* Buttons ------------------------------ */ .button { - background-color: #3869D4; - border-top: 10px solid #3869D4; - border-right: 18px solid #3869D4; - border-bottom: 10px solid #3869D4; - border-left: 18px solid #3869D4; + background-color: #6E56FF; + border-top: 10px solid #6E56FF; + border-right: 18px solid #6E56FF; + border-bottom: 10px solid #6E56FF; + border-left: 18px solid #6E56FF; display: inline-block; color: #FFF !important; text-decoration: none !important; diff --git a/packages/worker/src/constants/templates/core.hbs b/packages/worker/src/constants/templates/core.hbs index 0b8b8cbde7..07c755b1fb 100644 --- a/packages/worker/src/constants/templates/core.hbs +++ b/packages/worker/src/constants/templates/core.hbs @@ -16,15 +16,11 @@ cellspacing="0" > Budibase Logo - - Budibase - diff --git a/packages/worker/src/constants/templates/index.ts b/packages/worker/src/constants/templates/index.ts index 1feac62040..6dd3f556a6 100644 --- a/packages/worker/src/constants/templates/index.ts +++ b/packages/worker/src/constants/templates/index.ts @@ -56,12 +56,12 @@ export async function getTemplates({ id, }: { ownerId?: string; type?: string; id?: string } = {}) { const db = tenancy.getGlobalDB() - const response = await db.allDocs( + const response = await db.allDocs