diff --git a/.github/workflows/budibase_ci.yml b/.github/workflows/budibase_ci.yml index 9509a22e99..9da52f8bc0 100644 --- a/.github/workflows/budibase_ci.yml +++ b/.github/workflows/budibase_ci.yml @@ -25,13 +25,13 @@ jobs: steps: - name: Checkout repo and submodules uses: actions/checkout@v3 - if: github.repository == github.event.pull_request.head.repo.full_name + if: github.repository == 'Budibase/budibase' with: submodules: true token: ${{ secrets.PERSONAL_ACCESS_TOKEN || github.token }} - name: Checkout repo only uses: actions/checkout@v3 - if: github.repository != github.event.pull_request.head.repo.full_name + if: github.repository != 'Budibase/budibase' - name: Use Node.js 14.x uses: actions/setup-node@v3 @@ -46,13 +46,13 @@ jobs: steps: - name: Checkout repo and submodules uses: actions/checkout@v3 - if: github.repository == github.event.pull_request.head.repo.full_name + if: github.repository == 'Budibase/budibase' with: submodules: true token: ${{ secrets.PERSONAL_ACCESS_TOKEN || github.token }} - name: Checkout repo only uses: actions/checkout@v3 - if: github.repository != github.event.pull_request.head.repo.full_name + if: github.repository != 'Budibase/budibase' - name: Use Node.js 14.x uses: actions/setup-node@v3 @@ -70,13 +70,13 @@ jobs: steps: - name: Checkout repo and submodules uses: actions/checkout@v3 - if: github.repository == github.event.pull_request.head.repo.full_name + if: github.repository == 'Budibase/budibase' with: submodules: true token: ${{ secrets.PERSONAL_ACCESS_TOKEN || github.token }} - name: Checkout repo only uses: actions/checkout@v3 - if: github.repository != github.event.pull_request.head.repo.full_name + if: github.repository != 'Budibase/budibase' - name: Use Node.js 14.x uses: actions/setup-node@v3 @@ -96,13 +96,13 @@ jobs: steps: - name: Checkout repo and submodules uses: actions/checkout@v3 - if: github.repository == github.event.pull_request.head.repo.full_name + if: github.repository == 'Budibase/budibase' with: submodules: true token: ${{ secrets.PERSONAL_ACCESS_TOKEN || github.token }} - name: Checkout repo only uses: actions/checkout@v3 - if: github.repository != github.event.pull_request.head.repo.full_name + if: github.repository != 'Budibase/budibase' - name: Use Node.js 14.x uses: actions/setup-node@v3 @@ -119,7 +119,7 @@ jobs: test-pro: runs-on: ubuntu-latest - if: github.repository == github.event.pull_request.head.repo.full_name + if: github.repository == 'Budibase/budibase' steps: - name: Checkout repo and submodules uses: actions/checkout@v3 @@ -140,13 +140,13 @@ jobs: steps: - name: Checkout repo and submodules uses: actions/checkout@v3 - if: github.repository == github.event.pull_request.head.repo.full_name + if: github.repository == 'Budibase/budibase' with: submodules: true token: ${{ secrets.PERSONAL_ACCESS_TOKEN || github.token }} - name: Checkout repo only uses: actions/checkout@v3 - if: github.repository != github.event.pull_request.head.repo.full_name + if: github.repository != 'Budibase/budibase' - name: Use Node.js 14.x uses: actions/setup-node@v3 @@ -154,7 +154,7 @@ jobs: node-version: 14.x cache: "yarn" - run: yarn - - run: yarn build + - run: yarn build --projects=@budibase/server,@budibase/worker,@budibase/client - name: Run tests run: | cd qa-core @@ -166,7 +166,7 @@ jobs: check-pro-submodule: runs-on: ubuntu-latest - if: github.repository == github.event.pull_request.head.repo.full_name + if: github.repository == 'Budibase/budibase' steps: - name: Checkout repo and submodules uses: actions/checkout@v3 @@ -190,6 +190,8 @@ jobs: base_commit=$(git rev-parse origin/develop) fi + echo "target_branch=$branch" + echo "target_branch=$branch" >> "$GITHUB_OUTPUT" echo "pro_commit=$pro_commit" echo "pro_commit=$pro_commit" >> "$GITHUB_OUTPUT" echo "base_commit=$base_commit" @@ -204,7 +206,7 @@ jobs: const baseCommit = '${{ steps.get_pro_commits.outputs.base_commit }}'; if (submoduleCommit !== baseCommit) { - console.error('Submodule commit does not match the latest commit on the develop branch.'); + console.error('Submodule commit does not match the latest commit on the "${{ steps.get_pro_commits.outputs.target_branch }}"" branch.'); console.error('Refer to the pro repo to merge your changes: https://github.com/Budibase/budibase-pro/blob/develop/docs/getting_started.md') process.exit(1); } else { diff --git a/.github/workflows/check_unreleased_changes.yml b/.github/workflows/check_unreleased_changes.yml new file mode 100644 index 0000000000..d558330545 --- /dev/null +++ b/.github/workflows/check_unreleased_changes.yml @@ -0,0 +1,29 @@ +name: check_unreleased_changes + +on: + pull_request: + branches: + - master + +jobs: + check_unreleased: + runs-on: ubuntu-latest + steps: + - name: Check for unreleased changes + env: + REPO: "Budibase/budibase" + TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + RELEASE_TIMESTAMP=$(curl -s -H "Authorization: token $TOKEN" \ + "https://api.github.com/repos/$REPO/releases/latest" | \ + jq -r .published_at) + COMMIT_TIMESTAMP=$(curl -s -H "Authorization: token $TOKEN" \ + "https://api.github.com/repos/$REPO/commits/master" | \ + jq -r .commit.committer.date) + RELEASE_SECONDS=$(date --date="$RELEASE_TIMESTAMP" "+%s") + COMMIT_SECONDS=$(date --date="$COMMIT_TIMESTAMP" "+%s") + if (( COMMIT_SECONDS > RELEASE_SECONDS )); then + echo "There are unreleased changes. Please release these changes before merging." + exit 1 + fi + echo "No unreleased changes detected." diff --git a/.github/workflows/release-singleimage.yml b/.github/workflows/release-singleimage.yml index 92f21bd649..5b75c20d29 100644 --- a/.github/workflows/release-singleimage.yml +++ b/.github/workflows/release-singleimage.yml @@ -1,4 +1,4 @@ -name: Deploy Budibase Single Container Image to DockerHub +name: release-singleimage on: workflow_dispatch: @@ -8,8 +8,8 @@ env: PERSONAL_ACCESS_TOKEN: ${{ secrets.PERSONAL_ACCESS_TOKEN }} REGISTRY_URL: registry.hub.docker.com jobs: - build: - name: "build" + build-amd64: + name: "build-amd64" runs-on: ubuntu-latest strategy: matrix: @@ -27,14 +27,12 @@ jobs: submodules: true token: ${{ secrets.PERSONAL_ACCESS_TOKEN }} fetch-depth: 0 - - name: Fail if tag is not in master run: | if ! git merge-base --is-ancestor ${{ github.sha }} origin/master; then echo "Tag is not in master. This pipeline can only execute tags that are present on the master branch" exit 1 fi - - name: Use Node.js ${{ matrix.node-version }} uses: actions/setup-node@v1 with: @@ -70,9 +68,139 @@ jobs: with: context: . push: true - platforms: linux/amd64,linux/arm64 - tags: budibase/budibase,budibase/budibase:${{ env.RELEASE_VERSION }} + platforms: linux/amd64 + tags: budibase/budibase,budibase/budibase:v${{ env.RELEASE_VERSION }} file: ./hosting/single/Dockerfile + + - name: Tag and release Budibase Azure App Service docker image + uses: docker/build-push-action@v2 + with: + context: . + push: true + platforms: linux/amd64 + build-args: TARGETBUILD=aas + tags: budibase/budibase-aas,budibase/budibase-aas:v${{ env.RELEASE_VERSION }} + file: ./hosting/single/Dockerfile + + build-arm64: + name: "build-arm64" + runs-on: ubuntu-latest + strategy: + matrix: + node-version: [14.x] + steps: + - name: Fail if not a tag + run: | + if [[ $GITHUB_REF != refs/tags/* ]]; then + echo "Workflow Dispatch can only be run on tags" + exit 1 + fi + - name: "Checkout" + uses: actions/checkout@v2 + with: + submodules: true + token: ${{ secrets.PERSONAL_ACCESS_TOKEN }} + fetch-depth: 0 + - name: Fail if tag is not in master + run: | + if ! git merge-base --is-ancestor ${{ github.sha }} origin/master; then + echo "Tag is not in master. This pipeline can only execute tags that are present on the master branch" + exit 1 + fi + - name: Use Node.js ${{ matrix.node-version }} + uses: actions/setup-node@v1 + with: + node-version: ${{ matrix.node-version }} + - name: Setup QEMU + uses: docker/setup-qemu-action@v1 + - name: Setup Docker Buildx + id: buildx + uses: docker/setup-buildx-action@v1 + - name: Run Yarn + run: yarn + - name: Update versions + run: ./scripts/updateVersions.sh + - name: Runt Yarn Lint + run: yarn lint + - name: Update versions + run: ./scripts/updateVersions.sh + - name: Run Yarn Build + run: yarn build:docker:pre + - name: Login to Docker Hub + uses: docker/login-action@v2 + with: + username: ${{ secrets.DOCKER_USERNAME }} + password: ${{ secrets.DOCKER_API_KEY }} + - name: Get the latest release version + id: version + run: | + release_version=$(cat lerna.json | jq -r '.version') + echo $release_version + echo "RELEASE_VERSION=$release_version" >> $GITHUB_ENV + - name: Tag and release Budibase service docker image + uses: docker/build-push-action@v2 + with: + context: . + push: true + platforms: linux/arm64 + tags: budibase/budibase,budibase/budibase:v${{ env.RELEASE_VERSION }} + file: ./hosting/single/Dockerfile + + build-aas: + name: "build-aas" + runs-on: ubuntu-latest + strategy: + matrix: + node-version: [14.x] + steps: + - name: Fail if not a tag + run: | + if [[ $GITHUB_REF != refs/tags/* ]]; then + echo "Workflow Dispatch can only be run on tags" + exit 1 + fi + - name: "Checkout" + uses: actions/checkout@v2 + with: + submodules: true + token: ${{ secrets.PERSONAL_ACCESS_TOKEN }} + fetch-depth: 0 + - name: Fail if tag is not in master + run: | + if ! git merge-base --is-ancestor ${{ github.sha }} origin/master; then + echo "Tag is not in master. This pipeline can only execute tags that are present on the master branch" + exit 1 + fi + - name: Use Node.js ${{ matrix.node-version }} + uses: actions/setup-node@v1 + with: + node-version: ${{ matrix.node-version }} + - name: Setup QEMU + uses: docker/setup-qemu-action@v1 + - name: Setup Docker Buildx + id: buildx + uses: docker/setup-buildx-action@v1 + - name: Run Yarn + run: yarn + - name: Update versions + run: ./scripts/updateVersions.sh + - name: Runt Yarn Lint + run: yarn lint + - name: Update versions + run: ./scripts/updateVersions.sh + - name: Run Yarn Build + run: yarn build:docker:pre + - name: Login to Docker Hub + uses: docker/login-action@v2 + with: + username: ${{ secrets.DOCKER_USERNAME }} + password: ${{ secrets.DOCKER_API_KEY }} + - name: Get the latest release version + id: version + run: | + release_version=$(cat lerna.json | jq -r '.version') + echo $release_version + echo "RELEASE_VERSION=$release_version" >> $GITHUB_ENV - name: Tag and release Budibase Azure App Service docker image uses: docker/build-push-action@v2 with: diff --git a/.gitignore b/.gitignore index b3dc8af0d4..22a7313e66 100644 --- a/.gitignore +++ b/.gitignore @@ -101,8 +101,6 @@ packages/builder/cypress.env.json packages/builder/cypress/reports stats.html -# TypeScript cache -*.tsbuildinfo # plugins budibase-component diff --git a/.tool-versions b/.tool-versions index da92e03885..9f2ea77b14 100644 --- a/.tool-versions +++ b/.tool-versions @@ -1,2 +1,3 @@ nodejs 14.21.3 -python 3.10.0 \ No newline at end of file +python 3.10.0 +yarn 1.22.19 \ No newline at end of file diff --git a/charts/budibase/templates/app-service-deployment.yaml b/charts/budibase/templates/app-service-deployment.yaml index 2b2589406a..e47dc0bb58 100644 --- a/charts/budibase/templates/app-service-deployment.yaml +++ b/charts/budibase/templates/app-service-deployment.yaml @@ -201,25 +201,24 @@ spec: image: budibase/apps:{{ .Values.globals.appVersion | default .Chart.AppVersion }} imagePullPolicy: Always + {{- if .Values.services.apps.startupProbe }} + {{- with .Values.services.apps.startupProbe }} + startupProbe: + {{- toYaml . | nindent 10 }} + {{- end }} + {{- end }} + {{- if .Values.services.apps.livenessProbe }} + {{- with .Values.services.apps.livenessProbe }} livenessProbe: - httpGet: - path: /health - port: {{ .Values.services.apps.port }} - initialDelaySeconds: 10 - periodSeconds: 5 - successThreshold: 1 - failureThreshold: 3 - timeoutSeconds: 3 + {{- toYaml . | nindent 10 }} + {{- end }} + {{- end }} + {{- if .Values.services.apps.readinessProbe }} + {{- with .Values.services.apps.readinessProbe }} readinessProbe: - httpGet: - path: /health - port: {{ .Values.services.apps.port }} - initialDelaySeconds: 5 - periodSeconds: 5 - successThreshold: 1 - failureThreshold: 3 - timeoutSeconds: 3 - + {{- toYaml . | nindent 10 }} + {{- end }} + {{- end }} name: bbapps ports: - containerPort: {{ .Values.services.apps.port }} diff --git a/charts/budibase/templates/proxy-service-deployment.yaml b/charts/budibase/templates/proxy-service-deployment.yaml index c087627100..53bba6232d 100644 --- a/charts/budibase/templates/proxy-service-deployment.yaml +++ b/charts/budibase/templates/proxy-service-deployment.yaml @@ -40,24 +40,24 @@ spec: - image: budibase/proxy:{{ .Values.globals.appVersion | default .Chart.AppVersion }} imagePullPolicy: Always name: proxy-service + {{- if .Values.services.proxy.startupProbe }} + {{- with .Values.services.proxy.startupProbe }} + startupProbe: + {{- toYaml . | nindent 10 }} + {{- end }} + {{- end }} + {{- if .Values.services.proxy.livenessProbe }} + {{- with .Values.services.proxy.livenessProbe }} livenessProbe: - httpGet: - path: /health - port: {{ .Values.services.proxy.port }} - initialDelaySeconds: 0 - periodSeconds: 5 - successThreshold: 1 - failureThreshold: 2 - timeoutSeconds: 3 + {{- toYaml . | nindent 10 }} + {{- end }} + {{- end }} + {{- if .Values.services.proxy.readinessProbe }} + {{- with .Values.services.proxy.readinessProbe }} readinessProbe: - httpGet: - path: /health - port: {{ .Values.services.proxy.port }} - initialDelaySeconds: 0 - periodSeconds: 5 - successThreshold: 1 - failureThreshold: 2 - timeoutSeconds: 3 + {{- toYaml . | nindent 10 }} + {{- end }} + {{- end }} ports: - containerPort: {{ .Values.services.proxy.port }} env: diff --git a/charts/budibase/templates/worker-service-deployment.yaml b/charts/budibase/templates/worker-service-deployment.yaml index 5fed80b355..124c667807 100644 --- a/charts/budibase/templates/worker-service-deployment.yaml +++ b/charts/budibase/templates/worker-service-deployment.yaml @@ -190,24 +190,24 @@ spec: {{ end }} image: budibase/worker:{{ .Values.globals.appVersion | default .Chart.AppVersion }} imagePullPolicy: Always + {{- if .Values.services.worker.startupProbe }} + {{- with .Values.services.worker.startupProbe }} + startupProbe: + {{- toYaml . | nindent 10 }} + {{- end }} + {{- end }} + {{- if .Values.services.worker.livenessProbe }} + {{- with .Values.services.worker.livenessProbe }} livenessProbe: - httpGet: - path: /health - port: {{ .Values.services.worker.port }} - initialDelaySeconds: 10 - periodSeconds: 5 - successThreshold: 1 - failureThreshold: 3 - timeoutSeconds: 3 + {{- toYaml . | nindent 10 }} + {{- end }} + {{- end }} + {{- if .Values.services.worker.readinessProbe }} + {{- with .Values.services.worker.readinessProbe }} readinessProbe: - httpGet: - path: /health - port: {{ .Values.services.worker.port }} - initialDelaySeconds: 5 - periodSeconds: 5 - successThreshold: 1 - failureThreshold: 3 - timeoutSeconds: 3 + {{- toYaml . | nindent 10 }} + {{- end }} + {{- end }} name: bbworker ports: - containerPort: {{ .Values.services.worker.port }} diff --git a/charts/budibase/values.yaml b/charts/budibase/values.yaml index 74e4c52654..12e21a8e9c 100644 --- a/charts/budibase/values.yaml +++ b/charts/budibase/values.yaml @@ -119,15 +119,37 @@ services: port: 10000 replicaCount: 1 upstreams: - apps: 'http://app-service.{{ .Release.Namespace }}.svc.{{ .Values.services.dns }}:{{ .Values.services.apps.port }}' - worker: 'http://worker-service.{{ .Release.Namespace }}.svc.{{ .Values.services.dns }}:{{ .Values.services.worker.port }}' - minio: 'http://minio-service.{{ .Release.Namespace }}.svc.{{ .Values.services.dns }}:{{ .Values.services.objectStore.port }}' - couchdb: 'http://{{ .Release.Name }}-svc-couchdb:{{ .Values.services.couchdb.port }}' + apps: "http://app-service.{{ .Release.Namespace }}.svc.{{ .Values.services.dns }}:{{ .Values.services.apps.port }}" + worker: "http://worker-service.{{ .Release.Namespace }}.svc.{{ .Values.services.dns }}:{{ .Values.services.worker.port }}" + minio: "http://minio-service.{{ .Release.Namespace }}.svc.{{ .Values.services.dns }}:{{ .Values.services.objectStore.port }}" + couchdb: "http://{{ .Release.Name }}-svc-couchdb:{{ .Values.services.couchdb.port }}" resources: {} -# annotations: -# co.elastic.logs/module: nginx -# co.elastic.logs/fileset.stdout: access -# co.elastic.logs/fileset.stderr: error + startupProbe: + httpGet: + path: /health + port: 10000 + scheme: HTTP + failureThreshold: 30 + periodSeconds: 3 + readinessProbe: + httpGet: + path: /health + port: 10000 + scheme: HTTP + enabled: true + periodSeconds: 3 + failureThreshold: 1 + livenessProbe: + httpGet: + path: /health + port: 10000 + scheme: HTTP + failureThreshold: 3 + periodSeconds: 5 + # annotations: + # co.elastic.logs/module: nginx + # co.elastic.logs/fileset.stdout: access + # co.elastic.logs/fileset.stderr: error apps: port: 4002 @@ -135,23 +157,67 @@ services: logLevel: info httpLogging: 1 resources: {} -# nodeDebug: "" # set the value of NODE_DEBUG -# annotations: -# co.elastic.logs/multiline.type: pattern -# co.elastic.logs/multiline.pattern: '^[[:space:]]' -# co.elastic.logs/multiline.negate: false -# co.elastic.logs/multiline.match: after + startupProbe: + httpGet: + path: /health + port: 4002 + scheme: HTTP + failureThreshold: 30 + periodSeconds: 3 + readinessProbe: + httpGet: + path: /health + port: 4002 + scheme: HTTP + enabled: true + periodSeconds: 3 + failureThreshold: 1 + livenessProbe: + httpGet: + path: /health + port: 4002 + scheme: HTTP + failureThreshold: 3 + periodSeconds: 5 + # nodeDebug: "" # set the value of NODE_DEBUG + # annotations: + # co.elastic.logs/multiline.type: pattern + # co.elastic.logs/multiline.pattern: '^[[:space:]]' + # co.elastic.logs/multiline.negate: false + # co.elastic.logs/multiline.match: after worker: port: 4003 replicaCount: 1 logLevel: info httpLogging: 1 resources: {} -# annotations: -# co.elastic.logs/multiline.type: pattern -# co.elastic.logs/multiline.pattern: '^[[:space:]]' -# co.elastic.logs/multiline.negate: false -# co.elastic.logs/multiline.match: after + startupProbe: + httpGet: + path: /health + port: 4003 + scheme: HTTP + failureThreshold: 30 + periodSeconds: 3 + readinessProbe: + httpGet: + path: /health + port: 4003 + scheme: HTTP + enabled: true + periodSeconds: 3 + failureThreshold: 1 + livenessProbe: + httpGet: + path: /health + port: 4003 + scheme: HTTP + failureThreshold: 3 + periodSeconds: 5 + # annotations: + # co.elastic.logs/multiline.type: pattern + # co.elastic.logs/multiline.pattern: '^[[:space:]]' + # co.elastic.logs/multiline.negate: false + # co.elastic.logs/multiline.match: after couchdb: enabled: true diff --git a/lerna.json b/lerna.json index 0e5fa45410..f687e46b24 100644 --- a/lerna.json +++ b/lerna.json @@ -1,5 +1,5 @@ { - "version": "2.8.22-alpha.3", + "version": "2.8.32-alpha.4", "npmClient": "yarn", "packages": [ "packages/*" diff --git a/nx.json b/nx.json index c2f44ef70d..8176bae82c 100644 --- a/nx.json +++ b/nx.json @@ -3,19 +3,10 @@ "default": { "runner": "nx-cloud", "options": { - "cacheableOperations": ["build", "test"], + "cacheableOperations": ["build", "test", "check:types"], "accessToken": "MmM4OGYxNzItMDBlYy00ZmE3LTk4MTYtNmJhYWMyZjBjZTUyfHJlYWQ=" } } }, - "targetDefaults": { - "dev:builder": { - "dependsOn": [ - { - "projects": ["@budibase/string-templates"], - "target": "build" - } - ] - } - } + "targetDefaults": {} } diff --git a/package.json b/package.json index 3afe279e00..d27af2e27d 100644 --- a/package.json +++ b/package.json @@ -6,8 +6,8 @@ "@nx/js": "16.4.3", "@rollup/plugin-json": "^4.0.2", "@typescript-eslint/parser": "5.45.0", - "esbuild": "^0.17.18", - "esbuild-node-externals": "^1.7.0", + "esbuild": "^0.18.17", + "esbuild-node-externals": "^1.8.0", "eslint": "^8.44.0", "eslint-plugin-cypress": "^2.11.3", "husky": "^8.0.3", @@ -36,7 +36,7 @@ "bootstrap": "./scripts/link-dependencies.sh && echo '***BOOTSTRAP ONLY REQUIRED FOR USE WITH ACCOUNT PORTAL***'", "build": "yarn nx run-many -t=build", "build:dev": "lerna run --stream prebuild && yarn nx run-many --target=build --output-style=dynamic --watch --preserveWatchOutput", - "check:types": "lerna run check:types --skip-nx-cache", + "check:types": "lerna run check:types", "backend:bootstrap": "./scripts/scopeBackend.sh && yarn run bootstrap", "backend:build": "./scripts/scopeBackend.sh 'lerna run --stream build'", "build:sdk": "lerna run --stream build:sdk", @@ -51,7 +51,7 @@ "kill-builder": "kill-port 3000", "kill-server": "kill-port 4001 4002", "kill-all": "yarn run kill-builder && yarn run kill-server", - "dev": "yarn run kill-all && lerna run --stream dev:builder --stream", + "dev": "yarn run kill-all && lerna run --stream dev:builder", "dev:noserver": "yarn run kill-builder && lerna run --stream dev:stack:up && lerna run --stream dev:builder --ignore @budibase/backend-core --ignore @budibase/server --ignore @budibase/worker", "dev:server": "yarn run kill-server && lerna run --stream dev:builder --scope @budibase/worker --scope @budibase/server", "dev:built": "yarn run kill-all && cd packages/server && yarn dev:stack:up && cd ../../ && lerna run --stream dev:built", @@ -108,5 +108,8 @@ "@budibase/string-templates": "0.0.0", "@budibase/types": "0.0.0" }, + "engines": { + "node": ">=14.0.0 <15.0.0" + }, "dependencies": {} } diff --git a/packages/backend-core/jest.config.ts b/packages/backend-core/jest.config.ts index 8d64b24a2f..3f1065ead2 100644 --- a/packages/backend-core/jest.config.ts +++ b/packages/backend-core/jest.config.ts @@ -1,8 +1,6 @@ import { Config } from "@jest/types" -const preset = require("ts-jest/jest-preset") const baseConfig: Config.InitialProjectOptions = { - ...preset, preset: "@trendyol/jest-testcontainers", setupFiles: ["./tests/jestEnv.ts"], setupFilesAfterEnv: ["./tests/jestSetup.ts"], @@ -11,6 +9,7 @@ const baseConfig: Config.InitialProjectOptions = { }, moduleNameMapper: { "@budibase/types": "/../types/src", + "@budibase/shared-core": ["/../shared-core/src"], }, } diff --git a/packages/backend-core/package.json b/packages/backend-core/package.json index 7f3c064c92..4631b090fe 100644 --- a/packages/backend-core/package.json +++ b/packages/backend-core/package.json @@ -16,14 +16,15 @@ "prepack": "cp package.json dist", "build": "tsc -p tsconfig.build.json", "build:dev": "yarn prebuild && tsc --build --watch --preserveWatchOutput", + "check:types": "tsc -p tsconfig.json --noEmit --paths null", "test": "bash scripts/test.sh", "test:watch": "jest --watchAll" }, "dependencies": { "@budibase/nano": "10.1.2", "@budibase/pouchdb-replication-stream": "1.2.10", + "@budibase/shared-core": "0.0.0", "@budibase/types": "0.0.0", - "@shopify/jest-koa-mocks": "5.0.1", "@techpass/passport-openidconnect": "0.3.2", "aws-cloudfront-sign": "2.2.0", "aws-sdk": "2.1030.0", @@ -58,12 +59,13 @@ "uuid": "8.3.2" }, "devDependencies": { - "@jest/test-sequencer": "29.5.0", - "@swc/core": "^1.3.25", - "@swc/jest": "^0.2.24", + "@jest/test-sequencer": "29.6.2", + "@shopify/jest-koa-mocks": "5.1.1", + "@swc/core": "1.3.71", + "@swc/jest": "0.2.27", "@trendyol/jest-testcontainers": "^2.1.1", "@types/chance": "1.1.3", - "@types/jest": "29.5.0", + "@types/jest": "29.5.3", "@types/koa": "2.13.4", "@types/lodash": "4.14.180", "@types/node": "14.18.20", @@ -75,32 +77,16 @@ "@types/uuid": "8.3.4", "chance": "1.1.8", "ioredis-mock": "8.7.0", - "jest": "29.5.0", - "jest-environment-node": "29.5.0", - "jest-serial-runner": "^1.2.1", + "jest": "29.6.2", + "jest-environment-node": "29.6.2", + "jest-serial-runner": "1.2.1", "koa": "2.13.4", "nodemon": "2.0.16", "pino-pretty": "10.0.0", "pouchdb-adapter-memory": "7.2.2", "timekeeper": "2.2.0", - "ts-jest": "29.0.5", "ts-node": "10.8.1", "tsconfig-paths": "4.0.0", "typescript": "4.7.3" - }, - "nx": { - "targets": { - "build": { - "dependsOn": [ - { - "projects": [ - "@budibase/types" - ], - "target": "build" - } - ] - } - } - }, - "gitHead": "d1836a898cab3f8ab80ee6d8f42be1a9eed7dcdc" + } } diff --git a/packages/backend-core/scripts/test.sh b/packages/backend-core/scripts/test.sh index 3d8240e65a..7d19ec96cc 100644 --- a/packages/backend-core/scripts/test.sh +++ b/packages/backend-core/scripts/test.sh @@ -8,6 +8,6 @@ then jest --coverage --runInBand --forceExit else # --maxWorkers performs better in development - echo "jest --coverage --forceExit" - jest --coverage --forceExit + echo "jest --coverage --detectOpenHandles" + jest --coverage --detectOpenHandles fi \ No newline at end of file diff --git a/packages/backend-core/src/cache/tests/writethrough.spec.ts b/packages/backend-core/src/cache/tests/writethrough.spec.ts index 92b073ed64..97d3ece7a6 100644 --- a/packages/backend-core/src/cache/tests/writethrough.spec.ts +++ b/packages/backend-core/src/cache/tests/writethrough.spec.ts @@ -36,7 +36,7 @@ describe("writethrough", () => { _id: docId, value: 1, }) - const output = await db.get(response.id) + const output = await db.get(response.id) current = output expect(output.value).toBe(1) }) @@ -45,7 +45,7 @@ describe("writethrough", () => { it("second put shouldn't update DB", async () => { await config.doInTenant(async () => { const response = await writethrough.put({ ...current, value: 2 }) - const output = await db.get(response.id) + const output = await db.get(response.id) expect(current._rev).toBe(output._rev) expect(output.value).toBe(1) }) @@ -55,7 +55,7 @@ describe("writethrough", () => { await config.doInTenant(async () => { tk.freeze(Date.now() + DELAY + 1) const response = await writethrough.put({ ...current, value: 3 }) - const output = await db.get(response.id) + const output = await db.get(response.id) expect(response.rev).not.toBe(current._rev) expect(output.value).toBe(3) @@ -79,7 +79,7 @@ describe("writethrough", () => { expect.arrayContaining([current._rev, current._rev, newRev]) ) - const output = await db.get(current._id) + const output = await db.get(current._id) expect(output.value).toBe(4) expect(output._rev).toBe(newRev) @@ -107,7 +107,7 @@ describe("writethrough", () => { }) expect(res.ok).toBe(true) - const output = await db.get(id) + const output = await db.get(id) expect(output.value).toBe(3) expect(output._rev).toBe(res.rev) }) @@ -130,8 +130,8 @@ describe("writethrough", () => { const resp2 = await writethrough2.put({ _id: "db1", value: "second" }) expect(resp1.rev).toBeDefined() expect(resp2.rev).toBeDefined() - expect((await db.get("db1")).value).toBe("first") - expect((await db2.get("db1")).value).toBe("second") + expect((await db.get("db1")).value).toBe("first") + expect((await db2.get("db1")).value).toBe("second") }) }) }) diff --git a/packages/backend-core/src/constants/db.ts b/packages/backend-core/src/constants/db.ts index be49b9f261..83f8298f54 100644 --- a/packages/backend-core/src/constants/db.ts +++ b/packages/backend-core/src/constants/db.ts @@ -1,5 +1,5 @@ -export const SEPARATOR = "_" -export const UNICODE_MAX = "\ufff0" +import { prefixed, DocumentType } from "@budibase/types" +export { SEPARATOR, UNICODE_MAX, DocumentType } from "@budibase/types" /** * Can be used to create a few different forms of querying a view. @@ -14,8 +14,6 @@ export enum ViewName { USER_BY_APP = "by_app", USER_BY_EMAIL = "by_email2", BY_API_KEY = "by_api_key", - /** @deprecated - could be deleted */ - USER_BY_BUILDERS = "by_builders", LINK = "by_link", ROUTING = "screen_routes", AUTOMATION_LOGS = "automation_logs", @@ -36,42 +34,6 @@ export enum InternalTable { USER_METADATA = "ta_users", } -export enum DocumentType { - USER = "us", - GROUP = "gr", - WORKSPACE = "workspace", - CONFIG = "config", - TEMPLATE = "template", - APP = "app", - DEV = "dev", - APP_DEV = "app_dev", - APP_METADATA = "app_metadata", - ROLE = "role", - MIGRATIONS = "migrations", - DEV_INFO = "devinfo", - AUTOMATION_LOG = "log_au", - ACCOUNT_METADATA = "acc_metadata", - PLUGIN = "plg", - DATASOURCE = "datasource", - DATASOURCE_PLUS = "datasource_plus", - APP_BACKUP = "backup", - TABLE = "ta", - ROW = "ro", - AUTOMATION = "au", - LINK = "li", - WEBHOOK = "wh", - INSTANCE = "inst", - LAYOUT = "layout", - SCREEN = "screen", - QUERY = "query", - DEPLOYMENTS = "deployments", - METADATA = "metadata", - MEM_VIEW = "view", - USER_FLAG = "flag", - AUTOMATION_METADATA = "meta_au", - AUDIT_LOG = "al", -} - export const StaticDatabases = { GLOBAL: { name: "global-db", @@ -95,7 +57,7 @@ export const StaticDatabases = { }, } -export const APP_PREFIX = DocumentType.APP + SEPARATOR -export const APP_DEV = DocumentType.APP_DEV + SEPARATOR +export const APP_PREFIX = prefixed(DocumentType.APP) +export const APP_DEV = prefixed(DocumentType.APP_DEV) export const APP_DEV_PREFIX = APP_DEV export const BUDIBASE_DATASOURCE_TYPE = "budibase" diff --git a/packages/backend-core/src/db/constants.ts b/packages/backend-core/src/db/constants.ts new file mode 100644 index 0000000000..aea485e3e3 --- /dev/null +++ b/packages/backend-core/src/db/constants.ts @@ -0,0 +1,10 @@ +export const CONSTANT_INTERNAL_ROW_COLS = [ + "_id", + "_rev", + "type", + "createdAt", + "updatedAt", + "tableId", +] as const + +export const CONSTANT_EXTERNAL_ROW_COLS = ["_id", "_rev", "tableId"] as const diff --git a/packages/backend-core/src/db/couch/index.ts b/packages/backend-core/src/db/couch/index.ts index c731d20d6c..932efed3f7 100644 --- a/packages/backend-core/src/db/couch/index.ts +++ b/packages/backend-core/src/db/couch/index.ts @@ -2,3 +2,4 @@ export * from "./connections" export * from "./DatabaseImpl" export * from "./utils" export { init, getPouch, getPouchDB, closePouchDB } from "./pouchDB" +export * from "../constants" diff --git a/packages/backend-core/src/db/views.ts b/packages/backend-core/src/db/views.ts index fddb1ab34b..7f5ef29a0a 100644 --- a/packages/backend-core/src/db/views.ts +++ b/packages/backend-core/src/db/views.ts @@ -105,16 +105,6 @@ export const createApiKeyView = async () => { await createView(db, viewJs, ViewName.BY_API_KEY) } -export const createUserBuildersView = async () => { - const db = getGlobalDB() - const viewJs = `function(doc) { - if (doc.builder && doc.builder.global === true) { - emit(doc._id, doc._id) - } - }` - await createView(db, viewJs, ViewName.USER_BY_BUILDERS) -} - export interface QueryViewOptions { arrayResponse?: boolean } @@ -223,7 +213,6 @@ export const queryPlatformView = async ( const CreateFuncByName: any = { [ViewName.USER_BY_EMAIL]: createNewUserEmailView, [ViewName.BY_API_KEY]: createApiKeyView, - [ViewName.USER_BY_BUILDERS]: createUserBuildersView, [ViewName.USER_BY_APP]: createUserAppView, } diff --git a/packages/backend-core/src/environment.ts b/packages/backend-core/src/environment.ts index b8d2eb2a54..05fcbffd46 100644 --- a/packages/backend-core/src/environment.ts +++ b/packages/backend-core/src/environment.ts @@ -1,4 +1,5 @@ import { existsSync, readFileSync } from "fs" +import { ServiceType } from "@budibase/types" function isTest() { return isCypress() || isJest() @@ -83,10 +84,20 @@ function getPackageJsonFields(): { } } +function isWorker() { + return environment.SERVICE_TYPE === ServiceType.WORKER +} + +function isApps() { + return environment.SERVICE_TYPE === ServiceType.APPS +} + const environment = { isTest, isJest, isDev, + isWorker, + isApps, isProd: () => { return !isDev() }, @@ -153,6 +164,7 @@ const environment = { SMTP_FROM_ADDRESS: process.env.SMTP_FROM_ADDRESS, DISABLE_JWT_WARNING: process.env.DISABLE_JWT_WARNING, BLACKLIST_IPS: process.env.BLACKLIST_IPS, + SERVICE_TYPE: "unknown", /** * Enable to allow an admin user to login using a password. * This can be useful to prevent lockout when configuring SSO. diff --git a/packages/backend-core/src/events/identification.ts b/packages/backend-core/src/events/identification.ts index 948d3b692b..c7bc1c817b 100644 --- a/packages/backend-core/src/events/identification.ts +++ b/packages/backend-core/src/events/identification.ts @@ -21,6 +21,7 @@ import { processors } from "./processors" import { newid } from "../utils" import * as installation from "../installation" import * as configs from "../configs" +import * as users from "../users" import { withCache, TTL, CacheKey } from "../cache/generic" /** @@ -164,8 +165,8 @@ const identifyUser = async ( const id = user._id as string const tenantId = await getEventTenantId(user.tenantId) const type = IdentityType.USER - let builder = user.builder?.global || false - let admin = user.admin?.global || false + let builder = users.hasBuilderPermissions(user) + let admin = users.hasAdminPermissions(user) let providerType if (isSSOUser(user)) { providerType = user.providerType diff --git a/packages/backend-core/src/middleware/adminOnly.ts b/packages/backend-core/src/middleware/adminOnly.ts index dbe1e3a501..6b2ee87c01 100644 --- a/packages/backend-core/src/middleware/adminOnly.ts +++ b/packages/backend-core/src/middleware/adminOnly.ts @@ -1,10 +1,8 @@ -import { BBContext } from "@budibase/types" +import { UserCtx } from "@budibase/types" +import { isAdmin } from "../users" -export default async (ctx: BBContext, next: any) => { - if ( - !ctx.internal && - (!ctx.user || !ctx.user.admin || !ctx.user.admin.global) - ) { +export default async (ctx: UserCtx, next: any) => { + if (!ctx.internal && !isAdmin(ctx.user)) { ctx.throw(403, "Admin user only endpoint.") } return next() diff --git a/packages/backend-core/src/middleware/builderOnly.ts b/packages/backend-core/src/middleware/builderOnly.ts index a00fd63a22..8c1c54a44c 100644 --- a/packages/backend-core/src/middleware/builderOnly.ts +++ b/packages/backend-core/src/middleware/builderOnly.ts @@ -1,10 +1,19 @@ -import { BBContext } from "@budibase/types" +import { UserCtx } from "@budibase/types" +import { isBuilder, hasBuilderPermissions } from "../users" +import { getAppId } from "../context" +import env from "../environment" -export default async (ctx: BBContext, next: any) => { - if ( - !ctx.internal && - (!ctx.user || !ctx.user.builder || !ctx.user.builder.global) - ) { +export default async (ctx: UserCtx, next: any) => { + const appId = getAppId() + const builderFn = env.isWorker() + ? hasBuilderPermissions + : env.isApps() + ? isBuilder + : undefined + if (!builderFn) { + throw new Error("Service name unknown - middleware inactive.") + } + if (!ctx.internal && !builderFn(ctx.user, appId)) { ctx.throw(403, "Builder user only endpoint.") } return next() diff --git a/packages/backend-core/src/middleware/builderOrAdmin.ts b/packages/backend-core/src/middleware/builderOrAdmin.ts index 26bb3a1bda..c03e856233 100644 --- a/packages/backend-core/src/middleware/builderOrAdmin.ts +++ b/packages/backend-core/src/middleware/builderOrAdmin.ts @@ -1,12 +1,20 @@ -import { BBContext } from "@budibase/types" +import { UserCtx } from "@budibase/types" +import { isBuilder, isAdmin, hasBuilderPermissions } from "../users" +import { getAppId } from "../context" +import env from "../environment" -export default async (ctx: BBContext, next: any) => { - if ( - !ctx.internal && - (!ctx.user || !ctx.user.builder || !ctx.user.builder.global) && - (!ctx.user || !ctx.user.admin || !ctx.user.admin.global) - ) { - ctx.throw(403, "Builder user only endpoint.") +export default async (ctx: UserCtx, next: any) => { + const appId = getAppId() + const builderFn = env.isWorker() + ? hasBuilderPermissions + : env.isApps() + ? isBuilder + : undefined + if (!builderFn) { + throw new Error("Service name unknown - middleware inactive.") + } + if (!ctx.internal && !builderFn(ctx.user, appId) && !isAdmin(ctx.user)) { + ctx.throw(403, "Admin/Builder user only endpoint.") } return next() } diff --git a/packages/backend-core/src/middleware/tests/builder.spec.ts b/packages/backend-core/src/middleware/tests/builder.spec.ts new file mode 100644 index 0000000000..d350eff4f6 --- /dev/null +++ b/packages/backend-core/src/middleware/tests/builder.spec.ts @@ -0,0 +1,180 @@ +import adminOnly from "../adminOnly" +import builderOnly from "../builderOnly" +import builderOrAdmin from "../builderOrAdmin" +import { structures } from "../../../tests" +import { ContextUser, ServiceType } from "@budibase/types" +import { doInAppContext } from "../../context" +import env from "../../environment" +env._set("SERVICE_TYPE", ServiceType.APPS) + +const appId = "app_aaa" +const basicUser = structures.users.user() +const adminUser = structures.users.adminUser() +const adminOnlyUser = structures.users.adminOnlyUser() +const builderUser = structures.users.builderUser() +const appBuilderUser = structures.users.appBuilderUser(appId) + +function buildUserCtx(user: ContextUser) { + return { + internal: false, + user, + throw: jest.fn(), + } as any +} + +function passed(throwFn: jest.Func, nextFn: jest.Func) { + expect(throwFn).not.toBeCalled() + expect(nextFn).toBeCalled() +} + +function threw(throwFn: jest.Func) { + // cant check next, the throw function doesn't actually throw - so it still continues + expect(throwFn).toBeCalled() +} + +describe("adminOnly middleware", () => { + it("should allow admin user", () => { + const ctx = buildUserCtx(adminUser), + next = jest.fn() + adminOnly(ctx, next) + passed(ctx.throw, next) + }) + + it("should not allow basic user", () => { + const ctx = buildUserCtx(basicUser), + next = jest.fn() + adminOnly(ctx, next) + threw(ctx.throw) + }) + + it("should not allow builder user", () => { + const ctx = buildUserCtx(builderUser), + next = jest.fn() + adminOnly(ctx, next) + threw(ctx.throw) + }) +}) + +describe("builderOnly middleware", () => { + it("should allow builder user", () => { + const ctx = buildUserCtx(builderUser), + next = jest.fn() + builderOnly(ctx, next) + passed(ctx.throw, next) + }) + + it("should allow app builder user", () => { + const ctx = buildUserCtx(appBuilderUser), + next = jest.fn() + doInAppContext(appId, () => { + builderOnly(ctx, next) + }) + passed(ctx.throw, next) + }) + + it("should allow admin and builder user", () => { + const ctx = buildUserCtx(adminUser), + next = jest.fn() + builderOnly(ctx, next) + passed(ctx.throw, next) + }) + + it("should not allow admin user", () => { + const ctx = buildUserCtx(adminOnlyUser), + next = jest.fn() + builderOnly(ctx, next) + threw(ctx.throw) + }) + + it("should not allow app builder user to different app", () => { + const ctx = buildUserCtx(appBuilderUser), + next = jest.fn() + doInAppContext("app_bbb", () => { + builderOnly(ctx, next) + }) + threw(ctx.throw) + }) + + it("should not allow basic user", () => { + const ctx = buildUserCtx(basicUser), + next = jest.fn() + builderOnly(ctx, next) + threw(ctx.throw) + }) +}) + +describe("builderOrAdmin middleware", () => { + it("should allow builder user", () => { + const ctx = buildUserCtx(builderUser), + next = jest.fn() + builderOrAdmin(ctx, next) + passed(ctx.throw, next) + }) + + it("should allow builder and admin user", () => { + const ctx = buildUserCtx(adminUser), + next = jest.fn() + builderOrAdmin(ctx, next) + passed(ctx.throw, next) + }) + + it("should allow admin user", () => { + const ctx = buildUserCtx(adminOnlyUser), + next = jest.fn() + builderOrAdmin(ctx, next) + passed(ctx.throw, next) + }) + + it("should allow app builder user", () => { + const ctx = buildUserCtx(appBuilderUser), + next = jest.fn() + doInAppContext(appId, () => { + builderOrAdmin(ctx, next) + }) + passed(ctx.throw, next) + }) + + it("should not allow basic user", () => { + const ctx = buildUserCtx(basicUser), + next = jest.fn() + builderOrAdmin(ctx, next) + threw(ctx.throw) + }) +}) + +describe("check service difference", () => { + it("should not allow without app ID in apps", () => { + env._set("SERVICE_TYPE", ServiceType.APPS) + const appId = "app_a" + const ctx = buildUserCtx({ + ...basicUser, + builder: { + apps: [appId], + }, + }) + const next = jest.fn() + doInAppContext(appId, () => { + builderOnly(ctx, next) + }) + passed(ctx.throw, next) + doInAppContext("app_b", () => { + builderOnly(ctx, next) + }) + threw(ctx.throw) + }) + + it("should allow without app ID in worker", () => { + env._set("SERVICE_TYPE", ServiceType.WORKER) + const ctx = buildUserCtx({ + ...basicUser, + builder: { + apps: ["app_a"], + }, + }) + const next = jest.fn() + doInAppContext("app_b", () => { + builderOnly(ctx, next) + }) + passed(ctx.throw, next) + }) +}) diff --git a/packages/backend-core/src/security/permissions.ts b/packages/backend-core/src/security/permissions.ts index 6cacc12dd6..70dae57ae6 100644 --- a/packages/backend-core/src/security/permissions.ts +++ b/packages/backend-core/src/security/permissions.ts @@ -1,29 +1,12 @@ -const { flatten } = require("lodash") -const { cloneDeep } = require("lodash/fp") +import { PermissionType, PermissionLevel } from "@budibase/types" +export { PermissionType, PermissionLevel } from "@budibase/types" +import flatten from "lodash/flatten" +import cloneDeep from "lodash/fp/cloneDeep" export type RoleHierarchy = { permissionId: string }[] -export enum PermissionLevel { - READ = "read", - WRITE = "write", - EXECUTE = "execute", - ADMIN = "admin", -} - -// these are the global types, that govern the underlying default behaviour -export enum PermissionType { - APP = "app", - TABLE = "table", - USER = "user", - AUTOMATION = "automation", - WEBHOOK = "webhook", - BUILDER = "builder", - VIEW = "view", - QUERY = "query", -} - export class Permission { type: PermissionType level: PermissionLevel @@ -173,3 +156,4 @@ export function isPermissionLevelHigherThanRead(level: PermissionLevel) { // utility as a lot of things need simply the builder permission export const BUILDER = PermissionType.BUILDER +export const GLOBAL_BUILDER = PermissionType.GLOBAL_BUILDER diff --git a/packages/backend-core/src/security/roles.ts b/packages/backend-core/src/security/roles.ts index cf5c6bc406..081193b433 100644 --- a/packages/backend-core/src/security/roles.ts +++ b/packages/backend-core/src/security/roles.ts @@ -3,7 +3,7 @@ import { prefixRoleID, getRoleParams, DocumentType, SEPARATOR } from "../db" import { getAppDB } from "../context" import { doWithDB } from "../db" import { Screen, Role as RoleDoc } from "@budibase/types" -const { cloneDeep } = require("lodash/fp") +import cloneDeep from "lodash/fp/cloneDeep" export const BUILTIN_ROLE_IDS = { ADMIN: "ADMIN", diff --git a/packages/backend-core/src/security/tests/permissions.spec.ts b/packages/backend-core/src/security/tests/permissions.spec.ts index caf8bb29a6..39348646fb 100644 --- a/packages/backend-core/src/security/tests/permissions.spec.ts +++ b/packages/backend-core/src/security/tests/permissions.spec.ts @@ -1,4 +1,4 @@ -import { cloneDeep } from "lodash" +import cloneDeep from "lodash/cloneDeep" import * as permissions from "../permissions" import { BUILTIN_ROLE_IDS } from "../roles" diff --git a/packages/backend-core/src/users/db.ts b/packages/backend-core/src/users/db.ts new file mode 100644 index 0000000000..55cc97bb1c --- /dev/null +++ b/packages/backend-core/src/users/db.ts @@ -0,0 +1,460 @@ +import env from "../environment" +import * as eventHelpers from "./events" +import * as accounts from "../accounts" +import * as cache from "../cache" +import { getIdentity, getTenantId, getGlobalDB } from "../context" +import * as dbUtils from "../db" +import { EmailUnavailableError, HTTPError } from "../errors" +import * as platform from "../platform" +import * as sessions from "../security/sessions" +import * as usersCore from "./users" +import { + AllDocsResponse, + BulkUserCreated, + BulkUserDeleted, + RowResponse, + SaveUserOpts, + User, + Account, + isSSOUser, + isSSOAccount, + UserStatus, +} from "@budibase/types" +import * as accountSdk from "../accounts" +import { + validateUniqueUser, + getAccountHolderFromUserIds, + isAdmin, +} from "./utils" +import { searchExistingEmails } from "./lookup" +import { hash } from "../utils" + +type QuotaUpdateFn = (change: number, cb?: () => Promise) => Promise +type GroupUpdateFn = (groupId: string, userIds: string[]) => Promise +type FeatureFn = () => Promise +type QuotaFns = { addUsers: QuotaUpdateFn; removeUsers: QuotaUpdateFn } +type GroupFns = { addUsers: GroupUpdateFn } +type FeatureFns = { isSSOEnforced: FeatureFn; isAppBuildersEnabled: FeatureFn } + +const bulkDeleteProcessing = async (dbUser: User) => { + const userId = dbUser._id as string + await platform.users.removeUser(dbUser) + await eventHelpers.handleDeleteEvents(dbUser) + await cache.user.invalidateUser(userId) + await sessions.invalidateSessions(userId, { reason: "bulk-deletion" }) +} + +export class UserDB { + static quotas: QuotaFns + static groups: GroupFns + static features: FeatureFns + + static init(quotaFns: QuotaFns, groupFns: GroupFns, featureFns: FeatureFns) { + UserDB.quotas = quotaFns + UserDB.groups = groupFns + UserDB.features = featureFns + } + + static async isPreventPasswordActions(user: User, account?: Account) { + // when in maintenance mode we allow sso users with the admin role + // to perform any password action - this prevents lockout + if (env.ENABLE_SSO_MAINTENANCE_MODE && isAdmin(user)) { + return false + } + + // SSO is enforced for all users + if (await UserDB.features.isSSOEnforced()) { + return true + } + + // Check local sso + if (isSSOUser(user)) { + return true + } + + // Check account sso + if (!account) { + account = await accountSdk.getAccountByTenantId(getTenantId()) + } + return !!(account && account.email === user.email && isSSOAccount(account)) + } + + static async buildUser( + user: User, + opts: SaveUserOpts = { + hashPassword: true, + requirePassword: true, + }, + tenantId: string, + dbUser?: any, + account?: Account + ): Promise { + let { password, _id } = user + + // don't require a password if the db user doesn't already have one + if (dbUser && !dbUser.password) { + opts.requirePassword = false + } + + let hashedPassword + if (password) { + if (await UserDB.isPreventPasswordActions(user, account)) { + throw new HTTPError("Password change is disabled for this user", 400) + } + hashedPassword = opts.hashPassword ? await hash(password) : password + } else if (dbUser) { + hashedPassword = dbUser.password + } + + // passwords are never required if sso is enforced + const requirePasswords = + opts.requirePassword && !(await UserDB.features.isSSOEnforced()) + if (!hashedPassword && requirePasswords) { + throw "Password must be specified." + } + + _id = _id || dbUtils.generateGlobalUserID() + + const fullUser = { + createdAt: Date.now(), + ...dbUser, + ...user, + _id, + password: hashedPassword, + tenantId, + } + // make sure the roles object is always present + if (!fullUser.roles) { + fullUser.roles = {} + } + // add the active status to a user if its not provided + if (fullUser.status == null) { + fullUser.status = UserStatus.ACTIVE + } + + return fullUser + } + + static async allUsers() { + const db = getGlobalDB() + const response = await db.allDocs( + dbUtils.getGlobalUserParams(null, { + include_docs: true, + }) + ) + return response.rows.map((row: any) => row.doc) + } + + static async countUsersByApp(appId: string) { + let response: any = await usersCore.searchGlobalUsersByApp(appId, {}) + return { + userCount: response.length, + } + } + + static async getUsersByAppAccess(appId?: string) { + const opts: any = { + include_docs: true, + limit: 50, + } + let response: User[] = await usersCore.searchGlobalUsersByAppAccess( + appId, + opts + ) + return response + } + + static async getUserByEmail(email: string) { + return usersCore.getGlobalUserByEmail(email) + } + + /** + * Gets a user by ID from the global database, based on the current tenancy. + */ + static async getUser(userId: string) { + const user = await usersCore.getById(userId) + if (user) { + delete user.password + } + return user + } + + static async save(user: User, opts: SaveUserOpts = {}): Promise { + // default booleans to true + if (opts.hashPassword == null) { + opts.hashPassword = true + } + if (opts.requirePassword == null) { + opts.requirePassword = true + } + const tenantId = getTenantId() + const db = getGlobalDB() + + let { email, _id, userGroups = [], roles } = user + + if (!email && !_id) { + throw new Error("_id or email is required") + } + + if ( + user.builder?.apps?.length && + !(await UserDB.features.isAppBuildersEnabled()) + ) { + throw new Error("Unable to update app builders, please check license") + } + + let dbUser: User | undefined + if (_id) { + // try to get existing user from db + try { + dbUser = (await db.get(_id)) as User + if (email && dbUser.email !== email) { + throw "Email address cannot be changed" + } + email = dbUser.email + } catch (e: any) { + if (e.status === 404) { + // do nothing, save this new user with the id specified - required for SSO auth + } else { + throw e + } + } + } + + if (!dbUser && email) { + // no id was specified - load from email instead + dbUser = await usersCore.getGlobalUserByEmail(email) + if (dbUser && dbUser._id !== _id) { + throw new EmailUnavailableError(email) + } + } + + const change = dbUser ? 0 : 1 // no change if there is existing user + return UserDB.quotas.addUsers(change, async () => { + await validateUniqueUser(email, tenantId) + + let builtUser = await UserDB.buildUser(user, opts, tenantId, dbUser) + // don't allow a user to update its own roles/perms + if (opts.currentUserId && opts.currentUserId === dbUser?._id) { + builtUser = usersCore.cleanseUserObject(builtUser, dbUser) as User + } + + if (!dbUser && roles?.length) { + builtUser.roles = { ...roles } + } + + // make sure we set the _id field for a new user + // Also if this is a new user, associate groups with them + let groupPromises = [] + if (!_id) { + _id = builtUser._id! + + if (userGroups.length > 0) { + for (let groupId of userGroups) { + groupPromises.push(UserDB.groups.addUsers(groupId, [_id!])) + } + } + } + + try { + // save the user to db + let response = await db.put(builtUser) + builtUser._rev = response.rev + + await eventHelpers.handleSaveEvents(builtUser, dbUser) + await platform.users.addUser(tenantId, builtUser._id!, builtUser.email) + await cache.user.invalidateUser(response.id) + + await Promise.all(groupPromises) + + // finally returned the saved user from the db + return db.get(builtUser._id!) + } catch (err: any) { + if (err.status === 409) { + throw "User exists already" + } else { + throw err + } + } + }) + } + + static async bulkCreate( + newUsersRequested: User[], + groups: string[] + ): Promise { + const tenantId = getTenantId() + + let usersToSave: any[] = [] + let newUsers: any[] = [] + + const emails = newUsersRequested.map((user: User) => user.email) + const existingEmails = await searchExistingEmails(emails) + const unsuccessful: { email: string; reason: string }[] = [] + + for (const newUser of newUsersRequested) { + if ( + newUsers.find( + (x: User) => x.email.toLowerCase() === newUser.email.toLowerCase() + ) || + existingEmails.includes(newUser.email.toLowerCase()) + ) { + unsuccessful.push({ + email: newUser.email, + reason: `Unavailable`, + }) + continue + } + newUser.userGroups = groups + newUsers.push(newUser) + } + + const account = await accountSdk.getAccountByTenantId(tenantId) + return UserDB.quotas.addUsers(newUsers.length, async () => { + // create the promises array that will be called by bulkDocs + newUsers.forEach((user: any) => { + usersToSave.push( + UserDB.buildUser( + user, + { + hashPassword: true, + requirePassword: user.requirePassword, + }, + tenantId, + undefined, // no dbUser + account + ) + ) + }) + + const usersToBulkSave = await Promise.all(usersToSave) + await usersCore.bulkUpdateGlobalUsers(usersToBulkSave) + + // Post-processing of bulk added users, e.g. events and cache operations + for (const user of usersToBulkSave) { + // TODO: Refactor to bulk insert users into the info db + // instead of relying on looping tenant creation + await platform.users.addUser(tenantId, user._id, user.email) + await eventHelpers.handleSaveEvents(user, undefined) + } + + const saved = usersToBulkSave.map(user => { + return { + _id: user._id, + email: user.email, + } + }) + + // now update the groups + if (Array.isArray(saved) && groups) { + const groupPromises = [] + const createdUserIds = saved.map(user => user._id) + for (let groupId of groups) { + groupPromises.push(UserDB.groups.addUsers(groupId, createdUserIds)) + } + await Promise.all(groupPromises) + } + + return { + successful: saved, + unsuccessful, + } + }) + } + + static async bulkDelete(userIds: string[]): Promise { + const db = getGlobalDB() + + const response: BulkUserDeleted = { + successful: [], + unsuccessful: [], + } + + // remove the account holder from the delete request if present + const account = await getAccountHolderFromUserIds(userIds) + if (account) { + userIds = userIds.filter(u => u !== account.budibaseUserId) + // mark user as unsuccessful + response.unsuccessful.push({ + _id: account.budibaseUserId, + email: account.email, + reason: "Account holder cannot be deleted", + }) + } + + // Get users and delete + const allDocsResponse: AllDocsResponse = await db.allDocs({ + include_docs: true, + keys: userIds, + }) + const usersToDelete: User[] = allDocsResponse.rows.map( + (user: RowResponse) => { + return user.doc + } + ) + + // Delete from DB + const toDelete = usersToDelete.map(user => ({ + ...user, + _deleted: true, + })) + const dbResponse = await usersCore.bulkUpdateGlobalUsers(toDelete) + + await UserDB.quotas.removeUsers(toDelete.length) + for (let user of usersToDelete) { + await bulkDeleteProcessing(user) + } + + // Build Response + // index users by id + const userIndex: { [key: string]: User } = {} + usersToDelete.reduce((prev, current) => { + prev[current._id!] = current + return prev + }, userIndex) + + // add the successful and unsuccessful users to response + dbResponse.forEach(item => { + const email = userIndex[item.id].email + if (item.ok) { + response.successful.push({ _id: item.id, email }) + } else { + response.unsuccessful.push({ + _id: item.id, + email, + reason: "Database error", + }) + } + }) + + return response + } + + static async destroy(id: string) { + const db = getGlobalDB() + const dbUser = (await db.get(id)) as User + const userId = dbUser._id as string + + if (!env.SELF_HOSTED && !env.DISABLE_ACCOUNT_PORTAL) { + // root account holder can't be deleted from inside budibase + const email = dbUser.email + const account = await accounts.getAccount(email) + if (account) { + if (dbUser.userId === getIdentity()!._id) { + throw new HTTPError('Please visit "Account" to delete this user', 400) + } else { + throw new HTTPError("Account holder cannot be deleted", 400) + } + } + } + + await platform.users.removeUser(dbUser) + + await db.remove(userId, dbUser._rev) + + await UserDB.quotas.removeUsers(1) + await eventHelpers.handleDeleteEvents(dbUser) + await cache.user.invalidateUser(userId) + await sessions.invalidateSessions(userId, { reason: "deletion" }) + } +} diff --git a/packages/worker/src/sdk/users/events.ts b/packages/backend-core/src/users/events.ts similarity index 86% rename from packages/worker/src/sdk/users/events.ts rename to packages/backend-core/src/users/events.ts index 7d86182a3c..f170c9ffe9 100644 --- a/packages/worker/src/sdk/users/events.ts +++ b/packages/backend-core/src/users/events.ts @@ -1,15 +1,18 @@ -import env from "../../environment" -import { events, accounts, tenancy } from "@budibase/backend-core" +import env from "../environment" +import * as events from "../events" +import * as accounts from "../accounts" +import { getTenantId } from "../context" import { User, UserRoles, CloudAccount } from "@budibase/types" +import { hasBuilderPermissions, hasAdminPermissions } from "./utils" export const handleDeleteEvents = async (user: any) => { await events.user.deleted(user) - if (isBuilder(user)) { + if (hasBuilderPermissions(user)) { await events.user.permissionBuilderRemoved(user) } - if (isAdmin(user)) { + if (hasAdminPermissions(user)) { await events.user.permissionAdminRemoved(user) } } @@ -55,7 +58,7 @@ export const handleSaveEvents = async ( user: User, existingUser: User | undefined ) => { - const tenantId = tenancy.getTenantId() + const tenantId = getTenantId() let tenantAccount: CloudAccount | undefined if (!env.SELF_HOSTED && !env.DISABLE_ACCOUNT_PORTAL) { tenantAccount = await accounts.getAccountByTenantId(tenantId) @@ -103,23 +106,20 @@ export const handleSaveEvents = async ( await handleAppRoleEvents(user, existingUser) } -const isBuilder = (user: any) => user.builder && user.builder.global -const isAdmin = (user: any) => user.admin && user.admin.global - export const isAddingBuilder = (user: any, existingUser: any) => { - return isAddingPermission(user, existingUser, isBuilder) + return isAddingPermission(user, existingUser, hasBuilderPermissions) } export const isRemovingBuilder = (user: any, existingUser: any) => { - return isRemovingPermission(user, existingUser, isBuilder) + return isRemovingPermission(user, existingUser, hasBuilderPermissions) } const isAddingAdmin = (user: any, existingUser: any) => { - return isAddingPermission(user, existingUser, isAdmin) + return isAddingPermission(user, existingUser, hasAdminPermissions) } const isRemovingAdmin = (user: any, existingUser: any) => { - return isRemovingPermission(user, existingUser, isAdmin) + return isRemovingPermission(user, existingUser, hasAdminPermissions) } const isOnboardingComplete = (user: any, existingUser: any) => { diff --git a/packages/backend-core/src/users/index.ts b/packages/backend-core/src/users/index.ts new file mode 100644 index 0000000000..c11d2a2c62 --- /dev/null +++ b/packages/backend-core/src/users/index.ts @@ -0,0 +1,4 @@ +export * from "./users" +export * from "./utils" +export * from "./lookup" +export { UserDB } from "./db" diff --git a/packages/backend-core/src/users/lookup.ts b/packages/backend-core/src/users/lookup.ts new file mode 100644 index 0000000000..17d0e91d88 --- /dev/null +++ b/packages/backend-core/src/users/lookup.ts @@ -0,0 +1,102 @@ +import { + AccountMetadata, + PlatformUser, + PlatformUserByEmail, + User, +} from "@budibase/types" +import * as dbUtils from "../db" +import { ViewName } from "../constants" + +/** + * Apply a system-wide search on emails: + * - in tenant + * - cross tenant + * - accounts + * return an array of emails that match the supplied emails. + */ +export async function searchExistingEmails(emails: string[]) { + let matchedEmails: string[] = [] + + const existingTenantUsers = await getExistingTenantUsers(emails) + matchedEmails.push(...existingTenantUsers.map(user => user.email)) + + const existingPlatformUsers = await getExistingPlatformUsers(emails) + matchedEmails.push(...existingPlatformUsers.map(user => user._id!)) + + const existingAccounts = await getExistingAccounts(emails) + matchedEmails.push(...existingAccounts.map(account => account.email)) + + return [...new Set(matchedEmails.map(email => email.toLowerCase()))] +} + +// lookup, could be email or userId, either will return a doc +export async function getPlatformUser( + identifier: string +): Promise { + // use the view here and allow to find anyone regardless of casing + // Use lowercase to ensure email login is case insensitive + return (await dbUtils.queryPlatformView(ViewName.PLATFORM_USERS_LOWERCASE, { + keys: [identifier.toLowerCase()], + include_docs: true, + })) as PlatformUser +} + +export async function getExistingTenantUsers( + emails: string[] +): Promise { + const lcEmails = emails.map(email => email.toLowerCase()) + const params = { + keys: lcEmails, + include_docs: true, + } + + const opts = { + arrayResponse: true, + } + + return (await dbUtils.queryGlobalView( + ViewName.USER_BY_EMAIL, + params, + undefined, + opts + )) as User[] +} + +export async function getExistingPlatformUsers( + emails: string[] +): Promise { + const lcEmails = emails.map(email => email.toLowerCase()) + const params = { + keys: lcEmails, + include_docs: true, + } + + const opts = { + arrayResponse: true, + } + return (await dbUtils.queryPlatformView( + ViewName.PLATFORM_USERS_LOWERCASE, + params, + opts + )) as PlatformUserByEmail[] +} + +export async function getExistingAccounts( + emails: string[] +): Promise { + const lcEmails = emails.map(email => email.toLowerCase()) + const params = { + keys: lcEmails, + include_docs: true, + } + + const opts = { + arrayResponse: true, + } + + return (await dbUtils.queryPlatformView( + ViewName.ACCOUNT_BY_EMAIL, + params, + opts + )) as AccountMetadata[] +} diff --git a/packages/backend-core/src/users.ts b/packages/backend-core/src/users/users.ts similarity index 88% rename from packages/backend-core/src/users.ts rename to packages/backend-core/src/users/users.ts index b49058f546..a7e1389920 100644 --- a/packages/backend-core/src/users.ts +++ b/packages/backend-core/src/users/users.ts @@ -11,10 +11,16 @@ import { SEPARATOR, UNICODE_MAX, ViewName, -} from "./db" -import { BulkDocsResponse, SearchUsersRequest, User } from "@budibase/types" -import { getGlobalDB } from "./context" -import * as context from "./context" +} from "../db" +import { + BulkDocsResponse, + SearchUsersRequest, + User, + ContextUser, +} from "@budibase/types" +import { getGlobalDB } from "../context" +import * as context from "../context" +import { user as userCache } from "../cache" type GetOpts = { cleanup?: boolean } @@ -178,7 +184,7 @@ export const getGlobalUserByAppPage = (appId: string, user: User) => { * Performs a starts with search on the global email view. */ export const searchGlobalUsersByEmail = async ( - email: string, + email: string | unknown, opts: any, getOpts?: GetOpts ) => { @@ -248,3 +254,23 @@ export async function getUserCount() { }) return response.total_rows } + +// used to remove the builder/admin permissions, for processing the +// user as an app user (they may have some specific role/group +export function removePortalUserPermissions(user: User | ContextUser) { + delete user.admin + delete user.builder + return user +} + +export function cleanseUserObject(user: User | ContextUser, base?: User) { + delete user.admin + delete user.builder + delete user.roles + if (base) { + user.admin = base.admin + user.builder = base.builder + user.roles = base.roles + } + return user +} diff --git a/packages/backend-core/src/users/utils.ts b/packages/backend-core/src/users/utils.ts new file mode 100644 index 0000000000..af0e8e10c7 --- /dev/null +++ b/packages/backend-core/src/users/utils.ts @@ -0,0 +1,55 @@ +import { CloudAccount } from "@budibase/types" +import * as accountSdk from "../accounts" +import env from "../environment" +import { getPlatformUser } from "./lookup" +import { EmailUnavailableError } from "../errors" +import { getTenantId } from "../context" +import { sdk } from "@budibase/shared-core" +import { getAccountByTenantId } from "../accounts" + +// extract from shared-core to make easily accessible from backend-core +export const isBuilder = sdk.users.isBuilder +export const isAdmin = sdk.users.isAdmin +export const isGlobalBuilder = sdk.users.isGlobalBuilder +export const isAdminOrBuilder = sdk.users.isAdminOrBuilder +export const hasAdminPermissions = sdk.users.hasAdminPermissions +export const hasBuilderPermissions = sdk.users.hasBuilderPermissions +export const hasAppBuilderPermissions = sdk.users.hasAppBuilderPermissions + +export async function validateUniqueUser(email: string, tenantId: string) { + // check budibase users in other tenants + if (env.MULTI_TENANCY) { + const tenantUser = await getPlatformUser(email) + if (tenantUser != null && tenantUser.tenantId !== tenantId) { + throw new EmailUnavailableError(email) + } + } + + // check root account users in account portal + if (!env.SELF_HOSTED && !env.DISABLE_ACCOUNT_PORTAL) { + const account = await accountSdk.getAccount(email) + if (account && account.verified && account.tenantId !== tenantId) { + throw new EmailUnavailableError(email) + } + } +} + +/** + * For the given user id's, return the account holder if it is in the ids. + */ +export async function getAccountHolderFromUserIds( + userIds: string[] +): Promise { + if (!env.SELF_HOSTED && !env.DISABLE_ACCOUNT_PORTAL) { + const tenantId = getTenantId() + const account = await getAccountByTenantId(tenantId) + if (!account) { + throw new Error(`Account not found for tenantId=${tenantId}`) + } + + const budibaseUserId = account.budibaseUserId + if (userIds.includes(budibaseUserId)) { + return account + } + } +} diff --git a/packages/backend-core/tests/core/utilities/jestUtils.ts b/packages/backend-core/tests/core/utilities/jestUtils.ts index d84eac548c..4a3da8db8c 100644 --- a/packages/backend-core/tests/core/utilities/jestUtils.ts +++ b/packages/backend-core/tests/core/utilities/jestUtils.ts @@ -1,3 +1,5 @@ +import { db } from "../../../src" + export function expectFunctionWasCalledTimesWith( jestFunction: any, times: number, @@ -7,3 +9,22 @@ export function expectFunctionWasCalledTimesWith( jestFunction.mock.calls.filter((call: any) => call[0] === argument).length ).toBe(times) } + +export const expectAnyInternalColsAttributes: { + [K in (typeof db.CONSTANT_INTERNAL_ROW_COLS)[number]]: any +} = { + tableId: expect.anything(), + type: expect.anything(), + _id: expect.anything(), + _rev: expect.anything(), + createdAt: expect.anything(), + updatedAt: expect.anything(), +} + +export const expectAnyExternalColsAttributes: { + [K in (typeof db.CONSTANT_EXTERNAL_ROW_COLS)[number]]: any +} = { + tableId: expect.anything(), + _id: expect.anything(), + _rev: expect.anything(), +} diff --git a/packages/backend-core/tests/core/utilities/mocks/events.ts b/packages/backend-core/tests/core/utilities/mocks/events.ts index 81de1f8175..fef730768a 100644 --- a/packages/backend-core/tests/core/utilities/mocks/events.ts +++ b/packages/backend-core/tests/core/utilities/mocks/events.ts @@ -1,5 +1,3 @@ -import * as events from "../../../../src/events" - beforeAll(async () => { const processors = await import("../../../../src/events/processors") const events = await import("../../../../src/events") diff --git a/packages/backend-core/tests/core/utilities/mocks/licenses.ts b/packages/backend-core/tests/core/utilities/mocks/licenses.ts index 4272e78eb8..6747282040 100644 --- a/packages/backend-core/tests/core/utilities/mocks/licenses.ts +++ b/packages/backend-core/tests/core/utilities/mocks/licenses.ts @@ -1,5 +1,5 @@ import { Feature, License, Quotas } from "@budibase/types" -import _ from "lodash" +import cloneDeep from "lodash/cloneDeep" let CLOUD_FREE_LICENSE: License let UNLIMITED_LICENSE: License @@ -58,7 +58,7 @@ export const useCloudFree = () => { // FEATURES const useFeature = (feature: Feature) => { - const license = _.cloneDeep(UNLIMITED_LICENSE) + const license = cloneDeep(UNLIMITED_LICENSE) const opts: UseLicenseOpts = { features: [feature], } @@ -94,10 +94,14 @@ export const useSyncAutomations = () => { return useFeature(Feature.SYNC_AUTOMATIONS) } +export const useAppBuilders = () => { + return useFeature(Feature.APP_BUILDERS) +} + // QUOTAS export const setAutomationLogsQuota = (value: number) => { - const license = _.cloneDeep(UNLIMITED_LICENSE) + const license = cloneDeep(UNLIMITED_LICENSE) license.quotas.constant.automationLogRetentionDays.value = value return useLicense(license) } diff --git a/packages/backend-core/tests/core/utilities/structures/accounts.ts b/packages/backend-core/tests/core/utilities/structures/accounts.ts index 8476399aa3..67e4411ea3 100644 --- a/packages/backend-core/tests/core/utilities/structures/accounts.ts +++ b/packages/backend-core/tests/core/utilities/structures/accounts.ts @@ -11,7 +11,7 @@ import { CreateAccount, CreatePassswordAccount, } from "@budibase/types" -import _ from "lodash" +import sample from "lodash/sample" export const account = (partial: Partial = {}): Account => { return { @@ -46,13 +46,11 @@ export const cloudAccount = (): CloudAccount => { } function providerType(): AccountSSOProviderType { - return _.sample( - Object.values(AccountSSOProviderType) - ) as AccountSSOProviderType + return sample(Object.values(AccountSSOProviderType)) as AccountSSOProviderType } function provider(): AccountSSOProvider { - return _.sample(Object.values(AccountSSOProvider)) as AccountSSOProvider + return sample(Object.values(AccountSSOProvider)) as AccountSSOProvider } export function ssoAccount(account: Account = cloudAccount()): SSOAccount { diff --git a/packages/backend-core/tests/core/utilities/structures/scim.ts b/packages/backend-core/tests/core/utilities/structures/scim.ts index 741cff165e..80f41c605d 100644 --- a/packages/backend-core/tests/core/utilities/structures/scim.ts +++ b/packages/backend-core/tests/core/utilities/structures/scim.ts @@ -1,7 +1,6 @@ import { ScimCreateGroupRequest, ScimCreateUserRequest } from "@budibase/types" import { uuid } from "./common" import { generator } from "./generator" -import _ from "lodash" interface CreateUserRequestFields { externalId: string @@ -20,10 +19,10 @@ export function createUserRequest(userData?: Partial) { username: generator.name(), } - const { externalId, email, firstName, lastName, username } = _.assign( - defaultValues, - userData - ) + const { externalId, email, firstName, lastName, username } = { + ...defaultValues, + ...userData, + } let user: ScimCreateUserRequest = { schemas: [ diff --git a/packages/backend-core/tests/core/utilities/structures/sso.ts b/packages/backend-core/tests/core/utilities/structures/sso.ts index 9da9c82223..4d13635f09 100644 --- a/packages/backend-core/tests/core/utilities/structures/sso.ts +++ b/packages/backend-core/tests/core/utilities/structures/sso.ts @@ -15,7 +15,7 @@ import { generator } from "./generator" import { email, uuid } from "./common" import * as shared from "./shared" import { user } from "./shared" -import _ from "lodash" +import sample from "lodash/sample" export function OAuth(): OAuth2 { return { @@ -47,7 +47,7 @@ export function authDetails(userDoc?: User): SSOAuthDetails { } export function providerType(): SSOProviderType { - return _.sample(Object.values(SSOProviderType)) as SSOProviderType + return sample(Object.values(SSOProviderType)) as SSOProviderType } export function ssoProfile(user?: User): SSOProfile { diff --git a/packages/backend-core/tests/core/utilities/structures/users.ts b/packages/backend-core/tests/core/utilities/structures/users.ts index 7a6b4f0d80..0a4f2e8b54 100644 --- a/packages/backend-core/tests/core/utilities/structures/users.ts +++ b/packages/backend-core/tests/core/utilities/structures/users.ts @@ -1,5 +1,6 @@ import { AdminUser, + AdminOnlyUser, BuilderUser, SSOAuthDetails, SSOUser, @@ -21,6 +22,15 @@ export const adminUser = (userProps?: any): AdminUser => { } } +export const adminOnlyUser = (userProps?: any): AdminOnlyUser => { + return { + ...user(userProps), + admin: { + global: true, + }, + } +} + export const builderUser = (userProps?: any): BuilderUser => { return { ...user(userProps), @@ -30,6 +40,15 @@ export const builderUser = (userProps?: any): BuilderUser => { } } +export const appBuilderUser = (appId: string, userProps?: any): BuilderUser => { + return { + ...user(userProps), + builder: { + apps: [appId], + }, + } +} + export function ssoUser( opts: { user?: any; details?: SSOAuthDetails } = {} ): SSOUser { diff --git a/packages/backend-core/tsconfig.json b/packages/backend-core/tsconfig.json index 2b1419b051..128814b955 100644 --- a/packages/backend-core/tsconfig.json +++ b/packages/backend-core/tsconfig.json @@ -4,9 +4,9 @@ "composite": true, "baseUrl": ".", "paths": { - "@budibase/types": ["../types/src"] + "@budibase/types": ["../types/src"], + "@budibase/shared-core": ["../shared-core/src"] } }, - "exclude": ["node_modules", "dist"] } diff --git a/packages/bbui/package.json b/packages/bbui/package.json index b03c83d71b..8a9318ba94 100644 --- a/packages/bbui/package.json +++ b/packages/bbui/package.json @@ -85,7 +85,8 @@ "dayjs": "^1.10.4", "easymde": "^2.16.1", "svelte-flatpickr": "3.2.3", - "svelte-portal": "^1.0.0" + "svelte-portal": "^1.0.0", + "svelte-dnd-action": "^0.9.8" }, "resolutions": { "loader-utils": "1.4.1" @@ -96,13 +97,14 @@ "dependsOn": [ { "projects": [ - "@budibase/string-templates" + "@budibase/string-templates", + "@budibase/shared-core", + "@budibase/types" ], "target": "build" } ] } } - }, - "gitHead": "d1836a898cab3f8ab80ee6d8f42be1a9eed7dcdc" + } } diff --git a/packages/bbui/src/Actions/position_dropdown.js b/packages/bbui/src/Actions/position_dropdown.js index 01555446d9..8fa02bb8f3 100644 --- a/packages/bbui/src/Actions/position_dropdown.js +++ b/packages/bbui/src/Actions/position_dropdown.js @@ -64,7 +64,7 @@ export default function positionDropdown(element, opts) { // Apply styles Object.entries(styles).forEach(([style, value]) => { - if (value) { + if (value != null) { element.style[style] = `${value.toFixed(0)}px` } else { element.style[style] = null diff --git a/packages/bbui/src/IconPicker/IconPicker.svelte b/packages/bbui/src/IconPicker/IconPicker.svelte index 2b42da61b1..b3cc72daa3 100644 --- a/packages/bbui/src/IconPicker/IconPicker.svelte +++ b/packages/bbui/src/IconPicker/IconPicker.svelte @@ -1,5 +1,4 @@ + +
+
+ {#each options as option, idx (option.id)} +
+
+ +
+
+
openColorPickerPopover(idx, e.target)} + > + +
+ {#each colorsArray as color} +
handleColorChange(option.name, color, idx)} + style="--color:{color};" + class="circle circle-hover" + /> + {/each} +
+ +
+
+
+ handleNameChange(option.name, idx, e.target.value)} + value={option.name} + placeholder="Option name" + /> +
+
+ +
+
+ {/each} +
+
+ +
Add option
+
+
+ + diff --git a/packages/bbui/src/Popover/Popover.svelte b/packages/bbui/src/Popover/Popover.svelte index 9f951a6a7e..6706bf7a8b 100644 --- a/packages/bbui/src/Popover/Popover.svelte +++ b/packages/bbui/src/Popover/Popover.svelte @@ -21,6 +21,7 @@ export let offset = 5 export let customHeight export let animate = true + export let customZindex $: target = portalTarget || getContext(Context.PopoverRoot) || ".spectrum" @@ -77,8 +78,9 @@ }} on:keydown={handleEscape} class="spectrum-Popover is-open" + class:customZindex role="presentation" - style="height: {customHeight}" + style="height: {customHeight}; --customZindex: {customZindex};" transition:fly|local={{ y: -20, duration: animate ? 200 : 0 }} > @@ -92,4 +94,8 @@ border-color: var(--spectrum-global-color-gray-300); overflow: auto; } + + .customZindex { + z-index: var(--customZindex) !important; + } diff --git a/packages/bbui/src/index.js b/packages/bbui/src/index.js index 97762d2b3a..cda6b5acbf 100644 --- a/packages/bbui/src/index.js +++ b/packages/bbui/src/index.js @@ -84,7 +84,7 @@ export { default as IconSideNavItem } from "./IconSideNav/IconSideNavItem.svelte export { default as Slider } from "./Form/Slider.svelte" export { default as Accordion } from "./Accordion/Accordion.svelte" export { default as File } from "./Form/File.svelte" - +export { default as OptionSelectDnD } from "./OptionSelectDnD/OptionSelectDnD.svelte" // Renderers export { default as BoldRenderer } from "./Table/BoldRenderer.svelte" export { default as CodeRenderer } from "./Table/CodeRenderer.svelte" diff --git a/packages/builder/package.json b/packages/builder/package.json index 646bb144df..56834f79dd 100644 --- a/packages/builder/package.json +++ b/packages/builder/package.json @@ -101,14 +101,14 @@ "@rollup/plugin-replace": "^2.4.2", "@roxi/routify": "2.18.5", "@sveltejs/vite-plugin-svelte": "1.0.1", - "@testing-library/jest-dom": "^5.11.10", + "@testing-library/jest-dom": "5.17.0", "@testing-library/svelte": "^3.2.2", - "babel-jest": "^26.6.3", + "babel-jest": "29.6.2", "cypress": "^9.3.1", "cypress-multi-reporters": "^1.6.0", "cypress-terminal-report": "^1.4.1", "identity-obj-proxy": "^3.0.0", - "jest": "^26.6.3", + "jest": "29.6.2", "jsdom": "^21.1.1", "mochawesome": "^7.1.3", "mochawesome-merge": "^4.2.1", @@ -133,8 +133,21 @@ "dependsOn": [ { "projects": [ + "@budibase/shared-core", "@budibase/string-templates", - "@budibase/shared-core" + "@budibase/types" + ], + "target": "build" + } + ] + }, + "dev:builder": { + "dependsOn": [ + { + "projects": [ + "@budibase/shared-core", + "@budibase/string-templates", + "@budibase/types" ], "target": "build" } @@ -145,13 +158,13 @@ { "projects": [ "@budibase/shared-core", - "@budibase/string-templates" + "@budibase/string-templates", + "@budibase/types" ], "target": "build" } ] } } - }, - "gitHead": "115189f72a850bfb52b65ec61d932531bf327072" + } } diff --git a/packages/builder/src/builderStore/dataBinding.js b/packages/builder/src/builderStore/dataBinding.js index 9f383a4c83..ae8961f276 100644 --- a/packages/builder/src/builderStore/dataBinding.js +++ b/packages/builder/src/builderStore/dataBinding.js @@ -507,6 +507,7 @@ const getSelectedRowsBindings = asset => { readableBinding: `${table._instanceName}.Selected rows`, category: "Selected rows", icon: "ViewRow", + display: { name: table._instanceName }, })) ) @@ -522,6 +523,7 @@ const getSelectedRowsBindings = asset => { )}.${makePropSafe("selectedRows")}`, readableBinding: `${block._instanceName}.Selected rows`, category: "Selected rows", + display: { name: block._instanceName }, })) ) } diff --git a/packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte b/packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte index cece075860..6b57fe3d18 100644 --- a/packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte +++ b/packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte @@ -108,7 +108,10 @@ /****************************************************/ const getInputData = (testData, blockInputs) => { - let newInputData = testData || blockInputs + // Test data is not cloned for reactivity + let newInputData = testData || cloneDeep(blockInputs) + + // Ensures the app action fields are populated if (block.event === "app:trigger" && !newInputData?.fields) { newInputData = cloneDeep(blockInputs) } diff --git a/packages/builder/src/components/automation/SetupPanel/QueryParamSelector.svelte b/packages/builder/src/components/automation/SetupPanel/QueryParamSelector.svelte index 9c47178b0e..250d235266 100644 --- a/packages/builder/src/components/automation/SetupPanel/QueryParamSelector.svelte +++ b/packages/builder/src/components/automation/SetupPanel/QueryParamSelector.svelte @@ -50,6 +50,7 @@ type="string" {bindings} fillWidth={true} + updateOnChange={false} /> {/each}
diff --git a/packages/builder/src/components/backend/DataTable/DataTable.svelte b/packages/builder/src/components/backend/DataTable/DataTable.svelte index 33db9b60e3..38eb87aa73 100644 --- a/packages/builder/src/components/backend/DataTable/DataTable.svelte +++ b/packages/builder/src/components/backend/DataTable/DataTable.svelte @@ -1,5 +1,5 @@ - + field.name} getOptionValue={field => field.type} + getOptionIcon={field => field.icon} isOptionEnabled={option => { if (option.type == AUTO_TYPE) { return availableAutoColumnKeys?.length > 0 @@ -433,28 +439,6 @@ }} /> - {#if canBeRequired || canBeDisplay} -
- {#if canBeRequired} - - {/if} - {#if canBeDisplay} - - {/if} -
- {/if} - {#if editableColumn.type === "string"} {:else if editableColumn.type === "options"} - {:else if editableColumn.type === "longform"}
@@ -480,19 +464,28 @@ />
{:else if editableColumn.type === "array"} - {:else if editableColumn.type === "datetime" && !editableColumn.autocolumn} - - +
+
+ +
+
+ +
+
+ +
+
+ +
+
+ +
+
{#if datasource?.source !== "ORACLE" && datasource?.source !== "SQL_SERVER"}