diff --git a/.eslintrc.json b/.eslintrc.json index d94c749042..75584b8163 100644 --- a/.eslintrc.json +++ b/.eslintrc.json @@ -5,7 +5,7 @@ "jest": true, "node": true }, - "parser": "babel-eslint", + "parser": "@babel/eslint-parser", "parserOptions": { "ecmaVersion": 2019, "sourceType": "module", @@ -18,17 +18,23 @@ "*.spec.js", "bundle.js" ], - "plugins": ["svelte3"], "extends": ["eslint:recommended"], "overrides": [ { - "files": ["*.svelte"], - "processor": "svelte3/svelte3" + "files": ["**/*.svelte"], + "extends": "plugin:svelte/recommended", + "parser": "svelte-eslint-parser", + "parserOptions": { + "parser": "@babel/eslint-parser", + "ecmaVersion": 2019, + "sourceType": "module", + "allowImportExportEverywhere": true + } + }, { "files": ["**/*.ts"], "parser": "@typescript-eslint/parser", - "plugins": [], "extends": ["eslint:recommended"], "rules": { "no-unused-vars": "off", @@ -41,7 +47,8 @@ } ], "rules": { - "no-self-assign": "off" + "no-self-assign": "off", + "no-unused-vars": ["error", { "varsIgnorePattern": "^_", "argsIgnorePattern": "^_", "destructuredArrayIgnorePattern": "^_" }] }, "globals": { "GeolocationPositionError": true diff --git a/.github/stale.yml b/.github/stale.yml deleted file mode 100644 index 2a2c10cb7d..0000000000 --- a/.github/stale.yml +++ /dev/null @@ -1,19 +0,0 @@ -# Configuration for probot-stale - https://github.com/probot/stale -# Number of days of inactivity before an Issue or Pull Request becomes stale -daysUntilStale: 60 -# Number of days of inactivity before an Issue or Pull Request with the stale label is closed. -# Set to false to disable. If disabled, issues still need to be closed manually, but will remain marked as stale. -daysUntilClose: false -# Issues with these labels will never be considered stale -exemptLabels: - - pinned - - security - - roadmap -# Label to use when marking an issue as stale -staleLabel: stale -# Comment to post when marking an issue as stale. Set to `false` to disable -markComment: > - This issue has been automatically marked as stale because it has not had - recent activity. -# Comment to post when closing a stale issue. Set to `false` to disable -closeComment: false diff --git a/.github/workflows/budibase_ci.yml b/.github/workflows/budibase_ci.yml index f2e7f2eda9..b84fd54fae 100644 --- a/.github/workflows/budibase_ci.yml +++ b/.github/workflows/budibase_ci.yml @@ -12,9 +12,6 @@ on: - master - develop pull_request: - branches: - - master - - develop workflow_dispatch: env: @@ -157,12 +154,12 @@ jobs: node-version: 14.x cache: "yarn" - run: yarn - - run: yarn build + - run: yarn build --projects=@budibase/server,@budibase/worker,@budibase/client - name: Run tests run: | cd qa-core yarn setup - yarn test:ci + yarn serve:test:self:ci env: BB_ADMIN_USER_EMAIL: admin BB_ADMIN_USER_PASSWORD: admin @@ -185,7 +182,7 @@ jobs: pro_commit=$(git rev-parse HEAD) branch="${{ github.base_ref || github.ref_name }}" - echo "Running on branch `$branch` (base_ref=${{ github.base_ref }}, ref_name=${{ github.head_ref }})" + echo "Running on branch '$branch' (base_ref=${{ github.base_ref }}, ref_name=${{ github.head_ref }})" if [[ $branch == "master" ]]; then base_commit=$(git rev-parse origin/master) diff --git a/.github/workflows/release-master.yml b/.github/workflows/release-master.yml index 3c33dfcd86..7f8b8f1d55 100644 --- a/.github/workflows/release-master.yml +++ b/.github/workflows/release-master.yml @@ -34,7 +34,6 @@ jobs: exit 1 fi - - uses: actions/setup-node@v1 with: node-version: 14.x @@ -58,9 +57,12 @@ jobs: echo //registry.npmjs.org/:_authToken=${NPM_TOKEN} >> .npmrc yarn release - - name: "Get Previous tag" - id: previoustag - uses: "WyriHaximus/github-action-get-previous-tag@v1" + - name: "Get Current tag" + id: currenttag + run: | + version=v$(./scripts/getCurrentVersion.sh) + echo 'Using tag $version' + echo "::set-output name=tag::$resversionult" - name: Build/release Docker images run: | @@ -69,7 +71,7 @@ jobs: env: DOCKER_USER: ${{ secrets.DOCKER_USERNAME }} DOCKER_PASSWORD: ${{ secrets.DOCKER_API_KEY }} - BUDIBASE_RELEASE_VERSION: ${{ steps.previoustag.outputs.tag }} + BUDIBASE_RELEASE_VERSION: ${{ steps.currenttag.outputs.tag }} release-helm-chart: needs: [release-images] diff --git a/.github/workflows/stale_bot.yml b/.github/workflows/stale_bot.yml new file mode 100644 index 0000000000..8cda3a9342 --- /dev/null +++ b/.github/workflows/stale_bot.yml @@ -0,0 +1,29 @@ +name: Close stale issues and PRs # https://github.com/actions/stale +on: + workflow_dispatch: + schedule: + - cron: '30 1 * * *' # 1:30 every morning + +jobs: + stale: + runs-on: ubuntu-latest + steps: + - uses: actions/stale@v8 + with: + # stale rules + days-before-stale: 60 + days-before-pr-stale: 7 + stale-issue-label: stale + stale-issue-message: "This issue has been automatically marked as stale because it has not had any activity for 60 days." + + # close rules + # days after being marked as stale to close + days-before-close: 30 + close-issue-label: closed-stale + close-issue-message: This issue has been automatically closed it has not had any activity in 90 days." + days-before-pr-close: 7 + + # exemptions + exempt-pr-labels: pinned,security,roadmap + + diff --git a/.gitignore b/.gitignore index b3dc8af0d4..22a7313e66 100644 --- a/.gitignore +++ b/.gitignore @@ -101,8 +101,6 @@ packages/builder/cypress.env.json packages/builder/cypress/reports stats.html -# TypeScript cache -*.tsbuildinfo # plugins budibase-component diff --git a/.tool-versions b/.tool-versions index 094292d096..9f2ea77b14 100644 --- a/.tool-versions +++ b/.tool-versions @@ -1,2 +1,3 @@ -nodejs 14.20.1 -python 3.10.0 \ No newline at end of file +nodejs 14.21.3 +python 3.10.0 +yarn 1.22.19 \ No newline at end of file diff --git a/babel.config.json b/babel.config.json new file mode 100644 index 0000000000..a6c42326bb --- /dev/null +++ b/babel.config.json @@ -0,0 +1,3 @@ +{ + "presets": [["@babel/preset-env", { "targets": { "node": "current" } }]] +} diff --git a/charts/budibase/templates/app-service-deployment.yaml b/charts/budibase/templates/app-service-deployment.yaml index 2b2589406a..e47dc0bb58 100644 --- a/charts/budibase/templates/app-service-deployment.yaml +++ b/charts/budibase/templates/app-service-deployment.yaml @@ -201,25 +201,24 @@ spec: image: budibase/apps:{{ .Values.globals.appVersion | default .Chart.AppVersion }} imagePullPolicy: Always + {{- if .Values.services.apps.startupProbe }} + {{- with .Values.services.apps.startupProbe }} + startupProbe: + {{- toYaml . | nindent 10 }} + {{- end }} + {{- end }} + {{- if .Values.services.apps.livenessProbe }} + {{- with .Values.services.apps.livenessProbe }} livenessProbe: - httpGet: - path: /health - port: {{ .Values.services.apps.port }} - initialDelaySeconds: 10 - periodSeconds: 5 - successThreshold: 1 - failureThreshold: 3 - timeoutSeconds: 3 + {{- toYaml . | nindent 10 }} + {{- end }} + {{- end }} + {{- if .Values.services.apps.readinessProbe }} + {{- with .Values.services.apps.readinessProbe }} readinessProbe: - httpGet: - path: /health - port: {{ .Values.services.apps.port }} - initialDelaySeconds: 5 - periodSeconds: 5 - successThreshold: 1 - failureThreshold: 3 - timeoutSeconds: 3 - + {{- toYaml . | nindent 10 }} + {{- end }} + {{- end }} name: bbapps ports: - containerPort: {{ .Values.services.apps.port }} diff --git a/charts/budibase/templates/proxy-service-deployment.yaml b/charts/budibase/templates/proxy-service-deployment.yaml index e4825ea5d0..53bba6232d 100644 --- a/charts/budibase/templates/proxy-service-deployment.yaml +++ b/charts/budibase/templates/proxy-service-deployment.yaml @@ -40,6 +40,24 @@ spec: - image: budibase/proxy:{{ .Values.globals.appVersion | default .Chart.AppVersion }} imagePullPolicy: Always name: proxy-service + {{- if .Values.services.proxy.startupProbe }} + {{- with .Values.services.proxy.startupProbe }} + startupProbe: + {{- toYaml . | nindent 10 }} + {{- end }} + {{- end }} + {{- if .Values.services.proxy.livenessProbe }} + {{- with .Values.services.proxy.livenessProbe }} + livenessProbe: + {{- toYaml . | nindent 10 }} + {{- end }} + {{- end }} + {{- if .Values.services.proxy.readinessProbe }} + {{- with .Values.services.proxy.readinessProbe }} + readinessProbe: + {{- toYaml . | nindent 10 }} + {{- end }} + {{- end }} ports: - containerPort: {{ .Values.services.proxy.port }} env: diff --git a/charts/budibase/templates/worker-service-deployment.yaml b/charts/budibase/templates/worker-service-deployment.yaml index 5fed80b355..124c667807 100644 --- a/charts/budibase/templates/worker-service-deployment.yaml +++ b/charts/budibase/templates/worker-service-deployment.yaml @@ -190,24 +190,24 @@ spec: {{ end }} image: budibase/worker:{{ .Values.globals.appVersion | default .Chart.AppVersion }} imagePullPolicy: Always + {{- if .Values.services.worker.startupProbe }} + {{- with .Values.services.worker.startupProbe }} + startupProbe: + {{- toYaml . | nindent 10 }} + {{- end }} + {{- end }} + {{- if .Values.services.worker.livenessProbe }} + {{- with .Values.services.worker.livenessProbe }} livenessProbe: - httpGet: - path: /health - port: {{ .Values.services.worker.port }} - initialDelaySeconds: 10 - periodSeconds: 5 - successThreshold: 1 - failureThreshold: 3 - timeoutSeconds: 3 + {{- toYaml . | nindent 10 }} + {{- end }} + {{- end }} + {{- if .Values.services.worker.readinessProbe }} + {{- with .Values.services.worker.readinessProbe }} readinessProbe: - httpGet: - path: /health - port: {{ .Values.services.worker.port }} - initialDelaySeconds: 5 - periodSeconds: 5 - successThreshold: 1 - failureThreshold: 3 - timeoutSeconds: 3 + {{- toYaml . | nindent 10 }} + {{- end }} + {{- end }} name: bbworker ports: - containerPort: {{ .Values.services.worker.port }} diff --git a/charts/budibase/values.yaml b/charts/budibase/values.yaml index 74e4c52654..12e21a8e9c 100644 --- a/charts/budibase/values.yaml +++ b/charts/budibase/values.yaml @@ -119,15 +119,37 @@ services: port: 10000 replicaCount: 1 upstreams: - apps: 'http://app-service.{{ .Release.Namespace }}.svc.{{ .Values.services.dns }}:{{ .Values.services.apps.port }}' - worker: 'http://worker-service.{{ .Release.Namespace }}.svc.{{ .Values.services.dns }}:{{ .Values.services.worker.port }}' - minio: 'http://minio-service.{{ .Release.Namespace }}.svc.{{ .Values.services.dns }}:{{ .Values.services.objectStore.port }}' - couchdb: 'http://{{ .Release.Name }}-svc-couchdb:{{ .Values.services.couchdb.port }}' + apps: "http://app-service.{{ .Release.Namespace }}.svc.{{ .Values.services.dns }}:{{ .Values.services.apps.port }}" + worker: "http://worker-service.{{ .Release.Namespace }}.svc.{{ .Values.services.dns }}:{{ .Values.services.worker.port }}" + minio: "http://minio-service.{{ .Release.Namespace }}.svc.{{ .Values.services.dns }}:{{ .Values.services.objectStore.port }}" + couchdb: "http://{{ .Release.Name }}-svc-couchdb:{{ .Values.services.couchdb.port }}" resources: {} -# annotations: -# co.elastic.logs/module: nginx -# co.elastic.logs/fileset.stdout: access -# co.elastic.logs/fileset.stderr: error + startupProbe: + httpGet: + path: /health + port: 10000 + scheme: HTTP + failureThreshold: 30 + periodSeconds: 3 + readinessProbe: + httpGet: + path: /health + port: 10000 + scheme: HTTP + enabled: true + periodSeconds: 3 + failureThreshold: 1 + livenessProbe: + httpGet: + path: /health + port: 10000 + scheme: HTTP + failureThreshold: 3 + periodSeconds: 5 + # annotations: + # co.elastic.logs/module: nginx + # co.elastic.logs/fileset.stdout: access + # co.elastic.logs/fileset.stderr: error apps: port: 4002 @@ -135,23 +157,67 @@ services: logLevel: info httpLogging: 1 resources: {} -# nodeDebug: "" # set the value of NODE_DEBUG -# annotations: -# co.elastic.logs/multiline.type: pattern -# co.elastic.logs/multiline.pattern: '^[[:space:]]' -# co.elastic.logs/multiline.negate: false -# co.elastic.logs/multiline.match: after + startupProbe: + httpGet: + path: /health + port: 4002 + scheme: HTTP + failureThreshold: 30 + periodSeconds: 3 + readinessProbe: + httpGet: + path: /health + port: 4002 + scheme: HTTP + enabled: true + periodSeconds: 3 + failureThreshold: 1 + livenessProbe: + httpGet: + path: /health + port: 4002 + scheme: HTTP + failureThreshold: 3 + periodSeconds: 5 + # nodeDebug: "" # set the value of NODE_DEBUG + # annotations: + # co.elastic.logs/multiline.type: pattern + # co.elastic.logs/multiline.pattern: '^[[:space:]]' + # co.elastic.logs/multiline.negate: false + # co.elastic.logs/multiline.match: after worker: port: 4003 replicaCount: 1 logLevel: info httpLogging: 1 resources: {} -# annotations: -# co.elastic.logs/multiline.type: pattern -# co.elastic.logs/multiline.pattern: '^[[:space:]]' -# co.elastic.logs/multiline.negate: false -# co.elastic.logs/multiline.match: after + startupProbe: + httpGet: + path: /health + port: 4003 + scheme: HTTP + failureThreshold: 30 + periodSeconds: 3 + readinessProbe: + httpGet: + path: /health + port: 4003 + scheme: HTTP + enabled: true + periodSeconds: 3 + failureThreshold: 1 + livenessProbe: + httpGet: + path: /health + port: 4003 + scheme: HTTP + failureThreshold: 3 + periodSeconds: 5 + # annotations: + # co.elastic.logs/multiline.type: pattern + # co.elastic.logs/multiline.pattern: '^[[:space:]]' + # co.elastic.logs/multiline.negate: false + # co.elastic.logs/multiline.match: after couchdb: enabled: true diff --git a/docs/CONTRIBUTING.md b/docs/CONTRIBUTING.md index ac35929be1..2fb4c36fa8 100644 --- a/docs/CONTRIBUTING.md +++ b/docs/CONTRIBUTING.md @@ -231,18 +231,33 @@ An overview of the CI pipelines can be found [here](../.github/workflows/README. ### Pro -@budibase/pro is the closed source package that supports licensed features in budibase. By default the package will be pulled from NPM and will not normally need to be touched in local development. If you require to update code inside the pro package it can be cloned to the same root level as budibase, e.g. +@budibase/pro is the closed source package that supports licensed features in budibase. By default the package will be pulled from NPM and will not normally need to be touched in local development. If you need to make an update to pro and have access to the repo, then you can update your submodule within the mono-repo by running `git submodule update --init` - from here you can use normal submodule flow to develop a change within pro. + +Once you have updated to use the pro submodule, it will be linked into all of your local dependencies by NX as with all other monorepo packages. If you have been using the NPM version of `@budibase/pro` then you may need to run a `git reset --hard` to fix all of the pro versions back to `0.0.0` to be monorepo aware. + +From here - to develop a change in pro, you can follow the below flow: ``` -. -|_ budibase -|_ budibase-pro +# enter the pro submodule +cd packages/pro +# get the base branch you are working from (same as monorepo) +git fetch +git checkout +# create a branch, named the same as the branch in your monorepo +git checkout -b +... make changes +# commit the changes you've made, with a message for pro +git commit +# within the monorepo, add the pro reference to your branch, commit it with a message like "Update pro ref" +cd ../.. +git add packages/pro +git commit ``` +From here, you will have created a branch in the pro repository and commited the reference to your branch on the monorepo. When you eventually PR this work back into the mainline branch, you will need to first merge your pro PR to the pro mainline, then go into your PR in the monorepo and update the reference again to the new mainline. + Note that only budibase maintainers will be able to access the pro repo. -By default, NX will make sure that dependencies are replaced with local source aware version. This is achieved using the `yarn link` command. To see specifically how dependencies are linked see [scripts/link-dependencies.sh](../scripts/link-dependencies.sh). The same link script is used to link dependencies to account-portal in local dev. - ### Troubleshooting Sometimes, things go wrong. This can be due to incompatible updates on the budibase platform. To clear down your development environment and start again follow **Step 6. Cleanup**, then proceed from **Step 3. Install and Build** in the setup guide above to create a fresh Budibase installation. diff --git a/hosting/.env b/hosting/.env index c2b6d55eef..8a0756c0e3 100644 --- a/hosting/.env +++ b/hosting/.env @@ -28,3 +28,4 @@ BB_ADMIN_USER_PASSWORD= # A path that is watched for plugin bundles. Any bundles found are imported automatically/ PLUGINS_DIR= +ROLLING_LOG_MAX_SIZE= \ No newline at end of file diff --git a/lerna.json b/lerna.json index fc2a7b96b3..5f0bed7e2a 100644 --- a/lerna.json +++ b/lerna.json @@ -1,9 +1,7 @@ { "version": "2.8.31", "npmClient": "yarn", - "packages": [ - "packages/*" - ], + "packages": ["packages/*"], "useNx": true, "command": { "publish": { @@ -19,4 +17,4 @@ "loadEnvFiles": false } } -} \ No newline at end of file +} diff --git a/nx.json b/nx.json index fc0712eed4..8176bae82c 100644 --- a/nx.json +++ b/nx.json @@ -1,20 +1,12 @@ { "tasksRunnerOptions": { "default": { - "runner": "nx/tasks-runners/default", + "runner": "nx-cloud", "options": { - "cacheableOperations": ["build", "test"] + "cacheableOperations": ["build", "test", "check:types"], + "accessToken": "MmM4OGYxNzItMDBlYy00ZmE3LTk4MTYtNmJhYWMyZjBjZTUyfHJlYWQ=" } } }, - "targetDefaults": { - "dev:builder": { - "dependsOn": [ - { - "projects": ["@budibase/string-templates"], - "target": "build" - } - ] - } - } + "targetDefaults": {} } diff --git a/package.json b/package.json index 95edf63a03..d27af2e27d 100644 --- a/package.json +++ b/package.json @@ -3,27 +3,32 @@ "private": true, "devDependencies": { "@esbuild-plugins/tsconfig-paths": "^0.1.2", - "@nx/js": "16.2.1", + "@nx/js": "16.4.3", "@rollup/plugin-json": "^4.0.2", "@typescript-eslint/parser": "5.45.0", - "babel-eslint": "^10.0.3", "esbuild": "^0.18.17", "esbuild-node-externals": "^1.8.0", - "eslint": "^7.28.0", + "eslint": "^8.44.0", "eslint-plugin-cypress": "^2.11.3", - "eslint-plugin-svelte3": "^3.2.0", "husky": "^8.0.3", "js-yaml": "^4.1.0", "kill-port": "^1.6.1", - "lerna": "7.0.2", + "lerna": "7.1.1", "madge": "^6.0.0", "minimist": "^1.2.8", + "nx": "16.4.3", + "nx-cloud": "16.0.5", "prettier": "2.8.8", "prettier-plugin-svelte": "^2.3.0", "rimraf": "^3.0.2", "rollup-plugin-replace": "^2.2.0", "svelte": "^3.38.2", - "typescript": "4.7.3" + "typescript": "4.7.3", + "@babel/core": "^7.22.5", + "@babel/eslint-parser": "^7.22.5", + "@babel/preset-env": "^7.22.5", + "eslint-plugin-svelte": "^2.32.2", + "svelte-eslint-parser": "^0.32.0" }, "scripts": { "preinstall": "node scripts/syncProPackage.js", @@ -31,7 +36,7 @@ "bootstrap": "./scripts/link-dependencies.sh && echo '***BOOTSTRAP ONLY REQUIRED FOR USE WITH ACCOUNT PORTAL***'", "build": "yarn nx run-many -t=build", "build:dev": "lerna run --stream prebuild && yarn nx run-many --target=build --output-style=dynamic --watch --preserveWatchOutput", - "check:types": "lerna run check:types --skip-nx-cache", + "check:types": "lerna run check:types", "backend:bootstrap": "./scripts/scopeBackend.sh && yarn run bootstrap", "backend:build": "./scripts/scopeBackend.sh 'lerna run --stream build'", "build:sdk": "lerna run --stream build:sdk", @@ -41,21 +46,21 @@ "restore": "yarn run clean && yarn run bootstrap && yarn run build", "nuke": "yarn run nuke:packages && yarn run nuke:docker", "nuke:packages": "yarn run restore", - "nuke:docker": "lerna run --stream --parallel dev:stack:nuke", + "nuke:docker": "lerna run --stream dev:stack:nuke", "clean": "lerna clean", "kill-builder": "kill-port 3000", "kill-server": "kill-port 4001 4002", "kill-all": "yarn run kill-builder && yarn run kill-server", - "dev": "yarn run kill-all && lerna run --stream --parallel dev:builder", - "dev:noserver": "yarn run kill-builder && lerna run --stream dev:stack:up && lerna run --stream --parallel dev:builder --ignore @budibase/backend-core --ignore @budibase/server --ignore @budibase/worker", - "dev:server": "yarn run kill-server && lerna run --stream --parallel dev:builder --scope @budibase/worker --scope @budibase/server", - "dev:built": "yarn run kill-all && cd packages/server && yarn dev:stack:up && cd ../../ && lerna run --stream --parallel dev:built", + "dev": "yarn run kill-all && lerna run --stream dev:builder", + "dev:noserver": "yarn run kill-builder && lerna run --stream dev:stack:up && lerna run --stream dev:builder --ignore @budibase/backend-core --ignore @budibase/server --ignore @budibase/worker", + "dev:server": "yarn run kill-server && lerna run --stream dev:builder --scope @budibase/worker --scope @budibase/server", + "dev:built": "yarn run kill-all && cd packages/server && yarn dev:stack:up && cd ../../ && lerna run --stream dev:built", "dev:docker": "yarn build:docker:pre && docker-compose -f hosting/docker-compose.build.yaml -f hosting/docker-compose.dev.yaml --env-file hosting/.env up --build --scale proxy-service=0", "test": "lerna run --stream test --stream", - "lint:eslint": "eslint packages && eslint qa-core", + "lint:eslint": "eslint packages qa-core --max-warnings=0", "lint:prettier": "prettier --check \"packages/**/*.{js,ts,svelte}\" && prettier --write \"examples/**/*.{js,ts,svelte}\" && prettier --check \"qa-core/**/*.{js,ts,svelte}\"", "lint": "yarn run lint:eslint && yarn run lint:prettier", - "lint:fix:eslint": "eslint --fix packages qa-core", + "lint:fix:eslint": "eslint --fix --max-warnings=0 packages qa-core", "lint:fix:prettier": "prettier --write \"packages/**/*.{js,ts,svelte}\" && prettier --write \"examples/**/*.{js,ts,svelte}\" && prettier --write \"qa-core/**/*.{js,ts,svelte}\"", "lint:fix": "yarn run lint:fix:prettier && yarn run lint:fix:eslint", "build:specs": "lerna run --stream specs", @@ -103,5 +108,8 @@ "@budibase/string-templates": "0.0.0", "@budibase/types": "0.0.0" }, + "engines": { + "node": ">=14.0.0 <15.0.0" + }, "dependencies": {} } diff --git a/packages/backend-core/jest.config.ts b/packages/backend-core/jest.config.ts index 8d64b24a2f..3f1065ead2 100644 --- a/packages/backend-core/jest.config.ts +++ b/packages/backend-core/jest.config.ts @@ -1,8 +1,6 @@ import { Config } from "@jest/types" -const preset = require("ts-jest/jest-preset") const baseConfig: Config.InitialProjectOptions = { - ...preset, preset: "@trendyol/jest-testcontainers", setupFiles: ["./tests/jestEnv.ts"], setupFilesAfterEnv: ["./tests/jestSetup.ts"], @@ -11,6 +9,7 @@ const baseConfig: Config.InitialProjectOptions = { }, moduleNameMapper: { "@budibase/types": "/../types/src", + "@budibase/shared-core": ["/../shared-core/src"], }, } diff --git a/packages/backend-core/package.json b/packages/backend-core/package.json index 4a1ed5c373..4631b090fe 100644 --- a/packages/backend-core/package.json +++ b/packages/backend-core/package.json @@ -16,14 +16,15 @@ "prepack": "cp package.json dist", "build": "tsc -p tsconfig.build.json", "build:dev": "yarn prebuild && tsc --build --watch --preserveWatchOutput", + "check:types": "tsc -p tsconfig.json --noEmit --paths null", "test": "bash scripts/test.sh", "test:watch": "jest --watchAll" }, "dependencies": { "@budibase/nano": "10.1.2", "@budibase/pouchdb-replication-stream": "1.2.10", + "@budibase/shared-core": "0.0.0", "@budibase/types": "0.0.0", - "@shopify/jest-koa-mocks": "5.0.1", "@techpass/passport-openidconnect": "0.3.2", "aws-cloudfront-sign": "2.2.0", "aws-sdk": "2.1030.0", @@ -51,18 +52,20 @@ "pouchdb": "7.3.0", "pouchdb-find": "7.2.2", "redlock": "4.2.0", + "rotating-file-stream": "3.1.0", "sanitize-s3-objectkey": "0.0.1", "semver": "7.3.7", "tar-fs": "2.1.1", "uuid": "8.3.2" }, "devDependencies": { - "@jest/test-sequencer": "29.5.0", - "@swc/core": "^1.3.25", - "@swc/jest": "^0.2.24", + "@jest/test-sequencer": "29.6.2", + "@shopify/jest-koa-mocks": "5.1.1", + "@swc/core": "1.3.71", + "@swc/jest": "0.2.27", "@trendyol/jest-testcontainers": "^2.1.1", "@types/chance": "1.1.3", - "@types/jest": "29.5.0", + "@types/jest": "29.5.3", "@types/koa": "2.13.4", "@types/lodash": "4.14.180", "@types/node": "14.18.20", @@ -74,32 +77,16 @@ "@types/uuid": "8.3.4", "chance": "1.1.8", "ioredis-mock": "8.7.0", - "jest": "29.5.0", - "jest-environment-node": "29.5.0", - "jest-serial-runner": "^1.2.1", + "jest": "29.6.2", + "jest-environment-node": "29.6.2", + "jest-serial-runner": "1.2.1", "koa": "2.13.4", "nodemon": "2.0.16", "pino-pretty": "10.0.0", "pouchdb-adapter-memory": "7.2.2", "timekeeper": "2.2.0", - "ts-jest": "29.0.5", "ts-node": "10.8.1", "tsconfig-paths": "4.0.0", "typescript": "4.7.3" - }, - "nx": { - "targets": { - "build": { - "dependsOn": [ - { - "projects": [ - "@budibase/types" - ], - "target": "build" - } - ] - } - } - }, - "gitHead": "d1836a898cab3f8ab80ee6d8f42be1a9eed7dcdc" + } } diff --git a/packages/backend-core/scripts/test.sh b/packages/backend-core/scripts/test.sh index 3d8240e65a..7d19ec96cc 100644 --- a/packages/backend-core/scripts/test.sh +++ b/packages/backend-core/scripts/test.sh @@ -8,6 +8,6 @@ then jest --coverage --runInBand --forceExit else # --maxWorkers performs better in development - echo "jest --coverage --forceExit" - jest --coverage --forceExit + echo "jest --coverage --detectOpenHandles" + jest --coverage --detectOpenHandles fi \ No newline at end of file diff --git a/packages/backend-core/src/auth/auth.ts b/packages/backend-core/src/auth/auth.ts index fb2fd2cf51..0100a2d0e2 100644 --- a/packages/backend-core/src/auth/auth.ts +++ b/packages/backend-core/src/auth/auth.ts @@ -159,7 +159,7 @@ export async function updateUserOAuth(userId: string, oAuthConfig: any) { try { const db = getGlobalDB() - const dbUser = await db.get(userId) + const dbUser = await db.get(userId) //Do not overwrite the refresh token if a valid one is not provided. if (typeof details.refreshToken !== "string") { diff --git a/packages/backend-core/src/cache/appMetadata.ts b/packages/backend-core/src/cache/appMetadata.ts index 5b66c356d3..0c320ec776 100644 --- a/packages/backend-core/src/cache/appMetadata.ts +++ b/packages/backend-core/src/cache/appMetadata.ts @@ -2,9 +2,14 @@ import { getAppClient } from "../redis/init" import { doWithDB, DocumentType } from "../db" import { Database, App } from "@budibase/types" -const AppState = { - INVALID: "invalid", +export enum AppState { + INVALID = "invalid", } + +export interface DeletedApp { + state: AppState +} + const EXPIRY_SECONDS = 3600 /** @@ -31,7 +36,7 @@ function isInvalid(metadata?: { state: string }) { * @param {string} appId the id of the app to get metadata from. * @returns {object} the app metadata. */ -export async function getAppMetadata(appId: string) { +export async function getAppMetadata(appId: string): Promise { const client = await getAppClient() // try cache let metadata = await client.get(appId) @@ -61,11 +66,8 @@ export async function getAppMetadata(appId: string) { } await client.store(appId, metadata, expiry) } - // we've stored in the cache an object to tell us that it is currently invalid - if (isInvalid(metadata)) { - throw { status: 404, message: "No app metadata found" } - } - return metadata as App + + return metadata } /** diff --git a/packages/backend-core/src/cache/tests/writethrough.spec.ts b/packages/backend-core/src/cache/tests/writethrough.spec.ts index 92b073ed64..97d3ece7a6 100644 --- a/packages/backend-core/src/cache/tests/writethrough.spec.ts +++ b/packages/backend-core/src/cache/tests/writethrough.spec.ts @@ -36,7 +36,7 @@ describe("writethrough", () => { _id: docId, value: 1, }) - const output = await db.get(response.id) + const output = await db.get(response.id) current = output expect(output.value).toBe(1) }) @@ -45,7 +45,7 @@ describe("writethrough", () => { it("second put shouldn't update DB", async () => { await config.doInTenant(async () => { const response = await writethrough.put({ ...current, value: 2 }) - const output = await db.get(response.id) + const output = await db.get(response.id) expect(current._rev).toBe(output._rev) expect(output.value).toBe(1) }) @@ -55,7 +55,7 @@ describe("writethrough", () => { await config.doInTenant(async () => { tk.freeze(Date.now() + DELAY + 1) const response = await writethrough.put({ ...current, value: 3 }) - const output = await db.get(response.id) + const output = await db.get(response.id) expect(response.rev).not.toBe(current._rev) expect(output.value).toBe(3) @@ -79,7 +79,7 @@ describe("writethrough", () => { expect.arrayContaining([current._rev, current._rev, newRev]) ) - const output = await db.get(current._id) + const output = await db.get(current._id) expect(output.value).toBe(4) expect(output._rev).toBe(newRev) @@ -107,7 +107,7 @@ describe("writethrough", () => { }) expect(res.ok).toBe(true) - const output = await db.get(id) + const output = await db.get(id) expect(output.value).toBe(3) expect(output._rev).toBe(res.rev) }) @@ -130,8 +130,8 @@ describe("writethrough", () => { const resp2 = await writethrough2.put({ _id: "db1", value: "second" }) expect(resp1.rev).toBeDefined() expect(resp2.rev).toBeDefined() - expect((await db.get("db1")).value).toBe("first") - expect((await db2.get("db1")).value).toBe("second") + expect((await db.get("db1")).value).toBe("first") + expect((await db2.get("db1")).value).toBe("second") }) }) }) diff --git a/packages/backend-core/src/cache/user.ts b/packages/backend-core/src/cache/user.ts index b514c3af9b..8281bfca62 100644 --- a/packages/backend-core/src/cache/user.ts +++ b/packages/backend-core/src/cache/user.ts @@ -12,7 +12,7 @@ const EXPIRY_SECONDS = 3600 */ async function populateFromDB(userId: string, tenantId: string) { const db = tenancy.getTenantDB(tenantId) - const user = await db.get(userId) + const user = await db.get(userId) user.budibaseAccess = true if (!env.SELF_HOSTED && !env.DISABLE_ACCOUNT_PORTAL) { const account = await accounts.getAccount(user.email) diff --git a/packages/backend-core/src/constants/db.ts b/packages/backend-core/src/constants/db.ts index be49b9f261..83f8298f54 100644 --- a/packages/backend-core/src/constants/db.ts +++ b/packages/backend-core/src/constants/db.ts @@ -1,5 +1,5 @@ -export const SEPARATOR = "_" -export const UNICODE_MAX = "\ufff0" +import { prefixed, DocumentType } from "@budibase/types" +export { SEPARATOR, UNICODE_MAX, DocumentType } from "@budibase/types" /** * Can be used to create a few different forms of querying a view. @@ -14,8 +14,6 @@ export enum ViewName { USER_BY_APP = "by_app", USER_BY_EMAIL = "by_email2", BY_API_KEY = "by_api_key", - /** @deprecated - could be deleted */ - USER_BY_BUILDERS = "by_builders", LINK = "by_link", ROUTING = "screen_routes", AUTOMATION_LOGS = "automation_logs", @@ -36,42 +34,6 @@ export enum InternalTable { USER_METADATA = "ta_users", } -export enum DocumentType { - USER = "us", - GROUP = "gr", - WORKSPACE = "workspace", - CONFIG = "config", - TEMPLATE = "template", - APP = "app", - DEV = "dev", - APP_DEV = "app_dev", - APP_METADATA = "app_metadata", - ROLE = "role", - MIGRATIONS = "migrations", - DEV_INFO = "devinfo", - AUTOMATION_LOG = "log_au", - ACCOUNT_METADATA = "acc_metadata", - PLUGIN = "plg", - DATASOURCE = "datasource", - DATASOURCE_PLUS = "datasource_plus", - APP_BACKUP = "backup", - TABLE = "ta", - ROW = "ro", - AUTOMATION = "au", - LINK = "li", - WEBHOOK = "wh", - INSTANCE = "inst", - LAYOUT = "layout", - SCREEN = "screen", - QUERY = "query", - DEPLOYMENTS = "deployments", - METADATA = "metadata", - MEM_VIEW = "view", - USER_FLAG = "flag", - AUTOMATION_METADATA = "meta_au", - AUDIT_LOG = "al", -} - export const StaticDatabases = { GLOBAL: { name: "global-db", @@ -95,7 +57,7 @@ export const StaticDatabases = { }, } -export const APP_PREFIX = DocumentType.APP + SEPARATOR -export const APP_DEV = DocumentType.APP_DEV + SEPARATOR +export const APP_PREFIX = prefixed(DocumentType.APP) +export const APP_DEV = prefixed(DocumentType.APP_DEV) export const APP_DEV_PREFIX = APP_DEV export const BUDIBASE_DATASOURCE_TYPE = "budibase" diff --git a/packages/backend-core/src/constants/misc.ts b/packages/backend-core/src/constants/misc.ts index ba2533cf4a..0c68798164 100644 --- a/packages/backend-core/src/constants/misc.ts +++ b/packages/backend-core/src/constants/misc.ts @@ -20,6 +20,8 @@ export enum Header { TYPE = "x-budibase-type", PREVIEW_ROLE = "x-budibase-role", TENANT_ID = "x-budibase-tenant-id", + VERIFICATION_CODE = "x-budibase-verification-code", + RETURN_VERIFICATION_CODE = "x-budibase-return-verification-code", TOKEN = "x-budibase-token", CSRF_TOKEN = "x-csrf-token", CORRELATION_ID = "x-budibase-correlation-id", diff --git a/packages/backend-core/src/db/constants.ts b/packages/backend-core/src/db/constants.ts new file mode 100644 index 0000000000..aea485e3e3 --- /dev/null +++ b/packages/backend-core/src/db/constants.ts @@ -0,0 +1,10 @@ +export const CONSTANT_INTERNAL_ROW_COLS = [ + "_id", + "_rev", + "type", + "createdAt", + "updatedAt", + "tableId", +] as const + +export const CONSTANT_EXTERNAL_ROW_COLS = ["_id", "_rev", "tableId"] as const diff --git a/packages/backend-core/src/db/couch/index.ts b/packages/backend-core/src/db/couch/index.ts index c731d20d6c..932efed3f7 100644 --- a/packages/backend-core/src/db/couch/index.ts +++ b/packages/backend-core/src/db/couch/index.ts @@ -2,3 +2,4 @@ export * from "./connections" export * from "./DatabaseImpl" export * from "./utils" export { init, getPouch, getPouchDB, closePouchDB } from "./pouchDB" +export * from "../constants" diff --git a/packages/backend-core/src/db/searchIndexes/searchIndexes.ts b/packages/backend-core/src/db/searchIndexes/searchIndexes.ts index f03259b47f..b953e3516e 100644 --- a/packages/backend-core/src/db/searchIndexes/searchIndexes.ts +++ b/packages/backend-core/src/db/searchIndexes/searchIndexes.ts @@ -5,7 +5,7 @@ export async function createUserIndex() { const db = getGlobalDB() let designDoc try { - designDoc = await db.get("_design/database") + designDoc = await db.get("_design/database") } catch (err: any) { if (err.status === 404) { designDoc = { _id: "_design/database" } diff --git a/packages/backend-core/src/db/utils.ts b/packages/backend-core/src/db/utils.ts index 6034296996..4ebf8392b5 100644 --- a/packages/backend-core/src/db/utils.ts +++ b/packages/backend-core/src/db/utils.ts @@ -2,7 +2,7 @@ import env from "../environment" import { DEFAULT_TENANT_ID, SEPARATOR, DocumentType } from "../constants" import { getTenantId, getGlobalDBName } from "../context" import { doWithDB, directCouchAllDbs } from "./db" -import { getAppMetadata } from "../cache/appMetadata" +import { AppState, DeletedApp, getAppMetadata } from "../cache/appMetadata" import { isDevApp, isDevAppID, getProdAppID } from "../docIds/conversions" import { App, Database } from "@budibase/types" import { getStartEndKeyURL } from "../docIds" @@ -101,7 +101,9 @@ export async function getAllApps({ const response = await Promise.allSettled(appPromises) const apps = response .filter( - (result: any) => result.status === "fulfilled" && result.value != null + (result: any) => + result.status === "fulfilled" && + result.value?.state !== AppState.INVALID ) .map(({ value }: any) => value) if (!all) { @@ -126,7 +128,11 @@ export async function getAppsByIDs(appIds: string[]) { ) // have to list the apps which exist, some may have been deleted return settled - .filter(promise => promise.status === "fulfilled") + .filter( + promise => + promise.status === "fulfilled" && + (promise.value as DeletedApp).state !== AppState.INVALID + ) .map(promise => (promise as PromiseFulfilledResult).value) } diff --git a/packages/backend-core/src/db/views.ts b/packages/backend-core/src/db/views.ts index fddb1ab34b..7f5ef29a0a 100644 --- a/packages/backend-core/src/db/views.ts +++ b/packages/backend-core/src/db/views.ts @@ -105,16 +105,6 @@ export const createApiKeyView = async () => { await createView(db, viewJs, ViewName.BY_API_KEY) } -export const createUserBuildersView = async () => { - const db = getGlobalDB() - const viewJs = `function(doc) { - if (doc.builder && doc.builder.global === true) { - emit(doc._id, doc._id) - } - }` - await createView(db, viewJs, ViewName.USER_BY_BUILDERS) -} - export interface QueryViewOptions { arrayResponse?: boolean } @@ -223,7 +213,6 @@ export const queryPlatformView = async ( const CreateFuncByName: any = { [ViewName.USER_BY_EMAIL]: createNewUserEmailView, [ViewName.BY_API_KEY]: createApiKeyView, - [ViewName.USER_BY_BUILDERS]: createUserBuildersView, [ViewName.USER_BY_APP]: createUserAppView, } diff --git a/packages/backend-core/src/environment.ts b/packages/backend-core/src/environment.ts index c0785ef419..05fcbffd46 100644 --- a/packages/backend-core/src/environment.ts +++ b/packages/backend-core/src/environment.ts @@ -1,4 +1,5 @@ import { existsSync, readFileSync } from "fs" +import { ServiceType } from "@budibase/types" function isTest() { return isCypress() || isJest() @@ -47,7 +48,10 @@ function httpLogging() { return process.env.HTTP_LOGGING } -function findVersion() { +function getPackageJsonFields(): { + VERSION: string + SERVICE_NAME: string +} { function findFileInAncestors( fileName: string, currentDir: string @@ -69,17 +73,31 @@ function findVersion() { try { const packageJsonFile = findFileInAncestors("package.json", process.cwd()) const content = readFileSync(packageJsonFile!, "utf-8") - return JSON.parse(content).version + const parsedContent = JSON.parse(content) + return { + VERSION: parsedContent.version, + SERVICE_NAME: parsedContent.name, + } } catch { // throwing an error here is confusing/causes backend-core to be hard to import - return undefined + return { VERSION: "", SERVICE_NAME: "" } } } +function isWorker() { + return environment.SERVICE_TYPE === ServiceType.WORKER +} + +function isApps() { + return environment.SERVICE_TYPE === ServiceType.APPS +} + const environment = { isTest, isJest, isDev, + isWorker, + isApps, isProd: () => { return !isDev() }, @@ -146,6 +164,7 @@ const environment = { SMTP_FROM_ADDRESS: process.env.SMTP_FROM_ADDRESS, DISABLE_JWT_WARNING: process.env.DISABLE_JWT_WARNING, BLACKLIST_IPS: process.env.BLACKLIST_IPS, + SERVICE_TYPE: "unknown", /** * Enable to allow an admin user to login using a password. * This can be useful to prevent lockout when configuring SSO. @@ -154,7 +173,7 @@ const environment = { ENABLE_SSO_MAINTENANCE_MODE: selfHosted ? process.env.ENABLE_SSO_MAINTENANCE_MODE : false, - VERSION: findVersion(), + ...getPackageJsonFields(), DISABLE_PINO_LOGGER: process.env.DISABLE_PINO_LOGGER, OFFLINE_MODE: process.env.OFFLINE_MODE, _set(key: any, value: any) { @@ -162,6 +181,7 @@ const environment = { // @ts-ignore environment[key] = value }, + ROLLING_LOG_MAX_SIZE: process.env.ROLLING_LOG_MAX_SIZE || "10M", } // clean up any environment variable edge cases diff --git a/packages/backend-core/src/events/identification.ts b/packages/backend-core/src/events/identification.ts index 948d3b692b..c7bc1c817b 100644 --- a/packages/backend-core/src/events/identification.ts +++ b/packages/backend-core/src/events/identification.ts @@ -21,6 +21,7 @@ import { processors } from "./processors" import { newid } from "../utils" import * as installation from "../installation" import * as configs from "../configs" +import * as users from "../users" import { withCache, TTL, CacheKey } from "../cache/generic" /** @@ -164,8 +165,8 @@ const identifyUser = async ( const id = user._id as string const tenantId = await getEventTenantId(user.tenantId) const type = IdentityType.USER - let builder = user.builder?.global || false - let admin = user.admin?.global || false + let builder = users.hasBuilderPermissions(user) + let admin = users.hasAdminPermissions(user) let providerType if (isSSOUser(user)) { providerType = user.providerType diff --git a/packages/backend-core/src/logging/index.ts b/packages/backend-core/src/logging/index.ts index b87062c478..0824fa681b 100644 --- a/packages/backend-core/src/logging/index.ts +++ b/packages/backend-core/src/logging/index.ts @@ -1,6 +1,4 @@ export * as correlation from "./correlation/correlation" export { logger } from "./pino/logger" export * from "./alerts" - -// turn off or on context logging i.e. tenantId, appId etc -export let LOG_CONTEXT = true +export * as system from "./system" diff --git a/packages/backend-core/src/logging/pino/logger.ts b/packages/backend-core/src/logging/pino/logger.ts index c96bc83e04..7c444a3a59 100644 --- a/packages/backend-core/src/logging/pino/logger.ts +++ b/packages/backend-core/src/logging/pino/logger.ts @@ -1,37 +1,60 @@ -import env from "../../environment" import pino, { LoggerOptions } from "pino" +import pinoPretty from "pino-pretty" + +import { IdentityType } from "@budibase/types" +import env from "../../environment" import * as context from "../../context" import * as correlation from "../correlation" -import { IdentityType } from "@budibase/types" -import { LOG_CONTEXT } from "../index" + +import { localFileDestination } from "../system" // LOGGER let pinoInstance: pino.Logger | undefined if (!env.DISABLE_PINO_LOGGER) { + const level = env.LOG_LEVEL const pinoOptions: LoggerOptions = { - level: env.LOG_LEVEL, + level, formatters: { - level: label => { - return { level: label.toUpperCase() } + level: level => { + return { level: level.toUpperCase() } }, bindings: () => { - return {} + if (env.SELF_HOSTED) { + // "service" is being injected in datadog using the pod names, + // so we should leave it blank to allow the default behaviour if it's not running self-hosted + return { + service: env.SERVICE_NAME, + } + } else { + return {} + } }, }, timestamp: () => `,"timestamp":"${new Date(Date.now()).toISOString()}"`, } - if (env.isDev()) { - pinoOptions.transport = { - target: "pino-pretty", - options: { - singleLine: true, - }, - } + const destinations: pino.StreamEntry[] = [] + + destinations.push( + env.isDev() + ? { + stream: pinoPretty({ singleLine: true }), + level: level as pino.Level, + } + : { stream: process.stdout, level: level as pino.Level } + ) + + if (env.SELF_HOSTED) { + destinations.push({ + stream: localFileDestination(), + level: level as pino.Level, + }) } - pinoInstance = pino(pinoOptions) + pinoInstance = destinations.length + ? pino(pinoOptions, pino.multistream(destinations)) + : pino(pinoOptions) // CONSOLE OVERRIDES @@ -83,15 +106,13 @@ if (!env.DISABLE_PINO_LOGGER) { let contextObject = {} - if (LOG_CONTEXT) { - contextObject = { - tenantId: getTenantId(), - appId: getAppId(), - automationId: getAutomationId(), - identityId: identity?._id, - identityType: identity?.type, - correlationId: correlation.getId(), - } + contextObject = { + tenantId: getTenantId(), + appId: getAppId(), + automationId: getAutomationId(), + identityId: identity?._id, + identityType: identity?.type, + correlationId: correlation.getId(), } const mergingObject: any = { diff --git a/packages/backend-core/src/logging/system.ts b/packages/backend-core/src/logging/system.ts new file mode 100644 index 0000000000..d918c6efd6 --- /dev/null +++ b/packages/backend-core/src/logging/system.ts @@ -0,0 +1,81 @@ +import fs from "fs" +import path from "path" +import * as rfs from "rotating-file-stream" + +import env from "../environment" +import { budibaseTempDir } from "../objectStore" + +const logsFileName = `budibase.log` +const budibaseLogsHistoryFileName = "budibase-logs-history.txt" + +const logsPath = path.join(budibaseTempDir(), "systemlogs") + +function getFullPath(fileName: string) { + return path.join(logsPath, fileName) +} + +export function getSingleFileMaxSizeInfo(totalMaxSize: string) { + const regex = /(\d+)([A-Za-z])/ + const match = totalMaxSize?.match(regex) + if (!match) { + console.warn(`totalMaxSize does not have a valid value`, { + totalMaxSize, + }) + return undefined + } + + const size = +match[1] + const unit = match[2] + if (size === 1) { + switch (unit) { + case "B": + return { size: `${size}B`, totalHistoryFiles: 1 } + case "K": + return { size: `${(size * 1000) / 2}B`, totalHistoryFiles: 1 } + case "M": + return { size: `${(size * 1000) / 2}K`, totalHistoryFiles: 1 } + case "G": + return { size: `${(size * 1000) / 2}M`, totalHistoryFiles: 1 } + default: + return undefined + } + } + + if (size % 2 === 0) { + return { size: `${size / 2}${unit}`, totalHistoryFiles: 1 } + } + + return { size: `1${unit}`, totalHistoryFiles: size - 1 } +} + +export function localFileDestination() { + const fileInfo = getSingleFileMaxSizeInfo(env.ROLLING_LOG_MAX_SIZE) + const outFile = rfs.createStream(logsFileName, { + // As we have a rolling size, we want to half the max size + size: fileInfo?.size, + path: logsPath, + maxFiles: fileInfo?.totalHistoryFiles || 1, + immutable: true, + history: budibaseLogsHistoryFileName, + initialRotation: false, + }) + + return outFile +} + +export function getLogReadStream() { + const streams = [] + const historyFile = getFullPath(budibaseLogsHistoryFileName) + if (fs.existsSync(historyFile)) { + const fileContent = fs.readFileSync(historyFile, "utf-8") + const historyFiles = fileContent.split("\n") + for (const historyFile of historyFiles.filter(x => x)) { + streams.push(fs.readFileSync(historyFile)) + } + } + + streams.push(fs.readFileSync(getFullPath(logsFileName))) + + const combinedContent = Buffer.concat(streams) + return combinedContent +} diff --git a/packages/backend-core/src/logging/tests/system.spec.ts b/packages/backend-core/src/logging/tests/system.spec.ts new file mode 100644 index 0000000000..b84d8e8456 --- /dev/null +++ b/packages/backend-core/src/logging/tests/system.spec.ts @@ -0,0 +1,61 @@ +import { getSingleFileMaxSizeInfo } from "../system" + +describe("system", () => { + describe("getSingleFileMaxSizeInfo", () => { + it.each([ + ["100B", "50B"], + ["200K", "100K"], + ["20M", "10M"], + ["4G", "2G"], + ])( + "Halving even number (%s) returns halved size and 1 history file (%s)", + (totalValue, expectedMaxSize) => { + const result = getSingleFileMaxSizeInfo(totalValue) + expect(result).toEqual({ + size: expectedMaxSize, + totalHistoryFiles: 1, + }) + } + ) + + it.each([ + ["5B", "1B", 4], + ["17K", "1K", 16], + ["21M", "1M", 20], + ["3G", "1G", 2], + ])( + "Halving an odd number (%s) returns as many files as size (-1) (%s)", + (totalValue, expectedMaxSize, totalHistoryFiles) => { + const result = getSingleFileMaxSizeInfo(totalValue) + expect(result).toEqual({ + size: expectedMaxSize, + totalHistoryFiles, + }) + } + ) + + it.each([ + ["1B", "1B"], + ["1K", "500B"], + ["1M", "500K"], + ["1G", "500M"], + ])( + "Halving '%s' returns halved unit (%s)", + (totalValue, expectedMaxSize) => { + const result = getSingleFileMaxSizeInfo(totalValue) + expect(result).toEqual({ + size: expectedMaxSize, + totalHistoryFiles: 1, + }) + } + ) + + it.each([[undefined], [""], ["50"], ["wrongvalue"]])( + "Halving wrongly formatted value ('%s') returns undefined", + totalValue => { + const result = getSingleFileMaxSizeInfo(totalValue!) + expect(result).toBeUndefined() + } + ) + }) +}) diff --git a/packages/backend-core/src/middleware/adminOnly.ts b/packages/backend-core/src/middleware/adminOnly.ts index dbe1e3a501..6b2ee87c01 100644 --- a/packages/backend-core/src/middleware/adminOnly.ts +++ b/packages/backend-core/src/middleware/adminOnly.ts @@ -1,10 +1,8 @@ -import { BBContext } from "@budibase/types" +import { UserCtx } from "@budibase/types" +import { isAdmin } from "../users" -export default async (ctx: BBContext, next: any) => { - if ( - !ctx.internal && - (!ctx.user || !ctx.user.admin || !ctx.user.admin.global) - ) { +export default async (ctx: UserCtx, next: any) => { + if (!ctx.internal && !isAdmin(ctx.user)) { ctx.throw(403, "Admin user only endpoint.") } return next() diff --git a/packages/backend-core/src/middleware/builderOnly.ts b/packages/backend-core/src/middleware/builderOnly.ts index a00fd63a22..8c1c54a44c 100644 --- a/packages/backend-core/src/middleware/builderOnly.ts +++ b/packages/backend-core/src/middleware/builderOnly.ts @@ -1,10 +1,19 @@ -import { BBContext } from "@budibase/types" +import { UserCtx } from "@budibase/types" +import { isBuilder, hasBuilderPermissions } from "../users" +import { getAppId } from "../context" +import env from "../environment" -export default async (ctx: BBContext, next: any) => { - if ( - !ctx.internal && - (!ctx.user || !ctx.user.builder || !ctx.user.builder.global) - ) { +export default async (ctx: UserCtx, next: any) => { + const appId = getAppId() + const builderFn = env.isWorker() + ? hasBuilderPermissions + : env.isApps() + ? isBuilder + : undefined + if (!builderFn) { + throw new Error("Service name unknown - middleware inactive.") + } + if (!ctx.internal && !builderFn(ctx.user, appId)) { ctx.throw(403, "Builder user only endpoint.") } return next() diff --git a/packages/backend-core/src/middleware/builderOrAdmin.ts b/packages/backend-core/src/middleware/builderOrAdmin.ts index 26bb3a1bda..c03e856233 100644 --- a/packages/backend-core/src/middleware/builderOrAdmin.ts +++ b/packages/backend-core/src/middleware/builderOrAdmin.ts @@ -1,12 +1,20 @@ -import { BBContext } from "@budibase/types" +import { UserCtx } from "@budibase/types" +import { isBuilder, isAdmin, hasBuilderPermissions } from "../users" +import { getAppId } from "../context" +import env from "../environment" -export default async (ctx: BBContext, next: any) => { - if ( - !ctx.internal && - (!ctx.user || !ctx.user.builder || !ctx.user.builder.global) && - (!ctx.user || !ctx.user.admin || !ctx.user.admin.global) - ) { - ctx.throw(403, "Builder user only endpoint.") +export default async (ctx: UserCtx, next: any) => { + const appId = getAppId() + const builderFn = env.isWorker() + ? hasBuilderPermissions + : env.isApps() + ? isBuilder + : undefined + if (!builderFn) { + throw new Error("Service name unknown - middleware inactive.") + } + if (!ctx.internal && !builderFn(ctx.user, appId) && !isAdmin(ctx.user)) { + ctx.throw(403, "Admin/Builder user only endpoint.") } return next() } diff --git a/packages/backend-core/src/middleware/tests/builder.spec.ts b/packages/backend-core/src/middleware/tests/builder.spec.ts new file mode 100644 index 0000000000..d350eff4f6 --- /dev/null +++ b/packages/backend-core/src/middleware/tests/builder.spec.ts @@ -0,0 +1,180 @@ +import adminOnly from "../adminOnly" +import builderOnly from "../builderOnly" +import builderOrAdmin from "../builderOrAdmin" +import { structures } from "../../../tests" +import { ContextUser, ServiceType } from "@budibase/types" +import { doInAppContext } from "../../context" +import env from "../../environment" +env._set("SERVICE_TYPE", ServiceType.APPS) + +const appId = "app_aaa" +const basicUser = structures.users.user() +const adminUser = structures.users.adminUser() +const adminOnlyUser = structures.users.adminOnlyUser() +const builderUser = structures.users.builderUser() +const appBuilderUser = structures.users.appBuilderUser(appId) + +function buildUserCtx(user: ContextUser) { + return { + internal: false, + user, + throw: jest.fn(), + } as any +} + +function passed(throwFn: jest.Func, nextFn: jest.Func) { + expect(throwFn).not.toBeCalled() + expect(nextFn).toBeCalled() +} + +function threw(throwFn: jest.Func) { + // cant check next, the throw function doesn't actually throw - so it still continues + expect(throwFn).toBeCalled() +} + +describe("adminOnly middleware", () => { + it("should allow admin user", () => { + const ctx = buildUserCtx(adminUser), + next = jest.fn() + adminOnly(ctx, next) + passed(ctx.throw, next) + }) + + it("should not allow basic user", () => { + const ctx = buildUserCtx(basicUser), + next = jest.fn() + adminOnly(ctx, next) + threw(ctx.throw) + }) + + it("should not allow builder user", () => { + const ctx = buildUserCtx(builderUser), + next = jest.fn() + adminOnly(ctx, next) + threw(ctx.throw) + }) +}) + +describe("builderOnly middleware", () => { + it("should allow builder user", () => { + const ctx = buildUserCtx(builderUser), + next = jest.fn() + builderOnly(ctx, next) + passed(ctx.throw, next) + }) + + it("should allow app builder user", () => { + const ctx = buildUserCtx(appBuilderUser), + next = jest.fn() + doInAppContext(appId, () => { + builderOnly(ctx, next) + }) + passed(ctx.throw, next) + }) + + it("should allow admin and builder user", () => { + const ctx = buildUserCtx(adminUser), + next = jest.fn() + builderOnly(ctx, next) + passed(ctx.throw, next) + }) + + it("should not allow admin user", () => { + const ctx = buildUserCtx(adminOnlyUser), + next = jest.fn() + builderOnly(ctx, next) + threw(ctx.throw) + }) + + it("should not allow app builder user to different app", () => { + const ctx = buildUserCtx(appBuilderUser), + next = jest.fn() + doInAppContext("app_bbb", () => { + builderOnly(ctx, next) + }) + threw(ctx.throw) + }) + + it("should not allow basic user", () => { + const ctx = buildUserCtx(basicUser), + next = jest.fn() + builderOnly(ctx, next) + threw(ctx.throw) + }) +}) + +describe("builderOrAdmin middleware", () => { + it("should allow builder user", () => { + const ctx = buildUserCtx(builderUser), + next = jest.fn() + builderOrAdmin(ctx, next) + passed(ctx.throw, next) + }) + + it("should allow builder and admin user", () => { + const ctx = buildUserCtx(adminUser), + next = jest.fn() + builderOrAdmin(ctx, next) + passed(ctx.throw, next) + }) + + it("should allow admin user", () => { + const ctx = buildUserCtx(adminOnlyUser), + next = jest.fn() + builderOrAdmin(ctx, next) + passed(ctx.throw, next) + }) + + it("should allow app builder user", () => { + const ctx = buildUserCtx(appBuilderUser), + next = jest.fn() + doInAppContext(appId, () => { + builderOrAdmin(ctx, next) + }) + passed(ctx.throw, next) + }) + + it("should not allow basic user", () => { + const ctx = buildUserCtx(basicUser), + next = jest.fn() + builderOrAdmin(ctx, next) + threw(ctx.throw) + }) +}) + +describe("check service difference", () => { + it("should not allow without app ID in apps", () => { + env._set("SERVICE_TYPE", ServiceType.APPS) + const appId = "app_a" + const ctx = buildUserCtx({ + ...basicUser, + builder: { + apps: [appId], + }, + }) + const next = jest.fn() + doInAppContext(appId, () => { + builderOnly(ctx, next) + }) + passed(ctx.throw, next) + doInAppContext("app_b", () => { + builderOnly(ctx, next) + }) + threw(ctx.throw) + }) + + it("should allow without app ID in worker", () => { + env._set("SERVICE_TYPE", ServiceType.WORKER) + const ctx = buildUserCtx({ + ...basicUser, + builder: { + apps: ["app_a"], + }, + }) + const next = jest.fn() + doInAppContext("app_b", () => { + builderOnly(ctx, next) + }) + passed(ctx.throw, next) + }) +}) diff --git a/packages/backend-core/src/security/permissions.ts b/packages/backend-core/src/security/permissions.ts index 6cacc12dd6..70dae57ae6 100644 --- a/packages/backend-core/src/security/permissions.ts +++ b/packages/backend-core/src/security/permissions.ts @@ -1,29 +1,12 @@ -const { flatten } = require("lodash") -const { cloneDeep } = require("lodash/fp") +import { PermissionType, PermissionLevel } from "@budibase/types" +export { PermissionType, PermissionLevel } from "@budibase/types" +import flatten from "lodash/flatten" +import cloneDeep from "lodash/fp/cloneDeep" export type RoleHierarchy = { permissionId: string }[] -export enum PermissionLevel { - READ = "read", - WRITE = "write", - EXECUTE = "execute", - ADMIN = "admin", -} - -// these are the global types, that govern the underlying default behaviour -export enum PermissionType { - APP = "app", - TABLE = "table", - USER = "user", - AUTOMATION = "automation", - WEBHOOK = "webhook", - BUILDER = "builder", - VIEW = "view", - QUERY = "query", -} - export class Permission { type: PermissionType level: PermissionLevel @@ -173,3 +156,4 @@ export function isPermissionLevelHigherThanRead(level: PermissionLevel) { // utility as a lot of things need simply the builder permission export const BUILDER = PermissionType.BUILDER +export const GLOBAL_BUILDER = PermissionType.GLOBAL_BUILDER diff --git a/packages/backend-core/src/security/roles.ts b/packages/backend-core/src/security/roles.ts index cf5c6bc406..081193b433 100644 --- a/packages/backend-core/src/security/roles.ts +++ b/packages/backend-core/src/security/roles.ts @@ -3,7 +3,7 @@ import { prefixRoleID, getRoleParams, DocumentType, SEPARATOR } from "../db" import { getAppDB } from "../context" import { doWithDB } from "../db" import { Screen, Role as RoleDoc } from "@budibase/types" -const { cloneDeep } = require("lodash/fp") +import cloneDeep from "lodash/fp/cloneDeep" export const BUILTIN_ROLE_IDS = { ADMIN: "ADMIN", diff --git a/packages/backend-core/src/security/tests/permissions.spec.ts b/packages/backend-core/src/security/tests/permissions.spec.ts index caf8bb29a6..39348646fb 100644 --- a/packages/backend-core/src/security/tests/permissions.spec.ts +++ b/packages/backend-core/src/security/tests/permissions.spec.ts @@ -1,4 +1,4 @@ -import { cloneDeep } from "lodash" +import cloneDeep from "lodash/cloneDeep" import * as permissions from "../permissions" import { BUILTIN_ROLE_IDS } from "../roles" diff --git a/packages/backend-core/src/users/db.ts b/packages/backend-core/src/users/db.ts new file mode 100644 index 0000000000..55cc97bb1c --- /dev/null +++ b/packages/backend-core/src/users/db.ts @@ -0,0 +1,460 @@ +import env from "../environment" +import * as eventHelpers from "./events" +import * as accounts from "../accounts" +import * as cache from "../cache" +import { getIdentity, getTenantId, getGlobalDB } from "../context" +import * as dbUtils from "../db" +import { EmailUnavailableError, HTTPError } from "../errors" +import * as platform from "../platform" +import * as sessions from "../security/sessions" +import * as usersCore from "./users" +import { + AllDocsResponse, + BulkUserCreated, + BulkUserDeleted, + RowResponse, + SaveUserOpts, + User, + Account, + isSSOUser, + isSSOAccount, + UserStatus, +} from "@budibase/types" +import * as accountSdk from "../accounts" +import { + validateUniqueUser, + getAccountHolderFromUserIds, + isAdmin, +} from "./utils" +import { searchExistingEmails } from "./lookup" +import { hash } from "../utils" + +type QuotaUpdateFn = (change: number, cb?: () => Promise) => Promise +type GroupUpdateFn = (groupId: string, userIds: string[]) => Promise +type FeatureFn = () => Promise +type QuotaFns = { addUsers: QuotaUpdateFn; removeUsers: QuotaUpdateFn } +type GroupFns = { addUsers: GroupUpdateFn } +type FeatureFns = { isSSOEnforced: FeatureFn; isAppBuildersEnabled: FeatureFn } + +const bulkDeleteProcessing = async (dbUser: User) => { + const userId = dbUser._id as string + await platform.users.removeUser(dbUser) + await eventHelpers.handleDeleteEvents(dbUser) + await cache.user.invalidateUser(userId) + await sessions.invalidateSessions(userId, { reason: "bulk-deletion" }) +} + +export class UserDB { + static quotas: QuotaFns + static groups: GroupFns + static features: FeatureFns + + static init(quotaFns: QuotaFns, groupFns: GroupFns, featureFns: FeatureFns) { + UserDB.quotas = quotaFns + UserDB.groups = groupFns + UserDB.features = featureFns + } + + static async isPreventPasswordActions(user: User, account?: Account) { + // when in maintenance mode we allow sso users with the admin role + // to perform any password action - this prevents lockout + if (env.ENABLE_SSO_MAINTENANCE_MODE && isAdmin(user)) { + return false + } + + // SSO is enforced for all users + if (await UserDB.features.isSSOEnforced()) { + return true + } + + // Check local sso + if (isSSOUser(user)) { + return true + } + + // Check account sso + if (!account) { + account = await accountSdk.getAccountByTenantId(getTenantId()) + } + return !!(account && account.email === user.email && isSSOAccount(account)) + } + + static async buildUser( + user: User, + opts: SaveUserOpts = { + hashPassword: true, + requirePassword: true, + }, + tenantId: string, + dbUser?: any, + account?: Account + ): Promise { + let { password, _id } = user + + // don't require a password if the db user doesn't already have one + if (dbUser && !dbUser.password) { + opts.requirePassword = false + } + + let hashedPassword + if (password) { + if (await UserDB.isPreventPasswordActions(user, account)) { + throw new HTTPError("Password change is disabled for this user", 400) + } + hashedPassword = opts.hashPassword ? await hash(password) : password + } else if (dbUser) { + hashedPassword = dbUser.password + } + + // passwords are never required if sso is enforced + const requirePasswords = + opts.requirePassword && !(await UserDB.features.isSSOEnforced()) + if (!hashedPassword && requirePasswords) { + throw "Password must be specified." + } + + _id = _id || dbUtils.generateGlobalUserID() + + const fullUser = { + createdAt: Date.now(), + ...dbUser, + ...user, + _id, + password: hashedPassword, + tenantId, + } + // make sure the roles object is always present + if (!fullUser.roles) { + fullUser.roles = {} + } + // add the active status to a user if its not provided + if (fullUser.status == null) { + fullUser.status = UserStatus.ACTIVE + } + + return fullUser + } + + static async allUsers() { + const db = getGlobalDB() + const response = await db.allDocs( + dbUtils.getGlobalUserParams(null, { + include_docs: true, + }) + ) + return response.rows.map((row: any) => row.doc) + } + + static async countUsersByApp(appId: string) { + let response: any = await usersCore.searchGlobalUsersByApp(appId, {}) + return { + userCount: response.length, + } + } + + static async getUsersByAppAccess(appId?: string) { + const opts: any = { + include_docs: true, + limit: 50, + } + let response: User[] = await usersCore.searchGlobalUsersByAppAccess( + appId, + opts + ) + return response + } + + static async getUserByEmail(email: string) { + return usersCore.getGlobalUserByEmail(email) + } + + /** + * Gets a user by ID from the global database, based on the current tenancy. + */ + static async getUser(userId: string) { + const user = await usersCore.getById(userId) + if (user) { + delete user.password + } + return user + } + + static async save(user: User, opts: SaveUserOpts = {}): Promise { + // default booleans to true + if (opts.hashPassword == null) { + opts.hashPassword = true + } + if (opts.requirePassword == null) { + opts.requirePassword = true + } + const tenantId = getTenantId() + const db = getGlobalDB() + + let { email, _id, userGroups = [], roles } = user + + if (!email && !_id) { + throw new Error("_id or email is required") + } + + if ( + user.builder?.apps?.length && + !(await UserDB.features.isAppBuildersEnabled()) + ) { + throw new Error("Unable to update app builders, please check license") + } + + let dbUser: User | undefined + if (_id) { + // try to get existing user from db + try { + dbUser = (await db.get(_id)) as User + if (email && dbUser.email !== email) { + throw "Email address cannot be changed" + } + email = dbUser.email + } catch (e: any) { + if (e.status === 404) { + // do nothing, save this new user with the id specified - required for SSO auth + } else { + throw e + } + } + } + + if (!dbUser && email) { + // no id was specified - load from email instead + dbUser = await usersCore.getGlobalUserByEmail(email) + if (dbUser && dbUser._id !== _id) { + throw new EmailUnavailableError(email) + } + } + + const change = dbUser ? 0 : 1 // no change if there is existing user + return UserDB.quotas.addUsers(change, async () => { + await validateUniqueUser(email, tenantId) + + let builtUser = await UserDB.buildUser(user, opts, tenantId, dbUser) + // don't allow a user to update its own roles/perms + if (opts.currentUserId && opts.currentUserId === dbUser?._id) { + builtUser = usersCore.cleanseUserObject(builtUser, dbUser) as User + } + + if (!dbUser && roles?.length) { + builtUser.roles = { ...roles } + } + + // make sure we set the _id field for a new user + // Also if this is a new user, associate groups with them + let groupPromises = [] + if (!_id) { + _id = builtUser._id! + + if (userGroups.length > 0) { + for (let groupId of userGroups) { + groupPromises.push(UserDB.groups.addUsers(groupId, [_id!])) + } + } + } + + try { + // save the user to db + let response = await db.put(builtUser) + builtUser._rev = response.rev + + await eventHelpers.handleSaveEvents(builtUser, dbUser) + await platform.users.addUser(tenantId, builtUser._id!, builtUser.email) + await cache.user.invalidateUser(response.id) + + await Promise.all(groupPromises) + + // finally returned the saved user from the db + return db.get(builtUser._id!) + } catch (err: any) { + if (err.status === 409) { + throw "User exists already" + } else { + throw err + } + } + }) + } + + static async bulkCreate( + newUsersRequested: User[], + groups: string[] + ): Promise { + const tenantId = getTenantId() + + let usersToSave: any[] = [] + let newUsers: any[] = [] + + const emails = newUsersRequested.map((user: User) => user.email) + const existingEmails = await searchExistingEmails(emails) + const unsuccessful: { email: string; reason: string }[] = [] + + for (const newUser of newUsersRequested) { + if ( + newUsers.find( + (x: User) => x.email.toLowerCase() === newUser.email.toLowerCase() + ) || + existingEmails.includes(newUser.email.toLowerCase()) + ) { + unsuccessful.push({ + email: newUser.email, + reason: `Unavailable`, + }) + continue + } + newUser.userGroups = groups + newUsers.push(newUser) + } + + const account = await accountSdk.getAccountByTenantId(tenantId) + return UserDB.quotas.addUsers(newUsers.length, async () => { + // create the promises array that will be called by bulkDocs + newUsers.forEach((user: any) => { + usersToSave.push( + UserDB.buildUser( + user, + { + hashPassword: true, + requirePassword: user.requirePassword, + }, + tenantId, + undefined, // no dbUser + account + ) + ) + }) + + const usersToBulkSave = await Promise.all(usersToSave) + await usersCore.bulkUpdateGlobalUsers(usersToBulkSave) + + // Post-processing of bulk added users, e.g. events and cache operations + for (const user of usersToBulkSave) { + // TODO: Refactor to bulk insert users into the info db + // instead of relying on looping tenant creation + await platform.users.addUser(tenantId, user._id, user.email) + await eventHelpers.handleSaveEvents(user, undefined) + } + + const saved = usersToBulkSave.map(user => { + return { + _id: user._id, + email: user.email, + } + }) + + // now update the groups + if (Array.isArray(saved) && groups) { + const groupPromises = [] + const createdUserIds = saved.map(user => user._id) + for (let groupId of groups) { + groupPromises.push(UserDB.groups.addUsers(groupId, createdUserIds)) + } + await Promise.all(groupPromises) + } + + return { + successful: saved, + unsuccessful, + } + }) + } + + static async bulkDelete(userIds: string[]): Promise { + const db = getGlobalDB() + + const response: BulkUserDeleted = { + successful: [], + unsuccessful: [], + } + + // remove the account holder from the delete request if present + const account = await getAccountHolderFromUserIds(userIds) + if (account) { + userIds = userIds.filter(u => u !== account.budibaseUserId) + // mark user as unsuccessful + response.unsuccessful.push({ + _id: account.budibaseUserId, + email: account.email, + reason: "Account holder cannot be deleted", + }) + } + + // Get users and delete + const allDocsResponse: AllDocsResponse = await db.allDocs({ + include_docs: true, + keys: userIds, + }) + const usersToDelete: User[] = allDocsResponse.rows.map( + (user: RowResponse) => { + return user.doc + } + ) + + // Delete from DB + const toDelete = usersToDelete.map(user => ({ + ...user, + _deleted: true, + })) + const dbResponse = await usersCore.bulkUpdateGlobalUsers(toDelete) + + await UserDB.quotas.removeUsers(toDelete.length) + for (let user of usersToDelete) { + await bulkDeleteProcessing(user) + } + + // Build Response + // index users by id + const userIndex: { [key: string]: User } = {} + usersToDelete.reduce((prev, current) => { + prev[current._id!] = current + return prev + }, userIndex) + + // add the successful and unsuccessful users to response + dbResponse.forEach(item => { + const email = userIndex[item.id].email + if (item.ok) { + response.successful.push({ _id: item.id, email }) + } else { + response.unsuccessful.push({ + _id: item.id, + email, + reason: "Database error", + }) + } + }) + + return response + } + + static async destroy(id: string) { + const db = getGlobalDB() + const dbUser = (await db.get(id)) as User + const userId = dbUser._id as string + + if (!env.SELF_HOSTED && !env.DISABLE_ACCOUNT_PORTAL) { + // root account holder can't be deleted from inside budibase + const email = dbUser.email + const account = await accounts.getAccount(email) + if (account) { + if (dbUser.userId === getIdentity()!._id) { + throw new HTTPError('Please visit "Account" to delete this user', 400) + } else { + throw new HTTPError("Account holder cannot be deleted", 400) + } + } + } + + await platform.users.removeUser(dbUser) + + await db.remove(userId, dbUser._rev) + + await UserDB.quotas.removeUsers(1) + await eventHelpers.handleDeleteEvents(dbUser) + await cache.user.invalidateUser(userId) + await sessions.invalidateSessions(userId, { reason: "deletion" }) + } +} diff --git a/packages/worker/src/sdk/users/events.ts b/packages/backend-core/src/users/events.ts similarity index 86% rename from packages/worker/src/sdk/users/events.ts rename to packages/backend-core/src/users/events.ts index 7d86182a3c..f170c9ffe9 100644 --- a/packages/worker/src/sdk/users/events.ts +++ b/packages/backend-core/src/users/events.ts @@ -1,15 +1,18 @@ -import env from "../../environment" -import { events, accounts, tenancy } from "@budibase/backend-core" +import env from "../environment" +import * as events from "../events" +import * as accounts from "../accounts" +import { getTenantId } from "../context" import { User, UserRoles, CloudAccount } from "@budibase/types" +import { hasBuilderPermissions, hasAdminPermissions } from "./utils" export const handleDeleteEvents = async (user: any) => { await events.user.deleted(user) - if (isBuilder(user)) { + if (hasBuilderPermissions(user)) { await events.user.permissionBuilderRemoved(user) } - if (isAdmin(user)) { + if (hasAdminPermissions(user)) { await events.user.permissionAdminRemoved(user) } } @@ -55,7 +58,7 @@ export const handleSaveEvents = async ( user: User, existingUser: User | undefined ) => { - const tenantId = tenancy.getTenantId() + const tenantId = getTenantId() let tenantAccount: CloudAccount | undefined if (!env.SELF_HOSTED && !env.DISABLE_ACCOUNT_PORTAL) { tenantAccount = await accounts.getAccountByTenantId(tenantId) @@ -103,23 +106,20 @@ export const handleSaveEvents = async ( await handleAppRoleEvents(user, existingUser) } -const isBuilder = (user: any) => user.builder && user.builder.global -const isAdmin = (user: any) => user.admin && user.admin.global - export const isAddingBuilder = (user: any, existingUser: any) => { - return isAddingPermission(user, existingUser, isBuilder) + return isAddingPermission(user, existingUser, hasBuilderPermissions) } export const isRemovingBuilder = (user: any, existingUser: any) => { - return isRemovingPermission(user, existingUser, isBuilder) + return isRemovingPermission(user, existingUser, hasBuilderPermissions) } const isAddingAdmin = (user: any, existingUser: any) => { - return isAddingPermission(user, existingUser, isAdmin) + return isAddingPermission(user, existingUser, hasAdminPermissions) } const isRemovingAdmin = (user: any, existingUser: any) => { - return isRemovingPermission(user, existingUser, isAdmin) + return isRemovingPermission(user, existingUser, hasAdminPermissions) } const isOnboardingComplete = (user: any, existingUser: any) => { diff --git a/packages/backend-core/src/users/index.ts b/packages/backend-core/src/users/index.ts new file mode 100644 index 0000000000..c11d2a2c62 --- /dev/null +++ b/packages/backend-core/src/users/index.ts @@ -0,0 +1,4 @@ +export * from "./users" +export * from "./utils" +export * from "./lookup" +export { UserDB } from "./db" diff --git a/packages/backend-core/src/users/lookup.ts b/packages/backend-core/src/users/lookup.ts new file mode 100644 index 0000000000..17d0e91d88 --- /dev/null +++ b/packages/backend-core/src/users/lookup.ts @@ -0,0 +1,102 @@ +import { + AccountMetadata, + PlatformUser, + PlatformUserByEmail, + User, +} from "@budibase/types" +import * as dbUtils from "../db" +import { ViewName } from "../constants" + +/** + * Apply a system-wide search on emails: + * - in tenant + * - cross tenant + * - accounts + * return an array of emails that match the supplied emails. + */ +export async function searchExistingEmails(emails: string[]) { + let matchedEmails: string[] = [] + + const existingTenantUsers = await getExistingTenantUsers(emails) + matchedEmails.push(...existingTenantUsers.map(user => user.email)) + + const existingPlatformUsers = await getExistingPlatformUsers(emails) + matchedEmails.push(...existingPlatformUsers.map(user => user._id!)) + + const existingAccounts = await getExistingAccounts(emails) + matchedEmails.push(...existingAccounts.map(account => account.email)) + + return [...new Set(matchedEmails.map(email => email.toLowerCase()))] +} + +// lookup, could be email or userId, either will return a doc +export async function getPlatformUser( + identifier: string +): Promise { + // use the view here and allow to find anyone regardless of casing + // Use lowercase to ensure email login is case insensitive + return (await dbUtils.queryPlatformView(ViewName.PLATFORM_USERS_LOWERCASE, { + keys: [identifier.toLowerCase()], + include_docs: true, + })) as PlatformUser +} + +export async function getExistingTenantUsers( + emails: string[] +): Promise { + const lcEmails = emails.map(email => email.toLowerCase()) + const params = { + keys: lcEmails, + include_docs: true, + } + + const opts = { + arrayResponse: true, + } + + return (await dbUtils.queryGlobalView( + ViewName.USER_BY_EMAIL, + params, + undefined, + opts + )) as User[] +} + +export async function getExistingPlatformUsers( + emails: string[] +): Promise { + const lcEmails = emails.map(email => email.toLowerCase()) + const params = { + keys: lcEmails, + include_docs: true, + } + + const opts = { + arrayResponse: true, + } + return (await dbUtils.queryPlatformView( + ViewName.PLATFORM_USERS_LOWERCASE, + params, + opts + )) as PlatformUserByEmail[] +} + +export async function getExistingAccounts( + emails: string[] +): Promise { + const lcEmails = emails.map(email => email.toLowerCase()) + const params = { + keys: lcEmails, + include_docs: true, + } + + const opts = { + arrayResponse: true, + } + + return (await dbUtils.queryPlatformView( + ViewName.ACCOUNT_BY_EMAIL, + params, + opts + )) as AccountMetadata[] +} diff --git a/packages/backend-core/src/users.ts b/packages/backend-core/src/users/users.ts similarity index 86% rename from packages/backend-core/src/users.ts rename to packages/backend-core/src/users/users.ts index 166136df3c..a7e1389920 100644 --- a/packages/backend-core/src/users.ts +++ b/packages/backend-core/src/users/users.ts @@ -11,10 +11,16 @@ import { SEPARATOR, UNICODE_MAX, ViewName, -} from "./db" -import { BulkDocsResponse, SearchUsersRequest, User } from "@budibase/types" -import { getGlobalDB } from "./context" -import * as context from "./context" +} from "../db" +import { + BulkDocsResponse, + SearchUsersRequest, + User, + ContextUser, +} from "@budibase/types" +import { getGlobalDB } from "../context" +import * as context from "../context" +import { user as userCache } from "../cache" type GetOpts = { cleanup?: boolean } @@ -67,9 +73,9 @@ export const bulkUpdateGlobalUsers = async (users: User[]) => { export async function getById(id: string, opts?: GetOpts): Promise { const db = context.getGlobalDB() - let user = await db.get(id) + let user = await db.get(id) if (opts?.cleanup) { - user = removeUserPassword(user) + user = removeUserPassword(user) as User } return user } @@ -178,7 +184,7 @@ export const getGlobalUserByAppPage = (appId: string, user: User) => { * Performs a starts with search on the global email view. */ export const searchGlobalUsersByEmail = async ( - email: string, + email: string | unknown, opts: any, getOpts?: GetOpts ) => { @@ -248,3 +254,23 @@ export async function getUserCount() { }) return response.total_rows } + +// used to remove the builder/admin permissions, for processing the +// user as an app user (they may have some specific role/group +export function removePortalUserPermissions(user: User | ContextUser) { + delete user.admin + delete user.builder + return user +} + +export function cleanseUserObject(user: User | ContextUser, base?: User) { + delete user.admin + delete user.builder + delete user.roles + if (base) { + user.admin = base.admin + user.builder = base.builder + user.roles = base.roles + } + return user +} diff --git a/packages/backend-core/src/users/utils.ts b/packages/backend-core/src/users/utils.ts new file mode 100644 index 0000000000..af0e8e10c7 --- /dev/null +++ b/packages/backend-core/src/users/utils.ts @@ -0,0 +1,55 @@ +import { CloudAccount } from "@budibase/types" +import * as accountSdk from "../accounts" +import env from "../environment" +import { getPlatformUser } from "./lookup" +import { EmailUnavailableError } from "../errors" +import { getTenantId } from "../context" +import { sdk } from "@budibase/shared-core" +import { getAccountByTenantId } from "../accounts" + +// extract from shared-core to make easily accessible from backend-core +export const isBuilder = sdk.users.isBuilder +export const isAdmin = sdk.users.isAdmin +export const isGlobalBuilder = sdk.users.isGlobalBuilder +export const isAdminOrBuilder = sdk.users.isAdminOrBuilder +export const hasAdminPermissions = sdk.users.hasAdminPermissions +export const hasBuilderPermissions = sdk.users.hasBuilderPermissions +export const hasAppBuilderPermissions = sdk.users.hasAppBuilderPermissions + +export async function validateUniqueUser(email: string, tenantId: string) { + // check budibase users in other tenants + if (env.MULTI_TENANCY) { + const tenantUser = await getPlatformUser(email) + if (tenantUser != null && tenantUser.tenantId !== tenantId) { + throw new EmailUnavailableError(email) + } + } + + // check root account users in account portal + if (!env.SELF_HOSTED && !env.DISABLE_ACCOUNT_PORTAL) { + const account = await accountSdk.getAccount(email) + if (account && account.verified && account.tenantId !== tenantId) { + throw new EmailUnavailableError(email) + } + } +} + +/** + * For the given user id's, return the account holder if it is in the ids. + */ +export async function getAccountHolderFromUserIds( + userIds: string[] +): Promise { + if (!env.SELF_HOSTED && !env.DISABLE_ACCOUNT_PORTAL) { + const tenantId = getTenantId() + const account = await getAccountByTenantId(tenantId) + if (!account) { + throw new Error(`Account not found for tenantId=${tenantId}`) + } + + const budibaseUserId = account.budibaseUserId + if (userIds.includes(budibaseUserId)) { + return account + } + } +} diff --git a/packages/backend-core/tests/core/utilities/jestUtils.ts b/packages/backend-core/tests/core/utilities/jestUtils.ts index d84eac548c..4a3da8db8c 100644 --- a/packages/backend-core/tests/core/utilities/jestUtils.ts +++ b/packages/backend-core/tests/core/utilities/jestUtils.ts @@ -1,3 +1,5 @@ +import { db } from "../../../src" + export function expectFunctionWasCalledTimesWith( jestFunction: any, times: number, @@ -7,3 +9,22 @@ export function expectFunctionWasCalledTimesWith( jestFunction.mock.calls.filter((call: any) => call[0] === argument).length ).toBe(times) } + +export const expectAnyInternalColsAttributes: { + [K in (typeof db.CONSTANT_INTERNAL_ROW_COLS)[number]]: any +} = { + tableId: expect.anything(), + type: expect.anything(), + _id: expect.anything(), + _rev: expect.anything(), + createdAt: expect.anything(), + updatedAt: expect.anything(), +} + +export const expectAnyExternalColsAttributes: { + [K in (typeof db.CONSTANT_EXTERNAL_ROW_COLS)[number]]: any +} = { + tableId: expect.anything(), + _id: expect.anything(), + _rev: expect.anything(), +} diff --git a/packages/backend-core/tests/core/utilities/mocks/events.ts b/packages/backend-core/tests/core/utilities/mocks/events.ts index 81de1f8175..fef730768a 100644 --- a/packages/backend-core/tests/core/utilities/mocks/events.ts +++ b/packages/backend-core/tests/core/utilities/mocks/events.ts @@ -1,5 +1,3 @@ -import * as events from "../../../../src/events" - beforeAll(async () => { const processors = await import("../../../../src/events/processors") const events = await import("../../../../src/events") diff --git a/packages/backend-core/tests/core/utilities/mocks/licenses.ts b/packages/backend-core/tests/core/utilities/mocks/licenses.ts index 4272e78eb8..6747282040 100644 --- a/packages/backend-core/tests/core/utilities/mocks/licenses.ts +++ b/packages/backend-core/tests/core/utilities/mocks/licenses.ts @@ -1,5 +1,5 @@ import { Feature, License, Quotas } from "@budibase/types" -import _ from "lodash" +import cloneDeep from "lodash/cloneDeep" let CLOUD_FREE_LICENSE: License let UNLIMITED_LICENSE: License @@ -58,7 +58,7 @@ export const useCloudFree = () => { // FEATURES const useFeature = (feature: Feature) => { - const license = _.cloneDeep(UNLIMITED_LICENSE) + const license = cloneDeep(UNLIMITED_LICENSE) const opts: UseLicenseOpts = { features: [feature], } @@ -94,10 +94,14 @@ export const useSyncAutomations = () => { return useFeature(Feature.SYNC_AUTOMATIONS) } +export const useAppBuilders = () => { + return useFeature(Feature.APP_BUILDERS) +} + // QUOTAS export const setAutomationLogsQuota = (value: number) => { - const license = _.cloneDeep(UNLIMITED_LICENSE) + const license = cloneDeep(UNLIMITED_LICENSE) license.quotas.constant.automationLogRetentionDays.value = value return useLicense(license) } diff --git a/packages/backend-core/tests/core/utilities/structures/accounts.ts b/packages/backend-core/tests/core/utilities/structures/accounts.ts index 8476399aa3..67e4411ea3 100644 --- a/packages/backend-core/tests/core/utilities/structures/accounts.ts +++ b/packages/backend-core/tests/core/utilities/structures/accounts.ts @@ -11,7 +11,7 @@ import { CreateAccount, CreatePassswordAccount, } from "@budibase/types" -import _ from "lodash" +import sample from "lodash/sample" export const account = (partial: Partial = {}): Account => { return { @@ -46,13 +46,11 @@ export const cloudAccount = (): CloudAccount => { } function providerType(): AccountSSOProviderType { - return _.sample( - Object.values(AccountSSOProviderType) - ) as AccountSSOProviderType + return sample(Object.values(AccountSSOProviderType)) as AccountSSOProviderType } function provider(): AccountSSOProvider { - return _.sample(Object.values(AccountSSOProvider)) as AccountSSOProvider + return sample(Object.values(AccountSSOProvider)) as AccountSSOProvider } export function ssoAccount(account: Account = cloudAccount()): SSOAccount { diff --git a/packages/backend-core/tests/core/utilities/structures/scim.ts b/packages/backend-core/tests/core/utilities/structures/scim.ts index 741cff165e..80f41c605d 100644 --- a/packages/backend-core/tests/core/utilities/structures/scim.ts +++ b/packages/backend-core/tests/core/utilities/structures/scim.ts @@ -1,7 +1,6 @@ import { ScimCreateGroupRequest, ScimCreateUserRequest } from "@budibase/types" import { uuid } from "./common" import { generator } from "./generator" -import _ from "lodash" interface CreateUserRequestFields { externalId: string @@ -20,10 +19,10 @@ export function createUserRequest(userData?: Partial) { username: generator.name(), } - const { externalId, email, firstName, lastName, username } = _.assign( - defaultValues, - userData - ) + const { externalId, email, firstName, lastName, username } = { + ...defaultValues, + ...userData, + } let user: ScimCreateUserRequest = { schemas: [ diff --git a/packages/backend-core/tests/core/utilities/structures/sso.ts b/packages/backend-core/tests/core/utilities/structures/sso.ts index 9da9c82223..4d13635f09 100644 --- a/packages/backend-core/tests/core/utilities/structures/sso.ts +++ b/packages/backend-core/tests/core/utilities/structures/sso.ts @@ -15,7 +15,7 @@ import { generator } from "./generator" import { email, uuid } from "./common" import * as shared from "./shared" import { user } from "./shared" -import _ from "lodash" +import sample from "lodash/sample" export function OAuth(): OAuth2 { return { @@ -47,7 +47,7 @@ export function authDetails(userDoc?: User): SSOAuthDetails { } export function providerType(): SSOProviderType { - return _.sample(Object.values(SSOProviderType)) as SSOProviderType + return sample(Object.values(SSOProviderType)) as SSOProviderType } export function ssoProfile(user?: User): SSOProfile { diff --git a/packages/backend-core/tests/core/utilities/structures/users.ts b/packages/backend-core/tests/core/utilities/structures/users.ts index 7a6b4f0d80..0a4f2e8b54 100644 --- a/packages/backend-core/tests/core/utilities/structures/users.ts +++ b/packages/backend-core/tests/core/utilities/structures/users.ts @@ -1,5 +1,6 @@ import { AdminUser, + AdminOnlyUser, BuilderUser, SSOAuthDetails, SSOUser, @@ -21,6 +22,15 @@ export const adminUser = (userProps?: any): AdminUser => { } } +export const adminOnlyUser = (userProps?: any): AdminOnlyUser => { + return { + ...user(userProps), + admin: { + global: true, + }, + } +} + export const builderUser = (userProps?: any): BuilderUser => { return { ...user(userProps), @@ -30,6 +40,15 @@ export const builderUser = (userProps?: any): BuilderUser => { } } +export const appBuilderUser = (appId: string, userProps?: any): BuilderUser => { + return { + ...user(userProps), + builder: { + apps: [appId], + }, + } +} + export function ssoUser( opts: { user?: any; details?: SSOAuthDetails } = {} ): SSOUser { diff --git a/packages/backend-core/tsconfig.json b/packages/backend-core/tsconfig.json index 2b1419b051..128814b955 100644 --- a/packages/backend-core/tsconfig.json +++ b/packages/backend-core/tsconfig.json @@ -4,9 +4,9 @@ "composite": true, "baseUrl": ".", "paths": { - "@budibase/types": ["../types/src"] + "@budibase/types": ["../types/src"], + "@budibase/shared-core": ["../shared-core/src"] } }, - "exclude": ["node_modules", "dist"] } diff --git a/packages/bbui/package.json b/packages/bbui/package.json index b03c83d71b..8a9318ba94 100644 --- a/packages/bbui/package.json +++ b/packages/bbui/package.json @@ -85,7 +85,8 @@ "dayjs": "^1.10.4", "easymde": "^2.16.1", "svelte-flatpickr": "3.2.3", - "svelte-portal": "^1.0.0" + "svelte-portal": "^1.0.0", + "svelte-dnd-action": "^0.9.8" }, "resolutions": { "loader-utils": "1.4.1" @@ -96,13 +97,14 @@ "dependsOn": [ { "projects": [ - "@budibase/string-templates" + "@budibase/string-templates", + "@budibase/shared-core", + "@budibase/types" ], "target": "build" } ] } } - }, - "gitHead": "d1836a898cab3f8ab80ee6d8f42be1a9eed7dcdc" + } } diff --git a/packages/bbui/src/Button/Button.svelte b/packages/bbui/src/Button/Button.svelte index efd5f33bd2..9e49d84d44 100644 --- a/packages/bbui/src/Button/Button.svelte +++ b/packages/bbui/src/Button/Button.svelte @@ -1,6 +1,7 @@ - + {/if} + {#if $$slots} + + {/if} + + diff --git a/packages/bbui/src/FancyForm/FancyButtonRadio.svelte b/packages/bbui/src/FancyForm/FancyButtonRadio.svelte index 510fd8efb8..3048b09555 100644 --- a/packages/bbui/src/FancyForm/FancyButtonRadio.svelte +++ b/packages/bbui/src/FancyForm/FancyButtonRadio.svelte @@ -15,8 +15,6 @@ const dispatch = createEventDispatcher() - $: placeholder = !value - const extractProperty = (value, property) => { if (value && typeof value === "object") { return value[property] diff --git a/packages/bbui/src/Form/Core/CheckboxGroup.svelte b/packages/bbui/src/Form/Core/CheckboxGroup.svelte index 640d5d99cd..2b8a1e438a 100644 --- a/packages/bbui/src/Form/Core/CheckboxGroup.svelte +++ b/packages/bbui/src/Form/Core/CheckboxGroup.svelte @@ -12,23 +12,24 @@ export let getOptionValue = option => option const dispatch = createEventDispatcher() + const onChange = e => { - let tempValue = value - let isChecked = e.target.checked - if (!tempValue.includes(e.target.value) && isChecked) { - tempValue.push(e.target.value) + const optionValue = e.target.value + if (e.target.checked && !value.includes(optionValue)) { + dispatch("change", [...value, optionValue]) + } else { + dispatch( + "change", + value.filter(x => x !== optionValue) + ) } - value = tempValue - dispatch( - "change", - tempValue.filter(val => val !== e.target.value || isChecked) - ) }
{#if options && Array.isArray(options)} {#each options as option} + {@const optionValue = getOptionValue(option)}
{:else if variables.length}
- {#each variables as variable, idx} + {#each variables as variable}
  • - //import { createEventDispatcher } from "svelte" import "@spectrum-css/popover/dist/index-vars.css" import clickOutside from "../Actions/click_outside" import { fly } from "svelte/transition" diff --git a/packages/bbui/src/Link/Link.svelte b/packages/bbui/src/Link/Link.svelte index ea9496ba72..5eaeceaf76 100644 --- a/packages/bbui/src/Link/Link.svelte +++ b/packages/bbui/src/Link/Link.svelte @@ -1,6 +1,7 @@ dispatch("click") && e.stopPropagation()} + on:click={onClick} {href} {target} {download} + class:disabled class:spectrum-Link--primary={primary} class:spectrum-Link--secondary={secondary} class:spectrum-Link--overBackground={overBackground} class:spectrum-Link--quiet={quiet} - class="spectrum-Link spectrum-Link--size{size}"> + + {#if tooltip} +
    + +
    + {/if} + + + diff --git a/packages/bbui/src/Modal/ModalContent.svelte b/packages/bbui/src/Modal/ModalContent.svelte index 226414ee11..3ca584504c 100644 --- a/packages/bbui/src/Modal/ModalContent.svelte +++ b/packages/bbui/src/Modal/ModalContent.svelte @@ -1,3 +1,7 @@ + + + +
    +
    + {#each options as option, idx (option.id)} +
    +
    + +
    +
    +
    openColorPickerPopover(idx, e.target)} + > + +
    + {#each colorsArray as color} +
    handleColorChange(option.name, color, idx)} + style="--color:{color};" + class="circle circle-hover" + /> + {/each} +
    + +
    +
    +
    + handleNameChange(option.name, idx, e.target.value)} + value={option.name} + placeholder="Option name" + /> +
    +
    + +
    +
    + {/each} +
    +
    + +
    Add option
    +
    +
    + + diff --git a/packages/bbui/src/Popover/Popover.svelte b/packages/bbui/src/Popover/Popover.svelte index 9f951a6a7e..6706bf7a8b 100644 --- a/packages/bbui/src/Popover/Popover.svelte +++ b/packages/bbui/src/Popover/Popover.svelte @@ -21,6 +21,7 @@ export let offset = 5 export let customHeight export let animate = true + export let customZindex $: target = portalTarget || getContext(Context.PopoverRoot) || ".spectrum" @@ -77,8 +78,9 @@ }} on:keydown={handleEscape} class="spectrum-Popover is-open" + class:customZindex role="presentation" - style="height: {customHeight}" + style="height: {customHeight}; --customZindex: {customZindex};" transition:fly|local={{ y: -20, duration: animate ? 200 : 0 }} > @@ -92,4 +94,8 @@ border-color: var(--spectrum-global-color-gray-300); overflow: auto; } + + .customZindex { + z-index: var(--customZindex) !important; + } diff --git a/packages/bbui/src/Table/CellRenderer.svelte b/packages/bbui/src/Table/CellRenderer.svelte index 5004401d91..c64b975884 100644 --- a/packages/bbui/src/Table/CellRenderer.svelte +++ b/packages/bbui/src/Table/CellRenderer.svelte @@ -29,7 +29,6 @@ $: type = getType(schema) $: customRenderer = customRenderers?.find(x => x.column === schema?.name) $: renderer = customRenderer?.component ?? typeMap[type] ?? StringRenderer - $: width = schema?.width || "150px" $: cellValue = getCellValue(value, schema.template) const getType = schema => { diff --git a/packages/bbui/src/Table/Table.svelte b/packages/bbui/src/Table/Table.svelte index f8528aac36..3a86295178 100644 --- a/packages/bbui/src/Table/Table.svelte +++ b/packages/bbui/src/Table/Table.svelte @@ -379,7 +379,7 @@
    {/if} {#if sortedRows?.length} - {#each sortedRows as row, idx} + {#each sortedRows as row}
    {#if showEditColumn}
    + export const TooltipPosition = { + Top: "top", + Right: "right", + Bottom: "bottom", + Left: "left", + } + export const TooltipType = { + Default: "default", + Info: "info", + Positive: "positive", + Negative: "negative", + } + + + + +
    (hovered = true)} + on:mouseleave={() => (hovered = false)} +> + +
    + +{#if visible && text && left != null && top != null} + + + {text} + + + +{/if} + + diff --git a/packages/bbui/src/Tooltip/TempTooltip.svelte b/packages/bbui/src/Tooltip/TempTooltip.svelte new file mode 100644 index 0000000000..0d590b1ec6 --- /dev/null +++ b/packages/bbui/src/Tooltip/TempTooltip.svelte @@ -0,0 +1,39 @@ + + + + + diff --git a/packages/bbui/src/index.js b/packages/bbui/src/index.js index d26b938dd5..cda6b5acbf 100644 --- a/packages/bbui/src/index.js +++ b/packages/bbui/src/index.js @@ -36,13 +36,19 @@ export { default as Layout } from "./Layout/Layout.svelte" export { default as Page } from "./Layout/Page.svelte" export { default as Link } from "./Link/Link.svelte" export { default as Tooltip } from "./Tooltip/Tooltip.svelte" +export { default as TempTooltip } from "./Tooltip/TempTooltip.svelte" +export { + default as AbsTooltip, + TooltipPosition, + TooltipType, +} from "./Tooltip/AbsTooltip.svelte" export { default as TooltipWrapper } from "./Tooltip/TooltipWrapper.svelte" export { default as Menu } from "./Menu/Menu.svelte" export { default as MenuSection } from "./Menu/Section.svelte" export { default as MenuSeparator } from "./Menu/Separator.svelte" export { default as MenuItem } from "./Menu/Item.svelte" export { default as Modal } from "./Modal/Modal.svelte" -export { default as ModalContent } from "./Modal/ModalContent.svelte" +export { default as ModalContent, keepOpen } from "./Modal/ModalContent.svelte" export { default as NotificationDisplay } from "./Notification/NotificationDisplay.svelte" export { default as Notification } from "./Notification/Notification.svelte" export { default as SideNavigation } from "./SideNavigation/Navigation.svelte" @@ -78,7 +84,7 @@ export { default as IconSideNavItem } from "./IconSideNav/IconSideNavItem.svelte export { default as Slider } from "./Form/Slider.svelte" export { default as Accordion } from "./Accordion/Accordion.svelte" export { default as File } from "./Form/File.svelte" - +export { default as OptionSelectDnD } from "./OptionSelectDnD/OptionSelectDnD.svelte" // Renderers export { default as BoldRenderer } from "./Table/BoldRenderer.svelte" export { default as CodeRenderer } from "./Table/CodeRenderer.svelte" diff --git a/packages/builder/package.json b/packages/builder/package.json index 646bb144df..56834f79dd 100644 --- a/packages/builder/package.json +++ b/packages/builder/package.json @@ -101,14 +101,14 @@ "@rollup/plugin-replace": "^2.4.2", "@roxi/routify": "2.18.5", "@sveltejs/vite-plugin-svelte": "1.0.1", - "@testing-library/jest-dom": "^5.11.10", + "@testing-library/jest-dom": "5.17.0", "@testing-library/svelte": "^3.2.2", - "babel-jest": "^26.6.3", + "babel-jest": "29.6.2", "cypress": "^9.3.1", "cypress-multi-reporters": "^1.6.0", "cypress-terminal-report": "^1.4.1", "identity-obj-proxy": "^3.0.0", - "jest": "^26.6.3", + "jest": "29.6.2", "jsdom": "^21.1.1", "mochawesome": "^7.1.3", "mochawesome-merge": "^4.2.1", @@ -133,8 +133,21 @@ "dependsOn": [ { "projects": [ + "@budibase/shared-core", "@budibase/string-templates", - "@budibase/shared-core" + "@budibase/types" + ], + "target": "build" + } + ] + }, + "dev:builder": { + "dependsOn": [ + { + "projects": [ + "@budibase/shared-core", + "@budibase/string-templates", + "@budibase/types" ], "target": "build" } @@ -145,13 +158,13 @@ { "projects": [ "@budibase/shared-core", - "@budibase/string-templates" + "@budibase/string-templates", + "@budibase/types" ], "target": "build" } ] } } - }, - "gitHead": "115189f72a850bfb52b65ec61d932531bf327072" + } } diff --git a/packages/builder/src/builderStore/dataBinding.js b/packages/builder/src/builderStore/dataBinding.js index e9c8643bce..bbe116721a 100644 --- a/packages/builder/src/builderStore/dataBinding.js +++ b/packages/builder/src/builderStore/dataBinding.js @@ -491,6 +491,7 @@ const getSelectedRowsBindings = asset => { readableBinding: `${table._instanceName}.Selected rows`, category: "Selected rows", icon: "ViewRow", + display: { name: table._instanceName }, })) ) @@ -506,6 +507,7 @@ const getSelectedRowsBindings = asset => { )}.${makePropSafe("selectedRows")}`, readableBinding: `${block._instanceName}.Selected rows`, category: "Selected rows", + display: { name: block._instanceName }, })) ) } diff --git a/packages/builder/src/builderStore/index.js b/packages/builder/src/builderStore/index.js index 9dca6a64e6..2ca8057b48 100644 --- a/packages/builder/src/builderStore/index.js +++ b/packages/builder/src/builderStore/index.js @@ -3,6 +3,7 @@ import { getAutomationStore } from "./store/automation" import { getTemporalStore } from "./store/temporal" import { getThemeStore } from "./store/theme" import { getUserStore } from "./store/users" +import { getDeploymentStore } from "./store/deployments" import { derived } from "svelte/store" import { findComponent, findComponentPath } from "./componentUtils" import { RoleUtils } from "@budibase/frontend-core" @@ -14,6 +15,7 @@ export const automationStore = getAutomationStore() export const themeStore = getThemeStore() export const temporalStore = getTemporalStore() export const userStore = getUserStore() +export const deploymentStore = getDeploymentStore() // Setup history for screens export const screenHistoryStore = createHistoryStore({ @@ -118,3 +120,24 @@ export const selectedAutomation = derived(automationStore, $automationStore => { x => x._id === $automationStore.selectedAutomationId ) }) + +// Derive map of resource IDs to other users. +// We only ever care about a single user in each resource, so if multiple users +// share the same datasource we can just overwrite them. +export const userSelectedResourceMap = derived(userStore, $userStore => { + let map = {} + $userStore.forEach(user => { + const resource = user.builderMetadata?.selectedResourceId + if (resource) { + if (!map[resource]) { + map[resource] = [] + } + map[resource].push(user) + } + }) + return map +}) + +export const isOnlyUser = derived(userStore, $userStore => { + return $userStore.length < 2 +}) diff --git a/packages/builder/src/builderStore/store/automation/index.js b/packages/builder/src/builderStore/store/automation/index.js index 9e5516c512..4ebf0515d6 100644 --- a/packages/builder/src/builderStore/store/automation/index.js +++ b/packages/builder/src/builderStore/store/automation/index.js @@ -248,4 +248,36 @@ const automationActions = store => ({ } await store.actions.save(newAutomation) }, + replace: async (automationId, automation) => { + if (!automation) { + store.update(state => { + // Remove the automation + state.automations = state.automations.filter( + x => x._id !== automationId + ) + // Select a new automation if required + if (automationId === state.selectedAutomationId) { + store.actions.select(state.automations[0]?._id) + } + return state + }) + } else { + const index = get(store).automations.findIndex( + x => x._id === automation._id + ) + if (index === -1) { + // Automation addition + store.update(state => ({ + ...state, + automations: [...state.automations, automation], + })) + } else { + // Automation update + store.update(state => { + state.automations[index] = automation + return state + }) + } + } + }, }) diff --git a/packages/builder/src/builderStore/store/deployments.js b/packages/builder/src/builderStore/store/deployments.js new file mode 100644 index 0000000000..9f7a68cb46 --- /dev/null +++ b/packages/builder/src/builderStore/store/deployments.js @@ -0,0 +1,22 @@ +import { writable } from "svelte/store" +import { API } from "api" +import { notifications } from "@budibase/bbui" + +export const getDeploymentStore = () => { + let store = writable([]) + + const load = async () => { + try { + store.set(await API.getAppDeployments()) + } catch (err) { + notifications.error("Error fetching deployments") + } + } + + return { + subscribe: store.subscribe, + actions: { + load, + }, + } +} diff --git a/packages/builder/src/builderStore/store/frontend.js b/packages/builder/src/builderStore/store/frontend.js index 5de58f02e7..f312a58e97 100644 --- a/packages/builder/src/builderStore/store/frontend.js +++ b/packages/builder/src/builderStore/store/frontend.js @@ -38,6 +38,7 @@ import { import { makePropSafe as safe } from "@budibase/string-templates" import { getComponentFieldOptions } from "helpers/formFields" import { createBuilderWebsocket } from "builderStore/websocket" +import { BuilderSocketEvent } from "@budibase/shared-core" const INITIAL_FRONTEND_STATE = { initialised: false, @@ -353,6 +354,33 @@ export const getFrontendStore = () => { } return await sequentialScreenPatch(patchFn, screenId) }, + replace: async (screenId, screen) => { + if (!screenId) { + return + } + if (!screen) { + // Screen deletion + store.update(state => ({ + ...state, + screens: state.screens.filter(x => x._id !== screenId), + })) + } else { + const index = get(store).screens.findIndex(x => x._id === screen._id) + if (index === -1) { + // Screen addition + store.update(state => ({ + ...state, + screens: [...state.screens, screen], + })) + } else { + // Screen update + store.update(state => { + state.screens[index] = screen + return state + }) + } + } + }, delete: async screens => { const screensToDelete = Array.isArray(screens) ? screens : [screens] @@ -1305,7 +1333,7 @@ export const getFrontendStore = () => { links: { save: async (url, title) => { const navigation = get(store).navigation - let links = [...navigation?.links] + let links = [...(navigation?.links ?? [])] // Skip if we have an identical link if (links.find(link => link.url === url && link.text === title)) { @@ -1365,6 +1393,21 @@ export const getFrontendStore = () => { }) }, }, + websocket: { + selectResource: id => { + websocket.emit(BuilderSocketEvent.SelectResource, { + resourceId: id, + }) + }, + }, + metadata: { + replace: metadata => { + store.update(state => ({ + ...state, + ...metadata, + })) + }, + }, } return store diff --git a/packages/builder/src/builderStore/websocket.js b/packages/builder/src/builderStore/websocket.js index af6d58ee7f..6121831c38 100644 --- a/packages/builder/src/builderStore/websocket.js +++ b/packages/builder/src/builderStore/websocket.js @@ -1,10 +1,17 @@ import { createWebsocket } from "@budibase/frontend-core" -import { userStore, store } from "builderStore" +import { + userStore, + store, + deploymentStore, + automationStore, +} from "builderStore" import { datasources, tables } from "stores/backend" import { get } from "svelte/store" import { auth } from "stores/portal" import { SocketEvent, BuilderSocketEvent } from "@budibase/shared-core" +import { apps } from "stores/portal" import { notifications } from "@budibase/bbui" +import { helpers } from "@budibase/shared-core" export const createBuilderWebsocket = appId => { const socket = createWebsocket("/socket/builder") @@ -31,7 +38,6 @@ export const createBuilderWebsocket = appId => { }) socket.onOther(BuilderSocketEvent.LockTransfer, ({ userId }) => { if (userId === get(auth)?.user?._id) { - notifications.success("You can now edit screens and automations") store.update(state => ({ ...state, hasLock: true, @@ -39,15 +45,37 @@ export const createBuilderWebsocket = appId => { } }) - // Table events + // Data section events socket.onOther(BuilderSocketEvent.TableChange, ({ id, table }) => { tables.replaceTable(id, table) }) - - // Datasource events socket.onOther(BuilderSocketEvent.DatasourceChange, ({ id, datasource }) => { datasources.replaceDatasource(id, datasource) }) + // Design section events + socket.onOther(BuilderSocketEvent.ScreenChange, ({ id, screen }) => { + store.actions.screens.replace(id, screen) + }) + socket.onOther(BuilderSocketEvent.AppMetadataChange, ({ metadata }) => { + store.actions.metadata.replace(metadata) + }) + socket.onOther( + BuilderSocketEvent.AppPublishChange, + async ({ user, published }) => { + await apps.load() + if (published) { + await deploymentStore.actions.load() + } + const verb = published ? "published" : "unpublished" + notifications.success(`${helpers.getUserLabel(user)} ${verb} this app`) + } + ) + + // Automations + socket.onOther(BuilderSocketEvent.AutomationChange, ({ id, automation }) => { + automationStore.actions.replace(id, automation) + }) + return socket } diff --git a/packages/builder/src/components/automation/AutomationBuilder/FlowChart/ActionModal.svelte b/packages/builder/src/components/automation/AutomationBuilder/FlowChart/ActionModal.svelte index 96dc8f4686..15dd864168 100644 --- a/packages/builder/src/components/automation/AutomationBuilder/FlowChart/ActionModal.svelte +++ b/packages/builder/src/components/automation/AutomationBuilder/FlowChart/ActionModal.svelte @@ -168,7 +168,7 @@ Plugins
    - {#each Object.entries(plugins) as [idx, action]} + {#each Object.entries(plugins) as [_, action]}
    + {@html html}
    diff --git a/packages/builder/src/components/automation/AutomationPanel/AutomationList.svelte b/packages/builder/src/components/automation/AutomationPanel/AutomationList.svelte index 80d65a5cb6..cce0f4eeab 100644 --- a/packages/builder/src/components/automation/AutomationPanel/AutomationList.svelte +++ b/packages/builder/src/components/automation/AutomationPanel/AutomationList.svelte @@ -1,6 +1,10 @@
    - {#each $automationStore.automations.sort(aut => aut.name) as automation, idx} + {#each $automationStore.automations.sort(aut => aut.name) as automation} 0} icon="ShareAndroid" text={automation.name} selected={automation._id === selectedAutomationId} on:click={() => selectAutomation(automation._id)} + selectedBy={$userSelectedResourceMap[automation._id]} > @@ -40,6 +44,5 @@ flex-direction: column; justify-content: flex-start; align-items: stretch; - margin: 0 calc(-1 * var(--spacing-xl)); } diff --git a/packages/builder/src/components/automation/AutomationPanel/AutomationPanel.svelte b/packages/builder/src/components/automation/AutomationPanel/AutomationPanel.svelte index 85e6a5faa3..fc52b7323a 100644 --- a/packages/builder/src/components/automation/AutomationPanel/AutomationPanel.svelte +++ b/packages/builder/src/components/automation/AutomationPanel/AutomationPanel.svelte @@ -11,8 +11,8 @@ - + diff --git a/packages/builder/src/components/automation/AutomationPanel/CreateAutomationModal.svelte b/packages/builder/src/components/automation/AutomationPanel/CreateAutomationModal.svelte index 5fb27eaaf3..647a8081cf 100644 --- a/packages/builder/src/components/automation/AutomationPanel/CreateAutomationModal.svelte +++ b/packages/builder/src/components/automation/AutomationPanel/CreateAutomationModal.svelte @@ -71,7 +71,7 @@
    - {#each triggers as [idx, trigger]} + {#each triggers as [_, trigger]}
    { - let newInputData = testData || blockInputs + // Test data is not cloned for reactivity + let newInputData = testData || cloneDeep(blockInputs) + + // Ensures the app action fields are populated if (block.event === "app:trigger" && !newInputData?.fields) { newInputData = cloneDeep(blockInputs) } diff --git a/packages/builder/src/components/automation/SetupPanel/QueryParamSelector.svelte b/packages/builder/src/components/automation/SetupPanel/QueryParamSelector.svelte index 9c47178b0e..250d235266 100644 --- a/packages/builder/src/components/automation/SetupPanel/QueryParamSelector.svelte +++ b/packages/builder/src/components/automation/SetupPanel/QueryParamSelector.svelte @@ -50,6 +50,7 @@ type="string" {bindings} fillWidth={true} + updateOnChange={false} /> {/each}
    diff --git a/packages/builder/src/components/backend/DataTable/DataTable.svelte b/packages/builder/src/components/backend/DataTable/DataTable.svelte index d4c994dae5..38eb87aa73 100644 --- a/packages/builder/src/components/backend/DataTable/DataTable.svelte +++ b/packages/builder/src/components/backend/DataTable/DataTable.svelte @@ -64,6 +64,13 @@ + + + + + + + {#if isInternal} @@ -77,9 +84,8 @@ {:else} {/if} + - - {#if isUsersTable} {:else} diff --git a/packages/builder/src/components/backend/DataTable/Table.svelte b/packages/builder/src/components/backend/DataTable/Table.svelte index 1164ab717d..4569586762 100644 --- a/packages/builder/src/components/backend/DataTable/Table.svelte +++ b/packages/builder/src/components/backend/DataTable/Table.svelte @@ -3,8 +3,6 @@ import { goto, params } from "@roxi/routify" import { Table, Heading, Layout } from "@budibase/bbui" import Spinner from "components/common/Spinner.svelte" - import CreateEditRow from "./modals/CreateEditRow.svelte" - import CreateEditUser from "./modals/CreateEditUser.svelte" import { TableNames, UNEDITABLE_USER_FIELDS, @@ -33,7 +31,6 @@ $: selectedRows, dispatch("selectionUpdated", selectedRows) $: isUsersTable = tableId === TableNames.USERS $: data && resetSelectedRows() - $: editRowComponent = isUsersTable ? CreateEditUser : CreateEditRow $: { UNSORTABLE_TYPES.forEach(type => { Object.values(schema || {}).forEach(col => { @@ -112,6 +109,7 @@ {disableSorting} {customPlaceholder} allowEditRows={allowEditing} + allowEditColumns={allowEditing} showAutoColumns={!hideAutocolumns} {allowClickRows} on:clickrelationship={e => selectRelationship(e.detail)} diff --git a/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte b/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte index f319f09b16..7c3e13f39a 100644 --- a/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte +++ b/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte @@ -7,18 +7,18 @@ Toggle, RadioGroup, DatePicker, - ModalContent, - Context, Modal, notifications, + OptionSelectDnD, + Layout, } from "@budibase/bbui" - import { createEventDispatcher } from "svelte" + import { createEventDispatcher, getContext } from "svelte" import { cloneDeep } from "lodash/fp" import { tables, datasources } from "stores/backend" import { TableNames, UNEDITABLE_USER_FIELDS } from "constants" import { FIELDS, - RelationshipTypes, + RelationshipType, ALLOWABLE_STRING_OPTIONS, ALLOWABLE_NUMBER_OPTIONS, ALLOWABLE_STRING_TYPES, @@ -26,12 +26,10 @@ SWITCHABLE_TYPES, } from "constants/backend" import { getAutoColumnInformation, buildAutoColumn } from "builderStore/utils" - import ValuesList from "components/common/ValuesList.svelte" import ConfirmDialog from "components/common/ConfirmDialog.svelte" import { truncate } from "lodash" import ModalBindableInput from "components/common/bindings/ModalBindableInput.svelte" import { getBindings } from "components/backend/DataTable/formula" - import { getContext } from "svelte" import JSONSchemaModal from "./JSONSchemaModal.svelte" import { ValidColumnNameRegex } from "@budibase/shared-core" @@ -45,11 +43,11 @@ const dispatch = createEventDispatcher() const PROHIBITED_COLUMN_NAMES = ["type", "_id", "_rev", "tableId"] - const { hide } = getContext(Context.Modal) - let fieldDefinitions = cloneDeep(FIELDS) + const { dispatch: gridDispatch } = getContext("grid") export let field + let fieldDefinitions = cloneDeep(FIELDS) let originalName let linkEditDisabled let primaryDisplay @@ -58,15 +56,13 @@ let table = $tables.selected let confirmDeleteDialog - let deletion let savingColumn let deleteColName let jsonSchemaModal - + let allowedTypes = [] let editableColumn = { type: "string", constraints: fieldDefinitions.STRING.constraints, - // Initial value for column name in other table for linked records fieldName: $tables.selected.name, } @@ -84,7 +80,23 @@ primaryDisplay = $tables.selected.primaryDisplay == null || $tables.selected.primaryDisplay === editableColumn.name + } else if (!savingColumn) { + let highestNumber = 0 + Object.keys(table.schema).forEach(columnName => { + const columnNumber = extractColumnNumber(columnName) + if (columnNumber > highestNumber) { + highestNumber = columnNumber + } + return highestNumber + }) + + if (highestNumber >= 1) { + editableColumn.name = `Column 0${highestNumber + 1}` + } else { + editableColumn.name = "Column 01" + } } + allowedTypes = getAllowedTypes() } $: initialiseField(field, savingColumn) @@ -183,9 +195,11 @@ indexes, }) dispatch("updatecolumns") + gridDispatch("close-edit-column") + if ( saveColumn.type === LINK_TYPE && - saveColumn.relationshipType === RelationshipTypes.MANY_TO_MANY + saveColumn.relationshipType === RelationshipType.MANY_TO_MANY ) { // Fetching the new tables tables.fetch() @@ -204,6 +218,7 @@ function cancelEdit() { editableColumn.name = originalName + gridDispatch("close-edit-column") } async function deleteColumn() { @@ -215,9 +230,8 @@ await tables.deleteField(editableColumn) notifications.success(`Column ${editableColumn.name} deleted`) confirmDeleteDialog.hide() - hide() - deletion = false dispatch("updatecolumns") + gridDispatch("close-edit-column") } } catch (error) { notifications.error(`Error deleting column: ${error.message}`) @@ -240,7 +254,7 @@ // Default relationships many to many if (editableColumn.type === LINK_TYPE) { - editableColumn.relationshipType = RelationshipTypes.MANY_TO_MANY + editableColumn.relationshipType = RelationshipType.MANY_TO_MANY } if (editableColumn.type === FORMULA_TYPE) { editableColumn.formulaType = "dynamic" @@ -253,27 +267,22 @@ required = req } - function onChangePrimaryDisplay(e) { - const isPrimary = e.detail - // primary display is always required - if (isPrimary) { - editableColumn.constraints.presence = { allowEmpty: false } - } - } - function openJsonSchemaEditor() { jsonSchemaModal.show() } function confirmDelete() { confirmDeleteDialog.show() - deletion = true } function hideDeleteDialog() { confirmDeleteDialog.hide() deleteColName = "" - deletion = false + } + + function extractColumnNumber(columnName) { + const match = columnName.match(/Column (\d+)/) + return match ? parseInt(match[1]) : 0 } function getRelationshipOptions(field) { @@ -290,17 +299,17 @@ { name: `Many ${thisName} rows → many ${linkName} rows`, alt: `Many ${table.name} rows → many ${linkTable.name} rows`, - value: RelationshipTypes.MANY_TO_MANY, + value: RelationshipType.MANY_TO_MANY, }, { name: `One ${linkName} row → many ${thisName} rows`, alt: `One ${linkTable.name} rows → many ${table.name} rows`, - value: RelationshipTypes.ONE_TO_MANY, + value: RelationshipType.ONE_TO_MANY, }, { name: `One ${thisName} row → many ${linkName} rows`, alt: `One ${table.name} rows → many ${linkTable.name} rows`, - value: RelationshipTypes.MANY_TO_ONE, + value: RelationshipType.MANY_TO_ONE, }, ] } @@ -406,15 +415,8 @@ } - + field.name} getOptionValue={field => field.type} + getOptionIcon={field => field.icon} isOptionEnabled={option => { if (option.type == AUTO_TYPE) { return availableAutoColumnKeys?.length > 0 @@ -437,28 +439,6 @@ }} /> - {#if canBeRequired || canBeDisplay} -
    - {#if canBeRequired} - - {/if} - {#if canBeDisplay} - - {/if} -
    - {/if} - {#if editableColumn.type === "string"} {:else if editableColumn.type === "options"} - {:else if editableColumn.type === "longform"}
    @@ -484,19 +464,28 @@ />
    {:else if editableColumn.type === "array"} - {:else if editableColumn.type === "datetime" && !editableColumn.autocolumn} - - +
    +
    + +
    +
    + +
    +
    + +
    +
    + +
    +
    + +
    +
    {#if datasource?.source !== "ORACLE" && datasource?.source !== "SQL_SERVER"}