diff --git a/.eslintignore b/.eslintignore index 8d4c64d960..94984a446f 100644 --- a/.eslintignore +++ b/.eslintignore @@ -6,9 +6,11 @@ packages/server/coverage packages/worker/coverage packages/backend-core/coverage packages/server/client +packages/server/coverage packages/builder/.routify packages/sdk/sdk packages/account-portal/packages/server/build packages/account-portal/packages/ui/.routify packages/account-portal/packages/ui/build -**/*.ivm.bundle.js \ No newline at end of file +**/*.ivm.bundle.js +packages/server/build/oldClientVersions/**/** diff --git a/.eslintrc.json b/.eslintrc.json index 3de9d13046..525072dc6c 100644 --- a/.eslintrc.json +++ b/.eslintrc.json @@ -35,16 +35,40 @@ { "files": ["**/*.ts"], "parser": "@typescript-eslint/parser", + "plugins": ["@typescript-eslint"], "extends": ["eslint:recommended"], + "globals": { + "NodeJS": true + }, "rules": { "no-unused-vars": "off", - "no-inner-declarations": "off", - "no-case-declarations": "off", - "no-useless-escape": "off", - "no-undef": "off", - "no-prototype-builtins": "off", - "local-rules/no-budibase-imports": "error", - "local-rules/no-test-com": "error" + "@typescript-eslint/no-unused-vars": "error", + "local-rules/no-budibase-imports": "error" + } + }, + { + "files": ["**/*.spec.ts"], + "parser": "@typescript-eslint/parser", + "plugins": ["jest", "@typescript-eslint"], + "extends": ["eslint:recommended", "plugin:jest/recommended"], + "env": { + "jest/globals": true + }, + "globals": { + "NodeJS": true + }, + "rules": { + "no-unused-vars": "off", + "@typescript-eslint/no-unused-vars": "error", + "local-rules/no-test-com": "error", + "local-rules/email-domain-example-com": "error", + "no-console": "warn", + // We have a lot of tests that don't have assertions, they use our test + // API client that does the assertions for them + "jest/expect-expect": "off", + // We do this in some tests where the behaviour of internal tables + // differs to external, but the API is broadly the same + "jest/no-conditional-expect": "off" } }, { diff --git a/.github/workflows/budibase_ci.yml b/.github/workflows/budibase_ci.yml index 3060660d47..4ae0766242 100644 --- a/.github/workflows/budibase_ci.yml +++ b/.github/workflows/budibase_ci.yml @@ -66,7 +66,8 @@ jobs: # Run build all the projects - name: Build run: | - yarn build + yarn build:oss + yarn build:account-portal # Check the types of the projects built via esbuild - name: Check types run: | @@ -107,9 +108,9 @@ jobs: - name: Test run: | if ${{ env.USE_NX_AFFECTED }}; then - yarn test --ignore=@budibase/worker --ignore=@budibase/server --ignore=@budibase/pro --since=${{ env.NX_BASE_BRANCH }} + yarn test --ignore=@budibase/worker --ignore=@budibase/server --since=${{ env.NX_BASE_BRANCH }} else - yarn test --ignore=@budibase/worker --ignore=@budibase/server --ignore=@budibase/pro + yarn test --ignore=@budibase/worker --ignore=@budibase/server fi test-worker: @@ -138,6 +139,8 @@ jobs: test-server: runs-on: ubuntu-latest + env: + DEBUG: testcontainers,testcontainers:exec,testcontainers:build,testcontainers:pull steps: - name: Checkout repo uses: actions/checkout@v4 @@ -151,7 +154,19 @@ jobs: with: node-version: 20.x cache: yarn + + - name: Pull testcontainers images + run: | + docker pull mcr.microsoft.com/mssql/server:2022-latest + docker pull mysql:8.3 + docker pull postgres:16.1-bullseye + docker pull mongo:7.0-jammy + docker pull mariadb:lts + docker pull testcontainers/ryuk:0.5.1 + docker pull budibase/couchdb + - run: yarn --frozen-lockfile + - name: Test server run: | if ${{ env.USE_NX_AFFECTED }}; then @@ -160,60 +175,6 @@ jobs: yarn test --scope=@budibase/server fi - test-pro: - runs-on: ubuntu-latest - if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == 'Budibase/budibase' - steps: - - name: Checkout repo and submodules - uses: actions/checkout@v4 - with: - submodules: true - token: ${{ secrets.PERSONAL_ACCESS_TOKEN || github.token }} - fetch-depth: 0 - - - name: Use Node.js 20.x - uses: actions/setup-node@v4 - with: - node-version: 20.x - cache: yarn - - run: yarn --frozen-lockfile - - name: Test - run: | - if ${{ env.USE_NX_AFFECTED }}; then - yarn test --scope=@budibase/pro --since=${{ env.NX_BASE_BRANCH }} - else - yarn test --scope=@budibase/pro - fi - - integration-test: - runs-on: ubuntu-latest - steps: - - name: Checkout repo - uses: actions/checkout@v4 - with: - submodules: ${{ env.IS_OSS_CONTRIBUTOR == 'false' }} - token: ${{ secrets.PERSONAL_ACCESS_TOKEN || github.token }} - - - name: Use Node.js 20.x - uses: actions/setup-node@v4 - with: - node-version: 20.x - cache: yarn - - run: yarn --frozen-lockfile - - name: Build packages - run: yarn build --scope @budibase/server --scope @budibase/worker - - name: Build backend-core for OSS contributor (required for pro) - if: ${{ env.IS_OSS_CONTRIBUTOR == 'true' }} - run: yarn build --scope @budibase/backend-core - - name: Run tests - run: | - cd qa-core - yarn setup - yarn serve:test:self:ci - env: - BB_ADMIN_USER_EMAIL: admin - BB_ADMIN_USER_PASSWORD: admin - check-pro-submodule: runs-on: ubuntu-latest if: inputs.run_as_oss != true && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == 'Budibase/budibase') @@ -242,27 +203,34 @@ jobs: echo "pro_commit=$pro_commit" echo "pro_commit=$pro_commit" >> "$GITHUB_OUTPUT" echo "base_commit=$base_commit" - echo "base_commit=$base_commit" >> "$GITHUB_OUTPUT" + + base_commit_excluding_merges=$(git log --no-merges -n 1 --format=format:%H $base_commit) + echo "base_commit_excluding_merges=$base_commit_excluding_merges" + echo "base_commit_excluding_merges=$base_commit_excluding_merges" >> "$GITHUB_OUTPUT" else echo "Nothing to do - branch to branch merge." fi - - name: Check submodule merged to base branch - if: ${{ steps.get_pro_commits.outputs.base_commit != '' }} - uses: actions/github-script@v7 - with: - github-token: ${{ secrets.GITHUB_TOKEN }} - script: | - const submoduleCommit = '${{ steps.get_pro_commits.outputs.pro_commit }}'; - const baseCommit = '${{ steps.get_pro_commits.outputs.base_commit }}'; + - name: Check submodule merged and latest on base branch + if: ${{ steps.get_pro_commits.outputs.base_commit_excluding_merges != '' }} + run: | + cd packages/pro + base_commit_excluding_merges='${{ steps.get_pro_commits.outputs.base_commit_excluding_merges }}' + pro_commit='${{ steps.get_pro_commits.outputs.pro_commit }}' - if (submoduleCommit !== baseCommit) { - console.error('Submodule commit does not match the latest commit on the "${{ steps.get_pro_commits.outputs.target_branch }}" branch.'); - console.error('Refer to the pro repo to merge your changes: https://github.com/Budibase/budibase-pro/blob/master/docs/getting_started.md') - process.exit(1); - } else { - console.log('All good, the submodule had been merged and setup correctly!') - } + any_commit=$(git log --no-merges $base_commit_excluding_merges...$pro_commit) + + if [ -n "$any_commit" ]; then + echo $any_commit + + echo "An error occurred: " + echo 'Submodule commit does not match the latest commit on the "${{ steps.get_pro_commits.outputs.target_branch }}" branch.' + echo 'Refer to the pro repo to merge your changes: https://github.com/Budibase/budibase-pro/blob/master/docs/getting_started.md' + + exit 1 + else + echo 'All good, the submodule had been merged and setup correctly!' + fi check-accountportal-submodule: runs-on: ubuntu-latest @@ -275,7 +243,15 @@ jobs: token: ${{ secrets.PERSONAL_ACCESS_TOKEN || github.token }} fetch-depth: 0 - - name: Check account portal commit + - uses: dorny/paths-filter@v3 + id: changes + with: + filters: | + src: + - packages/account-portal/** + + - if: steps.changes.outputs.src == 'true' + name: Check account portal commit id: get_accountportal_commits run: | cd packages/account-portal diff --git a/.gitignore b/.gitignore index 3eb705dbbf..b68ddd975f 100644 --- a/.gitignore +++ b/.gitignore @@ -5,6 +5,9 @@ packages/server/runtime_apps/ bb-airgapped.tar.gz *.iml +packages/server/build/oldClientVersions/**/* +packages/builder/src/components/deploy/clientVersions.json + # Logs logs *.log @@ -66,7 +69,6 @@ typings/ # dotenv environment variables file .env -!qa-core/.env !hosting/.env # parcel-bundler cache (https://parceljs.org/) @@ -107,3 +109,4 @@ budibase-component budibase-datasource *.iml +.nx \ No newline at end of file diff --git a/.vscode/launch.json b/.vscode/launch.json index cfd8d7b155..2fda61345b 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -1,4 +1,3 @@ - { // Use IntelliSense to learn about possible attributes. // Hover to view descriptions of existing attributes. @@ -20,6 +19,13 @@ "runtimeArgs": ["--nolazy", "-r", "ts-node/register/transpile-only"], "args": ["${workspaceFolder}/packages/worker/src/index.ts"], "cwd": "${workspaceFolder}/packages/worker" + }, + { + "type": "chrome", + "request": "launch", + "name": "Launch Chrome against localhost", + "url": "http://localhost:10000", + "webRoot": "${workspaceFolder}" } ], "compounds": [ diff --git a/charts/budibase/README.md b/charts/budibase/README.md index 342011bdb1..dea7d1dbae 100644 --- a/charts/budibase/README.md +++ b/charts/budibase/README.md @@ -140,7 +140,7 @@ $ helm install --create-namespace --namespace budibase budibase . -f values.yaml | ingress.className | string | `""` | What ingress class to use. | | ingress.enabled | bool | `true` | Whether to create an Ingress resource pointing to the Budibase proxy. | | ingress.hosts | list | `[]` | Standard hosts block for the Ingress resource. Defaults to pointing to the Budibase proxy. | -| nameOverride | string | `""` | Override the name of the deploymen. Defaults to {{ .Chart.Name }}. | +| nameOverride | string | `""` | Override the name of the deployment. Defaults to {{ .Chart.Name }}. | | service.port | int | `10000` | Port to expose on the service. | | service.type | string | `"ClusterIP"` | Service type for the service that points to the main Budibase proxy pod. | | serviceAccount.annotations | object | `{}` | Annotations to add to the service account | diff --git a/charts/budibase/values.yaml b/charts/budibase/values.yaml index 09262df463..19b6c22d6c 100644 --- a/charts/budibase/values.yaml +++ b/charts/budibase/values.yaml @@ -1,6 +1,6 @@ # -- Passed to all pods created by this chart. Should not ordinarily need to be changed. imagePullSecrets: [] -# -- Override the name of the deploymen. Defaults to {{ .Chart.Name }}. +# -- Override the name of the deployment. Defaults to {{ .Chart.Name }}. nameOverride: "" serviceAccount: diff --git a/eslint-local-rules/index.js b/eslint-local-rules/index.js index 71bb5068da..a4866bc1f8 100644 --- a/eslint-local-rules/index.js +++ b/eslint-local-rules/index.js @@ -7,11 +7,12 @@ module.exports = { if ( /^@budibase\/[^/]+\/.*$/.test(importPath) && - importPath !== "@budibase/backend-core/tests" + importPath !== "@budibase/backend-core/tests" && + importPath !== "@budibase/string-templates/test/utils" ) { context.report({ node, - message: `Importing from @budibase is not allowed, except for @budibase/backend-core/tests.`, + message: `Importing from @budibase is not allowed, except for @budibase/backend-core/tests and @budibase/string-templates/test/utils.`, }) } }, @@ -24,11 +25,9 @@ module.exports = { docs: { description: "disallow the use of 'test.com' in strings and replace it with 'example.com'", - category: "Possible Errors", - recommended: false, }, - schema: [], // no options - fixable: "code", // Indicates that this rule supports automatic fixing + schema: [], + fixable: "code", }, create: function (context) { return { @@ -51,4 +50,39 @@ module.exports = { } }, }, + "email-domain-example-com": { + meta: { + type: "problem", + docs: { + description: + "enforce using the example.com domain for generator.email calls", + }, + fixable: "code", + schema: [], + }, + create: function (context) { + return { + CallExpression(node) { + if ( + node.callee.type === "MemberExpression" && + node.callee.object.name === "generator" && + node.callee.property.name === "email" && + node.arguments.length === 0 + ) { + context.report({ + node, + message: + "Prefer using generator.email with the domain \"{ domain: 'example.com' }\".", + fix: function (fixer) { + return fixer.replaceText( + node, + 'generator.email({ domain: "example.com" })' + ) + }, + }) + } + }, + } + }, + }, } diff --git a/globalSetup.ts b/globalSetup.ts new file mode 100644 index 0000000000..4cb542a3c3 --- /dev/null +++ b/globalSetup.ts @@ -0,0 +1,25 @@ +import { GenericContainer, Wait } from "testcontainers" + +export default async function setup() { + await new GenericContainer("budibase/couchdb") + .withExposedPorts(5984) + .withEnvironment({ + COUCHDB_PASSWORD: "budibase", + COUCHDB_USER: "budibase", + }) + .withCopyContentToContainer([ + { + content: ` + [log] + level = warn + `, + target: "/opt/couchdb/etc/local.d/test-couchdb.ini", + }, + ]) + .withWaitStrategy( + Wait.forSuccessfulCommand( + "curl http://budibase:budibase@localhost:5984/_up" + ).withStartupTimeout(20000) + ) + .start() +} diff --git a/hosting/single/Dockerfile b/hosting/single/Dockerfile index ee98b0729d..be01056b53 100644 --- a/hosting/single/Dockerfile +++ b/hosting/single/Dockerfile @@ -12,8 +12,6 @@ COPY .yarnrc . COPY packages/server/package.json packages/server/package.json COPY packages/worker/package.json packages/worker/package.json -# string-templates does not get bundled during the esbuild process, so we want to use the local version -COPY packages/string-templates/package.json packages/string-templates/package.json COPY scripts/removeWorkspaceDependencies.sh scripts/removeWorkspaceDependencies.sh @@ -26,7 +24,7 @@ RUN ./scripts/removeWorkspaceDependencies.sh packages/worker/package.json RUN echo '' > scripts/syncProPackage.js RUN jq 'del(.scripts.postinstall)' package.json > temp.json && mv temp.json package.json RUN ./scripts/removeWorkspaceDependencies.sh package.json -RUN --mount=type=cache,target=/root/.yarn YARN_CACHE_FOLDER=/root/.yarn yarn install --production +RUN --mount=type=cache,target=/root/.yarn YARN_CACHE_FOLDER=/root/.yarn yarn install --production --frozen-lockfile # copy the actual code COPY packages/server/dist packages/server/dist @@ -35,7 +33,6 @@ COPY packages/server/client packages/server/client COPY packages/server/builder packages/server/builder COPY packages/worker/dist packages/worker/dist COPY packages/worker/pm2.config.js packages/worker/pm2.config.js -COPY packages/string-templates packages/string-templates FROM budibase/couchdb:v3.3.3 as runner @@ -52,11 +49,11 @@ RUN apt-get update && \ # Install postgres client for pg_dump utils RUN apt install -y software-properties-common apt-transport-https ca-certificates gnupg \ - && curl -fsSl https://www.postgresql.org/media/keys/ACCC4CF8.asc | gpg --dearmor | tee /usr/share/keyrings/postgresql.gpg > /dev/null \ - && echo deb [arch=amd64,arm64,ppc64el signed-by=/usr/share/keyrings/postgresql.gpg] http://apt.postgresql.org/pub/repos/apt/ $(lsb_release -cs)-pgdg main | tee /etc/apt/sources.list.d/postgresql.list \ - && apt update -y \ - && apt install postgresql-client-15 -y \ - && apt remove software-properties-common apt-transport-https gpg -y + && curl -fsSl https://www.postgresql.org/media/keys/ACCC4CF8.asc | gpg --dearmor | tee /usr/share/keyrings/postgresql.gpg > /dev/null \ + && echo deb [arch=amd64,arm64,ppc64el signed-by=/usr/share/keyrings/postgresql.gpg] http://apt.postgresql.org/pub/repos/apt/ $(lsb_release -cs)-pgdg main | tee /etc/apt/sources.list.d/postgresql.list \ + && apt update -y \ + && apt install postgresql-client-15 -y \ + && apt remove software-properties-common apt-transport-https gpg -y # We use pm2 in order to run multiple node processes in a single container RUN npm install --global pm2 @@ -100,9 +97,6 @@ COPY --from=build /app/node_modules /node_modules COPY --from=build /app/package.json /package.json COPY --from=build /app/packages/server /app COPY --from=build /app/packages/worker /worker -COPY --from=build /app/packages/string-templates /string-templates - -RUN cd /string-templates && yarn link && cd ../app && yarn link @budibase/string-templates && cd ../worker && yarn link @budibase/string-templates EXPOSE 80 diff --git a/jestTestcontainersConfigGenerator.js b/jestTestcontainersConfigGenerator.js deleted file mode 100644 index 1e39ed771f..0000000000 --- a/jestTestcontainersConfigGenerator.js +++ /dev/null @@ -1,16 +0,0 @@ -module.exports = () => { - return { - couchdb: { - image: "budibase/couchdb", - ports: [5984], - env: { - COUCHDB_PASSWORD: "budibase", - COUCHDB_USER: "budibase", - }, - wait: { - type: "ports", - timeout: 20000, - } - } - } -} diff --git a/lerna.json b/lerna.json index c06173fe04..93b103ee00 100644 --- a/lerna.json +++ b/lerna.json @@ -1,5 +1,5 @@ { - "version": "2.21.0", + "version": "2.22.15", "npmClient": "yarn", "packages": [ "packages/*", diff --git a/package.json b/package.json index 0a20f01d52..c927002c88 100644 --- a/package.json +++ b/package.json @@ -12,6 +12,7 @@ "esbuild-node-externals": "^1.8.0", "eslint": "^8.52.0", "eslint-plugin-import": "^2.29.0", + "eslint-plugin-jest": "^27.9.0", "eslint-plugin-local-rules": "^2.0.0", "eslint-plugin-svelte": "^2.34.0", "husky": "^8.0.3", @@ -25,12 +26,16 @@ "svelte": "^4.2.10", "svelte-eslint-parser": "^0.33.1", "typescript": "5.2.2", + "typescript-eslint": "^7.3.1", "yargs": "^17.7.2" }, "scripts": { "preinstall": "node scripts/syncProPackage.js", + "get-past-client-version": "node scripts/getPastClientVersion.js", "setup": "git config submodule.recurse true && git submodule update && node ./hosting/scripts/setup.js && yarn && yarn build && yarn dev", "build": "NODE_OPTIONS=--max-old-space-size=1500 lerna run build --stream", + "build:oss": "NODE_OPTIONS=--max-old-space-size=1500 lerna run build --stream --ignore @budibase/account-portal --ignore @budibase/account-portal-server --ignore @budibase/account-portal-ui", + "build:account-portal": "NODE_OPTIONS=--max-old-space-size=1500 lerna run build --stream --scope @budibase/account-portal --scope @budibase/account-portal-server --scope @budibase/account-portal-ui", "build:dev": "lerna run --stream prebuild && yarn nx run-many --target=build --output-style=dynamic --watch --preserveWatchOutput", "check:types": "lerna run check:types", "build:sdk": "lerna run --stream build:sdk", @@ -53,11 +58,11 @@ "dev:built": "yarn run kill-all && cd packages/server && yarn dev:stack:up && cd ../../ && lerna run --stream dev:built", "dev:docker": "yarn build --scope @budibase/server --scope @budibase/worker && docker-compose -f hosting/docker-compose.build.yaml -f hosting/docker-compose.dev.yaml --env-file hosting/.env up --build --scale proxy-service=0", "test": "lerna run --stream test --stream", - "lint:eslint": "eslint packages qa-core --max-warnings=0", - "lint:prettier": "prettier --check \"packages/**/*.{js,ts,svelte}\" && prettier --write \"examples/**/*.{js,ts,svelte}\" && prettier --check \"qa-core/**/*.{js,ts,svelte}\"", + "lint:eslint": "eslint packages --max-warnings=0", + "lint:prettier": "prettier --check \"packages/**/*.{js,ts,svelte}\" && prettier --write \"examples/**/*.{js,ts,svelte}\"", "lint": "yarn run lint:eslint && yarn run lint:prettier", - "lint:fix:eslint": "eslint --fix --max-warnings=0 packages qa-core", - "lint:fix:prettier": "prettier --write \"packages/**/*.{js,ts,svelte}\" && prettier --write \"examples/**/*.{js,ts,svelte}\" && prettier --write \"qa-core/**/*.{js,ts,svelte}\"", + "lint:fix:eslint": "eslint --fix --max-warnings=0 packages", + "lint:fix:prettier": "prettier --write \"packages/**/*.{js,ts,svelte}\" && prettier --write \"examples/**/*.{js,ts,svelte}\"", "lint:fix": "yarn run lint:fix:eslint && yarn run lint:fix:prettier", "build:specs": "lerna run --stream specs", "build:docker:airgap": "node hosting/scripts/airgapped/airgappedDockerBuild", diff --git a/packages/account-portal b/packages/account-portal index 806b6fd5c1..360ad2dc29 160000 --- a/packages/account-portal +++ b/packages/account-portal @@ -1 +1 @@ -Subproject commit 806b6fd5c11c284ebf4a01627d75db939f0f8152 +Subproject commit 360ad2dc29c3f1fd5a1182ae258c45666b7f5eb1 diff --git a/packages/backend-core/jest-testcontainers-config.js b/packages/backend-core/jest-testcontainers-config.js deleted file mode 100644 index 8ac0f0cd9d..0000000000 --- a/packages/backend-core/jest-testcontainers-config.js +++ /dev/null @@ -1,8 +0,0 @@ -const { join } = require("path") -require("dotenv").config({ - path: join(__dirname, "..", "..", "hosting", ".env"), -}) - -const jestTestcontainersConfigGenerator = require("../../jestTestcontainersConfigGenerator") - -module.exports = jestTestcontainersConfigGenerator() diff --git a/packages/backend-core/jest.config.ts b/packages/backend-core/jest.config.ts index 3f1065ead2..c944b0d7e1 100644 --- a/packages/backend-core/jest.config.ts +++ b/packages/backend-core/jest.config.ts @@ -1,8 +1,8 @@ import { Config } from "@jest/types" const baseConfig: Config.InitialProjectOptions = { - preset: "@trendyol/jest-testcontainers", setupFiles: ["./tests/jestEnv.ts"], + globalSetup: "./../../globalSetup.ts", setupFilesAfterEnv: ["./tests/jestSetup.ts"], transform: { "^.+\\.ts?$": "@swc/jest", diff --git a/packages/backend-core/package.json b/packages/backend-core/package.json index 3f8c34f823..030fec8728 100644 --- a/packages/backend-core/package.json +++ b/packages/backend-core/package.json @@ -60,14 +60,13 @@ "@shopify/jest-koa-mocks": "5.1.1", "@swc/core": "1.3.71", "@swc/jest": "0.2.27", - "@trendyol/jest-testcontainers": "^2.1.1", "@types/chance": "1.1.3", "@types/cookies": "0.7.8", "@types/jest": "29.5.5", "@types/lodash": "4.14.200", "@types/node-fetch": "2.6.4", "@types/pouchdb": "6.4.0", - "@types/redlock": "4.0.3", + "@types/redlock": "4.0.7", "@types/semver": "7.3.7", "@types/tar-fs": "2.0.1", "@types/uuid": "8.3.4", @@ -78,6 +77,7 @@ "jest-serial-runner": "1.2.1", "pino-pretty": "10.0.0", "pouchdb-adapter-memory": "7.2.2", + "testcontainers": "^10.7.2", "timekeeper": "2.2.0", "typescript": "5.2.2" }, diff --git a/packages/backend-core/scripts/test.sh b/packages/backend-core/scripts/test.sh index 7d19ec96cc..b9937e3a4a 100644 --- a/packages/backend-core/scripts/test.sh +++ b/packages/backend-core/scripts/test.sh @@ -4,10 +4,10 @@ set -e if [[ -n $CI ]] then # --runInBand performs better in ci where resources are limited - echo "jest --coverage --runInBand --forceExit" - jest --coverage --runInBand --forceExit + echo "jest --coverage --runInBand --forceExit $@" + jest --coverage --runInBand --forceExit $@ else # --maxWorkers performs better in development - echo "jest --coverage --detectOpenHandles" - jest --coverage --detectOpenHandles + echo "jest --coverage --forceExit --detectOpenHandles $@" + jest --coverage --forceExit --detectOpenHandles $@ fi \ No newline at end of file diff --git a/packages/backend-core/src/auth/auth.ts b/packages/backend-core/src/auth/auth.ts index 1951c7986c..87ac46cf1c 100644 --- a/packages/backend-core/src/auth/auth.ts +++ b/packages/backend-core/src/auth/auth.ts @@ -133,7 +133,7 @@ export async function refreshOAuthToken( configId?: string ): Promise { switch (providerType) { - case SSOProviderType.OIDC: + case SSOProviderType.OIDC: { if (!configId) { return { err: { data: "OIDC config id not provided" } } } @@ -142,12 +142,14 @@ export async function refreshOAuthToken( return { err: { data: "OIDC configuration not found" } } } return refreshOIDCAccessToken(oidcConfig, refreshToken) - case SSOProviderType.GOOGLE: + } + case SSOProviderType.GOOGLE: { let googleConfig = await configs.getGoogleConfig() if (!googleConfig) { return { err: { data: "Google configuration not found" } } } return refreshGoogleAccessToken(googleConfig, refreshToken) + } } } diff --git a/packages/backend-core/src/auth/tests/auth.spec.ts b/packages/backend-core/src/auth/tests/auth.spec.ts index 3ae691be58..a80e1ea739 100644 --- a/packages/backend-core/src/auth/tests/auth.spec.ts +++ b/packages/backend-core/src/auth/tests/auth.spec.ts @@ -8,7 +8,7 @@ describe("platformLogout", () => { await testEnv.withTenant(async () => { const ctx = structures.koa.newContext() await auth.platformLogout({ ctx, userId: "test" }) - expect(events.auth.logout).toBeCalledTimes(1) + expect(events.auth.logout).toHaveBeenCalledTimes(1) }) }) }) diff --git a/packages/backend-core/src/cache/base/index.ts b/packages/backend-core/src/cache/base/index.ts index 264984c6a5..433941b5c7 100644 --- a/packages/backend-core/src/cache/base/index.ts +++ b/packages/backend-core/src/cache/base/index.ts @@ -23,6 +23,18 @@ export default class BaseCache { return client.keys(pattern) } + async exists(key: string, opts = { useTenancy: true }) { + key = opts.useTenancy ? generateTenantKey(key) : key + const client = await this.getClient() + return client.exists(key) + } + + async scan(key: string, opts = { useTenancy: true }) { + key = opts.useTenancy ? generateTenantKey(key) : key + const client = await this.getClient() + return client.scan(key) + } + /** * Read only from the cache. */ @@ -32,6 +44,15 @@ export default class BaseCache { return client.get(key) } + /** + * Read only from the cache. + */ + async bulkGet(keys: string[], opts = { useTenancy: true }) { + keys = opts.useTenancy ? keys.map(key => generateTenantKey(key)) : keys + const client = await this.getClient() + return client.bulkGet(keys) + } + /** * Write to the cache. */ @@ -46,6 +67,25 @@ export default class BaseCache { await client.store(key, value, ttl) } + /** + * Bulk write to the cache. + */ + async bulkStore( + data: Record, + ttl: number | null = null, + opts = { useTenancy: true } + ) { + if (opts.useTenancy) { + data = Object.entries(data).reduce((acc, [key, value]) => { + acc[generateTenantKey(key)] = value + return acc + }, {} as Record) + } + + const client = await this.getClient() + await client.bulkStore(data, ttl) + } + /** * Remove from cache. */ @@ -55,15 +95,24 @@ export default class BaseCache { return client.delete(key) } + /** + * Remove from cache. + */ + async bulkDelete(keys: string[], opts = { useTenancy: true }) { + keys = opts.useTenancy ? keys.map(key => generateTenantKey(key)) : keys + const client = await this.getClient() + return client.bulkDelete(keys) + } + /** * Read from the cache. Write to the cache if not exists. */ - async withCache( + async withCache( key: string, - ttl: number, - fetchFn: any, + ttl: number | null = null, + fetchFn: () => Promise | T, opts = { useTenancy: true } - ) { + ): Promise { const cachedValue = await this.get(key, opts) if (cachedValue) { return cachedValue @@ -80,7 +129,7 @@ export default class BaseCache { } } - async bustCache(key: string, opts = { client: null }) { + async bustCache(key: string) { const client = await this.getClient() try { await client.delete(generateTenantKey(key)) @@ -89,4 +138,13 @@ export default class BaseCache { throw err } } + + /** + * Delete the entry if the provided value matches the stored one. + */ + async deleteIfValue(key: string, value: any, opts = { useTenancy: true }) { + key = opts.useTenancy ? generateTenantKey(key) : key + const client = await this.getClient() + await client.deleteIfValue(key, value) + } } diff --git a/packages/backend-core/src/cache/docWritethrough.ts b/packages/backend-core/src/cache/docWritethrough.ts new file mode 100644 index 0000000000..05f13a0d91 --- /dev/null +++ b/packages/backend-core/src/cache/docWritethrough.ts @@ -0,0 +1,105 @@ +import { AnyDocument, Database } from "@budibase/types" + +import { JobQueue, Queue, createQueue } from "../queue" +import * as dbUtils from "../db" + +interface ProcessDocMessage { + dbName: string + docId: string + data: Record +} + +const PERSIST_MAX_ATTEMPTS = 100 +let processor: DocWritethroughProcessor | undefined + +export class DocWritethroughProcessor { + private static _queue: Queue + + public static get queue() { + if (!DocWritethroughProcessor._queue) { + DocWritethroughProcessor._queue = createQueue( + JobQueue.DOC_WRITETHROUGH_QUEUE, + { + jobOptions: { + attempts: PERSIST_MAX_ATTEMPTS, + }, + } + ) + } + + return DocWritethroughProcessor._queue + } + + init() { + DocWritethroughProcessor.queue.process(async message => { + try { + await this.persistToDb(message.data) + } catch (err: any) { + if (err.status === 409) { + // If we get a 409, it means that another job updated it meanwhile. We want to retry it to persist it again. + throw new Error( + `Conflict persisting message ${message.id}. Attempt ${message.attemptsMade}` + ) + } + + throw err + } + }) + return this + } + + private async persistToDb({ + dbName, + docId, + data, + }: { + dbName: string + docId: string + data: Record + }) { + const db = dbUtils.getDB(dbName) + let doc: AnyDocument | undefined + try { + doc = await db.get(docId) + } catch { + doc = { _id: docId } + } + + doc = { ...doc, ...data } + await db.put(doc) + } +} + +export class DocWritethrough { + private db: Database + private _docId: string + + constructor(db: Database, docId: string) { + this.db = db + this._docId = docId + } + + get docId() { + return this._docId + } + + async patch(data: Record) { + await DocWritethroughProcessor.queue.add({ + dbName: this.db.name, + docId: this.docId, + data, + }) + } +} + +export function init(): DocWritethroughProcessor { + processor = new DocWritethroughProcessor().init() + return processor +} + +export function getProcessor(): DocWritethroughProcessor { + if (!processor) { + return init() + } + return processor +} diff --git a/packages/backend-core/src/cache/generic.ts b/packages/backend-core/src/cache/generic.ts index 3ac323a8d4..2d6d8b9472 100644 --- a/packages/backend-core/src/cache/generic.ts +++ b/packages/backend-core/src/cache/generic.ts @@ -26,7 +26,8 @@ export const store = (...args: Parameters) => GENERIC.store(...args) export const destroy = (...args: Parameters) => GENERIC.delete(...args) -export const withCache = (...args: Parameters) => - GENERIC.withCache(...args) +export const withCache = ( + ...args: Parameters> +) => GENERIC.withCache(...args) export const bustCache = (...args: Parameters) => GENERIC.bustCache(...args) diff --git a/packages/backend-core/src/cache/index.ts b/packages/backend-core/src/cache/index.ts index 4fa986e4e2..3b25108634 100644 --- a/packages/backend-core/src/cache/index.ts +++ b/packages/backend-core/src/cache/index.ts @@ -5,3 +5,4 @@ export * as writethrough from "./writethrough" export * as invite from "./invite" export * as passwordReset from "./passwordReset" export * from "./generic" +export * as docWritethrough from "./docWritethrough" diff --git a/packages/backend-core/src/cache/invite.ts b/packages/backend-core/src/cache/invite.ts index e43ebc4aa8..e3d698bcc6 100644 --- a/packages/backend-core/src/cache/invite.ts +++ b/packages/backend-core/src/cache/invite.ts @@ -1,5 +1,5 @@ import * as utils from "../utils" -import { Duration, DurationType } from "../utils" +import { Duration } from "../utils" import env from "../environment" import { getTenantId } from "../context" import * as redis from "../redis/init" diff --git a/packages/backend-core/src/cache/tests/docWritethrough.spec.ts b/packages/backend-core/src/cache/tests/docWritethrough.spec.ts new file mode 100644 index 0000000000..47b3f0672f --- /dev/null +++ b/packages/backend-core/src/cache/tests/docWritethrough.spec.ts @@ -0,0 +1,294 @@ +import tk from "timekeeper" + +import _ from "lodash" +import { DBTestConfiguration, generator, structures } from "../../../tests" +import { getDB } from "../../db" + +import { + DocWritethrough, + DocWritethroughProcessor, + init, +} from "../docWritethrough" + +import InMemoryQueue from "../../queue/inMemoryQueue" + +const initialTime = Date.now() + +async function waitForQueueCompletion() { + const queue: InMemoryQueue = DocWritethroughProcessor.queue as never + await queue.waitForCompletion() +} + +describe("docWritethrough", () => { + beforeAll(() => { + init() + }) + + const config = new DBTestConfiguration() + + const db = getDB(structures.db.id()) + let documentId: string + let docWritethrough: DocWritethrough + + describe("patch", () => { + function generatePatchObject(fieldCount: number) { + const keys = generator.unique(() => generator.guid(), fieldCount) + return keys.reduce((acc, c) => { + acc[c] = generator.word() + return acc + }, {} as Record) + } + + beforeEach(async () => { + jest.clearAllMocks() + documentId = structures.uuid() + docWritethrough = new DocWritethrough(db, documentId) + }) + + it("patching will not persist until the messages are persisted", async () => { + await config.doInTenant(async () => { + await docWritethrough.patch(generatePatchObject(2)) + await docWritethrough.patch(generatePatchObject(2)) + + expect(await db.exists(documentId)).toBe(false) + }) + }) + + it("patching will persist when the messages are persisted", async () => { + await config.doInTenant(async () => { + const patch1 = generatePatchObject(2) + const patch2 = generatePatchObject(2) + await docWritethrough.patch(patch1) + await docWritethrough.patch(patch2) + + await waitForQueueCompletion() + + // This will not be persisted + const patch3 = generatePatchObject(3) + await docWritethrough.patch(patch3) + + expect(await db.get(documentId)).toEqual({ + _id: documentId, + ...patch1, + ...patch2, + _rev: expect.stringMatching(/2-.+/), + createdAt: new Date(initialTime).toISOString(), + updatedAt: new Date(initialTime).toISOString(), + }) + }) + }) + + it("patching will persist keeping the previous data", async () => { + await config.doInTenant(async () => { + const patch1 = generatePatchObject(2) + const patch2 = generatePatchObject(2) + await docWritethrough.patch(patch1) + await docWritethrough.patch(patch2) + + await waitForQueueCompletion() + + const patch3 = generatePatchObject(3) + await docWritethrough.patch(patch3) + + await waitForQueueCompletion() + + expect(await db.get(documentId)).toEqual( + expect.objectContaining({ + _id: documentId, + ...patch1, + ...patch2, + ...patch3, + }) + ) + }) + }) + + it("date audit fields are set correctly when persisting", async () => { + await config.doInTenant(async () => { + const patch1 = generatePatchObject(2) + const patch2 = generatePatchObject(2) + await docWritethrough.patch(patch1) + const date1 = new Date() + await waitForQueueCompletion() + await docWritethrough.patch(patch2) + + tk.travel(Date.now() + 100) + const date2 = new Date() + await waitForQueueCompletion() + + expect(date1).not.toEqual(date2) + expect(await db.get(documentId)).toEqual( + expect.objectContaining({ + createdAt: date1.toISOString(), + updatedAt: date2.toISOString(), + }) + ) + }) + }) + + it("concurrent patches will override keys", async () => { + await config.doInTenant(async () => { + const patch1 = generatePatchObject(2) + await docWritethrough.patch(patch1) + await waitForQueueCompletion() + const patch2 = generatePatchObject(1) + await docWritethrough.patch(patch2) + + const keyToOverride = _.sample(Object.keys(patch1))! + expect(await db.get(documentId)).toEqual( + expect.objectContaining({ + [keyToOverride]: patch1[keyToOverride], + }) + ) + + await waitForQueueCompletion() + + const patch3 = { + ...generatePatchObject(3), + [keyToOverride]: generator.word(), + } + await docWritethrough.patch(patch3) + await waitForQueueCompletion() + + expect(await db.get(documentId)).toEqual( + expect.objectContaining({ + ...patch1, + ...patch2, + ...patch3, + }) + ) + }) + }) + + it("concurrent patches to different docWritethrough will not pollute each other", async () => { + await config.doInTenant(async () => { + const secondDocWritethrough = new DocWritethrough( + db, + structures.db.id() + ) + + const doc1Patch = generatePatchObject(2) + await docWritethrough.patch(doc1Patch) + const doc2Patch = generatePatchObject(1) + await secondDocWritethrough.patch(doc2Patch) + + await waitForQueueCompletion() + + const doc1Patch2 = generatePatchObject(3) + await docWritethrough.patch(doc1Patch2) + const doc2Patch2 = generatePatchObject(3) + await secondDocWritethrough.patch(doc2Patch2) + await waitForQueueCompletion() + + expect(await db.get(docWritethrough.docId)).toEqual( + expect.objectContaining({ + ...doc1Patch, + ...doc1Patch2, + }) + ) + + expect(await db.get(secondDocWritethrough.docId)).toEqual( + expect.objectContaining({ + ...doc2Patch, + ...doc2Patch2, + }) + ) + }) + }) + + it("cached values are persisted only once", async () => { + await config.doInTenant(async () => { + const initialPatch = generatePatchObject(5) + + await docWritethrough.patch(initialPatch) + await waitForQueueCompletion() + + expect(await db.get(documentId)).toEqual( + expect.objectContaining(initialPatch) + ) + + await db.remove(await db.get(documentId)) + + await waitForQueueCompletion() + const extraPatch = generatePatchObject(5) + await docWritethrough.patch(extraPatch) + await waitForQueueCompletion() + + expect(await db.get(documentId)).toEqual( + expect.objectContaining(extraPatch) + ) + expect(await db.get(documentId)).not.toEqual( + expect.objectContaining(initialPatch) + ) + }) + }) + + it("concurrent calls will not cause conflicts", async () => { + async function parallelPatch(count: number) { + const patches = Array.from({ length: count }).map(() => + generatePatchObject(1) + ) + await Promise.all(patches.map(p => docWritethrough.patch(p))) + + return patches.reduce((acc, c) => { + acc = { ...acc, ...c } + return acc + }, {}) + } + const queueMessageSpy = jest.spyOn(DocWritethroughProcessor.queue, "add") + + await config.doInTenant(async () => { + let patches = await parallelPatch(5) + expect(queueMessageSpy).toHaveBeenCalledTimes(5) + + await waitForQueueCompletion() + expect(await db.get(documentId)).toEqual( + expect.objectContaining(patches) + ) + + patches = { ...patches, ...(await parallelPatch(40)) } + expect(queueMessageSpy).toHaveBeenCalledTimes(45) + + await waitForQueueCompletion() + expect(await db.get(documentId)).toEqual( + expect.objectContaining(patches) + ) + + patches = { ...patches, ...(await parallelPatch(10)) } + expect(queueMessageSpy).toHaveBeenCalledTimes(55) + + await waitForQueueCompletion() + expect(await db.get(documentId)).toEqual( + expect.objectContaining(patches) + ) + }) + }) + + // This is not yet supported + // eslint-disable-next-line jest/no-disabled-tests + it.skip("patches will execute in order", async () => { + let incrementalValue = 0 + const keyToOverride = generator.word() + async function incrementalPatches(count: number) { + for (let i = 0; i < count; i++) { + await docWritethrough.patch({ [keyToOverride]: incrementalValue++ }) + } + } + + await config.doInTenant(async () => { + await incrementalPatches(5) + + await waitForQueueCompletion() + expect(await db.get(documentId)).toEqual( + expect.objectContaining({ [keyToOverride]: 5 }) + ) + + await incrementalPatches(40) + await waitForQueueCompletion() + expect(await db.get(documentId)).toEqual( + expect.objectContaining({ [keyToOverride]: 45 }) + ) + }) + }) + }) +}) diff --git a/packages/backend-core/src/cache/tests/user.spec.ts b/packages/backend-core/src/cache/tests/user.spec.ts index 80e5bc3063..49a8d51c16 100644 --- a/packages/backend-core/src/cache/tests/user.spec.ts +++ b/packages/backend-core/src/cache/tests/user.spec.ts @@ -55,8 +55,8 @@ describe("user cache", () => { })), }) - expect(UserDB.bulkGet).toBeCalledTimes(1) - expect(UserDB.bulkGet).toBeCalledWith(userIdsToRequest) + expect(UserDB.bulkGet).toHaveBeenCalledTimes(1) + expect(UserDB.bulkGet).toHaveBeenCalledWith(userIdsToRequest) }) it("on a second all, all of them are retrieved from cache", async () => { @@ -82,7 +82,7 @@ describe("user cache", () => { ), }) - expect(UserDB.bulkGet).toBeCalledTimes(1) + expect(UserDB.bulkGet).toHaveBeenCalledTimes(1) }) it("when some users are cached, only the missing ones are retrieved from db", async () => { @@ -110,8 +110,8 @@ describe("user cache", () => { ), }) - expect(UserDB.bulkGet).toBeCalledTimes(1) - expect(UserDB.bulkGet).toBeCalledWith([ + expect(UserDB.bulkGet).toHaveBeenCalledTimes(1) + expect(UserDB.bulkGet).toHaveBeenCalledWith([ userIdsToRequest[1], userIdsToRequest[2], userIdsToRequest[4], diff --git a/packages/backend-core/src/cache/user.ts b/packages/backend-core/src/cache/user.ts index 313b9a4d4a..ecfa20f99e 100644 --- a/packages/backend-core/src/cache/user.ts +++ b/packages/backend-core/src/cache/user.ts @@ -6,7 +6,7 @@ import env from "../environment" import * as accounts from "../accounts" import { UserDB } from "../users" import { sdk } from "@budibase/shared-core" -import { User } from "@budibase/types" +import { User, UserMetadata } from "@budibase/types" const EXPIRY_SECONDS = 3600 @@ -15,7 +15,7 @@ const EXPIRY_SECONDS = 3600 */ async function populateFromDB(userId: string, tenantId: string) { const db = tenancy.getTenantDB(tenantId) - const user = await db.get(userId) + const user = await db.get(userId) user.budibaseAccess = true if (!env.SELF_HOSTED && !env.DISABLE_ACCOUNT_PORTAL) { const account = await accounts.getAccount(user.email) diff --git a/packages/backend-core/src/cache/writethrough.ts b/packages/backend-core/src/cache/writethrough.ts index 5cafe418d7..cd7409ca15 100644 --- a/packages/backend-core/src/cache/writethrough.ts +++ b/packages/backend-core/src/cache/writethrough.ts @@ -8,7 +8,7 @@ const DEFAULT_WRITE_RATE_MS = 10000 let CACHE: BaseCache | null = null interface CacheItem { - doc: any + doc: T lastWrite: number } diff --git a/packages/backend-core/src/constants/db.ts b/packages/backend-core/src/constants/db.ts index ac00483021..f4caac502e 100644 --- a/packages/backend-core/src/constants/db.ts +++ b/packages/backend-core/src/constants/db.ts @@ -57,6 +57,9 @@ export const StaticDatabases = { AUDIT_LOGS: { name: "audit-logs", }, + SCIM_LOGS: { + name: "scim-logs", + }, } export const APP_PREFIX = prefixed(DocumentType.APP) diff --git a/packages/backend-core/src/context/mainContext.ts b/packages/backend-core/src/context/mainContext.ts index 36fd5dcb48..6cea7efeba 100644 --- a/packages/backend-core/src/context/mainContext.ts +++ b/packages/backend-core/src/context/mainContext.ts @@ -10,7 +10,7 @@ import { StaticDatabases, DEFAULT_TENANT_ID, } from "../constants" -import { Database, IdentityContext } from "@budibase/types" +import { Database, IdentityContext, Snippet, App } from "@budibase/types" import { ContextMap } from "./types" let TEST_APP_ID: string | null = null @@ -35,6 +35,17 @@ export function getAuditLogDBName(tenantId?: string) { } } +export function getScimDBName(tenantId?: string) { + if (!tenantId) { + tenantId = getTenantId() + } + if (tenantId === DEFAULT_TENANT_ID) { + return StaticDatabases.SCIM_LOGS.name + } else { + return `${tenantId}${SEPARATOR}${StaticDatabases.SCIM_LOGS.name}` + } +} + export function baseGlobalDBName(tenantId: string | undefined | null) { if (!tenantId || tenantId === DEFAULT_TENANT_ID) { return StaticDatabases.GLOBAL.name @@ -111,10 +122,10 @@ export async function doInAutomationContext(params: { automationId: string task: () => T }): Promise { - const tenantId = getTenantIDFromAppID(params.appId) + await ensureSnippetContext() return newContext( { - tenantId, + tenantId: getTenantIDFromAppID(params.appId), appId: params.appId, automationId: params.automationId, }, @@ -270,6 +281,27 @@ export function doInScimContext(task: any) { return newContext(updates, task) } +export async function ensureSnippetContext() { + const ctx = getCurrentContext() + + // If we've already added snippets to context, continue + if (!ctx || ctx.snippets) { + return + } + + // Otherwise get snippets for this app and update context + let snippets: Snippet[] | undefined + const db = getAppDB() + if (db && !env.isTest()) { + const app = await db.get(DocumentType.APP_METADATA) + snippets = app.snippets + } + + // Always set snippets to a non-null value so that we can tell we've attempted + // to load snippets + ctx.snippets = snippets || [] +} + export function getEnvironmentVariables() { const context = Context.get() if (!context.environmentVariables) { diff --git a/packages/backend-core/src/context/tests/index.spec.ts b/packages/backend-core/src/context/tests/index.spec.ts index cfc820e169..2d89131549 100644 --- a/packages/backend-core/src/context/tests/index.spec.ts +++ b/packages/backend-core/src/context/tests/index.spec.ts @@ -246,7 +246,7 @@ describe("context", () => { context.doInAppMigrationContext(db.generateAppID(), async () => { await otherContextCall() }) - ).rejects.toThrowError( + ).rejects.toThrow( "The context cannot be changed, a migration is currently running" ) } diff --git a/packages/backend-core/src/context/types.ts b/packages/backend-core/src/context/types.ts index 6fb9f44fad..f297d3089f 100644 --- a/packages/backend-core/src/context/types.ts +++ b/packages/backend-core/src/context/types.ts @@ -1,5 +1,4 @@ -import { IdentityContext, VM } from "@budibase/types" -import { ExecutionTimeTracker } from "../timers" +import { IdentityContext, Snippet, VM } from "@budibase/types" // keep this out of Budibase types, don't want to expose context info export type ContextMap = { @@ -10,6 +9,7 @@ export type ContextMap = { isScim?: boolean automationId?: string isMigrating?: boolean - jsExecutionTracker?: ExecutionTimeTracker vm?: VM + cleanup?: (() => void | Promise)[] + snippets?: Snippet[] } diff --git a/packages/backend-core/src/db/Replication.ts b/packages/backend-core/src/db/Replication.ts index f91a37ce8f..735c2fa86e 100644 --- a/packages/backend-core/src/db/Replication.ts +++ b/packages/backend-core/src/db/Replication.ts @@ -1,66 +1,57 @@ +import PouchDB from "pouchdb" import { getPouchDB, closePouchDB } from "./couch" import { DocumentType } from "../constants" class Replication { - source: any - target: any - replication: any + source: PouchDB.Database + target: PouchDB.Database - /** - * - * @param source - the DB you want to replicate or rollback to - * @param target - the DB you want to replicate to, or rollback from - */ - constructor({ source, target }: any) { + constructor({ source, target }: { source: string; target: string }) { this.source = getPouchDB(source) this.target = getPouchDB(target) } - close() { - return Promise.all([closePouchDB(this.source), closePouchDB(this.target)]) + async close() { + await Promise.all([closePouchDB(this.source), closePouchDB(this.target)]) } - promisify(operation: any, opts = {}) { - return new Promise(resolve => { - operation(this.target, opts) - .on("denied", function (err: any) { + replicate(opts: PouchDB.Replication.ReplicateOptions = {}) { + return new Promise>(resolve => { + this.source.replicate + .to(this.target, opts) + .on("denied", function (err) { // a document failed to replicate (e.g. due to permissions) throw new Error(`Denied: Document failed to replicate ${err}`) }) - .on("complete", function (info: any) { + .on("complete", function (info) { return resolve(info) }) - .on("error", function (err: any) { - throw new Error(`Replication Error: ${err}`) + .on("error", function (err) { + throw err }) }) } - /** - * Two way replication operation, intended to be promise based. - * @param opts - PouchDB replication options - */ - sync(opts = {}) { - this.replication = this.promisify(this.source.sync, opts) - return this.replication - } + appReplicateOpts( + opts: PouchDB.Replication.ReplicateOptions = {} + ): PouchDB.Replication.ReplicateOptions { + if (typeof opts.filter === "string") { + return opts + } - /** - * One way replication operation, intended to be promise based. - * @param opts - PouchDB replication options - */ - replicate(opts = {}) { - this.replication = this.promisify(this.source.replicate.to, opts) - return this.replication - } + const filter = opts.filter + delete opts.filter - appReplicateOpts() { return { - filter: (doc: any) => { + ...opts, + filter: (doc: any, params: any) => { if (doc._id && doc._id.startsWith(DocumentType.AUTOMATION_LOG)) { return false } - return doc._id !== DocumentType.APP_METADATA + if (doc._id === DocumentType.APP_METADATA) { + return false + } + return filter ? filter(doc, params) : true }, } } @@ -75,10 +66,6 @@ class Replication { // take the opportunity to remove deleted tombstones await this.replicate() } - - cancel() { - this.replication.cancel() - } } export default Replication diff --git a/packages/backend-core/src/db/couch/DatabaseImpl.ts b/packages/backend-core/src/db/couch/DatabaseImpl.ts index 7e7c997cbe..416313f520 100644 --- a/packages/backend-core/src/db/couch/DatabaseImpl.ts +++ b/packages/backend-core/src/db/couch/DatabaseImpl.ts @@ -70,7 +70,15 @@ export class DatabaseImpl implements Database { DatabaseImpl.nano = buildNano(couchInfo) } - async exists() { + exists(docId?: string) { + if (docId === undefined) { + return this.dbExists() + } + + return this.docExists(docId) + } + + private async dbExists() { const response = await directCouchUrlCall({ url: `${this.couchInfo.url}/${this.name}`, method: "HEAD", @@ -79,6 +87,15 @@ export class DatabaseImpl implements Database { return response.status === 200 } + private async docExists(id: string): Promise { + try { + await this.performCall(db => () => db.head(id)) + return true + } catch { + return false + } + } + private nano() { return this.instanceNano || DatabaseImpl.nano } diff --git a/packages/backend-core/src/db/instrumentation.ts b/packages/backend-core/src/db/instrumentation.ts index 03010d4c92..795f30d7cd 100644 --- a/packages/backend-core/src/db/instrumentation.ts +++ b/packages/backend-core/src/db/instrumentation.ts @@ -24,9 +24,12 @@ export class DDInstrumentedDatabase implements Database { return this.db.name } - exists(): Promise { + exists(docId?: string): Promise { return tracer.trace("db.exists", span => { - span?.addTags({ db_name: this.name }) + span?.addTags({ db_name: this.name, doc_id: docId }) + if (docId) { + return this.db.exists(docId) + } return this.db.exists() }) } diff --git a/packages/backend-core/src/db/lucene.ts b/packages/backend-core/src/db/lucene.ts index f982ee67d0..987d750d45 100644 --- a/packages/backend-core/src/db/lucene.ts +++ b/packages/backend-core/src/db/lucene.ts @@ -10,10 +10,6 @@ interface SearchResponse { totalRows: number } -interface PaginatedSearchResponse extends SearchResponse { - hasNextPage: boolean -} - export type SearchParams = { tableId?: string sort?: string @@ -247,7 +243,7 @@ export class QueryBuilder { } // Escape characters if (!this.#noEscaping && escape && originalType === "string") { - value = `${value}`.replace(/[ \/#+\-&|!(){}\]^"~*?:\\]/g, "\\$&") + value = `${value}`.replace(/[ /#+\-&|!(){}\]^"~*?:\\]/g, "\\$&") } // Wrap in quotes diff --git a/packages/backend-core/src/db/searchIndexes/searchIndexes.ts b/packages/backend-core/src/db/searchIndexes/searchIndexes.ts index b953e3516e..8742d405f2 100644 --- a/packages/backend-core/src/db/searchIndexes/searchIndexes.ts +++ b/packages/backend-core/src/db/searchIndexes/searchIndexes.ts @@ -34,12 +34,12 @@ export async function createUserIndex() { } let idxKey = prev != null ? `${prev}.${key}` : key if (typeof input[key] === "string") { + // @ts-expect-error index is available in a CouchDB map function // eslint-disable-next-line no-undef - // @ts-ignore index(idxKey, input[key].toLowerCase(), { facet: true }) } else if (typeof input[key] !== "object") { + // @ts-expect-error index is available in a CouchDB map function // eslint-disable-next-line no-undef - // @ts-ignore index(idxKey, input[key], { facet: true }) } else { idx(input[key], idxKey) diff --git a/packages/backend-core/src/db/tests/DatabaseImpl.spec.ts b/packages/backend-core/src/db/tests/DatabaseImpl.spec.ts new file mode 100644 index 0000000000..586f13f417 --- /dev/null +++ b/packages/backend-core/src/db/tests/DatabaseImpl.spec.ts @@ -0,0 +1,55 @@ +import _ from "lodash" +import { AnyDocument } from "@budibase/types" +import { generator } from "../../../tests" +import { DatabaseImpl } from "../couch" +import { newid } from "../../utils" + +describe("DatabaseImpl", () => { + const database = new DatabaseImpl(generator.word()) + const documents: AnyDocument[] = [] + + beforeAll(async () => { + const docsToCreate = Array.from({ length: 10 }).map(() => ({ + _id: newid(), + })) + const createdDocs = await database.bulkDocs(docsToCreate) + + documents.push(...createdDocs.map((x: any) => ({ _id: x.id, _rev: x.rev }))) + }) + + describe("document exists", () => { + it("can check existing docs by id", async () => { + const existingDoc = _.sample(documents) + const result = await database.exists(existingDoc!._id!) + + expect(result).toBe(true) + }) + + it("can check non existing docs by id", async () => { + const result = await database.exists(newid()) + + expect(result).toBe(false) + }) + + it("can check an existing doc by id multiple times", async () => { + const existingDoc = _.sample(documents) + const id = existingDoc!._id! + + const results = [] + results.push(await database.exists(id)) + results.push(await database.exists(id)) + results.push(await database.exists(id)) + + expect(results).toEqual([true, true, true]) + }) + + it("returns false after the doc is deleted", async () => { + const existingDoc = _.sample(documents) + const id = existingDoc!._id! + expect(await database.exists(id)).toBe(true) + + await database.remove(existingDoc!) + expect(await database.exists(id)).toBe(false) + }) + }) +}) diff --git a/packages/backend-core/src/docUpdates/index.ts b/packages/backend-core/src/docUpdates/index.ts index 3971f8de12..bd34f4f0cd 100644 --- a/packages/backend-core/src/docUpdates/index.ts +++ b/packages/backend-core/src/docUpdates/index.ts @@ -17,13 +17,8 @@ export function init(processors: ProcessorMap) { // if not processing in this instance, kick it off if (!processingPromise) { processingPromise = asyncEventQueue.process(async job => { - const { event, identity, properties, timestamp } = job.data - await documentProcessor.processEvent( - event, - identity, - properties, - timestamp - ) + const { event, identity, properties } = job.data + await documentProcessor.processEvent(event, identity, properties) }) } } diff --git a/packages/backend-core/src/environment.ts b/packages/backend-core/src/environment.ts index b3179cbeea..2da2a77d67 100644 --- a/packages/backend-core/src/environment.ts +++ b/packages/backend-core/src/environment.ts @@ -186,6 +186,7 @@ const environment = { environment[key] = value }, ROLLING_LOG_MAX_SIZE: process.env.ROLLING_LOG_MAX_SIZE || "10M", + DISABLE_SCIM_CALLS: process.env.DISABLE_SCIM_CALLS, } // clean up any environment variable edge cases diff --git a/packages/backend-core/src/events/events.ts b/packages/backend-core/src/events/events.ts index f02b9fdf32..92b81553b0 100644 --- a/packages/backend-core/src/events/events.ts +++ b/packages/backend-core/src/events/events.ts @@ -1,4 +1,4 @@ -import { Event } from "@budibase/types" +import { Event, Identity } from "@budibase/types" import { processors } from "./processors" import identification from "./identification" import * as backfill from "./backfill" @@ -7,12 +7,19 @@ import { publishAsyncEvent } from "./asyncEvents" export const publishEvent = async ( event: Event, properties: any, - timestamp?: string | number + timestamp?: string | number, + identityOverride?: Identity ) => { // in future this should use async events via a distributed queue. - const identity = await identification.getCurrentIdentity() + const identity = + identityOverride || (await identification.getCurrentIdentity()) + + // Backfilling is get from the user cache, but when we override the identity cache is not available. Overrides are + // normally performed in automatic actions or operations in async flows (BPM) where the user session is not available. + const backfilling = identityOverride + ? false + : await backfill.isBackfillingEvent(event) - const backfilling = await backfill.isBackfillingEvent(event) // no backfill - send the event and exit if (!backfilling) { // send off async events if required diff --git a/packages/backend-core/src/events/processors/AuditLogsProcessor.ts b/packages/backend-core/src/events/processors/AuditLogsProcessor.ts index 94b4e1b09f..3dd2ab9d10 100644 --- a/packages/backend-core/src/events/processors/AuditLogsProcessor.ts +++ b/packages/backend-core/src/events/processors/AuditLogsProcessor.ts @@ -1,7 +1,6 @@ import { Event, Identity, - Group, IdentityType, AuditLogQueueEvent, AuditLogFn, @@ -79,11 +78,11 @@ export default class AuditLogsProcessor implements EventProcessor { } } - async identify(identity: Identity, timestamp?: string | number) { + async identify() { // no-op } - async identifyGroup(group: Group, timestamp?: string | number) { + async identifyGroup() { // no-op } diff --git a/packages/backend-core/src/events/processors/LoggingProcessor.ts b/packages/backend-core/src/events/processors/LoggingProcessor.ts index 0f4d02b99c..9f2dc5a473 100644 --- a/packages/backend-core/src/events/processors/LoggingProcessor.ts +++ b/packages/backend-core/src/events/processors/LoggingProcessor.ts @@ -8,8 +8,7 @@ export default class LoggingProcessor implements EventProcessor { async processEvent( event: Event, identity: Identity, - properties: any, - timestamp?: string + properties: any ): Promise { if (skipLogging) { return @@ -17,14 +16,14 @@ export default class LoggingProcessor implements EventProcessor { console.log(`[audit] [identityType=${identity.type}] ${event}`, properties) } - async identify(identity: Identity, timestamp?: string | number) { + async identify(identity: Identity) { if (skipLogging) { return } console.log(`[audit] identified`, identity) } - async identifyGroup(group: Group, timestamp?: string | number) { + async identifyGroup(group: Group) { if (skipLogging) { return } diff --git a/packages/backend-core/src/events/processors/async/DocumentUpdateProcessor.ts b/packages/backend-core/src/events/processors/async/DocumentUpdateProcessor.ts index 54304ee21b..92afcdc637 100644 --- a/packages/backend-core/src/events/processors/async/DocumentUpdateProcessor.ts +++ b/packages/backend-core/src/events/processors/async/DocumentUpdateProcessor.ts @@ -14,12 +14,7 @@ export default class DocumentUpdateProcessor implements EventProcessor { this.processors = processors } - async processEvent( - event: Event, - identity: Identity, - properties: any, - timestamp?: string | number - ) { + async processEvent(event: Event, identity: Identity, properties: any) { const tenantId = identity.realTenantId const docId = getDocumentId(event, properties) if (!tenantId || !docId) { diff --git a/packages/backend-core/src/events/publishers/account.ts b/packages/backend-core/src/events/publishers/account.ts index d337e404ef..99767962dd 100644 --- a/packages/backend-core/src/events/publishers/account.ts +++ b/packages/backend-core/src/events/publishers/account.ts @@ -5,13 +5,19 @@ import { AccountCreatedEvent, AccountDeletedEvent, AccountVerifiedEvent, + Identity, } from "@budibase/types" -async function created(account: Account) { +async function created(account: Account, identityOverride?: Identity) { const properties: AccountCreatedEvent = { tenantId: account.tenantId, } - await publishEvent(Event.ACCOUNT_CREATED, properties) + await publishEvent( + Event.ACCOUNT_CREATED, + properties, + undefined, + identityOverride + ) } async function deleted(account: Account) { diff --git a/packages/backend-core/src/events/publishers/app.ts b/packages/backend-core/src/events/publishers/app.ts index d08d59b5f1..af26b09e72 100644 --- a/packages/backend-core/src/events/publishers/app.ts +++ b/packages/backend-core/src/events/publishers/app.ts @@ -13,6 +13,7 @@ import { AppVersionRevertedEvent, AppRevertedEvent, AppExportedEvent, + AppDuplicatedEvent, } from "@budibase/types" const created = async (app: App, timestamp?: string | number) => { @@ -77,6 +78,17 @@ async function fileImported(app: App) { await publishEvent(Event.APP_FILE_IMPORTED, properties) } +async function duplicated(app: App, duplicateAppId: string) { + const properties: AppDuplicatedEvent = { + duplicateAppId, + appId: app.appId, + audited: { + name: app.name, + }, + } + await publishEvent(Event.APP_DUPLICATED, properties) +} + async function templateImported(app: App, templateKey: string) { const properties: AppTemplateImportedEvent = { appId: app.appId, @@ -147,6 +159,7 @@ export default { published, unpublished, fileImported, + duplicated, templateImported, versionUpdated, versionReverted, diff --git a/packages/backend-core/src/logging/pino/logger.ts b/packages/backend-core/src/logging/pino/logger.ts index 7a051e7f12..0a8470a453 100644 --- a/packages/backend-core/src/logging/pino/logger.ts +++ b/packages/backend-core/src/logging/pino/logger.ts @@ -10,6 +10,18 @@ import { formats } from "dd-trace/ext" import { localFileDestination } from "../system" +function isPlainObject(obj: any) { + return typeof obj === "object" && obj !== null && !(obj instanceof Error) +} + +function isError(obj: any) { + return obj instanceof Error +} + +function isMessage(obj: any) { + return typeof obj === "string" +} + // LOGGER let pinoInstance: pino.Logger | undefined @@ -71,23 +83,11 @@ if (!env.DISABLE_PINO_LOGGER) { err?: Error } - function isPlainObject(obj: any) { - return typeof obj === "object" && obj !== null && !(obj instanceof Error) - } - - function isError(obj: any) { - return obj instanceof Error - } - - function isMessage(obj: any) { - return typeof obj === "string" - } - /** * Backwards compatibility between console logging statements * and pino logging requirements. */ - function getLogParams(args: any[]): [MergingObject, string] { + const getLogParams = (args: any[]): [MergingObject, string] => { let error = undefined let objects: any[] = [] let message = "" diff --git a/packages/backend-core/src/middleware/errorHandling.ts b/packages/backend-core/src/middleware/errorHandling.ts index ebdd4107e9..2b8f7195ed 100644 --- a/packages/backend-core/src/middleware/errorHandling.ts +++ b/packages/backend-core/src/middleware/errorHandling.ts @@ -1,5 +1,6 @@ import { APIError } from "@budibase/types" import * as errors from "../errors" +import environment from "../environment" export async function errorHandling(ctx: any, next: any) { try { @@ -14,15 +15,19 @@ export async function errorHandling(ctx: any, next: any) { console.error(err) } - const error = errors.getPublicError(err) - const body: APIError = { + let error: APIError = { message: err.message, status: status, validationErrors: err.validation, - error, + error: errors.getPublicError(err), } - ctx.body = body + if (environment.isTest() && ctx.headers["x-budibase-include-stacktrace"]) { + // @ts-ignore + error.stack = err.stack + } + + ctx.body = error } } diff --git a/packages/backend-core/src/middleware/matchers.ts b/packages/backend-core/src/middleware/matchers.ts index efbdec2dbe..757d93a60d 100644 --- a/packages/backend-core/src/middleware/matchers.ts +++ b/packages/backend-core/src/middleware/matchers.ts @@ -11,7 +11,6 @@ export const buildMatcherRegex = ( return patterns.map(pattern => { let route = pattern.route const method = pattern.method - const strict = pattern.strict ? pattern.strict : false // if there is a param in the route // use a wildcard pattern @@ -24,24 +23,17 @@ export const buildMatcherRegex = ( } } - return { regex: new RegExp(route), method, strict, route } + return { regex: new RegExp(route), method, route } }) } export const matches = (ctx: BBContext, options: RegexMatcher[]) => { - return options.find(({ regex, method, strict, route }) => { - let urlMatch - if (strict) { - urlMatch = ctx.request.url === route - } else { - urlMatch = regex.test(ctx.request.url) - } - + return options.find(({ regex, method }) => { + const urlMatch = regex.test(ctx.request.url) const methodMatch = method === "ALL" ? true : ctx.request.method.toLowerCase() === method.toLowerCase() - return urlMatch && methodMatch }) } diff --git a/packages/backend-core/src/middleware/passport/datasource/google.ts b/packages/backend-core/src/middleware/passport/datasource/google.ts index ab4ffee9d2..7f768f1623 100644 --- a/packages/backend-core/src/middleware/passport/datasource/google.ts +++ b/packages/backend-core/src/middleware/passport/datasource/google.ts @@ -3,7 +3,7 @@ import { Cookie } from "../../../constants" import * as configs from "../../../configs" import * as cache from "../../../cache" import * as utils from "../../../utils" -import { UserCtx, SSOProfile, DatasourceAuthCookie } from "@budibase/types" +import { UserCtx, SSOProfile } from "@budibase/types" import { ssoSaveUserNoOp } from "../sso/sso" const GoogleStrategy = require("passport-google-oauth").OAuth2Strategy diff --git a/packages/backend-core/src/middleware/passport/sso/sso.ts b/packages/backend-core/src/middleware/passport/sso/sso.ts index 2fc1184722..ee84f03dae 100644 --- a/packages/backend-core/src/middleware/passport/sso/sso.ts +++ b/packages/backend-core/src/middleware/passport/sso/sso.ts @@ -5,7 +5,6 @@ import * as context from "../../../context" import fetch from "node-fetch" import { SaveSSOUserFunction, - SaveUserOpts, SSOAuthDetails, SSOUser, User, @@ -14,10 +13,8 @@ import { // no-op function for user save // - this allows datasource auth and access token refresh to work correctly // - prefer no-op over an optional argument to ensure function is provided to login flows -export const ssoSaveUserNoOp: SaveSSOUserFunction = ( - user: SSOUser, - opts: SaveUserOpts -) => Promise.resolve(user) +export const ssoSaveUserNoOp: SaveSSOUserFunction = (user: SSOUser) => + Promise.resolve(user) /** * Common authentication logic for third parties. e.g. OAuth, OIDC. diff --git a/packages/backend-core/src/middleware/passport/sso/tests/sso.spec.ts b/packages/backend-core/src/middleware/passport/sso/tests/sso.spec.ts index d3486a5b14..ea9584c284 100644 --- a/packages/backend-core/src/middleware/passport/sso/tests/sso.spec.ts +++ b/packages/backend-core/src/middleware/passport/sso/tests/sso.spec.ts @@ -114,11 +114,11 @@ describe("sso", () => { // tenant id added ssoUser.tenantId = context.getTenantId() - expect(mockSaveUser).toBeCalledWith(ssoUser, { + expect(mockSaveUser).toHaveBeenCalledWith(ssoUser, { hashPassword: false, requirePassword: false, }) - expect(mockDone).toBeCalledWith(null, ssoUser) + expect(mockDone).toHaveBeenCalledWith(null, ssoUser) }) }) }) @@ -159,11 +159,11 @@ describe("sso", () => { // existing id preserved ssoUser._id = existingUser._id - expect(mockSaveUser).toBeCalledWith(ssoUser, { + expect(mockSaveUser).toHaveBeenCalledWith(ssoUser, { hashPassword: false, requirePassword: false, }) - expect(mockDone).toBeCalledWith(null, ssoUser) + expect(mockDone).toHaveBeenCalledWith(null, ssoUser) }) }) @@ -187,11 +187,11 @@ describe("sso", () => { // existing id preserved ssoUser._id = existingUser._id - expect(mockSaveUser).toBeCalledWith(ssoUser, { + expect(mockSaveUser).toHaveBeenCalledWith(ssoUser, { hashPassword: false, requirePassword: false, }) - expect(mockDone).toBeCalledWith(null, ssoUser) + expect(mockDone).toHaveBeenCalledWith(null, ssoUser) }) }) }) diff --git a/packages/backend-core/src/middleware/tests/builder.spec.ts b/packages/backend-core/src/middleware/tests/builder.spec.ts index 0514dc13f0..0f35b0b833 100644 --- a/packages/backend-core/src/middleware/tests/builder.spec.ts +++ b/packages/backend-core/src/middleware/tests/builder.spec.ts @@ -24,13 +24,13 @@ function buildUserCtx(user: ContextUser) { } function passed(throwFn: jest.Func, nextFn: jest.Func) { - expect(throwFn).not.toBeCalled() - expect(nextFn).toBeCalled() + expect(throwFn).not.toHaveBeenCalled() + expect(nextFn).toHaveBeenCalled() } function threw(throwFn: jest.Func) { // cant check next, the throw function doesn't actually throw - so it still continues - expect(throwFn).toBeCalled() + expect(throwFn).toHaveBeenCalled() } describe("adminOnly middleware", () => { diff --git a/packages/backend-core/src/middleware/tests/matchers.spec.ts b/packages/backend-core/src/middleware/tests/matchers.spec.ts index c39bbb6dd3..1b79db2e68 100644 --- a/packages/backend-core/src/middleware/tests/matchers.spec.ts +++ b/packages/backend-core/src/middleware/tests/matchers.spec.ts @@ -34,23 +34,6 @@ describe("matchers", () => { expect(!!matchers.matches(ctx, built)).toBe(true) }) - it("doesn't wildcard path with strict", () => { - const pattern = [ - { - route: "/api/tests", - method: "POST", - strict: true, - }, - ] - const ctx = structures.koa.newContext() - ctx.request.url = "/api/tests/id/something/else" - ctx.request.method = "POST" - - const built = matchers.buildMatcherRegex(pattern) - - expect(!!matchers.matches(ctx, built)).toBe(false) - }) - it("matches with param", () => { const pattern = [ { @@ -67,23 +50,6 @@ describe("matchers", () => { expect(!!matchers.matches(ctx, built)).toBe(true) }) - // TODO: Support the below behaviour - // Strict does not work when a param is present - // it("matches with param with strict", () => { - // const pattern = [{ - // route: "/api/tests/:testId", - // method: "GET", - // strict: true - // }] - // const ctx = structures.koa.newContext() - // ctx.request.url = "/api/tests/id" - // ctx.request.method = "GET" - // - // const built = matchers.buildMatcherRegex(pattern) - // - // expect(!!matchers.matches(ctx, built)).toBe(true) - // }) - it("doesn't match by path", () => { const pattern = [ { diff --git a/packages/backend-core/src/migrations/migrations.ts b/packages/backend-core/src/migrations/migrations.ts index c750bc4882..3f033b8cdb 100644 --- a/packages/backend-core/src/migrations/migrations.ts +++ b/packages/backend-core/src/migrations/migrations.ts @@ -45,10 +45,6 @@ export const runMigration = async ( options: MigrationOptions = {} ) => { const migrationType = migration.type - let tenantId: string | undefined - if (migrationType !== MigrationType.INSTALLATION) { - tenantId = context.getTenantId() - } const migrationName = migration.name const silent = migration.silent diff --git a/packages/backend-core/src/objectStore/buckets/tests/app.spec.ts b/packages/backend-core/src/objectStore/buckets/tests/app.spec.ts index cbbbee6255..4a132ce54d 100644 --- a/packages/backend-core/src/objectStore/buckets/tests/app.spec.ts +++ b/packages/backend-core/src/objectStore/buckets/tests/app.spec.ts @@ -126,7 +126,7 @@ describe("app", () => { it("gets url with embedded minio", async () => { testEnv.withMinio() - await testEnv.withTenant(tenantId => { + await testEnv.withTenant(() => { const url = getAppFileUrl() expect(url).toBe( "/files/signed/prod-budi-app-assets/app_123/attachments/image.jpeg" @@ -136,7 +136,7 @@ describe("app", () => { it("gets url with custom S3", async () => { testEnv.withS3() - await testEnv.withTenant(tenantId => { + await testEnv.withTenant(() => { const url = getAppFileUrl() expect(url).toBe( "http://s3.example.com/prod-budi-app-assets/app_123/attachments/image.jpeg" @@ -146,7 +146,7 @@ describe("app", () => { it("gets url with cloudfront + s3", async () => { testEnv.withCloudfront() - await testEnv.withTenant(tenantId => { + await testEnv.withTenant(() => { const url = getAppFileUrl() // omit rest of signed params expect( diff --git a/packages/backend-core/src/platform/tests/tenants.spec.ts b/packages/backend-core/src/platform/tests/tenants.spec.ts index b2ab75c954..e22003fd45 100644 --- a/packages/backend-core/src/platform/tests/tenants.spec.ts +++ b/packages/backend-core/src/platform/tests/tenants.spec.ts @@ -3,7 +3,7 @@ import { DBTestConfiguration } from "../../../tests/extra" import * as tenants from "../tenants" describe("tenants", () => { - const config = new DBTestConfiguration() + new DBTestConfiguration() describe("addTenant", () => { it("concurrently adds multiple tenants safely", async () => { diff --git a/packages/backend-core/src/platform/users.ts b/packages/backend-core/src/platform/users.ts index 6f030afb7c..ccaad76b19 100644 --- a/packages/backend-core/src/platform/users.ts +++ b/packages/backend-core/src/platform/users.ts @@ -20,7 +20,7 @@ export async function lookupTenantId(userId: string) { return user.tenantId } -async function getUserDoc(emailOrId: string): Promise { +export async function getUserDoc(emailOrId: string): Promise { const db = getPlatformDB() return db.get(emailOrId) } @@ -79,6 +79,17 @@ async function addUserDoc(emailOrId: string, newDocFn: () => PlatformUser) { } } +export async function addSsoUser( + ssoId: string, + email: string, + userId: string, + tenantId: string +) { + return addUserDoc(ssoId, () => + newUserSsoIdDoc(ssoId, email, userId, tenantId) + ) +} + export async function addUser( tenantId: string, userId: string, @@ -91,9 +102,7 @@ export async function addUser( ] if (ssoId) { - promises.push( - addUserDoc(ssoId, () => newUserSsoIdDoc(ssoId, email, userId, tenantId)) - ) + promises.push(addSsoUser(ssoId, email, userId, tenantId)) } await Promise.all(promises) diff --git a/packages/backend-core/src/queue/constants.ts b/packages/backend-core/src/queue/constants.ts index eb4f21aced..a095c6c769 100644 --- a/packages/backend-core/src/queue/constants.ts +++ b/packages/backend-core/src/queue/constants.ts @@ -4,4 +4,5 @@ export enum JobQueue { AUDIT_LOG = "auditLogQueue", SYSTEM_EVENT_QUEUE = "systemEventQueue", APP_MIGRATION = "appMigration", + DOC_WRITETHROUGH_QUEUE = "docWritethroughQueue", } diff --git a/packages/backend-core/src/queue/inMemoryQueue.ts b/packages/backend-core/src/queue/inMemoryQueue.ts index c05bbffbe9..87e43b324d 100644 --- a/packages/backend-core/src/queue/inMemoryQueue.ts +++ b/packages/backend-core/src/queue/inMemoryQueue.ts @@ -1,5 +1,14 @@ import events from "events" -import { timeout } from "../utils" +import { newid, timeout } from "../utils" +import { Queue, QueueOptions, JobOptions } from "./queue" + +interface JobMessage { + id: string + timestamp: number + queue: string + data: any + opts?: JobOptions +} /** * Bull works with a Job wrapper around all messages that contains a lot more information about @@ -10,12 +19,13 @@ import { timeout } from "../utils" * @returns A new job which can now be put onto the queue, this is mostly an * internal structure so that an in memory queue can be easily swapped for a Bull queue. */ -function newJob(queue: string, message: any) { +function newJob(queue: string, message: any, opts?: JobOptions): JobMessage { return { + id: newid(), timestamp: Date.now(), queue: queue, data: message, - opts: {}, + opts, } } @@ -24,26 +34,29 @@ function newJob(queue: string, message: any) { * It is relatively simple, using an event emitter internally to register when messages are available * to the consumers - in can support many inputs and many consumers. */ -class InMemoryQueue { +class InMemoryQueue implements Partial { _name: string - _opts?: any - _messages: any[] - _emitter: EventEmitter + _opts?: QueueOptions + _messages: JobMessage[] + _queuedJobIds: Set + _emitter: NodeJS.EventEmitter _runCount: number _addCount: number + /** * The constructor the queue, exactly the same as that of Bulls. * @param name The name of the queue which is being configured. * @param opts This is not used by the in memory queue as there is no real use * case when in memory, but is the same API as Bull */ - constructor(name: string, opts?: any) { + constructor(name: string, opts?: QueueOptions) { this._name = name this._opts = opts this._messages = [] this._emitter = new events.EventEmitter() this._runCount = 0 this._addCount = 0 + this._queuedJobIds = new Set() } /** @@ -55,22 +68,42 @@ class InMemoryQueue { * note this is incredibly limited compared to Bull as in reality the Job would contain * a lot more information about the queue and current status of Bull cluster. */ - process(func: any) { + async process(func: any) { this._emitter.on("message", async () => { if (this._messages.length <= 0) { return } let msg = this._messages.shift() + let resp = func(msg) + + async function retryFunc(fnc: any) { + try { + await fnc + } catch (e: any) { + await new Promise(r => setTimeout(() => r(), 50)) + + await retryFunc(func(msg)) + } + } + if (resp.then != null) { - await resp + try { + await retryFunc(resp) + } catch (e: any) { + console.error(e) + } } this._runCount++ + const jobId = msg?.opts?.jobId?.toString() + if (jobId && msg?.opts?.removeOnComplete) { + this._queuedJobIds.delete(jobId) + } }) } async isReady() { - return true + return this as any } // simply puts a message to the queue and emits to the queue for processing @@ -83,27 +116,45 @@ class InMemoryQueue { * @param repeat serves no purpose for the import queue. */ // eslint-disable-next-line no-unused-vars - add(msg: any, repeat: boolean) { - if (typeof msg !== "object") { + async add(data: any, opts?: JobOptions) { + const jobId = opts?.jobId?.toString() + if (jobId && this._queuedJobIds.has(jobId)) { + console.log(`Ignoring already queued job ${jobId}`) + return + } + + if (typeof data !== "object") { throw "Queue only supports carrying JSON." } - this._messages.push(newJob(this._name, msg)) - this._addCount++ - this._emitter.emit("message") + if (jobId) { + this._queuedJobIds.add(jobId) + } + + const pushMessage = () => { + this._messages.push(newJob(this._name, data, opts)) + this._addCount++ + this._emitter.emit("message") + } + + const delay = opts?.delay + if (delay) { + setTimeout(pushMessage, delay) + } else { + pushMessage() + } + return {} as any } /** * replicating the close function from bull, which waits for jobs to finish. */ - async close() { - return [] - } + async close() {} /** * This removes a cron which has been implemented, this is part of Bull API. * @param cronJobId The cron which is to be removed. */ - removeRepeatableByKey(cronJobId: string) { + async removeRepeatableByKey(cronJobId: string) { // TODO: implement for testing console.log(cronJobId) } @@ -111,12 +162,12 @@ class InMemoryQueue { /** * Implemented for tests */ - getRepeatableJobs() { + async getRepeatableJobs() { return [] } - // eslint-disable-next-line no-unused-vars - removeJobs(pattern: string) { + // eslint-disable-next-line @typescript-eslint/no-unused-vars + async removeJobs(pattern: string) { // no-op } @@ -128,18 +179,22 @@ class InMemoryQueue { } async getJob() { - return {} + return null } on() { // do nothing - return this + return this as any } async waitForCompletion() { do { await timeout(50) - } while (this._addCount < this._runCount) + } while (this.hasRunningJobs()) + } + + hasRunningJobs() { + return this._addCount > this._runCount } } diff --git a/packages/backend-core/src/queue/listeners.ts b/packages/backend-core/src/queue/listeners.ts index 063a01bd2f..cd25ff2254 100644 --- a/packages/backend-core/src/queue/listeners.ts +++ b/packages/backend-core/src/queue/listeners.ts @@ -88,6 +88,7 @@ enum QueueEventType { AUDIT_LOG_EVENT = "audit-log-event", SYSTEM_EVENT = "system-event", APP_MIGRATION = "app-migration", + DOC_WRITETHROUGH = "doc-writethrough", } const EventTypeMap: { [key in JobQueue]: QueueEventType } = { @@ -96,6 +97,7 @@ const EventTypeMap: { [key in JobQueue]: QueueEventType } = { [JobQueue.AUDIT_LOG]: QueueEventType.AUDIT_LOG_EVENT, [JobQueue.SYSTEM_EVENT_QUEUE]: QueueEventType.SYSTEM_EVENT, [JobQueue.APP_MIGRATION]: QueueEventType.APP_MIGRATION, + [JobQueue.DOC_WRITETHROUGH_QUEUE]: QueueEventType.DOC_WRITETHROUGH, } function logging(queue: Queue, jobQueue: JobQueue) { @@ -130,7 +132,7 @@ function logging(queue: Queue, jobQueue: JobQueue) { // A Job is waiting to be processed as soon as a worker is idling. console.info(...getLogParams(eventType, BullEvent.WAITING, { jobId })) }) - .on(BullEvent.ACTIVE, async (job: Job, jobPromise: any) => { + .on(BullEvent.ACTIVE, async (job: Job) => { // A job has started. You can use `jobPromise.cancel()`` to abort it. await doInJobContext(job, () => { console.info(...getLogParams(eventType, BullEvent.ACTIVE, { job })) diff --git a/packages/backend-core/src/queue/queue.ts b/packages/backend-core/src/queue/queue.ts index 0bcb25a35f..1838eed92f 100644 --- a/packages/backend-core/src/queue/queue.ts +++ b/packages/backend-core/src/queue/queue.ts @@ -7,6 +7,8 @@ import { addListeners, StalledFn } from "./listeners" import { Duration } from "../utils" import * as timers from "../timers" +export { QueueOptions, Queue, JobOptions } from "bull" + // the queue lock is held for 5 minutes const QUEUE_LOCK_MS = Duration.fromMinutes(5).toMs() // queue lock is refreshed every 30 seconds diff --git a/packages/backend-core/src/redis/init.ts b/packages/backend-core/src/redis/init.ts index f3bcee3209..44ba28a83c 100644 --- a/packages/backend-core/src/redis/init.ts +++ b/packages/backend-core/src/redis/init.ts @@ -9,7 +9,8 @@ let userClient: Client, lockClient: Client, socketClient: Client, inviteClient: Client, - passwordResetClient: Client + passwordResetClient: Client, + docWritethroughClient: Client export async function init() { userClient = await new Client(utils.Databases.USER_CACHE).init() @@ -24,6 +25,9 @@ export async function init() { utils.Databases.SOCKET_IO, utils.SelectableDatabase.SOCKET_IO ).init() + docWritethroughClient = await new Client( + utils.Databases.DOC_WRITE_THROUGH + ).init() } export async function shutdown() { @@ -36,6 +40,7 @@ export async function shutdown() { if (inviteClient) await inviteClient.finish() if (passwordResetClient) await passwordResetClient.finish() if (socketClient) await socketClient.finish() + if (docWritethroughClient) await docWritethroughClient.finish() } process.on("exit", async () => { @@ -104,3 +109,10 @@ export async function getPasswordResetClient() { } return passwordResetClient } + +export async function getDocWritethroughClient() { + if (!writethroughClient) { + await init() + } + return writethroughClient +} diff --git a/packages/backend-core/src/redis/redis.ts b/packages/backend-core/src/redis/redis.ts index d15453ba62..79f75421d3 100644 --- a/packages/backend-core/src/redis/redis.ts +++ b/packages/backend-core/src/redis/redis.ts @@ -1,5 +1,5 @@ import env from "../environment" -import Redis from "ioredis" +import Redis, { Cluster } from "ioredis" // mock-redis doesn't have any typing let MockRedis: any | undefined if (env.MOCK_REDIS) { @@ -28,7 +28,7 @@ const DEFAULT_SELECT_DB = SelectableDatabase.DEFAULT // for testing just generate the client once let CLOSED = false -let CLIENTS: { [key: number]: any } = {} +const CLIENTS: Record = {} let CONNECTED = false // mock redis always connected @@ -36,7 +36,7 @@ if (env.MOCK_REDIS) { CONNECTED = true } -function pickClient(selectDb: number): any { +function pickClient(selectDb: number) { return CLIENTS[selectDb] } @@ -201,12 +201,15 @@ class RedisWrapper { key = `${db}${SEPARATOR}${key}` let stream if (CLUSTERED) { - let node = this.getClient().nodes("master") + let node = (this.getClient() as never as Cluster).nodes("master") stream = node[0].scanStream({ match: key + "*", count: 100 }) } else { - stream = this.getClient().scanStream({ match: key + "*", count: 100 }) + stream = (this.getClient() as Redis).scanStream({ + match: key + "*", + count: 100, + }) } - return promisifyStream(stream, this.getClient()) + return promisifyStream(stream, this.getClient() as any) } async keys(pattern: string) { @@ -221,14 +224,16 @@ class RedisWrapper { async get(key: string) { const db = this._db - let response = await this.getClient().get(addDbPrefix(db, key)) + const response = await this.getClient().get(addDbPrefix(db, key)) // overwrite the prefixed key + // @ts-ignore if (response != null && response.key) { + // @ts-ignore response.key = key } // if its not an object just return the response try { - return JSON.parse(response) + return JSON.parse(response!) } catch (err) { return response } @@ -274,13 +279,37 @@ class RedisWrapper { } } + async bulkStore( + data: Record, + expirySeconds: number | null = null + ) { + const client = this.getClient() + + const dataToStore = Object.entries(data).reduce((acc, [key, value]) => { + acc[addDbPrefix(this._db, key)] = + typeof value === "object" ? JSON.stringify(value) : value + return acc + }, {} as Record) + + const pipeline = client.pipeline() + pipeline.mset(dataToStore) + + if (expirySeconds !== null) { + for (const key of Object.keys(dataToStore)) { + pipeline.expire(key, expirySeconds) + } + } + + await pipeline.exec() + } + async getTTL(key: string) { const db = this._db const prefixedKey = addDbPrefix(db, key) return this.getClient().ttl(prefixedKey) } - async setExpiry(key: string, expirySeconds: number | null) { + async setExpiry(key: string, expirySeconds: number) { const db = this._db const prefixedKey = addDbPrefix(db, key) await this.getClient().expire(prefixedKey, expirySeconds) @@ -291,10 +320,35 @@ class RedisWrapper { await this.getClient().del(addDbPrefix(db, key)) } + async bulkDelete(keys: string[]) { + const db = this._db + await this.getClient().del(keys.map(key => addDbPrefix(db, key))) + } + async clear() { let items = await this.scan() await Promise.all(items.map((obj: any) => this.delete(obj.key))) } + + async increment(key: string) { + const result = await this.getClient().incr(addDbPrefix(this._db, key)) + if (isNaN(result)) { + throw new Error(`Redis ${key} does not contain a number`) + } + return result + } + + async deleteIfValue(key: string, value: any) { + const client = this.getClient() + + const luaScript = ` + if redis.call('GET', KEYS[1]) == ARGV[1] then + redis.call('DEL', KEYS[1]) + end + ` + + await client.eval(luaScript, 1, addDbPrefix(this._db, key), value) + } } export default RedisWrapper diff --git a/packages/backend-core/src/redis/redlockImpl.ts b/packages/backend-core/src/redis/redlockImpl.ts index 7009dc6f55..adeb5b12ec 100644 --- a/packages/backend-core/src/redis/redlockImpl.ts +++ b/packages/backend-core/src/redis/redlockImpl.ts @@ -72,7 +72,7 @@ const OPTIONS: Record = { export async function newRedlock(opts: Redlock.Options = {}) { const options = { ...OPTIONS.DEFAULT, ...opts } const redisWrapper = await getLockClient() - const client = redisWrapper.getClient() + const client = redisWrapper.getClient() as any return new Redlock([client], options) } diff --git a/packages/backend-core/src/redis/tests/redis.spec.ts b/packages/backend-core/src/redis/tests/redis.spec.ts new file mode 100644 index 0000000000..4d11caf220 --- /dev/null +++ b/packages/backend-core/src/redis/tests/redis.spec.ts @@ -0,0 +1,203 @@ +import { GenericContainer, StartedTestContainer } from "testcontainers" +import { generator, structures } from "../../../tests" +import RedisWrapper from "../redis" +import { env } from "../.." + +jest.setTimeout(30000) + +describe("redis", () => { + let redis: RedisWrapper + let container: StartedTestContainer + + beforeAll(async () => { + const container = await new GenericContainer("redis") + .withExposedPorts(6379) + .start() + + env._set( + "REDIS_URL", + `${container.getHost()}:${container.getMappedPort(6379)}` + ) + env._set("MOCK_REDIS", 0) + env._set("REDIS_PASSWORD", 0) + }) + + afterAll(() => container?.stop()) + + beforeEach(async () => { + redis = new RedisWrapper(structures.db.id()) + await redis.init() + }) + + describe("store", () => { + it("a basic value can be persisted", async () => { + const key = structures.uuid() + const value = generator.word() + + await redis.store(key, value) + + expect(await redis.get(key)).toEqual(value) + }) + + it("objects can be persisted", async () => { + const key = structures.uuid() + const value = { [generator.word()]: generator.word() } + + await redis.store(key, value) + + expect(await redis.get(key)).toEqual(value) + }) + }) + + describe("bulkStore", () => { + function createRandomObject( + keyLength: number, + valueGenerator: () => any = () => generator.word() + ) { + return generator + .unique(() => generator.word(), keyLength) + .reduce((acc, key) => { + acc[key] = valueGenerator() + return acc + }, {} as Record) + } + + it("a basic object can be persisted", async () => { + const data = createRandomObject(10) + + await redis.bulkStore(data) + + for (const [key, value] of Object.entries(data)) { + expect(await redis.get(key)).toEqual(value) + } + + expect(await redis.keys("*")).toHaveLength(10) + }) + + it("a complex object can be persisted", async () => { + const data = { + ...createRandomObject(10, () => createRandomObject(5)), + ...createRandomObject(5), + } + + await redis.bulkStore(data) + + for (const [key, value] of Object.entries(data)) { + expect(await redis.get(key)).toEqual(value) + } + + expect(await redis.keys("*")).toHaveLength(15) + }) + + it("no TTL is set by default", async () => { + const data = createRandomObject(10) + + await redis.bulkStore(data) + + for (const [key, value] of Object.entries(data)) { + expect(await redis.get(key)).toEqual(value) + expect(await redis.getTTL(key)).toEqual(-1) + } + }) + + it("a bulk store can be persisted with TTL", async () => { + const ttl = 500 + const data = createRandomObject(8) + + await redis.bulkStore(data, ttl) + + for (const [key, value] of Object.entries(data)) { + expect(await redis.get(key)).toEqual(value) + expect(await redis.getTTL(key)).toEqual(ttl) + } + + expect(await redis.keys("*")).toHaveLength(8) + }) + + it("setting a TTL of -1 will not persist the key", async () => { + const ttl = -1 + const data = createRandomObject(5) + + await redis.bulkStore(data, ttl) + + for (const key of Object.keys(data)) { + expect(await redis.get(key)).toBe(null) + } + + expect(await redis.keys("*")).toHaveLength(0) + }) + }) + + describe("increment", () => { + it("can increment on a new key", async () => { + const key = structures.uuid() + const result = await redis.increment(key) + expect(result).toBe(1) + }) + + it("can increment multiple times", async () => { + const key = structures.uuid() + const results = [ + await redis.increment(key), + await redis.increment(key), + await redis.increment(key), + await redis.increment(key), + await redis.increment(key), + ] + expect(results).toEqual([1, 2, 3, 4, 5]) + }) + + it("can increment multiple times in parallel", async () => { + const key = structures.uuid() + const results = await Promise.all( + Array.from({ length: 100 }).map(() => redis.increment(key)) + ) + expect(results).toHaveLength(100) + expect(results).toEqual(Array.from({ length: 100 }).map((_, i) => i + 1)) + }) + + it("can increment existing set keys", async () => { + const key = structures.uuid() + await redis.store(key, 70) + await redis.increment(key) + + const result = await redis.increment(key) + expect(result).toBe(72) + }) + + it.each([ + generator.word(), + generator.bool(), + { [generator.word()]: generator.word() }, + ])("cannot increment if the store value is not a number", async value => { + const key = structures.uuid() + await redis.store(key, value) + + await expect(redis.increment(key)).rejects.toThrow( + "ERR value is not an integer or out of range" + ) + }) + }) + + describe("deleteIfValue", () => { + it("can delete if the value matches", async () => { + const key = structures.uuid() + const value = generator.word() + await redis.store(key, value) + + await redis.deleteIfValue(key, value) + + expect(await redis.get(key)).toBeNull() + }) + + it("will not delete if the value does not matches", async () => { + const key = structures.uuid() + const value = generator.word() + await redis.store(key, value) + + await redis.deleteIfValue(key, generator.word()) + + expect(await redis.get(key)).toEqual(value) + }) + }) +}) diff --git a/packages/backend-core/src/redis/tests/redlockImpl.spec.ts b/packages/backend-core/src/redis/tests/redlockImpl.spec.ts index a1e83d8e6c..e647b63bf5 100644 --- a/packages/backend-core/src/redis/tests/redlockImpl.spec.ts +++ b/packages/backend-core/src/redis/tests/redlockImpl.spec.ts @@ -96,8 +96,8 @@ describe("redlockImpl", () => { task: mockTask, executionTimeMs: lockTtl * 2, }) - ).rejects.toThrowError( - `Unable to fully release the lock on resource \"lock:${config.tenantId}_persist_writethrough\".` + ).rejects.toThrow( + `Unable to fully release the lock on resource "lock:${config.tenantId}_persist_writethrough".` ) } ) diff --git a/packages/backend-core/src/redis/utils.ts b/packages/backend-core/src/redis/utils.ts index 7b93458b52..7f84f11467 100644 --- a/packages/backend-core/src/redis/utils.ts +++ b/packages/backend-core/src/redis/utils.ts @@ -30,6 +30,7 @@ export enum Databases { LOCKS = "locks", SOCKET_IO = "socket_io", BPM_EVENTS = "bpmEvents", + DOC_WRITE_THROUGH = "docWriteThrough", } /** diff --git a/packages/backend-core/src/security/roles.ts b/packages/backend-core/src/security/roles.ts index 4f048c0a11..a64be6b319 100644 --- a/packages/backend-core/src/security/roles.ts +++ b/packages/backend-core/src/security/roles.ts @@ -84,25 +84,24 @@ export function getBuiltinRoles(): { [key: string]: RoleDoc } { return cloneDeep(BUILTIN_ROLES) } -export const BUILTIN_ROLE_ID_ARRAY = Object.values(BUILTIN_ROLES).map( - role => role._id -) +export function isBuiltin(role: string) { + return getBuiltinRole(role) !== undefined +} -export const BUILTIN_ROLE_NAME_ARRAY = Object.values(BUILTIN_ROLES).map( - role => role.name -) - -export function isBuiltin(role?: string) { - return BUILTIN_ROLE_ID_ARRAY.some(builtin => role?.includes(builtin)) +export function getBuiltinRole(roleId: string): Role | undefined { + const role = Object.values(BUILTIN_ROLES).find(role => + roleId.includes(role._id) + ) + if (!role) { + return undefined + } + return cloneDeep(role) } /** * Works through the inheritance ranks to see how far up the builtin stack this ID is. */ -export function builtinRoleToNumber(id?: string) { - if (!id) { - return 0 - } +export function builtinRoleToNumber(id: string) { const builtins = getBuiltinRoles() const MAX = Object.values(builtins).length + 1 if (id === BUILTIN_IDS.ADMIN || id === BUILTIN_IDS.BUILDER) { @@ -123,7 +122,7 @@ export function builtinRoleToNumber(id?: string) { /** * Converts any role to a number, but has to be async to get the roles from db. */ -export async function roleToNumber(id?: string) { +export async function roleToNumber(id: string) { if (isBuiltin(id)) { return builtinRoleToNumber(id) } @@ -131,7 +130,7 @@ export async function roleToNumber(id?: string) { defaultPublic: true, })) as RoleDoc[] for (let role of hierarchy) { - if (isBuiltin(role?.inherits)) { + if (role?.inherits && isBuiltin(role.inherits)) { return builtinRoleToNumber(role.inherits) + 1 } } @@ -161,35 +160,28 @@ export function lowerBuiltinRoleID(roleId1?: string, roleId2?: string): string { * @returns The role object, which may contain an "inherits" property. */ export async function getRole( - roleId?: string, + roleId: string, opts?: { defaultPublic?: boolean } -): Promise { - if (!roleId) { - return undefined - } - let role: any = {} +): Promise { // built in roles mostly come from the in-code implementation, // but can be extended by a doc stored about them (e.g. permissions) - if (isBuiltin(roleId)) { - role = cloneDeep( - Object.values(BUILTIN_ROLES).find(role => role._id === roleId) - ) - } else { + let role: RoleDoc | undefined = getBuiltinRole(roleId) + if (!role) { // make sure has the prefix (if it has it then it won't be added) roleId = prefixRoleID(roleId) } try { const db = getAppDB() - const dbRole = await db.get(getDBRoleID(roleId)) - role = Object.assign(role, dbRole) + const dbRole = await db.get(getDBRoleID(roleId)) + role = Object.assign(role || {}, dbRole) // finalise the ID - role._id = getExternalRoleID(role._id, role.version) + role._id = getExternalRoleID(role._id!, role.version) } catch (err) { if (!isBuiltin(roleId) && opts?.defaultPublic) { return cloneDeep(BUILTIN_ROLES.PUBLIC) } // only throw an error if there is no role at all - if (Object.keys(role).length === 0) { + if (!role || Object.keys(role).length === 0) { throw err } } @@ -200,7 +192,7 @@ export async function getRole( * Simple function to get all the roles based on the top level user role ID. */ async function getAllUserRoles( - userRoleId?: string, + userRoleId: string, opts?: { defaultPublic?: boolean } ): Promise { // admins have access to all roles @@ -226,7 +218,7 @@ async function getAllUserRoles( } export async function getUserRoleIdHierarchy( - userRoleId?: string + userRoleId: string ): Promise { const roles = await getUserRoleHierarchy(userRoleId) return roles.map(role => role._id!) @@ -241,7 +233,7 @@ export async function getUserRoleIdHierarchy( * highest level of access and the last being the lowest level. */ export async function getUserRoleHierarchy( - userRoleId?: string, + userRoleId: string, opts?: { defaultPublic?: boolean } ) { // special case, if they don't have a role then they are a public user @@ -265,9 +257,9 @@ export function checkForRoleResourceArray( return rolePerms } -export async function getAllRoleIds(appId?: string) { +export async function getAllRoleIds(appId: string): Promise { const roles = await getAllRoles(appId) - return roles.map(role => role._id) + return roles.map(role => role._id!) } /** diff --git a/packages/backend-core/src/tenancy/tests/tenancy.spec.ts b/packages/backend-core/src/tenancy/tests/tenancy.spec.ts index 95dd76a6dd..34e9f87064 100644 --- a/packages/backend-core/src/tenancy/tests/tenancy.spec.ts +++ b/packages/backend-core/src/tenancy/tests/tenancy.spec.ts @@ -158,8 +158,8 @@ describe("getTenantIDFromCtx", () => { ], } expect(getTenantIDFromCtx(ctx, mockOpts)).toBeUndefined() - expect(ctx.throw).toBeCalledTimes(1) - expect(ctx.throw).toBeCalledWith(403, "Tenant id not set") + expect(ctx.throw).toHaveBeenCalledTimes(1) + expect(ctx.throw).toHaveBeenCalledWith(403, "Tenant id not set") }) it("returns undefined if allowNoTenant is true", () => { diff --git a/packages/backend-core/src/timers/timers.ts b/packages/backend-core/src/timers/timers.ts index 9121c576cd..000be74821 100644 --- a/packages/backend-core/src/timers/timers.ts +++ b/packages/backend-core/src/timers/timers.ts @@ -20,41 +20,3 @@ export function cleanup() { } intervals = [] } - -export class ExecutionTimeoutError extends Error { - public readonly name = "ExecutionTimeoutError" -} - -export class ExecutionTimeTracker { - static withLimit(limitMs: number) { - return new ExecutionTimeTracker(limitMs) - } - - constructor(readonly limitMs: number) {} - - private totalTimeMs = 0 - - track(f: () => T): T { - this.checkLimit() - const start = process.hrtime.bigint() - try { - return f() - } finally { - const end = process.hrtime.bigint() - this.totalTimeMs += Number(end - start) / 1e6 - this.checkLimit() - } - } - - get elapsedMS() { - return this.totalTimeMs - } - - checkLimit() { - if (this.totalTimeMs > this.limitMs) { - throw new ExecutionTimeoutError( - `Execution time limit of ${this.limitMs}ms exceeded: ${this.totalTimeMs}ms` - ) - } - } -} diff --git a/packages/backend-core/src/users/db.ts b/packages/backend-core/src/users/db.ts index 136cb4b8ad..04d3264e6f 100644 --- a/packages/backend-core/src/users/db.ts +++ b/packages/backend-core/src/users/db.ts @@ -500,13 +500,13 @@ export class UserDB { static async createAdminUser( email: string, - password: string, tenantId: string, + password?: string, opts?: CreateAdminUserOpts ) { const user: User = { email: email, - password: password, + password, createdAt: Date.now(), roles: {}, builder: { diff --git a/packages/backend-core/src/users/test/utils.spec.ts b/packages/backend-core/src/users/test/utils.spec.ts index 0fe27f57a6..cb98b8972b 100644 --- a/packages/backend-core/src/users/test/utils.spec.ts +++ b/packages/backend-core/src/users/test/utils.spec.ts @@ -45,7 +45,7 @@ describe("Users", () => { ...{ _id: groupId, roles: { app1: "ADMIN" } }, } const users: User[] = [] - for (const _ of Array.from({ length: usersInGroup })) { + for (let i = 0; i < usersInGroup; i++) { const userId = `us_${generator.guid()}` const user: User = structures.users.user({ _id: userId, diff --git a/packages/backend-core/src/users/users.ts b/packages/backend-core/src/users/users.ts index 638da4a5b1..48920a3771 100644 --- a/packages/backend-core/src/users/users.ts +++ b/packages/backend-core/src/users/users.ts @@ -14,16 +14,16 @@ import { } from "../db" import { BulkDocsResponse, + ContextUser, + CouchFindOptions, + DatabaseQueryOpts, SearchQuery, SearchQueryOperators, SearchUsersRequest, User, - ContextUser, - DatabaseQueryOpts, - CouchFindOptions, } from "@budibase/types" -import { getGlobalDB } from "../context" import * as context from "../context" +import { getGlobalDB } from "../context" import { isCreator } from "./utils" import { UserDB } from "./db" @@ -48,6 +48,7 @@ export function isSupportedUserSearch(query: SearchQuery) { const allowed = [ { op: SearchQueryOperators.STRING, key: "email" }, { op: SearchQueryOperators.EQUAL, key: "_id" }, + { op: SearchQueryOperators.ONE_OF, key: "_id" }, ] for (let [key, operation] of Object.entries(query)) { if (typeof operation !== "object") { @@ -285,6 +286,10 @@ export async function paginatedUsers({ } else if (query?.string?.email) { userList = await searchGlobalUsersByEmail(query?.string?.email, opts) property = "email" + } else if (query?.oneOf?._id) { + userList = await bulkGetGlobalUsersById(query?.oneOf?._id, { + cleanup: true, + }) } else { // no search, query allDocs const response = await db.allDocs(getGlobalUserParams(null, opts)) diff --git a/packages/backend-core/tests/core/utilities/mocks/events.ts b/packages/backend-core/tests/core/utilities/mocks/events.ts index fef730768a..96f351de10 100644 --- a/packages/backend-core/tests/core/utilities/mocks/events.ts +++ b/packages/backend-core/tests/core/utilities/mocks/events.ts @@ -15,6 +15,7 @@ beforeAll(async () => { jest.spyOn(events.app, "created") jest.spyOn(events.app, "updated") + jest.spyOn(events.app, "duplicated") jest.spyOn(events.app, "deleted") jest.spyOn(events.app, "published") jest.spyOn(events.app, "unpublished") diff --git a/packages/backend-core/tests/core/utilities/structures/accounts.ts b/packages/backend-core/tests/core/utilities/structures/accounts.ts index 515f94db1e..7dcc2de116 100644 --- a/packages/backend-core/tests/core/utilities/structures/accounts.ts +++ b/packages/backend-core/tests/core/utilities/structures/accounts.ts @@ -18,7 +18,7 @@ export const account = (partial: Partial = {}): Account => { return { accountId: uuid(), tenantId: generator.word(), - email: generator.email(), + email: generator.email({ domain: "example.com" }), tenantName: generator.word(), hosting: Hosting.SELF, createdAt: Date.now(), diff --git a/packages/backend-core/tests/core/utilities/structures/scim.ts b/packages/backend-core/tests/core/utilities/structures/scim.ts index 80f41c605d..f424b2881a 100644 --- a/packages/backend-core/tests/core/utilities/structures/scim.ts +++ b/packages/backend-core/tests/core/utilities/structures/scim.ts @@ -13,7 +13,7 @@ interface CreateUserRequestFields { export function createUserRequest(userData?: Partial) { const defaultValues = { externalId: uuid(), - email: generator.email(), + email: `${uuid()}@example.com`, firstName: generator.first(), lastName: generator.last(), username: generator.name(), diff --git a/packages/backend-core/tests/core/utilities/structures/userGroups.ts b/packages/backend-core/tests/core/utilities/structures/userGroups.ts index 4dc870a00a..4af3f72e51 100644 --- a/packages/backend-core/tests/core/utilities/structures/userGroups.ts +++ b/packages/backend-core/tests/core/utilities/structures/userGroups.ts @@ -3,7 +3,7 @@ import { generator } from "./generator" export function userGroup(): UserGroup { return { - name: generator.word(), + name: generator.guid(), icon: generator.word(), color: generator.word(), } diff --git a/packages/backend-core/tests/core/utilities/testContainerUtils.ts b/packages/backend-core/tests/core/utilities/testContainerUtils.ts index 7da6cbc777..5d4f5a3c11 100644 --- a/packages/backend-core/tests/core/utilities/testContainerUtils.ts +++ b/packages/backend-core/tests/core/utilities/testContainerUtils.ts @@ -1,80 +1,58 @@ +import { DatabaseImpl } from "../../../src/db" import { execSync } from "child_process" -let dockerPsResult: string | undefined - -function formatDockerPsResult(serverName: string, port: number) { - const lines = dockerPsResult?.split("\n") - let first = true - if (!lines) { - return null - } - for (let line of lines) { - if (first) { - first = false - continue - } - let toLookFor = serverName.split("-service")[0] - if (!line.includes(toLookFor)) { - continue - } - const regex = new RegExp(`0.0.0.0:([0-9]*)->${port}`, "g") - const found = line.match(regex) - if (found) { - return found[0].split(":")[1].split("->")[0] - } - } - return null +interface ContainerInfo { + Command: string + CreatedAt: string + ID: string + Image: string + Labels: string + LocalVolumes: string + Mounts: string + Names: string + Networks: string + Ports: string + RunningFor: string + Size: string + State: string + Status: string } -function getTestContainerSettings( - serverName: string, - key: string -): string | null { - const entry = Object.entries(global).find( - ([k]) => - k.includes(`${serverName.toUpperCase()}`) && - k.includes(`${key.toUpperCase()}`) - ) - if (!entry) { - return null - } - return entry[1] +function getTestcontainers(): ContainerInfo[] { + return execSync("docker ps --format json") + .toString() + .split("\n") + .filter(x => x.length > 0) + .map(x => JSON.parse(x) as ContainerInfo) + .filter(x => x.Labels.includes("org.testcontainers=true")) } -function getContainerInfo(containerName: string, port: number) { - let assignedPort = getTestContainerSettings( - containerName.toUpperCase(), - `PORT_${port}` - ) - if (!dockerPsResult) { - try { - const outputBuffer = execSync("docker ps") - dockerPsResult = outputBuffer.toString("utf8") - } catch (err) { - //no-op - } - } - const possiblePort = formatDockerPsResult(containerName, port) - if (possiblePort) { - assignedPort = possiblePort - } - const host = getTestContainerSettings(containerName.toUpperCase(), "IP") - return { - port: assignedPort, - host, - url: host && assignedPort && `http://${host}:${assignedPort}`, - } +function getContainerByImage(image: string) { + return getTestcontainers().find(x => x.Image.startsWith(image)) } -function getCouchConfig() { - return getContainerInfo("couchdb", 5984) +function getExposedPort(container: ContainerInfo, port: number) { + const match = container.Ports.match(new RegExp(`0.0.0.0:(\\d+)->${port}/tcp`)) + if (!match) { + return undefined + } + return parseInt(match[1]) } export function setupEnv(...envs: any[]) { - const couch = getCouchConfig() + const couch = getContainerByImage("budibase/couchdb") + if (!couch) { + throw new Error("CouchDB container not found") + } + + const couchPort = getExposedPort(couch, 5984) + if (!couchPort) { + throw new Error("CouchDB port not found") + } + const configs = [ - { key: "COUCH_DB_PORT", value: couch.port }, - { key: "COUCH_DB_URL", value: couch.url }, + { key: "COUCH_DB_PORT", value: `${couchPort}` }, + { key: "COUCH_DB_URL", value: `http://localhost:${couchPort}` }, ] for (const config of configs.filter(x => !!x.value)) { @@ -82,4 +60,7 @@ export function setupEnv(...envs: any[]) { env._set(config.key, config.value) } } + + // @ts-expect-error + DatabaseImpl.nano = undefined } diff --git a/packages/backend-core/tests/jestEnv.ts b/packages/backend-core/tests/jestEnv.ts index c2047118ec..2c797c9fff 100644 --- a/packages/backend-core/tests/jestEnv.ts +++ b/packages/backend-core/tests/jestEnv.ts @@ -4,3 +4,7 @@ process.env.NODE_ENV = "jest" process.env.MOCK_REDIS = "1" process.env.LOG_LEVEL = process.env.LOG_LEVEL || "error" process.env.REDIS_PASSWORD = "budibase" +process.env.COUCH_DB_PASSWORD = "budibase" +process.env.COUCH_DB_USER = "budibase" +process.env.API_ENCRYPTION_KEY = "testsecret" +process.env.JWT_SECRET = "testsecret" diff --git a/packages/bbui/rollup.config.js b/packages/bbui/rollup.config.js index e285d548d6..da274e0ba5 100644 --- a/packages/bbui/rollup.config.js +++ b/packages/bbui/rollup.config.js @@ -12,6 +12,13 @@ export default { format: "esm", file: "dist/bbui.es.js", }, + onwarn(warning, warn) { + // suppress eval warnings + if (warning.code === "EVAL") { + return + } + warn(warning) + }, plugins: [ resolve(), commonjs(), diff --git a/packages/bbui/src/ActionMenu/ActionMenu.svelte b/packages/bbui/src/ActionMenu/ActionMenu.svelte index 642ec4932a..c55d1cb43d 100644 --- a/packages/bbui/src/ActionMenu/ActionMenu.svelte +++ b/packages/bbui/src/ActionMenu/ActionMenu.svelte @@ -38,7 +38,7 @@
- + diff --git a/packages/bbui/src/Actions/click_outside.js b/packages/bbui/src/Actions/click_outside.js index 1961dca47c..eafca657f3 100644 --- a/packages/bbui/src/Actions/click_outside.js +++ b/packages/bbui/src/Actions/click_outside.js @@ -32,19 +32,30 @@ const handleClick = event => { return } + // Ignore clicks for drawers, unless the handler is registered from a drawer + const sourceInDrawer = handler.anchor.closest(".drawer-wrapper") != null + const clickInDrawer = event.target.closest(".drawer-wrapper") != null + if (clickInDrawer && !sourceInDrawer) { + return + } + + if (handler.allowedType && event.type !== handler.allowedType) { + return + } + handler.callback?.(event) }) } document.documentElement.addEventListener("click", handleClick, true) -document.documentElement.addEventListener("contextmenu", handleClick, true) +document.documentElement.addEventListener("mousedown", handleClick, true) /** * Adds or updates a click handler */ -const updateHandler = (id, element, anchor, callback) => { +const updateHandler = (id, element, anchor, callback, allowedType) => { let existingHandler = clickHandlers.find(x => x.id === id) if (!existingHandler) { - clickHandlers.push({ id, element, anchor, callback }) + clickHandlers.push({ id, element, anchor, callback, allowedType }) } else { existingHandler.callback = callback } @@ -68,9 +79,11 @@ const removeHandler = id => { export default (element, opts) => { const id = Math.random() const update = newOpts => { - const callback = newOpts?.callback || newOpts + const callback = + newOpts?.callback || (typeof newOpts === "function" ? newOpts : null) const anchor = newOpts?.anchor || element - updateHandler(id, element, anchor, callback) + const allowedType = newOpts?.allowedType || "click" + updateHandler(id, element, anchor, callback, allowedType) } update(opts) return { diff --git a/packages/bbui/src/Actions/position_dropdown.js b/packages/bbui/src/Actions/position_dropdown.js index cc169eac09..770d1bd507 100644 --- a/packages/bbui/src/Actions/position_dropdown.js +++ b/packages/bbui/src/Actions/position_dropdown.js @@ -15,6 +15,7 @@ export default function positionDropdown(element, opts) { align, maxHeight, maxWidth, + minWidth, useAnchorWidth, offset = 5, customUpdate, @@ -28,18 +29,26 @@ export default function positionDropdown(element, opts) { const elementBounds = element.getBoundingClientRect() let styles = { maxHeight: null, - minWidth: null, + minWidth, maxWidth, left: null, top: null, } if (typeof customUpdate === "function") { - styles = customUpdate(anchorBounds, elementBounds, styles) + styles = customUpdate(anchorBounds, elementBounds, { + ...styles, + offset: opts.offset, + }) } else { // Determine vertical styles - if (align === "right-outside") { - styles.top = anchorBounds.top + if (align === "right-outside" || align === "left-outside") { + styles.top = + anchorBounds.top + anchorBounds.height / 2 - elementBounds.height / 2 + styles.maxHeight = maxHeight + if (styles.top + elementBounds.height > window.innerHeight) { + styles.top = window.innerHeight - elementBounds.height + } } else if ( window.innerHeight - anchorBounds.bottom < (maxHeight || 100) diff --git a/packages/bbui/src/Drawer/Drawer.svelte b/packages/bbui/src/Drawer/Drawer.svelte index 8976bfb81e..89ee92726d 100644 --- a/packages/bbui/src/Drawer/Drawer.svelte +++ b/packages/bbui/src/Drawer/Drawer.svelte @@ -1,28 +1,111 @@ + + {#if visible} - -
- {#if !headless} + + +
+
+
0} + class:modal={$modal} + transition:drawerSlide|local + {style} + >
-
- {title} - - - -
+ {#if $$slots.title} + + {:else} +
{title || "Bindings"}
+ {/if}
+ {#if $resizable} + modal.set(!$modal)} + > + + + {/if}
- {/if} - -
+ +
+
+ {/if} diff --git a/packages/bbui/src/Drawer/DrawerContent.svelte b/packages/bbui/src/Drawer/DrawerContent.svelte index 944a3f4313..f7345afb11 100644 --- a/packages/bbui/src/Drawer/DrawerContent.svelte +++ b/packages/bbui/src/Drawer/DrawerContent.svelte @@ -1,4 +1,8 @@ -
+ + +
{#if $$slots.sidebar}
@@ -13,8 +17,8 @@ diff --git a/packages/bbui/src/Modal/Modal.svelte b/packages/bbui/src/Modal/Modal.svelte index f891d0584d..be9c338892 100644 --- a/packages/bbui/src/Modal/Modal.svelte +++ b/packages/bbui/src/Modal/Modal.svelte @@ -10,6 +10,7 @@ export let inline = false export let disableCancel = false export let autoFocus = true + export let zIndex = 999 const dispatch = createEventDispatcher() let visible = fixed || inline @@ -101,7 +102,11 @@ {#if visible} -
+
diff --git a/packages/bbui/src/Table/CellRenderer.svelte b/packages/bbui/src/Table/CellRenderer.svelte index 4ad6e22d7e..eff1178f6d 100644 --- a/packages/bbui/src/Table/CellRenderer.svelte +++ b/packages/bbui/src/Table/CellRenderer.svelte @@ -12,6 +12,7 @@ export let schema export let value export let customRenderers = [] + export let snippets let renderer const typeMap = { @@ -44,7 +45,7 @@ if (!template) { return value } - return processStringSync(template, { value }) + return processStringSync(template, { value, snippets }) } diff --git a/packages/bbui/src/Table/Table.svelte b/packages/bbui/src/Table/Table.svelte index 36668571bc..868f7b3a0b 100644 --- a/packages/bbui/src/Table/Table.svelte +++ b/packages/bbui/src/Table/Table.svelte @@ -42,6 +42,7 @@ export let customPlaceholder = false export let showHeaderBorder = true export let placeholderText = "No rows found" + export let snippets = [] const dispatch = createEventDispatcher() @@ -425,6 +426,7 @@ @@ -118,6 +120,9 @@ .abs-tooltip { display: contents; } + .spectrum-Tooltip.noWrap .spectrum-Tooltip-label { + width: max-content; + } .spectrum-Tooltip { position: absolute; z-index: 9999; diff --git a/packages/bbui/src/index.js b/packages/bbui/src/index.js index cda6b5acbf..d18c4d6910 100644 --- a/packages/bbui/src/index.js +++ b/packages/bbui/src/index.js @@ -19,7 +19,7 @@ export { default as ActionMenu } from "./ActionMenu/ActionMenu.svelte" export { default as Button } from "./Button/Button.svelte" export { default as ButtonGroup } from "./ButtonGroup/ButtonGroup.svelte" export { default as ClearButton } from "./ClearButton/ClearButton.svelte" -export { default as Icon, directions } from "./Icon/Icon.svelte" +export { default as Icon } from "./Icon/Icon.svelte" export { default as IconAvatar } from "./Icon/IconAvatar.svelte" export { default as Toggle } from "./Form/Toggle.svelte" export { default as RadioGroup } from "./Form/RadioGroup.svelte" diff --git a/packages/builder/.vscode/launch.json b/packages/builder/.vscode/launch.json deleted file mode 100644 index f6b35a0b63..0000000000 --- a/packages/builder/.vscode/launch.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - // Use IntelliSense to learn about possible attributes. - // Hover to view descriptions of existing attributes. - // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 - "version": "0.2.0", - "configurations": [ - { - "type": "chrome", - "request": "launch", - "name": "Launch Chrome against localhost", - "url": "http://localhost:3000", - "webRoot": "${workspaceFolder}" - } - ] -} \ No newline at end of file diff --git a/packages/builder/.vscode/settings.json b/packages/builder/.vscode/settings.json deleted file mode 100644 index 7148fd3b98..0000000000 --- a/packages/builder/.vscode/settings.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "javascript.format.enable": false, - "svelte.plugin.svelte.format.enable": false, - "html.format.enable": false, - "json.format.enable": false, - "editor.trimAutoWhitespace": false, - "sass.format.deleteWhitespace": false -} \ No newline at end of file diff --git a/packages/builder/package.json b/packages/builder/package.json index e66c747505..f61ac4fe26 100644 --- a/packages/builder/package.json +++ b/packages/builder/package.json @@ -66,10 +66,11 @@ "@spectrum-css/page": "^3.0.1", "@spectrum-css/vars": "^3.0.1", "@zerodevx/svelte-json-view": "^1.0.7", - "codemirror": "^5.59.0", + "codemirror": "^5.65.16", "dayjs": "^1.10.8", "downloadjs": "1.4.7", "fast-json-patch": "^3.1.1", + "json-format-highlight": "^1.0.4", "lodash": "4.17.21", "posthog-js": "^1.36.0", "remixicon": "2.5.0", diff --git a/packages/builder/src/analytics/index.js b/packages/builder/src/analytics/index.js index 6bb10acdb5..3a80a05d7f 100644 --- a/packages/builder/src/analytics/index.js +++ b/packages/builder/src/analytics/index.js @@ -9,13 +9,17 @@ const intercom = new IntercomClient(process.env.INTERCOM_TOKEN) class AnalyticsHub { constructor() { this.clients = [posthog, intercom] + this.initialised = false } async activate() { // Check analytics are enabled const analyticsStatus = await API.getAnalyticsStatus() - if (analyticsStatus.enabled) { - this.clients.forEach(client => client.init()) + if (analyticsStatus.enabled && !this.initialised) { + this.clients.forEach(client => { + client.init() + }) + this.initialised = true } } diff --git a/packages/builder/src/components/automation/AutomationPanel/AutomationPanel.svelte b/packages/builder/src/components/automation/AutomationPanel/AutomationPanel.svelte index 582db950fe..ac1c4f91cb 100644 --- a/packages/builder/src/components/automation/AutomationPanel/AutomationPanel.svelte +++ b/packages/builder/src/components/automation/AutomationPanel/AutomationPanel.svelte @@ -49,7 +49,7 @@
modal.show()} /> diff --git a/packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte b/packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte index 707317f9e6..0632993cf0 100644 --- a/packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte +++ b/packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte @@ -12,7 +12,6 @@ Drawer, Modal, notifications, - Icon, Checkbox, DatePicker, } from "@budibase/bbui" @@ -31,8 +30,8 @@ import Editor from "components/integration/QueryEditor.svelte" import ModalBindableInput from "components/common/bindings/ModalBindableInput.svelte" import CodeEditor from "components/common/CodeEditor/CodeEditor.svelte" - import BindingPicker from "components/common/bindings/BindingPicker.svelte" - import { BindingHelpers } from "components/common/bindings/utils" + import BindingSidePanel from "components/common/bindings/BindingSidePanel.svelte" + import { BindingHelpers, BindingType } from "components/common/bindings/utils" import { bindingsToCompletions, hbAutocomplete, @@ -52,11 +51,12 @@ export let testData export let schemaProperties export let isTestModal = false + let webhookModal let drawer - let fillWidth = true let inputData let insertAtPos, getCaretPosition + $: filters = lookForFilters(schemaProperties) || [] $: tempFilters = filters $: stepId = block.stepId @@ -80,7 +80,6 @@ }) $: editingJs = codeMode === EditorModes.JS $: requiredProperties = block.schema.inputs.required || [] - $: stepCompletions = codeMode === EditorModes.Handlebars ? [hbAutocomplete([...bindingsToCompletions(bindings, codeMode)])] @@ -377,12 +376,13 @@
{#each schemaProperties as [key, value]} {#if canShowField(key, value)} + {@const label = getFieldLabel(key, value)}
{#if key !== "fields" && value.type !== "boolean" && shouldRenderField(value)} {label} {/if}
@@ -415,8 +415,7 @@
{:else if value.type === "date"} {:else if value.customType === "filters"} Define filters - + @@ -450,7 +449,6 @@ {schemaFields} datasource={{ type: "table", tableId }} panel={AutomationBindingPanel} - fillWidth on:change={e => (tempFilters = e.detail)} /> @@ -463,19 +461,17 @@ {:else if value.customType === "email"} {#if isTestModal} onChange(e, key)} {bindings} - fillWidth updateOnChange={false} /> {:else}
-
+
{ @@ -563,24 +559,14 @@ autocompleteEnabled={codeMode !== EditorModes.JS} bind:getCaretPosition bind:insertAtPos - height={500} + placeholder={codeMode === EditorModes.Handlebars + ? "Add bindings by typing {{" + : null} /> -
- {#if codeMode === EditorModes.Handlebars} - -
-
- Add available bindings by typing - }} - -
-
- {/if} -
{#if editingJs}
- @@ -590,6 +576,7 @@ { js: true, dontDecode: true, + type: BindingType.RUNTIME, } )} mode="javascript" @@ -609,7 +596,7 @@ {:else if value.type === "string" || value.type === "number" || value.type === "integer"} {#if isTestModal} onChange(e, field)} type="string" {bindings} - fillWidth={true} updateOnChange={false} />
diff --git a/packages/builder/src/components/automation/SetupPanel/QueryParamSelector.svelte b/packages/builder/src/components/automation/SetupPanel/QueryParamSelector.svelte index 4674002d61..f46bca3090 100644 --- a/packages/builder/src/components/automation/SetupPanel/QueryParamSelector.svelte +++ b/packages/builder/src/components/automation/SetupPanel/QueryParamSelector.svelte @@ -52,7 +52,6 @@ on:change={e => onChange(e, field)} type="string" {bindings} - fillWidth={true} updateOnChange={false} />
diff --git a/packages/builder/src/components/automation/SetupPanel/RowSelector.svelte b/packages/builder/src/components/automation/SetupPanel/RowSelector.svelte index 6cc96d8c68..0d15df6c87 100644 --- a/packages/builder/src/components/automation/SetupPanel/RowSelector.svelte +++ b/packages/builder/src/components/automation/SetupPanel/RowSelector.svelte @@ -129,8 +129,7 @@ /> {:else} onChange(e, field)} type="string" bindings={parsedBindings} - fillWidth={true} allowJS={true} updateOnChange={false} + title={schema.name} /> {/if} diff --git a/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte b/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte index 1bad1ee221..cfc6e9a7be 100644 --- a/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte +++ b/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte @@ -36,6 +36,8 @@ import { ValidColumnNameRegex } from "@budibase/shared-core" import { FieldType, FieldSubtype, SourceName } from "@budibase/types" import RelationshipSelector from "components/common/RelationshipSelector.svelte" + import { RowUtils } from "@budibase/frontend-core" + import ServerBindingPanel from "components/common/bindings/ServerBindingPanel.svelte" const AUTO_TYPE = FIELDS.AUTO.type const FORMULA_TYPE = FIELDS.FORMULA.type @@ -49,43 +51,21 @@ const dispatch = createEventDispatcher() const PROHIBITED_COLUMN_NAMES = ["type", "_id", "_rev", "tableId"] - const { dispatch: gridDispatch } = getContext("grid") + const { dispatch: gridDispatch, rows } = getContext("grid") export let field let mounted = false - const fieldDefinitions = Object.values(FIELDS).reduce( - // Storing the fields by complex field id - (acc, field) => ({ - ...acc, - [makeFieldId(field.type, field.subtype)]: field, - }), - {} - ) - - function makeFieldId(type, subtype, autocolumn) { - // don't make field IDs for auto types - if (type === AUTO_TYPE || autocolumn) { - return type.toUpperCase() - } else { - return `${type}${subtype || ""}`.toUpperCase() - } - } - let originalName let linkEditDisabled let primaryDisplay let indexes = [...($tables.selected.indexes || [])] let isCreating = undefined - let relationshipPart1 = PrettyRelationshipDefinitions.Many let relationshipPart2 = PrettyRelationshipDefinitions.One - let relationshipTableIdPrimary = null let relationshipTableIdSecondary = null - let table = $tables.selected - let confirmDeleteDialog let savingColumn let deleteColName @@ -99,11 +79,6 @@ } let relationshipOpts1 = Object.values(PrettyRelationshipDefinitions) let relationshipOpts2 = Object.values(PrettyRelationshipDefinitions) - - $: if (primaryDisplay) { - editableColumn.constraints.presence = { allowEmpty: false } - } - let relationshipMap = { [RelationshipType.ONE_TO_MANY]: { part1: PrettyRelationshipDefinitions.MANY, @@ -118,7 +93,12 @@ part2: PrettyRelationshipDefinitions.MANY, }, } + let autoColumnInfo = getAutoColumnInformation() + $: rowGoldenSample = RowUtils.generateGoldenSample($rows) + $: if (primaryDisplay) { + editableColumn.constraints.presence = { allowEmpty: false } + } $: { // this parses any changes the user has made when creating a new internal relationship // into what we expect the schema to look like @@ -148,6 +128,76 @@ editableColumn.tableId = relationshipTableIdSecondary } } + $: initialiseField(field, savingColumn) + $: checkConstraints(editableColumn) + $: required = !!editableColumn?.constraints?.presence || primaryDisplay + $: uneditable = + $tables.selected?._id === TableNames.USERS && + UNEDITABLE_USER_FIELDS.includes(editableColumn.name) + $: invalid = + !editableColumn?.name || + (editableColumn?.type === LINK_TYPE && !editableColumn?.tableId) || + Object.keys(errors).length !== 0 + $: errors = checkErrors(editableColumn) + $: datasource = $datasources.list.find( + source => source._id === table?.sourceId + ) + $: tableAutoColumnsTypes = getTableAutoColumnTypes($tables?.selected) + $: availableAutoColumns = Object.keys(autoColumnInfo).reduce((acc, key) => { + if (!tableAutoColumnsTypes.includes(key)) { + acc[key] = autoColumnInfo[key] + } + return acc + }, {}) + $: availableAutoColumnKeys = availableAutoColumns + ? Object.keys(availableAutoColumns) + : [] + $: autoColumnOptions = editableColumn.autocolumn + ? autoColumnInfo + : availableAutoColumns + // used to select what different options can be displayed for column type + $: canBeDisplay = + editableColumn?.type !== LINK_TYPE && + editableColumn?.type !== AUTO_TYPE && + editableColumn?.type !== JSON_TYPE && + !editableColumn.autocolumn + $: canBeRequired = + editableColumn?.type !== LINK_TYPE && + !uneditable && + editableColumn?.type !== AUTO_TYPE && + !editableColumn.autocolumn + $: externalTable = table.sourceType === DB_TYPE_EXTERNAL + // in the case of internal tables the sourceId will just be undefined + $: tableOptions = $tables.list.filter( + opt => + opt.sourceType === table.sourceType && table.sourceId === opt.sourceId + ) + $: typeEnabled = + !originalName || + (originalName && + SWITCHABLE_TYPES.indexOf(editableColumn.type) !== -1 && + !editableColumn?.autocolumn) + + const fieldDefinitions = Object.values(FIELDS).reduce( + // Storing the fields by complex field id + (acc, field) => ({ + ...acc, + [makeFieldId(field.type, field.subtype)]: field, + }), + {} + ) + + function makeFieldId(type, subtype, autocolumn) { + // don't make field IDs for auto types + if (type === AUTO_TYPE || autocolumn) { + return type.toUpperCase() + } else if (type === FieldType.BB_REFERENCE) { + return `${type}${subtype || ""}`.toUpperCase() + } else { + return type.toUpperCase() + } + } + const initialiseField = (field, savingColumn) => { isCreating = !field if (field && !savingColumn) { @@ -187,22 +237,6 @@ } } - $: initialiseField(field, savingColumn) - - $: checkConstraints(editableColumn) - $: required = !!editableColumn?.constraints?.presence || primaryDisplay - $: uneditable = - $tables.selected?._id === TableNames.USERS && - UNEDITABLE_USER_FIELDS.includes(editableColumn.name) - $: invalid = - !editableColumn?.name || - (editableColumn?.type === LINK_TYPE && !editableColumn?.tableId) || - Object.keys(errors).length !== 0 - $: errors = checkErrors(editableColumn) - $: datasource = $datasources.list.find( - source => source._id === table?.sourceId - ) - const getTableAutoColumnTypes = table => { return Object.keys(table?.schema).reduce((acc, key) => { let fieldSchema = table?.schema[key] @@ -213,47 +247,6 @@ }, []) } - let autoColumnInfo = getAutoColumnInformation() - - $: tableAutoColumnsTypes = getTableAutoColumnTypes($tables?.selected) - $: availableAutoColumns = Object.keys(autoColumnInfo).reduce((acc, key) => { - if (!tableAutoColumnsTypes.includes(key)) { - acc[key] = autoColumnInfo[key] - } - return acc - }, {}) - - $: availableAutoColumnKeys = availableAutoColumns - ? Object.keys(availableAutoColumns) - : [] - - $: autoColumnOptions = editableColumn.autocolumn - ? autoColumnInfo - : availableAutoColumns - - // used to select what different options can be displayed for column type - $: canBeDisplay = - editableColumn?.type !== LINK_TYPE && - editableColumn?.type !== AUTO_TYPE && - editableColumn?.type !== JSON_TYPE && - !editableColumn.autocolumn - $: canBeRequired = - editableColumn?.type !== LINK_TYPE && - !uneditable && - editableColumn?.type !== AUTO_TYPE && - !editableColumn.autocolumn - $: externalTable = table.sourceType === DB_TYPE_EXTERNAL - // in the case of internal tables the sourceId will just be undefined - $: tableOptions = $tables.list.filter( - opt => - opt.sourceType === table.sourceType && table.sourceId === opt.sourceId - ) - $: typeEnabled = - !originalName || - (originalName && - SWITCHABLE_TYPES.indexOf(editableColumn.type) !== -1 && - !editableColumn?.autocolumn) - async function saveColumn() { savingColumn = true if (errors?.length) { @@ -479,7 +472,7 @@ newError.name = `Column name already in use.` } - if (fieldInfo.type === "auto" && !fieldInfo.subtype) { + if (fieldInfo.type === FieldType.AUTO && !fieldInfo.subtype) { newError.subtype = `Auto Column requires a type` } @@ -540,18 +533,18 @@ }} /> - {#if editableColumn.type === "string"} + {#if editableColumn.type === FieldType.STRING} - {:else if editableColumn.type === "options"} + {:else if editableColumn.type === FieldType.OPTIONS} - {:else if editableColumn.type === "longform"} + {:else if editableColumn.type === FieldType.LONGFORM}
@@ -569,12 +562,12 @@ text="Enable rich text support (markdown)" />
- {:else if editableColumn.type === "array"} + {:else if editableColumn.type === FieldType.ARRAY} - {:else if editableColumn.type === "datetime" && !editableColumn.autocolumn} + {:else if editableColumn.type === FieldType.DATETIME && !editableColumn.autocolumn}
@@ -613,7 +606,7 @@
{/if} - {:else if editableColumn.type === "number" && !editableColumn.autocolumn} + {:else if editableColumn.type === FieldType.NUMBER && !editableColumn.autocolumn}
@@ -638,7 +631,7 @@ />
- {:else if editableColumn.type === "link"} + {:else if editableColumn.type === FieldType.LINK}
{ @@ -689,6 +683,7 @@ }} bindings={getBindings({ table })} allowJS + context={rowGoldenSample} />
diff --git a/packages/builder/src/components/backend/Datasources/CreateEditRelationship.svelte b/packages/builder/src/components/backend/Datasources/CreateEditRelationship.svelte index 6b9524776c..b54ecbf9fd 100644 --- a/packages/builder/src/components/backend/Datasources/CreateEditRelationship.svelte +++ b/packages/builder/src/components/backend/Datasources/CreateEditRelationship.svelte @@ -40,8 +40,15 @@ part2: PrettyRelationshipDefinitions.MANY, }, } - let relationshipOpts1 = Object.values(PrettyRelationshipDefinitions) - let relationshipOpts2 = Object.values(PrettyRelationshipDefinitions) + $: relationshipOpts1 = + relationshipPart2 === PrettyRelationshipDefinitions.ONE + ? [PrettyRelationshipDefinitions.MANY] + : Object.values(PrettyRelationshipDefinitions) + + $: relationshipOpts2 = + relationshipPart1 === PrettyRelationshipDefinitions.ONE + ? [PrettyRelationshipDefinitions.MANY] + : Object.values(PrettyRelationshipDefinitions) let relationshipPart1 = PrettyRelationshipDefinitions.ONE let relationshipPart2 = PrettyRelationshipDefinitions.MANY diff --git a/packages/builder/src/components/backend/TableNavigator/popovers/EditTablePopover.svelte b/packages/builder/src/components/backend/TableNavigator/popovers/EditTablePopover.svelte index c2cda1f2d8..f2c726c8bf 100644 --- a/packages/builder/src/components/backend/TableNavigator/popovers/EditTablePopover.svelte +++ b/packages/builder/src/components/backend/TableNavigator/popovers/EditTablePopover.svelte @@ -28,7 +28,6 @@ let deleteTableName $: externalTable = table?.sourceType === DB_TYPE_EXTERNAL - $: allowDeletion = !externalTable || table?.created function showDeleteModal() { templateScreens = $screenStore.screens.filter( @@ -56,7 +55,7 @@ $goto(`./datasource/${table.datasourceId}`) } } catch (error) { - notifications.error("Error deleting table") + notifications.error(`Error deleting table - ${error.message}`) } } @@ -86,17 +85,15 @@ } -{#if allowDeletion} - -
- -
- {#if !externalTable} - Edit - {/if} - Delete - -{/if} + +
+ +
+ {#if !externalTable} + Edit + {/if} + Delete + { @@ -82,8 +83,8 @@ }) } - // For handlebars only. - const bindStyle = new MatchDecorator({ + // Match decoration for HBS bindings + const hbsMatchDeco = new MatchDecorator({ regexp: FIND_ANY_HBS_REGEX, decoration: () => { return Decoration.mark({ @@ -94,12 +95,11 @@ }) }, }) - - let plugin = ViewPlugin.define( + const hbsMatchDecoPlugin = ViewPlugin.define( view => ({ - decorations: bindStyle.createDeco(view), + decorations: hbsMatchDeco.createDeco(view), update(u) { - this.decorations = bindStyle.updateDeco(u, this.decorations) + this.decorations = hbsMatchDeco.updateDeco(u, this.decorations) }, }), { @@ -107,7 +107,29 @@ } ) - const dispatch = createEventDispatcher() + // Match decoration for snippets + const snippetMatchDeco = new MatchDecorator({ + regexp: /snippets\.[^\s(]+/g, + decoration: () => { + return Decoration.mark({ + tag: "span", + attributes: { + class: "snippet-wrap", + }, + }) + }, + }) + const snippetMatchDecoPlugin = ViewPlugin.define( + view => ({ + decorations: snippetMatchDeco.createDeco(view), + update(u) { + this.decorations = snippetMatchDeco.updateDeco(u, this.decorations) + }, + }), + { + decorations: v => v.decorations, + } + ) // Theming! let currentTheme = $themeStore?.theme @@ -117,7 +139,7 @@ const indentWithTabCustom = { key: "Tab", run: view => { - if (completionStatus(view.state) == "active") { + if (completionStatus(view.state) === "active") { acceptCompletion(view) return true } @@ -131,7 +153,7 @@ } const buildKeymap = () => { - const baseMap = [ + return [ ...closeBracketsKeymap, ...defaultKeymap, ...historyKeymap, @@ -139,43 +161,25 @@ ...completionKeymap, indentWithTabCustom, ] - return baseMap } const buildBaseExtensions = () => { return [ - ...(mode.name === "handlebars" ? [plugin] : []), - history(), drawSelection(), dropCursor(), bracketMatching(), closeBrackets(), - highlightActiveLine(), syntaxHighlighting(oneDarkHighlightStyle, { fallback: true }), - highlightActiveLineGutter(), highlightSpecialChars(), EditorView.lineWrapping, - EditorView.updateListener.of(v => { - const docStr = v.state.doc?.toString() - if (docStr === value) { - return - } - dispatch("change", docStr) - }), - keymap.of(buildKeymap()), - themeConfig.of([ - getDefaultTheme({ - height: editorHeight, - resize, - dark: isDark, - }), - ...(isDark ? [oneDark] : []), - ]), + themeConfig.of([...(isDark ? [oneDark] : [])]), ] } + // None of this is reactive, but it never has been, so we just assume most + // config flags aren't changed at runtime const buildExtensions = base => { - const complete = [...base] + let complete = [...base] if (autocompleteEnabled) { complete.push( @@ -183,7 +187,10 @@ override: [...completions], closeOnBlur: true, icons: false, - optionClass: () => "autocomplete-option", + optionClass: completion => + completion.simple + ? "autocomplete-option-simple" + : "autocomplete-option", }) ) complete.push( @@ -209,22 +216,49 @@ view.dispatch(tr) return true } - return false }) ) } - if (mode.name == "javascript") { + // JS only plugins + if (mode.name === "javascript") { + complete.push(snippetMatchDecoPlugin) complete.push(javascript()) - complete.push(highlightWhitespace()) - complete.push(lineNumbers()) - complete.push(foldGutter()) + if (!readonly) { + complete.push(highlightWhitespace()) + } + } + // HBS only plugins + else { + complete.push(hbsMatchDecoPlugin) } if (placeholder) { complete.push(placeholderFn(placeholder)) } + + if (readonly) { + complete.push(EditorState.readOnly.of(true)) + } else { + complete = [ + ...complete, + history(), + highlightActiveLine(), + highlightActiveLineGutter(), + lineNumbers(), + foldGutter(), + keymap.of(buildKeymap()), + EditorView.updateListener.of(v => { + const docStr = v.state.doc?.toString() + if (docStr === value) { + return + } + dispatch("change", docStr) + }), + ] + } + return complete } @@ -249,8 +283,6 @@ } } - $: editorHeight = typeof height === "number" ? `${height}px` : height - // Init when all elements are ready $: if (mounted && !isEditorInitialised) { isEditorInitialised = true @@ -265,14 +297,7 @@ // Issue theme compartment update editor.dispatch({ - effects: themeConfig.reconfigure([ - getDefaultTheme({ - height: editorHeight, - resize, - dark: isDark, - }), - ...(isDark ? [oneDark] : []), - ]), + effects: themeConfig.reconfigure([...(isDark ? [oneDark] : [])]), }) } } @@ -298,27 +323,207 @@
diff --git a/packages/builder/src/components/common/CodeEditor/index.js b/packages/builder/src/components/common/CodeEditor/index.js index 0d71a475f0..a04a140ad6 100644 --- a/packages/builder/src/components/common/CodeEditor/index.js +++ b/packages/builder/src/components/common/CodeEditor/index.js @@ -1,4 +1,3 @@ -import { EditorView } from "@codemirror/view" import { getManifest } from "@budibase/string-templates" import sanitizeHtml from "sanitize-html" import { groupBy } from "lodash" @@ -27,123 +26,33 @@ export const SECTIONS = { }, } -export const getDefaultTheme = opts => { - const { height, resize, dark } = opts - return EditorView.theme( - { - "&.cm-focused .cm-cursor": { - borderLeftColor: "var(--spectrum-alias-text-color)", - }, - "&": { - height: height ? `${height}` : "", - lineHeight: "1.3", - border: - "var(--spectrum-alias-border-size-thin) solid var(--spectrum-alias-border-color)", - borderRadius: "var(--border-radius-s)", - backgroundColor: - "var( --spectrum-textfield-m-background-color, var(--spectrum-global-color-gray-50) )", - resize: resize ? `${resize}` : "", - overflow: "hidden", - color: "var(--spectrum-alias-text-color)", - }, - "& .cm-tooltip.cm-tooltip-autocomplete > ul": { - fontFamily: - "var(--spectrum-alias-body-text-font-family, var(--spectrum-global-font-family-base))", - maxHeight: "16em", - }, - "& .cm-placeholder": { - color: "var(--spectrum-alias-text-color)", - fontStyle: "italic", - }, - "&.cm-focused": { - outline: "none", - borderColor: "var(--spectrum-alias-border-color-mouse-focus)", - }, - // AUTO COMPLETE - "& .cm-completionDetail": { - fontStyle: "unset", - textTransform: "uppercase", - fontSize: "10px", - backgroundColor: "var(--spectrum-global-color-gray-100)", - color: "var(--spectrum-global-color-gray-600)", - }, - "& .cm-completionLabel": { - marginLeft: - "calc(var(--spectrum-alias-workflow-icon-size-m) + var(--spacing-m))", - }, - "& .info-bubble": { - fontSize: "var(--font-size-s)", - display: "grid", - gridGap: "var(--spacing-s)", - gridTemplateColumns: "1fr", - color: "var(--spectrum-global-color-gray-800)", - }, - "& .cm-tooltip": { - marginLeft: "var(--spacing-s)", - border: "1px solid var(--spectrum-global-color-gray-300)", - borderRadius: - "var( --spectrum-popover-border-radius, var(--spectrum-alias-border-radius-regular) )", - backgroundColor: "var(--spectrum-global-color-gray-50)", - }, - // Section header - "& .info-section": { - display: "flex", - padding: "var(--spacing-s)", - gap: "var(--spacing-m)", - borderBottom: "1px solid var(--spectrum-global-color-gray-200)", - color: "var(--spectrum-global-color-gray-800)", - fontWeight: "bold", - }, - "& .info-section .spectrum-Icon": { - color: "var(--spectrum-global-color-gray-600)", - }, - // Autocomplete Option - "& .cm-tooltip.cm-tooltip-autocomplete .autocomplete-option": { - display: "flex", - justifyContent: "space-between", - alignItems: "center", - fontSize: "var(--spectrum-alias-font-size-default)", - padding: "var(--spacing-s)", - color: "var(--spectrum-global-color-gray-800)", - }, - "& .cm-tooltip-autocomplete ul li[aria-selected].autocomplete-option": { - backgroundColor: "var(--spectrum-global-color-gray-200)", - }, - "& .binding-wrap": { - color: "var(--spectrum-global-color-blue-700)", - fontFamily: "monospace", - }, - }, - { dark } - ) -} - export const buildHelperInfoNode = (completion, helper) => { const ele = document.createElement("div") ele.classList.add("info-bubble") const exampleNodeHtml = helper.example - ? `
${helper.example}
` + ? `
${helper.example}
` : "" const descriptionMarkup = sanitizeHtml(helper.description, { allowedTags: [], allowedAttributes: {}, }) - const descriptionNodeHtml = `
${descriptionMarkup}
` + const descriptionNodeHtml = `
${descriptionMarkup}
` ele.innerHTML = ` - ${exampleNodeHtml} ${descriptionNodeHtml} + ${exampleNodeHtml} ` return ele } const toSpectrumIcon = name => { return ` ` @@ -152,7 +61,9 @@ const toSpectrumIcon = name => { export const buildSectionHeader = (type, sectionName, icon, rank) => { const ele = document.createElement("div") ele.classList.add("info-section") - ele.classList.add(type) + if (type) { + ele.classList.add(type) + } ele.innerHTML = `${toSpectrumIcon(icon)}${sectionName}` return { name: sectionName, @@ -174,7 +85,7 @@ export const helpersToCompletion = (helpers, mode) => { }, type: "helper", section: helperSection, - detail: "FUNCTION", + detail: "Function", apply: (view, completion, from, to) => { insertBinding(view, from, to, key, mode) }, @@ -191,6 +102,29 @@ export const getHelperCompletions = mode => { }, []) } +export const snippetAutoComplete = snippets => { + return function myCompletions(context) { + if (!snippets?.length) { + return null + } + const word = context.matchBefore(/\w*/) + if (word.from == word.to && !context.explicit) { + return null + } + return { + from: word.from, + options: snippets.map(snippet => ({ + label: `snippets.${snippet.name}`, + type: "text", + simple: true, + apply: (view, completion, from, to) => { + insertSnippet(view, from, to, completion.label) + }, + })), + } + } +} + const bindingFilter = (options, query) => { return options.filter(completion => { const section_parsed = completion.section.name.toLowerCase() @@ -252,21 +186,12 @@ export const jsAutocomplete = baseCompletions => { } export const buildBindingInfoNode = (completion, binding) => { + if (!binding.valueHTML || binding.value == null) { + return null + } const ele = document.createElement("div") ele.classList.add("info-bubble") - - const exampleNodeHtml = binding.readableBinding - ? `
{{ ${binding.readableBinding} }}
` - : "" - - const descriptionNodeHtml = binding.description - ? `
${binding.description}
` - : "" - - ele.innerHTML = ` - ${exampleNodeHtml} - ${descriptionNodeHtml} - ` + ele.innerHTML = `
${binding.valueHTML}
` return ele } @@ -345,6 +270,20 @@ export const insertBinding = (view, from, to, text, mode) => { }) } +export const insertSnippet = (view, from, to, text) => { + let cursorPos = from + text.length + view.dispatch({ + changes: { + from, + to, + insert: text, + }, + selection: { + anchor: cursorPos, + }, + }) +} + export const bindingsToCompletions = (bindings, mode) => { const bindingByCategory = groupBy(bindings, "category") const categoryMeta = bindings?.reduce((acc, ele) => { @@ -374,7 +313,7 @@ export const bindingsToCompletions = (bindings, mode) => { ...bindingByCategory[catKey].reduce((acc, binding) => { let displayType = binding.fieldSchema?.type || binding.display?.type acc.push({ - label: binding.display?.name || "NO NAME", + label: binding.display?.name || binding.readableBinding || "NO NAME", info: completion => { return buildBindingInfoNode(completion, binding) }, diff --git a/packages/builder/src/components/common/HelpMenu.svelte b/packages/builder/src/components/common/HelpMenu.svelte index baff9a5a27..63156676d2 100644 --- a/packages/builder/src/components/common/HelpMenu.svelte +++ b/packages/builder/src/components/common/HelpMenu.svelte @@ -5,7 +5,7 @@ import { licensing } from "stores/portal" import { isPremiumOrAbove } from "helpers/planTitle" - $: premiumOrAboveLicense = isPremiumOrAbove($licensing?.license.plan.type) + $: premiumOrAboveLicense = isPremiumOrAbove($licensing?.license?.plan?.type) let show let hide diff --git a/packages/builder/src/components/common/NavHeader.svelte b/packages/builder/src/components/common/NavHeader.svelte index a163f354d2..2c67a3f9b3 100644 --- a/packages/builder/src/components/common/NavHeader.svelte +++ b/packages/builder/src/components/common/NavHeader.svelte @@ -59,7 +59,7 @@ class="searchButton" class:hide={search} > - +
- +
diff --git a/packages/builder/src/components/common/NavItem.svelte b/packages/builder/src/components/common/NavItem.svelte index 0819bcb86b..66b21e95a1 100644 --- a/packages/builder/src/components/common/NavItem.svelte +++ b/packages/builder/src/components/common/NavItem.svelte @@ -8,6 +8,7 @@ export let iconTooltip export let withArrow = false export let withActions = true + export let showActions = false export let indentLevel = 0 export let text export let border = true @@ -68,6 +69,8 @@ class:border class:selected class:withActions + class:showActions + class:actionsOpen={highlighted && withActions} class:scrollable class:highlighted class:selectedBy @@ -168,8 +171,10 @@ --avatars-background: var(--spectrum-global-color-gray-300); } .nav-item:hover .actions, - .hovering .actions { - visibility: visible; + .hovering .actions, + .nav-item.withActions.actionsOpen .actions, + .nav-item.withActions.showActions .actions { + opacity: 1; } .nav-item-content { flex: 1 1 auto; @@ -272,7 +277,6 @@ position: relative; display: grid; place-items: center; - visibility: hidden; order: 3; opacity: 0; width: 20px; diff --git a/packages/builder/src/components/common/bindings/BindingPanel.svelte b/packages/builder/src/components/common/bindings/BindingPanel.svelte index 7b567e052b..10d95a3e7e 100644 --- a/packages/builder/src/components/common/bindings/BindingPanel.svelte +++ b/packages/builder/src/components/common/bindings/BindingPanel.svelte @@ -1,74 +1,194 @@ - - + +
- { - if (selectedMode == mode) { - return true - } - - //Get the current mode value - const editorValue = usingJS ? decodeJSBinding(jsValue) : hbsValue - - if (editorValue) { - targetMode = selectedMode - return false - } - return true - }} - > - -
-
-
- {#if targetMode} -
-
- - {`Switch to ${targetMode}?`} - - This will discard anything in your binding -
- - -
-
-
- {/if} - -
-
-
- {#if !valid} -
- Current Handlebars syntax is invalid, please check the - guide - here - for more details. -
- {:else} - -
-
- Add available bindings by typing {{ or use the - menu on the right -
-
- {/if} -
-
- {#if $admin.isDev && allowJS} - { - convert() - targetMode = null - }} - > - Convert To JS - - {/if} - { - sidebar = !sidebar - }} - /> -
-
-
- - {#if sidebar} -
- -
- {/if} + {#if showTabBar} +
+
+ {#each editorModeOptions as editorMode} + changeMode(editorMode)} + > + {capitalise(editorMode)} + + {/each} +
+
+ {#each sidePanelOptions as panel} + changeSidePanel(panel)} + > + + + {/each}
- - {#if allowJS} - -
-
-
- {#if targetMode} -
-
- - {`Switch to ${targetMode}?`} - - This will discard anything in your binding -
- - -
-
-
- {/if} - - -
-
-
- -
-
- Add available bindings by typing $ or use the menu on - the right -
-
-
-
- { - sidebar = !sidebar - }} - /> -
-
-
- - {#if sidebar} -
- -
- {/if} -
- - {/if} -
- {#if typeof drawerActions?.hide === "function" && drawerActions?.headless} - - {/if} - {#if typeof bindingDrawerActions?.save === "function" && drawerActions?.headless} - - {/if}
- + {/if} +
+ {#if mode === Modes.Text} + {#key hbsCompletions} + + {/key} + {:else if mode === Modes.JavaScript} + {#key jsCompletions} + + {/key} + {/if} + {#if targetMode} +
+
+ + Switch to {targetMode}? + + This will discard anything in your binding +
+ + +
+
+
+ {/if} +
- - +
+ {#if sidePanel === SidePanels.Bindings} + + {:else if sidePanel === SidePanels.Evaluation} + + {:else if sidePanel === SidePanels.Snippets} + bindingHelpers.onSelectSnippet(snippet)} + {snippets} + /> + {/if} +
+
+ diff --git a/packages/builder/src/components/common/bindings/BindingPicker.svelte b/packages/builder/src/components/common/bindings/BindingPicker.svelte deleted file mode 100644 index 42057b382b..0000000000 --- a/packages/builder/src/components/common/bindings/BindingPicker.svelte +++ /dev/null @@ -1,399 +0,0 @@ - - - - - -
- {#if hoverTarget.title} -
{hoverTarget.title}
- {/if} - {#if hoverTarget.description} -
- - {@html hoverTarget.description} -
- {/if} - {#if hoverTarget.example} - 
{hoverTarget.example}
- {/if} -
- - - - - - - - - {#if selectedCategory} -
- { - selectedCategory = null - }} - > - Back - -
- {/if} - - {#if !selectedCategory} -
- - - - - { - search = null - }} - class:searching={search} - > - - -
- {/if} - - {#if !selectedCategory && !search} -
    - {#each categoryNames as categoryName} -
  • { - selectedCategory = categoryName - }} - > - - {categoryName} - -
    • - {/each} -
- {/if} - - {#if selectedCategory || search} - {#each filteredCategories as category} - {#if category.bindings?.length} -
-
- {category.name} -
-
    - {#each category.bindings as binding} -
  • { - popoverAnchor = e.target - if (!binding.description) { - return - } - hoverTarget = { - title: binding.display?.name || binding.fieldSchema?.name, - description: binding.description, - } - popover.show() - e.stopPropagation() - }} - on:mouseleave={() => { - popover.hide() - popoverAnchor = null - hoverTarget = null - }} - on:focus={() => {}} - on:blur={() => {}} - on:click={() => addBinding(binding)} - > - - {#if binding.display?.name} - {binding.display.name} - {:else if binding.fieldSchema?.name} - {binding.fieldSchema?.name} - {:else} - {binding.readableBinding} - {/if} - - - {#if binding.display?.type || binding.fieldSchema?.type} - - - {binding.display?.type || binding.fieldSchema?.type} - - - {/if} -
    • - {/each} -
-
- {/if} - {/each} - - {#if selectedCategory === "Helpers" || search} - {#if filteredHelpers?.length} -
-
Helpers
-
    - {#each filteredHelpers as helper} -
  • addHelper(helper, mode.name == "javascript")} - on:mouseenter={e => { - popoverAnchor = e.target - if (!helper.displayText && helper.description) { - return - } - hoverTarget = { - title: helper.displayText, - description: helper.description, - example: getHelperExample( - helper, - mode.name == "javascript" - ), - } - popover.show() - e.stopPropagation() - }} - on:mouseleave={() => { - popover.hide() - popoverAnchor = null - hoverTarget = null - }} - on:focus={() => {}} - on:blur={() => {}} - > - {helper.displayText} - - function - -
    • - {/each} -
-
- {/if} - {/if} - {/if} - - - diff --git a/packages/builder/src/components/common/bindings/BindingSidePanel.svelte b/packages/builder/src/components/common/bindings/BindingSidePanel.svelte new file mode 100644 index 0000000000..6ef2d35a6c --- /dev/null +++ b/packages/builder/src/components/common/bindings/BindingSidePanel.svelte @@ -0,0 +1,446 @@ + + + +
+ {#if hoverTarget.description} +
+ + {@html hoverTarget.description} +
+ {/if} + {#if hoverTarget.code} + + 
{@html hoverTarget.code}
+ {/if} +
+ + + + +
+ + {#if selectedCategory} +
+ (selectedCategory = null)} + /> + {selectedCategory} +
+ {/if} + + {#if !selectedCategory} +
+ {#if searching} +
+ +
+ + {:else} +
Bindings
+ + {/if} +
+ {/if} + {#if !selectedCategory && !search} +
    + {#each categoryNames as categoryName} +
  • { + selectedCategory = categoryName + }} + > + + {categoryName} + +
    • + {/each} +
+ {/if} + + {#if selectedCategory || search} + {#each filteredCategories as category} + {#if category.bindings?.length} +
+ {#if filteredCategories.length > 1} +
+ {category.name} +
+ {/if} +
    + {#each category.bindings as binding} +
  • showBindingPopover(binding, e.target)} + on:mouseleave={hidePopover} + on:click={() => addBinding(binding)} + > + + {#if binding.display?.name} + {binding.display.name} + {:else if binding.fieldSchema?.name} + {binding.fieldSchema?.name} + {:else} + {binding.readableBinding} + {/if} + + {#if binding.display?.type || binding.fieldSchema?.type} + + + {binding.display?.type || binding.fieldSchema?.type} + + + {/if} +
    • + {/each} +
+
+ {/if} + {/each} + + {#if selectedCategory === "Helpers" || search} + {#if filteredHelpers?.length} +
+
    + {#each filteredHelpers as helper} +
  • showHelperPopover(helper, e.target)} + on:mouseleave={hidePopover} + on:click={() => addHelper(helper, mode.name === "javascript")} + > + {helper.displayText} + + function + +
    • + {/each} +
+
+ {/if} + {/if} + {/if} + +
+ + diff --git a/packages/builder/src/components/common/bindings/ClientBindingPanel.svelte b/packages/builder/src/components/common/bindings/ClientBindingPanel.svelte index 7888f2e486..4e5789b563 100644 --- a/packages/builder/src/components/common/bindings/ClientBindingPanel.svelte +++ b/packages/builder/src/components/common/bindings/ClientBindingPanel.svelte @@ -1,12 +1,14 @@ - - - Add the objects on the left to enrich your text. - - - + + (tempValue = event.detail)} {bindings} {allowJS} diff --git a/packages/builder/src/components/common/bindings/DrawerBindableInput.svelte b/packages/builder/src/components/common/bindings/DrawerBindableInput.svelte index 80996a484d..d11ebcf87a 100644 --- a/packages/builder/src/components/common/bindings/DrawerBindableInput.svelte +++ b/packages/builder/src/components/common/bindings/DrawerBindableInput.svelte @@ -13,21 +13,21 @@ export let panel = ClientBindingPanel export let value = "" export let bindings = [] - export let title = "Bindings" + export let title export let placeholder export let label export let disabled = false - export let fillWidth export let allowJS = true export let allowHelpers = true export let updateOnChange = true - export let drawerLeft export let key export let disableBindings = false + export let forceModal = false + export let context = null const dispatch = createEventDispatcher() + let bindingDrawer - let valid = true let currentVal = value $: readableValue = runtimeToReadableBinding(bindings, value) @@ -88,27 +88,20 @@ - - Add the objects on the left to enrich your text. - - + (tempValue = event.detail)} {bindings} {allowJS} {allowHelpers} + {context} /> diff --git a/packages/builder/src/components/common/bindings/DrawerBindableSlot.svelte b/packages/builder/src/components/common/bindings/DrawerBindableSlot.svelte index 4cac24660f..8ce9dda209 100644 --- a/packages/builder/src/components/common/bindings/DrawerBindableSlot.svelte +++ b/packages/builder/src/components/common/bindings/DrawerBindableSlot.svelte @@ -16,7 +16,6 @@ export let placeholder export let label export let disabled = false - export let fillWidth export let allowJS = true export let allowHelpers = true export let updateOnChange = true @@ -26,7 +25,6 @@ const dispatch = createEventDispatcher() let bindingDrawer - let valid = true let currentVal = value $: readableValue = runtimeToReadableBinding(bindings, value) @@ -173,22 +171,14 @@ - - Add the objects on the left to enrich your text. - - + (tempValue = event.detail)} {bindings} diff --git a/packages/builder/src/components/common/bindings/EvaluationSidePanel.svelte b/packages/builder/src/components/common/bindings/EvaluationSidePanel.svelte new file mode 100644 index 0000000000..2c4e6a0991 --- /dev/null +++ b/packages/builder/src/components/common/bindings/EvaluationSidePanel.svelte @@ -0,0 +1,133 @@ + + +
+
+
+ {#if error} + +
Error
+ {#if evaluating} +
+ +
+ {/if} + + + {:else} +
Preview
+ {#if evaluating} +
+ +
+ {/if} + + {#if !empty} + + {/if} + {/if} +
+
+
+ {#if empty} + Your expression will be evaluated here + {:else} + + {@html highlightedResult} + {/if} +
+
+ + diff --git a/packages/builder/src/components/common/bindings/ModalBindableInput.svelte b/packages/builder/src/components/common/bindings/ModalBindableInput.svelte index 41245ceaec..3261dc6b74 100644 --- a/packages/builder/src/components/common/bindings/ModalBindableInput.svelte +++ b/packages/builder/src/components/common/bindings/ModalBindableInput.svelte @@ -1,115 +1,12 @@ - - -
- onChange(event.detail)} - {placeholder} - {updateOnChange} - /> -
- -
-
- - - - Add the objects on the left to enrich your text. - -
- (tempValue = e.detail)} - {bindings} - {allowJS} - /> -
- - - - + diff --git a/packages/builder/src/components/common/bindings/ServerBindingPanel.svelte b/packages/builder/src/components/common/bindings/ServerBindingPanel.svelte index 213e5bbf1d..df0da768e0 100644 --- a/packages/builder/src/components/common/bindings/ServerBindingPanel.svelte +++ b/packages/builder/src/components/common/bindings/ServerBindingPanel.svelte @@ -1,10 +1,11 @@ diff --git a/packages/builder/src/components/common/bindings/SnippetDrawer.svelte b/packages/builder/src/components/common/bindings/SnippetDrawer.svelte new file mode 100644 index 0000000000..d6b6f92b17 --- /dev/null +++ b/packages/builder/src/components/common/bindings/SnippetDrawer.svelte @@ -0,0 +1,160 @@ + + + + + {#if snippet} + {snippet.name} + {:else} +
+ Name + + {#if nameError} + + + + {/if} +
+ {/if} + + + {#if snippet} + + {/if} + + + + {#key key} + (code = e.detail)} + > +
+ +
+ + {/key} + + + + + + diff --git a/packages/builder/src/components/common/bindings/SnippetSidePanel.svelte b/packages/builder/src/components/common/bindings/SnippetSidePanel.svelte new file mode 100644 index 0000000000..c68699fc0f --- /dev/null +++ b/packages/builder/src/components/common/bindings/SnippetSidePanel.svelte @@ -0,0 +1,278 @@ + + + + +
+ +
+ {#if enableSnippets} + {#if searching} +
+ +
+ + {:else} +
Snippets
+ + + {/if} + {:else} +
+ Snippets + + Premium + +
+ {/if} +
+
+ {#if enableSnippets && filteredSnippets?.length} + {#each filteredSnippets as snippet} +
showSnippet(snippet, e.target)} + on:mouseleave={hidePopover} + on:click={() => addSnippet(snippet)} + > + {snippet.name} + editSnippet(e, snippet)} + /> +
+ {/each} + {:else} +
+ + Snippets let you create reusable JS functions and values that can + all be managed in one place + + {#if enableSnippets} + + {:else} + + {/if} +
+ {/if} +
+ +
+ + +
+ {#key hoveredSnippet} + + {/key} +
+ + + + + diff --git a/packages/builder/src/components/common/bindings/utils.js b/packages/builder/src/components/common/bindings/utils.js index a086cd0394..77e4a1dfb1 100644 --- a/packages/builder/src/components/common/bindings/utils.js +++ b/packages/builder/src/components/common/bindings/utils.js @@ -1,6 +1,11 @@ import { decodeJSBinding } from "@budibase/string-templates" import { hbInsert, jsInsert } from "components/common/CodeEditor" +export const BindingType = { + READABLE: "readableBinding", + RUNTIME: "runtimeBinding", +} + export class BindingHelpers { constructor(getCaretPosition, insertAtPos, { disableWrapping } = {}) { this.getCaretPosition = getCaretPosition @@ -25,17 +30,28 @@ export class BindingHelpers { } // Adds a data binding to the expression - onSelectBinding(value, binding, { js, dontDecode }) { + onSelectBinding( + value, + binding, + { js, dontDecode, type = BindingType.READABLE } + ) { const { start, end } = this.getCaretPosition() if (js) { const jsVal = dontDecode ? value : decodeJSBinding(value) - const insertVal = jsInsert(jsVal, start, end, binding.readableBinding, { + const insertVal = jsInsert(jsVal, start, end, binding[type], { disableWrapping: this.disableWrapping, }) this.insertAtPos({ start, end, value: insertVal }) } else { - const insertVal = hbInsert(value, start, end, binding.readableBinding) + const insertVal = hbInsert(value, start, end, binding[type]) this.insertAtPos({ start, end, value: insertVal }) } } + + // Adds a snippet to the expression + onSelectSnippet(snippet) { + const pos = this.getCaretPosition() + const { start, end } = pos + this.insertAtPos({ start, end, value: `snippets.${snippet.name}` }) + } } diff --git a/packages/builder/src/components/deploy/AppActions.svelte b/packages/builder/src/components/deploy/AppActions.svelte index 86427ff761..1d7bb4f65e 100644 --- a/packages/builder/src/components/deploy/AppActions.svelte +++ b/packages/builder/src/components/deploy/AppActions.svelte @@ -19,7 +19,7 @@ import ConfirmDialog from "components/common/ConfirmDialog.svelte" import analytics, { Events, EventSource } from "analytics" import { API } from "api" - import { apps } from "stores/portal" + import { appsStore } from "stores/portal" import { previewStore, builderStore, @@ -32,10 +32,14 @@ import TourWrap from "components/portal/onboarding/TourWrap.svelte" import { TOUR_STEP_KEYS } from "components/portal/onboarding/tours.js" import { goto } from "@roxi/routify" + import { onMount } from "svelte" + import PosthogClient from "../../analytics/PosthogClient" export let application export let loaded + const posthog = new PosthogClient(process.env.POSTHOG_TOKEN) + let unpublishModal let updateAppModal let revertModal @@ -44,8 +48,10 @@ let appActionPopoverOpen = false let appActionPopoverAnchor let publishing = false + let showNpsSurvey = false + let lastOpened - $: filteredApps = $apps.filter(app => app.devId === application) + $: filteredApps = $appsStore.apps.filter(app => app.devId === application) $: selectedApp = filteredApps?.length ? filteredApps[0] : null $: latestDeployments = $deploymentStore .filter(deployment => deployment.status === "SUCCESS") @@ -57,7 +63,7 @@ $appStore.version && $appStore.upgradableVersion !== $appStore.version $: canPublish = !publishing && loaded && $sortedScreens.length > 0 - $: lastDeployed = getLastDeployedString($deploymentStore) + $: lastDeployed = getLastDeployedString($deploymentStore, lastOpened) const initialiseApp = async () => { const applicationPkg = await API.fetchAppPackage($appStore.devId) @@ -97,6 +103,7 @@ type: "success", icon: "GlobeCheck", }) + showNpsSurvey = true await completePublish() } catch (error) { console.error(error) @@ -129,7 +136,7 @@ } try { await API.unpublishApp(selectedApp.prodId) - await apps.load() + await appsStore.load() notifications.send("App unpublished", { type: "success", icon: "GlobeStrike", @@ -141,12 +148,16 @@ const completePublish = async () => { try { - await apps.load() + await appsStore.load() await deploymentStore.load() } catch (err) { notifications.error("Error refreshing app") } } + + onMount(() => { + posthog.init() + }) @@ -164,9 +175,10 @@
{/if}
@@ -200,6 +212,7 @@ class="app-action-button publish app-action-popover" on:click={() => { if (!appActionPopoverOpen) { + lastOpened = new Date() appActionPopover.show() } else { appActionPopover.hide() @@ -209,7 +222,7 @@
- + Publish +{#if showNpsSurvey} +
+{/if} + diff --git a/packages/builder/src/components/deploy/RevertModalVersionSelect.svelte b/packages/builder/src/components/deploy/RevertModalVersionSelect.svelte new file mode 100644 index 0000000000..ed40a101d0 --- /dev/null +++ b/packages/builder/src/components/deploy/RevertModalVersionSelect.svelte @@ -0,0 +1,33 @@ + + +
+ ($validation.touched.name = true)} + on:change={nameToUrl($values.name)} + label="Name" + placeholder={defaultAppName} + /> + + ($validation.touched.url = true)} + on:change={tidyUrl($values.url)} + label="URL" + placeholder={$values.url + ? $values.url + : `/${resolveAppUrl($values.name)}`} + /> + {#if $values.url && $values.url !== "" && !$validation.errors.url} +
+ {appUrl} +
+ {/if} + + + + + diff --git a/packages/builder/src/components/start/ExportAppModal.svelte b/packages/builder/src/components/start/ExportAppModal.svelte index 734e4448a1..ec0cf42fe0 100644 --- a/packages/builder/src/components/start/ExportAppModal.svelte +++ b/packages/builder/src/components/start/ExportAppModal.svelte @@ -121,6 +121,7 @@ { - const applications = svelteGet(apps) + const applications = svelteGet(appsStore).apps appValidation.name(validation, { apps: applications, currentApp: { @@ -62,7 +62,7 @@ async function updateApp() { try { - await apps.update(app.appId, { + await appsStore.save(app.appId, { name: $values.name?.trim(), url: $values.url?.trim(), icon: { diff --git a/packages/builder/src/dataBinding.js b/packages/builder/src/dataBinding.js index 0442a67da9..8fdf71c156 100644 --- a/packages/builder/src/dataBinding.js +++ b/packages/builder/src/dataBinding.js @@ -1146,7 +1146,7 @@ export const getAllStateVariables = () => { "@budibase/standard-components/multistepformblockstep" ) - steps.forEach(step => { + steps?.forEach(step => { parseComponentSettings(stepDefinition, step) }) }) diff --git a/packages/builder/src/global.css b/packages/builder/src/global.css index bc1f55d9d3..adf4a47070 100644 --- a/packages/builder/src/global.css +++ b/packages/builder/src/global.css @@ -22,6 +22,7 @@ body { --grey-7: var(--spectrum-global-color-gray-700); --grey-8: var(--spectrum-global-color-gray-800); --grey-9: var(--spectrum-global-color-gray-900); + --spectrum-global-color-yellow-1000: #d8b500; color: var(--ink); background-color: var(--background-alt); diff --git a/packages/builder/src/helpers/components.js b/packages/builder/src/helpers/components.js index 4f4f3ed380..a03ebfdfa7 100644 --- a/packages/builder/src/helpers/components.js +++ b/packages/builder/src/helpers/components.js @@ -279,3 +279,11 @@ export const buildContextTreeLookupMap = rootComponent => { }) return map } + +// Get a flat list of ids for all descendants of a component +export const getChildIdsForComponent = component => { + return [ + component._id, + ...(component?._children ?? []).map(getChildIdsForComponent).flat(1), + ] +} diff --git a/packages/builder/src/helpers/duplicate.js b/packages/builder/src/helpers/duplicate.js index 1547fcd4d1..c2a924f97b 100644 --- a/packages/builder/src/helpers/duplicate.js +++ b/packages/builder/src/helpers/duplicate.js @@ -48,3 +48,53 @@ export const duplicateName = (name, allNames) => { return `${baseName} ${number}` } + +/** + * More flexible alternative to the above function, which handles getting the + * next sequential name from an array of existing items while accounting for + * any type of prefix, and being able to deeply retrieve that name from the + * existing item array. + * + * Examples with a prefix of "foo": + * [] => "foo" + * ["foo"] => "foo2" + * ["foo", "foo6"] => "foo7" + * + * Examples with a prefix of "foo " (space at the end): + * [] => "foo" + * ["foo"] => "foo 2" + * ["foo", "foo 6"] => "foo 7" + * + * @param items the array of existing items + * @param prefix the string prefix of each name, including any spaces desired + * @param getName optional function to extract the name for an item, if not a + * flat array of strings + */ +export const getSequentialName = (items, prefix, getName = x => x) => { + if (!prefix?.length || !getName) { + return null + } + const trimmedPrefix = prefix.trim() + if (!items?.length) { + return trimmedPrefix + } + let max = 0 + items.forEach(item => { + const name = getName(item) + if (typeof name !== "string" || !name.startsWith(trimmedPrefix)) { + return + } + const split = name.split(trimmedPrefix) + if (split.length !== 2) { + return + } + if (split[1].trim() === "") { + split[1] = "1" + } + const num = parseInt(split[1]) + if (num > max) { + max = num + } + }) + return max === 0 ? trimmedPrefix : `${prefix}${max + 1}` +} diff --git a/packages/builder/src/helpers/tests/duplicate.test.js b/packages/builder/src/helpers/tests/duplicate.test.js index 400abed0aa..7e51c5ff2a 100644 --- a/packages/builder/src/helpers/tests/duplicate.test.js +++ b/packages/builder/src/helpers/tests/duplicate.test.js @@ -1,5 +1,5 @@ import { expect, describe, it } from "vitest" -import { duplicateName } from "../duplicate" +import { duplicateName, getSequentialName } from "../duplicate" describe("duplicate", () => { describe("duplicates a name ", () => { @@ -40,3 +40,64 @@ describe("duplicate", () => { }) }) }) + +describe("getSequentialName", () => { + it("handles nullish items", async () => { + const name = getSequentialName(null, "foo", () => {}) + expect(name).toBe("foo") + }) + + it("handles nullish prefix", async () => { + const name = getSequentialName([], null, () => {}) + expect(name).toBe(null) + }) + + it("handles nullish getName function", async () => { + const name = getSequentialName([], "foo", null) + expect(name).toBe(null) + }) + + it("handles just the prefix", async () => { + const name = getSequentialName(["foo"], "foo", x => x) + expect(name).toBe("foo2") + }) + + it("handles continuous ranges", async () => { + const name = getSequentialName(["foo", "foo2", "foo3"], "foo", x => x) + expect(name).toBe("foo4") + }) + + it("handles discontinuous ranges", async () => { + const name = getSequentialName(["foo", "foo3"], "foo", x => x) + expect(name).toBe("foo4") + }) + + it("handles a space inside the prefix", async () => { + const name = getSequentialName(["foo", "foo 2", "foo 3"], "foo ", x => x) + expect(name).toBe("foo 4") + }) + + it("handles a space inside the prefix with just the prefix", async () => { + const name = getSequentialName(["foo"], "foo ", x => x) + expect(name).toBe("foo 2") + }) + + it("handles no matches", async () => { + const name = getSequentialName(["aaa", "bbb"], "foo", x => x) + expect(name).toBe("foo") + }) + + it("handles similar names", async () => { + const name = getSequentialName( + ["fooo1", "2foo", "a3foo4", "5foo5"], + "foo", + x => x + ) + expect(name).toBe("foo") + }) + + it("handles non-string names", async () => { + const name = getSequentialName([null, 4123, [], {}], "foo", x => x) + expect(name).toBe("foo") + }) +}) diff --git a/packages/builder/src/pages/builder/_layout.svelte b/packages/builder/src/pages/builder/_layout.svelte index 62d3951fb5..95ca05b87b 100644 --- a/packages/builder/src/pages/builder/_layout.svelte +++ b/packages/builder/src/pages/builder/_layout.svelte @@ -71,6 +71,10 @@ await auth.getSelf() await admin.init() + if ($admin.maintenance.length > 0) { + $redirect("./maintenance") + } + if ($auth.user) { await licensing.init() } diff --git a/packages/builder/src/pages/builder/app/[application]/_components/BuilderSidePanel.svelte b/packages/builder/src/pages/builder/app/[application]/_components/BuilderSidePanel.svelte index 7c1cc583e1..035dc5a2ef 100644 --- a/packages/builder/src/pages/builder/app/[application]/_components/BuilderSidePanel.svelte +++ b/packages/builder/src/pages/builder/app/[application]/_components/BuilderSidePanel.svelte @@ -15,7 +15,14 @@ FancySelect, } from "@budibase/bbui" import { builderStore, appStore, roles } from "stores/builder" - import { groups, licensing, apps, users, auth, admin } from "stores/portal" + import { + groups, + licensing, + appsStore, + users, + auth, + admin, + } from "stores/portal" import { fetchData, Constants, @@ -54,7 +61,7 @@ let inviteFailureResponse = "" $: validEmail = emailValidator(email) === true - $: prodAppId = apps.getProdAppID($appStore.appId) + $: prodAppId = appsStore.getProdAppID($appStore.appId) $: promptInvite = showInvite( filteredInvites, filteredUsers, @@ -122,10 +129,7 @@ filteredUsers = $usersFetch.rows .filter(user => user.email !== $auth.user.email) .map(user => { - const isAdminOrGlobalBuilder = sdk.users.isAdminOrGlobalBuilder( - user, - prodAppId - ) + const isAdminOrGlobalBuilder = sdk.users.isAdminOrGlobalBuilder(user) const isAppBuilder = user.builder?.apps?.includes(prodAppId) let role if (isAdminOrGlobalBuilder) { diff --git a/packages/builder/src/pages/builder/app/[application]/_layout.svelte b/packages/builder/src/pages/builder/app/[application]/_layout.svelte index dd66f5bc34..fd6a97560d 100644 --- a/packages/builder/src/pages/builder/app/[application]/_layout.svelte +++ b/packages/builder/src/pages/builder/app/[application]/_layout.svelte @@ -8,7 +8,7 @@ userStore, deploymentStore, } from "stores/builder" - import { auth, apps } from "stores/portal" + import { auth, appsStore } from "stores/portal" import { TENANT_FEATURE_FLAGS, isEnabled } from "helpers/featureFlags" import { Icon, @@ -52,7 +52,7 @@ const pkg = await API.fetchAppPackage(application) await initialise(pkg) - await apps.load() + await appsStore.load() await deploymentStore.load() loaded = true @@ -96,7 +96,7 @@ const release_date = new Date("2023-03-01T00:00:00.000Z") const onboarded = new Date($auth.user?.onboardedAt) if (onboarded < release_date) { - builderStore.startTour(TOUR_KEYS.FEATURE_ONBOARDING) + builderStore.setTour(TOUR_KEYS.FEATURE_ONBOARDING) } } } @@ -144,7 +144,7 @@ {#each $layout.children as { path, title }} - + - + diff --git a/packages/builder/src/pages/builder/app/[application]/automation/_layout.svelte b/packages/builder/src/pages/builder/app/[application]/automation/_layout.svelte index c4ee060149..57180625b1 100644 --- a/packages/builder/src/pages/builder/app/[application]/automation/_layout.svelte +++ b/packages/builder/src/pages/builder/app/[application]/automation/_layout.svelte @@ -40,7 +40,7 @@
-
+
{#if $automationStore.automations?.length} {:else} diff --git a/packages/builder/src/pages/builder/app/[application]/design/[screenId]/[componentId]/_components/Component/ComponentSettingsPanel.svelte b/packages/builder/src/pages/builder/app/[application]/design/[screenId]/[componentId]/_components/Component/ComponentSettingsPanel.svelte index 8b778a2ce7..7a5372e883 100644 --- a/packages/builder/src/pages/builder/app/[application]/design/[screenId]/[componentId]/_components/Component/ComponentSettingsPanel.svelte +++ b/packages/builder/src/pages/builder/app/[application]/design/[screenId]/[componentId]/_components/Component/ComponentSettingsPanel.svelte @@ -16,6 +16,14 @@ } from "dataBinding" import { ActionButton, notifications } from "@budibase/bbui" import { capitalise } from "helpers" + import TourWrap from "components/portal/onboarding/TourWrap.svelte" + import { TOUR_STEP_KEYS } from "components/portal/onboarding/tours.js" + + const { + BUILDER_FORM_CREATE_STEPS, + BUILDER_FORM_VIEW_UPDATE_STEPS, + BUILDER_FORM_ROW_ID, + } = TOUR_STEP_KEYS const onUpdateName = async value => { try { @@ -46,7 +54,6 @@ $: id = $selectedComponent?._id $: id, (section = tabs[0]) - $: componentName = getComponentName(componentInstance) @@ -92,13 +99,21 @@
{#if section == "settings"} - + + + {/if} {#if section == "styles"} import { helpers } from "@budibase/shared-core" import { DetailSummary, notifications } from "@budibase/bbui" - import { componentStore } from "stores/builder" + import { componentStore, builderStore } from "stores/builder" import PropertyControl from "components/design/settings/controls/PropertyControl.svelte" import ResetFieldsButton from "components/design/settings/controls/ResetFieldsButton.svelte" import EjectBlockButton from "components/design/settings/controls/EjectBlockButton.svelte" @@ -177,9 +177,7 @@ defaultValue={setting.defaultValue} nested={setting.nested} onChange={val => updateSetting(setting, val)} - highlighted={$componentStore.highlightedSettingKey === - setting.key} - propertyFocus={$componentStore.propertyFocus === setting.key} + propertyFocus={$builderStore.propertyFocus === setting.key} info={setting.info} disableBindings={setting.disableBindings} props={{ diff --git a/packages/builder/src/pages/builder/app/[application]/design/[screenId]/[componentId]/_components/Navigation/index.svelte b/packages/builder/src/pages/builder/app/[application]/design/[screenId]/[componentId]/_components/Navigation/index.svelte index 50e1ad0cf8..4db218f60b 100644 --- a/packages/builder/src/pages/builder/app/[application]/design/[screenId]/[componentId]/_components/Navigation/index.svelte +++ b/packages/builder/src/pages/builder/app/[application]/design/[screenId]/[componentId]/_components/Navigation/index.svelte @@ -24,6 +24,13 @@ navigationStore, } from "stores/builder" import { DefaultAppTheme } from "constants" + import BarButtonList from "/src/components/design/settings/controls/BarButtonList.svelte" + + $: alignmentOptions = [ + { value: "Left", barIcon: "TextAlignLeft" }, + { value: "Center", barIcon: "TextAlignCenter" }, + { value: "Right", barIcon: "TextAlignRight" }, + ] $: screenRouteOptions = $screenStore.screens .map(screen => screen.routing?.route) @@ -46,6 +53,10 @@ notifications.error("Error updating navigation settings") } } + + const updateTextAlign = textAlignValue => { + navigationStore.syncAppNavigation({ textAlign: textAlignValue }) + } update("title", e.detail)} updateOnChange={false} /> + +
+ +
+ {/if}
diff --git a/packages/builder/src/pages/builder/app/[application]/design/[screenId]/[componentId]/new/_components/componentStructure.json b/packages/builder/src/pages/builder/app/[application]/design/[screenId]/[componentId]/new/_components/componentStructure.json index 96e8faf93c..b66c9cca3a 100644 --- a/packages/builder/src/pages/builder/app/[application]/design/[screenId]/[componentId]/new/_components/componentStructure.json +++ b/packages/builder/src/pages/builder/app/[application]/design/[screenId]/[componentId]/new/_components/componentStructure.json @@ -3,8 +3,6 @@ "name": "Blocks", "icon": "Article", "children": [ - "gridblock", - "tableblock", "cardsblock", "repeaterblock", "formblock", @@ -16,7 +14,7 @@ { "name": "Layout", "icon": "ClassicGridView", - "children": ["container", "section", "grid", "sidepanel"] + "children": ["container", "section", "sidepanel"] }, { "name": "Data", @@ -24,7 +22,7 @@ "children": [ "dataprovider", "repeater", - "table", + "gridblock", "spreadsheet", "dynamicfilter", "daterangepicker" diff --git a/packages/builder/src/pages/builder/app/[application]/design/[screenId]/_components/AppPanel.svelte b/packages/builder/src/pages/builder/app/[application]/design/[screenId]/_components/AppPanel.svelte index 5867021386..4617814485 100644 --- a/packages/builder/src/pages/builder/app/[application]/design/[screenId]/_components/AppPanel.svelte +++ b/packages/builder/src/pages/builder/app/[application]/design/[screenId]/_components/AppPanel.svelte @@ -6,6 +6,7 @@
+
@@ -32,7 +33,17 @@ flex-direction: column; justify-content: flex-start; align-items: stretch; - padding: 9px var(--spacing-m); + padding: 9px 10px 12px 10px; + position: relative; + transition: width 360ms ease-out; + } + .drawer-container { + position: absolute; + height: 100%; + width: 100%; + overflow: hidden; + top: 0; + left: 0; } .header { display: flex; diff --git a/packages/builder/src/pages/builder/app/[application]/design/[screenId]/_components/AppPreview.svelte b/packages/builder/src/pages/builder/app/[application]/design/[screenId]/_components/AppPreview.svelte index fa126bbc99..0ce9e096f2 100644 --- a/packages/builder/src/pages/builder/app/[application]/design/[screenId]/_components/AppPreview.svelte +++ b/packages/builder/src/pages/builder/app/[application]/design/[screenId]/_components/AppPreview.svelte @@ -10,19 +10,15 @@ navigationStore, selectedScreen, hoverStore, + componentTreeNodesStore, + snippets, } from "stores/builder" import ConfirmDialog from "components/common/ConfirmDialog.svelte" - import { - ProgressCircle, - Layout, - Heading, - Body, - Icon, - notifications, - } from "@budibase/bbui" + import { Layout, Heading, Body, Icon, notifications } from "@budibase/bbui" import ErrorSVG from "@budibase/frontend-core/assets/error.svg?raw" import { findComponent, findComponentPath } from "helpers/components" import { isActive, goto } from "@roxi/routify" + import { ClientAppSkeleton } from "@budibase/frontend-core" let iframe let layout @@ -68,6 +64,7 @@ hostname: window.location.hostname, port: window.location.port, }, + snippets: $snippets, } // Refresh the preview when required @@ -130,6 +127,7 @@ error = event.error || "An unknown error occurred" } else if (type === "select-component" && data.id) { componentStore.select(data.id) + componentTreeNodesStore.makeNodeVisible(data.id) } else if (type === "hover-component") { hoverStore.hover(data.id, false) } else if (type === "update-prop") { @@ -174,7 +172,7 @@ } else if (type === "request-add-component") { toggleAddComponent() } else if (type === "highlight-setting") { - builderStore.highlightSetting(data.setting) + builderStore.highlightSetting(data.setting, "error") // Also scroll setting into view const selector = `#${data.setting}-prop-control` @@ -196,6 +194,16 @@ } else if (type === "add-parent-component") { const { componentId, parentType } = data await componentStore.addParent(componentId, parentType) + } else if (type === "provide-context") { + let context = data?.context + if (context) { + try { + context = JSON.parse(context) + } catch (error) { + context = null + } + } + previewStore.setSelectedComponentContext(context) } else { console.warn(`Client sent unknown event type: ${type}`) } @@ -240,8 +248,16 @@
{#if loading} -
- +
+
{:else if error}
@@ -258,8 +274,6 @@ bind:this={iframe} src="/app/preview" class:hidden={loading || error} - class:tablet={$previewStore.previewDevice === "tablet"} - class:mobile={$previewStore.previewDevice === "mobile"} />
diff --git a/packages/builder/src/pages/builder/app/[application]/design/_components/NewScreen/ScreenRoleModal.svelte b/packages/builder/src/pages/builder/app/[application]/design/_components/NewScreen/ScreenRoleModal.svelte index 5e34d32219..8605412c1c 100644 --- a/packages/builder/src/pages/builder/app/[application]/design/_components/NewScreen/ScreenRoleModal.svelte +++ b/packages/builder/src/pages/builder/app/[application]/design/_components/NewScreen/ScreenRoleModal.svelte @@ -9,6 +9,7 @@ export let onCancel export let screenUrl export let screenAccessRole + export let confirmText = "Done" let error @@ -40,7 +41,7 @@ import { Body } from "@budibase/bbui" import CreationPage from "components/common/CreationPage.svelte" - import blankImage from "./blank.png" - import tableImage from "./table.png" - import gridImage from "./grid.png" + import blankImage from "./images/blank.png" + import tableInline from "./images/tableInline.png" + import tableDetails from "./images/tableDetails.png" + import formImage from "./images/form.png" import CreateScreenModal from "./CreateScreenModal.svelte" import { screenStore } from "stores/builder" @@ -37,23 +38,33 @@
-
createScreenModal.show("table")}> +
createScreenModal.show("grid")}>
- +
- Table - View, edit and delete rows on a table + Table with inline editing + View, edit and delete rows inline
-
createScreenModal.show("grid")}> +
createScreenModal.show("gridDetails")}>
- +
- Grid - View and manipulate rows on a grid + Table with details panel + Manage your row details in a side panel +
+
+ +
createScreenModal.show("form")}> +
+ +
+
+ Form + Capture data from your users
@@ -102,6 +113,11 @@ width: 100%; } + .card .image { + min-height: 130px; + min-width: 235px; + } + .text { border: 1px solid var(--grey-4); border-radius: 0 0 4px 4px; diff --git a/packages/builder/src/pages/builder/app/[application]/settings/_layout.svelte b/packages/builder/src/pages/builder/app/[application]/settings/_layout.svelte index da4f743f04..67befddcb9 100644 --- a/packages/builder/src/pages/builder/app/[application]/settings/_layout.svelte +++ b/packages/builder/src/pages/builder/app/[application]/settings/_layout.svelte @@ -3,7 +3,7 @@ import { Page, Layout, AbsTooltip, TooltipPosition } from "@budibase/bbui" import { url, isActive } from "@roxi/routify" import DeleteModal from "components/deploy/DeleteModal.svelte" - import { isOnlyUser } from "stores/builder" + import { isOnlyUser, appStore } from "stores/builder" let deleteModal @@ -67,7 +67,11 @@
- + diff --git a/packages/builder/src/pages/builder/portal/_components/BudibaseLogo.svelte b/packages/builder/src/pages/builder/portal/_components/BudibaseLogo.svelte new file mode 100644 index 0000000000..b0dc4cda50 --- /dev/null +++ b/packages/builder/src/pages/builder/portal/_components/BudibaseLogo.svelte @@ -0,0 +1,15 @@ + + + + +Budibase Logo $goto("./apps")} /> + + diff --git a/packages/builder/src/pages/builder/portal/_layout.svelte b/packages/builder/src/pages/builder/portal/_layout.svelte index aad33f852a..18378c37e9 100644 --- a/packages/builder/src/pages/builder/portal/_layout.svelte +++ b/packages/builder/src/pages/builder/portal/_layout.svelte @@ -1,7 +1,7 @@ + + { + e.stopPropagation() + const userAppFavourites = new Set([...($auth.user.appFavourites || [])]) + let processedAppIds = [] + + if ($auth.user.appFavourites && app?.appId) { + if (userAppFavourites.has(app.appId)) { + userAppFavourites.delete(app.appId) + } else { + userAppFavourites.add(app.appId) + } + processedAppIds = [...userAppFavourites] + } else { + processedAppIds = [app.appId] + } + + await auth.updateSelf({ + appFavourites: processedAppIds, + }) + }} + disabled={!app} +/> diff --git a/packages/builder/src/pages/builder/portal/apps/[appId]/_layout.svelte b/packages/builder/src/pages/builder/portal/apps/[appId]/_layout.svelte index f1ed686a3a..871326eb91 100644 --- a/packages/builder/src/pages/builder/portal/apps/[appId]/_layout.svelte +++ b/packages/builder/src/pages/builder/portal/apps/[appId]/_layout.svelte @@ -1,8 +1,8 @@ + +
{#if $sideBarCollapsed} - sideBarCollapsed.set(false)} - > - Menu - +
sideBarCollapsed.set(false)}> + +
{:else} - sideBarCollapsed.set(true)} - > - Collapse - +
sideBarCollapsed.set(true)}> + +
{/if} {#if isBuilder} - $goto(`/builder/app/${app.devId}`)} > Edit - + {/if} - window.open(iframeUrl, "_blank")} - > - Fullscreen - +
+ +
+
window.open(iframeUrl, "_blank")}> + +
+
{#if noScreens}
@@ -78,11 +133,39 @@
{:else} -