diff --git a/.eslintignore b/.eslintignore index 94984a446f..2bc00912d2 100644 --- a/.eslintignore +++ b/.eslintignore @@ -9,8 +9,5 @@ packages/server/client packages/server/coverage packages/builder/.routify packages/sdk/sdk -packages/account-portal/packages/server/build -packages/account-portal/packages/ui/.routify -packages/account-portal/packages/ui/build **/*.ivm.bundle.js packages/server/build/oldClientVersions/**/** diff --git a/.eslintrc.json b/.eslintrc.json index a94ea87704..4e6a169dae 100644 --- a/.eslintrc.json +++ b/.eslintrc.json @@ -27,9 +27,8 @@ "extends": "plugin:svelte/recommended", "parser": "svelte-eslint-parser", "parserOptions": { - "parser": "@babel/eslint-parser", + "parser": "@typescript-eslint/parser", "ecmaVersion": 2019, - "sourceType": "module", "allowImportExportEverywhere": true } }, diff --git a/.github/workflows/budibase_ci.yml b/.github/workflows/budibase_ci.yml index e894a91703..1258bddcca 100644 --- a/.github/workflows/budibase_ci.yml +++ b/.github/workflows/budibase_ci.yml @@ -64,18 +64,15 @@ jobs: - run: yarn --frozen-lockfile # Run build all the projects - - name: Build OSS - run: yarn build:oss - - name: Build account portal - run: yarn build:account-portal - if: ${{ env.IS_OSS_CONTRIBUTOR == 'false' }} + - name: Build + run: yarn build # Check the types of the projects built via esbuild - name: Check types run: | if ${{ env.ONLY_AFFECTED_TASKS }}; then - yarn check:types --since=${{ env.NX_BASE_BRANCH }} --ignore @budibase/account-portal-server + yarn check:types --since=${{ env.NX_BASE_BRANCH }} else - yarn check:types --ignore @budibase/account-portal-server + yarn check:types fi helm-lint: @@ -117,9 +114,11 @@ jobs: - name: Test run: | if ${{ env.ONLY_AFFECTED_TASKS }}; then - yarn test --ignore=@budibase/worker --ignore=@budibase/server --since=${{ env.NX_BASE_BRANCH }} + yarn test -- --ignore=@budibase/worker --ignore=@budibase/server --ignore=@budibase/builder --no-prefix --since=${{ env.NX_BASE_BRANCH }} -- --verbose --reporters=default --reporters=github-actions + yarn test -- --scope=@budibase/builder --since=${{ env.NX_BASE_BRANCH }} else - yarn test --ignore=@budibase/worker --ignore=@budibase/server + yarn test -- --ignore=@budibase/worker --ignore=@budibase/server --ignore=@budibase/builder --no-prefix -- --verbose --reporters=default --reporters=github-actions + yarn test -- --scope=@budibase/builder --no-prefix fi test-worker: @@ -141,16 +140,22 @@ jobs: - name: Test worker run: | if ${{ env.ONLY_AFFECTED_TASKS }}; then - node scripts/run-affected.js --task=test --scope=@budibase/worker --since=${{ env.NX_BASE_BRANCH }} - else - yarn test --scope=@budibase/worker + AFFECTED=$(yarn --silent nx show projects --affected -t test --base=${{ env.NX_BASE_BRANCH }} -p @budibase/worker) + if [ -z "$AFFECTED" ]; then + echo "No affected tests to run" + exit 0 + fi fi + cd packages/worker + yarn test --verbose --reporters=default --reporters=github-actions + test-server: runs-on: ubuntu-latest strategy: matrix: - datasource: [mssql, mysql, postgres, mongodb, mariadb, oracle, none] + datasource: + [mssql, mysql, postgres, mongodb, mariadb, oracle, sqs, none] steps: - name: Checkout repo uses: actions/checkout@v4 @@ -213,7 +218,7 @@ jobs: fi cd packages/server - yarn test --filter $FILTER --passWithNoTests + yarn test --filter $FILTER --verbose --reporters=default --reporters=github-actions check-pro-submodule: runs-on: ubuntu-latest @@ -274,64 +279,6 @@ jobs: echo 'All good, the submodule had been merged and setup correctly!' fi - check-accountportal-submodule: - runs-on: ubuntu-latest - if: inputs.run_as_oss != true && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == 'Budibase/budibase') - steps: - - name: Checkout repo and submodules - uses: actions/checkout@v4 - with: - submodules: true - token: ${{ secrets.PERSONAL_ACCESS_TOKEN || github.token }} - fetch-depth: 0 - - - uses: dorny/paths-filter@v3 - id: changes - with: - filters: | - src: - - packages/account-portal/** - - - if: steps.changes.outputs.src == 'true' - name: Check account portal commit - id: get_accountportal_commits - run: | - cd packages/account-portal - accountportal_commit=$(git rev-parse HEAD) - - branch="${{ github.base_ref || github.ref_name }}" - echo "Running on branch '$branch' (base_ref=${{ github.base_ref }}, ref_name=${{ github.head_ref }})" - - base_commit=$(git rev-parse origin/master) - - if [[ ! -z $base_commit ]]; then - echo "target_branch=$branch" - echo "target_branch=$branch" >> "$GITHUB_OUTPUT" - echo "accountportal_commit=$accountportal_commit" - echo "accountportal_commit=$accountportal_commit" >> "$GITHUB_OUTPUT" - echo "base_commit=$base_commit" - echo "base_commit=$base_commit" >> "$GITHUB_OUTPUT" - else - echo "Nothing to do - branch to branch merge." - fi - - - name: Check submodule merged to base branch - if: ${{ steps.get_accountportal_commits.outputs.base_commit != '' }} - uses: actions/github-script@v7 - with: - github-token: ${{ secrets.GITHUB_TOKEN }} - script: | - const submoduleCommit = '${{ steps.get_accountportal_commits.outputs.accountportal_commit }}'; - const baseCommit = '${{ steps.get_accountportal_commits.outputs.base_commit }}'; - - if (submoduleCommit !== baseCommit) { - console.error('Submodule commit does not match the latest commit on the "${{ steps.get_accountportal_commits.outputs.target_branch }}" branch.'); - console.error('Refer to the account portal repo to merge your changes: https://github.com/Budibase/account-portal/blob/master/docs/index.md') - process.exit(1); - } else { - console.log('All good, the submodule had been merged and setup correctly!') - } - check-lockfile: runs-on: ubuntu-latest steps: diff --git a/.gitignore b/.gitignore index bac643e5df..21637edfbe 100644 --- a/.gitignore +++ b/.gitignore @@ -8,6 +8,7 @@ packages/server/build/oldClientVersions/**/* packages/builder/src/components/deploy/clientVersions.json packages/server/src/integrations/tests/utils/*.lock packages/builder/vite.config.mjs.timestamp* +packages/account-portal # Logs logs @@ -110,4 +111,4 @@ budibase-component budibase-datasource *.iml -.nx \ No newline at end of file +.nx diff --git a/.gitmodules b/.gitmodules index cb6d1c5dc8..2dd6ea53f2 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,6 +1,3 @@ [submodule "packages/pro"] path = packages/pro url = git@github.com:Budibase/budibase-pro.git -[submodule "packages/account-portal"] - path = packages/account-portal - url = git@github.com:Budibase/account-portal.git diff --git a/.prettierignore b/.prettierignore index 72cdc75a23..b1ee287391 100644 --- a/.prettierignore +++ b/.prettierignore @@ -9,8 +9,4 @@ packages/backend-core/coverage packages/builder/.routify packages/sdk/sdk packages/pro/coverage -packages/account-portal/packages/ui/build -packages/account-portal/packages/ui/.routify -packages/account-portal/packages/server/build -packages/account-portal/packages/server/coverage **/*.ivm.bundle.js \ No newline at end of file diff --git a/.vscode/launch.json b/.vscode/launch.json index 543d1e6179..2fda61345b 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -20,16 +20,6 @@ "args": ["${workspaceFolder}/packages/worker/src/index.ts"], "cwd": "${workspaceFolder}/packages/worker" }, - { - "name": "Camunda Worker", - "type": "node", - "request": "launch", - "runtimeArgs": ["--nolazy", "-r", "ts-node/register/transpile-only"], - "args": [ - "${workspaceFolder}/packages/account-portal/packages/server/src/v2/run.ts" - ], - "cwd": "${workspaceFolder}/packages/account-portal/packages/server" - }, { "type": "chrome", "request": "launch", diff --git a/charts/budibase/templates/secrets.yaml b/charts/budibase/templates/secrets.yaml index a44d2c0d9b..20701d4125 100644 --- a/charts/budibase/templates/secrets.yaml +++ b/charts/budibase/templates/secrets.yaml @@ -12,12 +12,12 @@ metadata: type: Opaque data: {{- if $existingSecret }} - internalApiKey: {{ index $existingSecret.data "internalApiKey" }} - jwtSecret: {{ index $existingSecret.data "jwtSecret" }} - objectStoreAccess: {{ index $existingSecret.data "objectStoreAccess" }} - objectStoreSecret: {{ index $existingSecret.data "objectStoreSecret" }} - bbEncryptionKey: {{ index $existingSecret.data "bbEncryptionKey" }} - apiEncryptionKey: {{ index $existingSecret.data "apiEncryptionKey" }} + internalApiKey: {{ index $existingSecret.data "internalApiKey" | quote }} + jwtSecret: {{ index $existingSecret.data "jwtSecret" | quote }} + objectStoreAccess: {{ index $existingSecret.data "objectStoreAccess" | quote }} + objectStoreSecret: {{ index $existingSecret.data "objectStoreSecret" | quote }} + bbEncryptionKey: {{ index $existingSecret.data "bbEncryptionKey" | quote }} + apiEncryptionKey: {{ index $existingSecret.data "apiEncryptionKey" | quote }} {{- else }} internalApiKey: {{ template "budibase.defaultsecret" .Values.globals.internalApiKey }} jwtSecret: {{ template "budibase.defaultsecret" .Values.globals.jwtSecret }} diff --git a/examples/nextjs-api-sales/yarn.lock b/examples/nextjs-api-sales/yarn.lock index 9acbdfdeb6..867835a6b7 100644 --- a/examples/nextjs-api-sales/yarn.lock +++ b/examples/nextjs-api-sales/yarn.lock @@ -423,9 +423,9 @@ core-js-pure@^3.20.2: integrity sha512-12VZfFIu+wyVbBebyHmRTuEE/tZrB4tJToWcwAMcsp3h4+sHR+fMJWbKpYiCRWlhFBq+KNyO8rIV9rTkeVmznQ== cross-spawn@^7.0.2: - version "7.0.3" - resolved "https://registry.yarnpkg.com/cross-spawn/-/cross-spawn-7.0.3.tgz#f73a85b9d5d41d045551c177e2882d4ac85728a6" - integrity sha512-iRDPJKUPVEND7dHPO8rkbOnPpyDygcDFtWjpeWNCgy8WP2rXcxXL8TskReQl6OrB2G7+UJrags1q15Fudc7G6w== + version "7.0.6" + resolved "https://registry.yarnpkg.com/cross-spawn/-/cross-spawn-7.0.6.tgz#8a58fe78f00dcd70c370451759dfbfaf03e8ee9f" + integrity sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA== dependencies: path-key "^3.1.0" shebang-command "^2.0.0" diff --git a/lerna.json b/lerna.json index e5b0013535..dc238bb392 100644 --- a/lerna.json +++ b/lerna.json @@ -1,12 +1,7 @@ { "$schema": "node_modules/lerna/schemas/lerna-schema.json", - "version": "3.2.5", + "version": "3.2.12", "npmClient": "yarn", - "packages": [ - "packages/*", - "!packages/account-portal", - "packages/account-portal/packages/*" - ], "concurrency": 20, "command": { "publish": { diff --git a/nx.json b/nx.json index fb05ea94d0..22b23a7874 100644 --- a/nx.json +++ b/nx.json @@ -2,7 +2,6 @@ "$schema": "./node_modules/nx/schemas/nx-schema.json", "tasksRunnerOptions": { "default": { - "runner": "nx-cloud", "options": { "cacheableOperations": ["build", "test", "check:types"] } diff --git a/package.json b/package.json index fc7e202e3d..e354f36d2a 100644 --- a/package.json +++ b/package.json @@ -9,6 +9,7 @@ "@types/node": "20.10.0", "@types/proper-lockfile": "^4.1.4", "@typescript-eslint/parser": "6.9.0", + "depcheck": "^1.4.7", "esbuild": "^0.18.17", "esbuild-node-externals": "^1.14.0", "eslint": "^8.52.0", @@ -24,22 +25,22 @@ "prettier": "2.8.8", "prettier-plugin-svelte": "^2.3.0", "proper-lockfile": "^4.1.2", - "svelte": "^4.2.10", + "svelte": "4.2.19", "svelte-eslint-parser": "^0.33.1", "typescript": "5.5.2", "typescript-eslint": "^7.3.1", - "yargs": "^17.7.2" + "yargs": "^17.7.2", + "cross-spawn": "7.0.6" }, "scripts": { "get-past-client-version": "node scripts/getPastClientVersion.js", "setup": "git config submodule.recurse true && git submodule update && node ./hosting/scripts/setup.js && yarn && yarn build && yarn dev", "build": "DISABLE_V8_COMPILE_CACHE=1 NODE_OPTIONS=--max-old-space-size=1500 lerna run build --stream", "build:apps": "DISABLE_V8_COMPILE_CACHE=1 yarn build --scope @budibase/server --scope @budibase/worker", - "build:oss": "DISABLE_V8_COMPILE_CACHE=1 NODE_OPTIONS=--max-old-space-size=1500 lerna run build --stream --ignore @budibase/account-portal-server --ignore @budibase/account-portal-ui", "build:cli": "yarn build --scope @budibase/cli", - "build:account-portal": "NODE_OPTIONS=--max-old-space-size=1500 lerna run build --stream --scope @budibase/account-portal-server --scope @budibase/account-portal-ui", "build:dev": "lerna run --stream prebuild && yarn nx run-many --target=build --output-style=dynamic --watch --preserveWatchOutput", - "check:types": "lerna run --concurrency 2 check:types --ignore @budibase/account-portal-server", + "check:types": "yarn check:dependencies && lerna run --concurrency 2 check:types", + "check:dependencies": "lerna run --concurrency 2 check:dependencies", "build:sdk": "lerna run --stream build:sdk", "deps:circular": "madge packages/server/dist/index.js packages/worker/src/index.ts packages/backend-core/dist/src/index.js packages/cli/src/index.js --circular", "release": "lerna publish from-package --yes --force-publish --no-git-tag-version --no-push --no-git-reset", @@ -52,15 +53,12 @@ "kill-server": "kill-port 4001 4002", "kill-accountportal": "kill-port 3001 4003", "kill-all": "yarn run kill-builder && yarn run kill-server && yarn kill-accountportal", - "dev": "yarn run kill-all && lerna run --parallel prebuild && lerna run --stream dev --ignore=@budibase/account-portal-ui --ignore @budibase/account-portal-server", - "dev:noserver": "yarn run kill-builder && lerna run --stream dev:stack:up --ignore @budibase/account-portal-server && lerna run --stream dev --ignore @budibase/backend-core --ignore @budibase/server --ignore @budibase/worker --ignore=@budibase/account-portal-ui --ignore @budibase/account-portal-server", + "dev": "yarn run kill-all && lerna run --parallel prebuild && lerna run --stream dev", + "dev:noserver": "yarn run kill-builder && lerna run --stream dev:stack:up && lerna run --stream dev --ignore @budibase/backend-core --ignore @budibase/server --ignore @budibase/worker", "dev:server": "yarn run kill-server && lerna run --stream dev --scope @budibase/worker --scope @budibase/server", - "dev:accountportal": "yarn kill-accountportal && lerna run dev --stream --scope @budibase/account-portal-ui --scope @budibase/account-portal-server", - "dev:camunda": "./scripts/deploy-camunda.sh", - "dev:all": "yarn run kill-all && lerna run --stream dev", "dev:built": "yarn run kill-all && cd packages/server && yarn dev:stack:up && cd ../../ && lerna run --stream dev:built", "dev:docker": "./scripts/devDocker.sh", - "test": "lerna run --concurrency 1 --stream test --stream", + "test": "lerna run --concurrency 1 --stream test", "test:containers:kill": "./scripts/killTestcontainers.sh", "lint:eslint": "eslint packages --max-warnings=0", "lint:prettier": "prettier --check \"packages/**/*.{js,ts,svelte}\" && prettier --write \"examples/**/*.{js,ts,svelte}\"", @@ -98,9 +96,7 @@ }, "workspaces": { "packages": [ - "packages/*", - "!packages/account-portal", - "packages/account-portal/packages/*" + "packages/*" ] }, "resolutions": { @@ -114,7 +110,7 @@ "semver": "7.5.3", "http-cache-semantics": "4.1.1", "msgpackr": "1.10.1", - "axios": "1.6.3", + "axios": "1.7.7", "xml2js": "0.6.2", "unset-value": "2.0.1", "passport": "0.6.0", @@ -124,6 +120,5 @@ }, "engines": { "node": ">=20.0.0 <21.0.0" - }, - "dependencies": {} + } } diff --git a/packages/account-portal b/packages/account-portal deleted file mode 160000 index 9bef5d1656..0000000000 --- a/packages/account-portal +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 9bef5d1656b4f3c991447ded6d65b0eba393a140 diff --git a/packages/backend-core/.npmignore b/packages/backend-core/.npmignore index 30bba85ce8..fb547825eb 100644 --- a/packages/backend-core/.npmignore +++ b/packages/backend-core/.npmignore @@ -1,6 +1,4 @@ * !dist/**/* dist/tsconfig.build.tsbuildinfo -!package.json -!src/** -!tests/** \ No newline at end of file +!package.json \ No newline at end of file diff --git a/packages/backend-core/package.json b/packages/backend-core/package.json index b68cba5fd9..a4381b4200 100644 --- a/packages/backend-core/package.json +++ b/packages/backend-core/package.json @@ -9,6 +9,13 @@ "./tests": "./dist/tests/index.js", "./*": "./dist/*.js" }, + "typesVersions": { + "*": { + "tests": [ + "dist/tests/index.d.ts" + ] + } + }, "author": "Budibase", "license": "GPL-3.0", "scripts": { @@ -17,6 +24,7 @@ "build": "tsc -p tsconfig.build.json --paths null && node ./scripts/build.js", "build:dev": "yarn prebuild && tsc --build --watch --preserveWatchOutput", "check:types": "tsc -p tsconfig.json --noEmit --paths null --target es2020", + "check:dependencies": "node ../../scripts/depcheck.js", "test": "bash scripts/test.sh", "test:watch": "jest --watchAll" }, @@ -25,17 +33,21 @@ "@budibase/pouchdb-replication-stream": "1.2.11", "@budibase/shared-core": "0.0.0", "@budibase/types": "0.0.0", + "@techpass/passport-openidconnect": "0.3.3", "aws-cloudfront-sign": "3.0.2", - "aws-sdk": "2.1030.0", + "aws-sdk": "2.1692.0", "bcrypt": "5.1.0", "bcryptjs": "2.4.3", "bull": "4.10.1", "correlation-id": "4.0.0", - "dd-trace": "5.2.0", + "dd-trace": "5.26.0", "dotenv": "16.0.1", + "google-auth-library": "^8.0.1", + "google-spreadsheet": "npm:@budibase/google-spreadsheet@4.1.5", "ioredis": "5.3.2", "joi": "17.6.0", "jsonwebtoken": "9.0.2", + "knex": "2.4.2", "koa-passport": "^6.0.0", "koa-pino-logger": "4.0.0", "lodash": "4.17.21", @@ -46,17 +58,17 @@ "pino": "8.11.0", "pino-http": "8.3.3", "posthog-node": "4.0.1", - "pouchdb": "7.3.0", - "pouchdb-find": "7.2.2", + "pouchdb": "9.0.0", + "pouchdb-find": "9.0.0", "redlock": "4.2.0", "rotating-file-stream": "3.1.0", "sanitize-s3-objectkey": "0.0.1", "semver": "^7.5.4", "tar-fs": "2.1.1", - "uuid": "^8.3.2", - "knex": "2.4.2" + "uuid": "^8.3.2" }, "devDependencies": { + "@jest/types": "^29.6.3", "@shopify/jest-koa-mocks": "5.1.1", "@swc/core": "1.3.71", "@swc/jest": "0.2.27", @@ -64,8 +76,9 @@ "@types/cookies": "0.7.8", "@types/jest": "29.5.5", "@types/lodash": "4.14.200", + "@types/node": "^22.9.0", "@types/node-fetch": "2.6.4", - "@types/pouchdb": "6.4.0", + "@types/pouchdb": "6.4.2", "@types/redlock": "4.0.7", "@types/semver": "7.3.7", "@types/tar-fs": "2.0.1", @@ -74,6 +87,7 @@ "ioredis-mock": "8.9.0", "jest": "29.7.0", "jest-serial-runner": "1.2.1", + "nock": "^13.5.6", "pino-pretty": "10.0.0", "pouchdb-adapter-memory": "7.2.2", "testcontainers": "^10.7.2", diff --git a/packages/backend-core/src/db/couch/DatabaseImpl.ts b/packages/backend-core/src/db/couch/DatabaseImpl.ts index 83b9b69d0b..371f3dc997 100644 --- a/packages/backend-core/src/db/couch/DatabaseImpl.ts +++ b/packages/backend-core/src/db/couch/DatabaseImpl.ts @@ -190,7 +190,7 @@ export class DatabaseImpl implements Database { } } - private async performCall(call: DBCallback): Promise { + private async performCall(call: DBCallback): Promise { const db = this.getDb() const fnc = await call(db) try { @@ -467,7 +467,7 @@ export class DatabaseImpl implements Database { } catch (err: any) { // didn't exist, don't worry if (err.statusCode === 404) { - return + return { ok: true } } else { throw new CouchDBError(err.message, err) } diff --git a/packages/backend-core/src/db/instrumentation.ts b/packages/backend-core/src/db/instrumentation.ts index e08bfc0362..0c0056d6ed 100644 --- a/packages/backend-core/src/db/instrumentation.ts +++ b/packages/backend-core/src/db/instrumentation.ts @@ -27,7 +27,7 @@ export class DDInstrumentedDatabase implements Database { exists(docId?: string): Promise { return tracer.trace("db.exists", span => { - span?.addTags({ db_name: this.name, doc_id: docId }) + span.addTags({ db_name: this.name, doc_id: docId }) if (docId) { return this.db.exists(docId) } @@ -37,15 +37,17 @@ export class DDInstrumentedDatabase implements Database { get(id?: string | undefined): Promise { return tracer.trace("db.get", span => { - span?.addTags({ db_name: this.name, doc_id: id }) + span.addTags({ db_name: this.name, doc_id: id }) return this.db.get(id) }) } tryGet(id?: string | undefined): Promise { - return tracer.trace("db.tryGet", span => { - span?.addTags({ db_name: this.name, doc_id: id }) - return this.db.tryGet(id) + return tracer.trace("db.tryGet", async span => { + span.addTags({ db_name: this.name, doc_id: id }) + const doc = await this.db.tryGet(id) + span.addTags({ doc_found: doc !== undefined }) + return doc }) } @@ -53,13 +55,15 @@ export class DDInstrumentedDatabase implements Database { ids: string[], opts?: { allowMissing?: boolean | undefined } | undefined ): Promise { - return tracer.trace("db.getMultiple", span => { - span?.addTags({ + return tracer.trace("db.getMultiple", async span => { + span.addTags({ db_name: this.name, num_docs: ids.length, allow_missing: opts?.allowMissing, }) - return this.db.getMultiple(ids, opts) + const docs = await this.db.getMultiple(ids, opts) + span.addTags({ num_docs_found: docs.length }) + return docs }) } @@ -69,12 +73,14 @@ export class DDInstrumentedDatabase implements Database { idOrDoc: string | Document, rev?: string ): Promise { - return tracer.trace("db.remove", span => { - span?.addTags({ db_name: this.name, doc_id: idOrDoc }) + return tracer.trace("db.remove", async span => { + span.addTags({ db_name: this.name, doc_id: idOrDoc, rev }) const isDocument = typeof idOrDoc === "object" const id = isDocument ? idOrDoc._id! : idOrDoc rev = isDocument ? idOrDoc._rev : rev - return this.db.remove(id, rev) + const resp = await this.db.remove(id, rev) + span.addTags({ ok: resp.ok }) + return resp }) } @@ -83,7 +89,11 @@ export class DDInstrumentedDatabase implements Database { opts?: { silenceErrors?: boolean } ): Promise { return tracer.trace("db.bulkRemove", span => { - span?.addTags({ db_name: this.name, num_docs: documents.length }) + span.addTags({ + db_name: this.name, + num_docs: documents.length, + silence_errors: opts?.silenceErrors, + }) return this.db.bulkRemove(documents, opts) }) } @@ -92,15 +102,21 @@ export class DDInstrumentedDatabase implements Database { document: AnyDocument, opts?: DatabasePutOpts | undefined ): Promise { - return tracer.trace("db.put", span => { - span?.addTags({ db_name: this.name, doc_id: document._id }) - return this.db.put(document, opts) + return tracer.trace("db.put", async span => { + span.addTags({ + db_name: this.name, + doc_id: document._id, + force: opts?.force, + }) + const resp = await this.db.put(document, opts) + span.addTags({ ok: resp.ok }) + return resp }) } bulkDocs(documents: AnyDocument[]): Promise { return tracer.trace("db.bulkDocs", span => { - span?.addTags({ db_name: this.name, num_docs: documents.length }) + span.addTags({ db_name: this.name, num_docs: documents.length }) return this.db.bulkDocs(documents) }) } @@ -108,9 +124,15 @@ export class DDInstrumentedDatabase implements Database { allDocs( params: DatabaseQueryOpts ): Promise> { - return tracer.trace("db.allDocs", span => { - span?.addTags({ db_name: this.name }) - return this.db.allDocs(params) + return tracer.trace("db.allDocs", async span => { + span.addTags({ db_name: this.name, ...params }) + const resp = await this.db.allDocs(params) + span.addTags({ + total_rows: resp.total_rows, + rows_length: resp.rows.length, + offset: resp.offset, + }) + return resp }) } @@ -118,57 +140,75 @@ export class DDInstrumentedDatabase implements Database { viewName: string, params: DatabaseQueryOpts ): Promise> { - return tracer.trace("db.query", span => { - span?.addTags({ db_name: this.name, view_name: viewName }) - return this.db.query(viewName, params) + return tracer.trace("db.query", async span => { + span.addTags({ db_name: this.name, view_name: viewName, ...params }) + const resp = await this.db.query(viewName, params) + span.addTags({ + total_rows: resp.total_rows, + rows_length: resp.rows.length, + offset: resp.offset, + }) + return resp }) } - destroy(): Promise { - return tracer.trace("db.destroy", span => { - span?.addTags({ db_name: this.name }) - return this.db.destroy() + destroy(): Promise { + return tracer.trace("db.destroy", async span => { + span.addTags({ db_name: this.name }) + const resp = await this.db.destroy() + span.addTags({ ok: resp.ok }) + return resp }) } - compact(): Promise { - return tracer.trace("db.compact", span => { - span?.addTags({ db_name: this.name }) - return this.db.compact() + compact(): Promise { + return tracer.trace("db.compact", async span => { + span.addTags({ db_name: this.name }) + const resp = await this.db.compact() + span.addTags({ ok: resp.ok }) + return resp }) } dump(stream: Writable, opts?: DatabaseDumpOpts | undefined): Promise { return tracer.trace("db.dump", span => { - span?.addTags({ db_name: this.name }) + span.addTags({ + db_name: this.name, + batch_limit: opts?.batch_limit, + batch_size: opts?.batch_size, + style: opts?.style, + timeout: opts?.timeout, + num_doc_ids: opts?.doc_ids?.length, + view: opts?.view, + }) return this.db.dump(stream, opts) }) } load(...args: any[]): Promise { return tracer.trace("db.load", span => { - span?.addTags({ db_name: this.name }) + span.addTags({ db_name: this.name, num_args: args.length }) return this.db.load(...args) }) } createIndex(...args: any[]): Promise { return tracer.trace("db.createIndex", span => { - span?.addTags({ db_name: this.name }) + span.addTags({ db_name: this.name, num_args: args.length }) return this.db.createIndex(...args) }) } deleteIndex(...args: any[]): Promise { return tracer.trace("db.deleteIndex", span => { - span?.addTags({ db_name: this.name }) + span.addTags({ db_name: this.name, num_args: args.length }) return this.db.deleteIndex(...args) }) } getIndexes(...args: any[]): Promise { return tracer.trace("db.getIndexes", span => { - span?.addTags({ db_name: this.name }) + span.addTags({ db_name: this.name, num_args: args.length }) return this.db.getIndexes(...args) }) } @@ -177,22 +217,27 @@ export class DDInstrumentedDatabase implements Database { sql: string, parameters?: SqlQueryBinding ): Promise { - return tracer.trace("db.sql", span => { - span?.addTags({ db_name: this.name }) - return this.db.sql(sql, parameters) + return tracer.trace("db.sql", async span => { + span.addTags({ db_name: this.name, num_bindings: parameters?.length }) + const resp = await this.db.sql(sql, parameters) + span.addTags({ num_rows: resp.length }) + return resp }) } sqlPurgeDocument(docIds: string[] | string): Promise { return tracer.trace("db.sqlPurgeDocument", span => { - span?.addTags({ db_name: this.name }) + span.addTags({ + db_name: this.name, + num_docs: Array.isArray(docIds) ? docIds.length : 1, + }) return this.db.sqlPurgeDocument(docIds) }) } sqlDiskCleanup(): Promise { return tracer.trace("db.sqlDiskCleanup", span => { - span?.addTags({ db_name: this.name }) + span.addTags({ db_name: this.name }) return this.db.sqlDiskCleanup() }) } diff --git a/packages/backend-core/src/environment.ts b/packages/backend-core/src/environment.ts index 2763944d2f..56d9cd6e10 100644 --- a/packages/backend-core/src/environment.ts +++ b/packages/backend-core/src/environment.ts @@ -19,6 +19,12 @@ function isDev() { return process.env.NODE_ENV !== "production" } +function parseIntSafe(number?: string) { + if (number) { + return parseInt(number) + } +} + let LOADED = false if (!LOADED && isDev() && !isTest()) { require("dotenv").config() @@ -231,6 +237,7 @@ const environment = { MIN_VERSION_WITHOUT_POWER_ROLE: process.env.MIN_VERSION_WITHOUT_POWER_ROLE || "3.0.0", DISABLE_CONTENT_SECURITY_POLICY: process.env.DISABLE_CONTENT_SECURITY_POLICY, + BSON_BUFFER_SIZE: parseIntSafe(process.env.BSON_BUFFER_SIZE), } export function setEnv(newEnvVars: Partial): () => void { diff --git a/packages/bbui/src/Button/Button.svelte b/packages/bbui/src/Button/Button.svelte index 0a8917c3c1..77653d0c42 100644 --- a/packages/bbui/src/Button/Button.svelte +++ b/packages/bbui/src/Button/Button.svelte @@ -3,7 +3,7 @@ import AbsTooltip from "../Tooltip/AbsTooltip.svelte" import { createEventDispatcher } from "svelte" - export let type + export let type = undefined export let disabled = false export let size = "M" export let cta = false @@ -16,8 +16,8 @@ export let active = false export let tooltip = undefined export let newStyles = true - export let id - export let ref + export let id = undefined + export let ref = undefined export let reverse = false const dispatch = createEventDispatcher() diff --git a/packages/bbui/src/Form/Core/DateRangePicker.svelte b/packages/bbui/src/Form/Core/DateRangePicker.svelte index 9084942ba7..72180a98d6 100644 --- a/packages/bbui/src/Form/Core/DateRangePicker.svelte +++ b/packages/bbui/src/Form/Core/DateRangePicker.svelte @@ -2,13 +2,6 @@ import CoreDatePicker from "./DatePicker/DatePicker.svelte" import Icon from "../../Icon/Icon.svelte" - export let value = null - export let disabled = false - export let readonly = false - export let error = null - export let appendTo = undefined - export let ignoreTimezones = false - let fromDate let toDate diff --git a/packages/bbui/src/Form/Search.svelte b/packages/bbui/src/Form/Search.svelte index 2fbda275eb..a085986d3e 100644 --- a/packages/bbui/src/Form/Search.svelte +++ b/packages/bbui/src/Form/Search.svelte @@ -10,7 +10,7 @@ export let disabled = false export let updateOnChange = true export let quiet = false - export let inputRef + export let inputRef = undefined export let helpText = null const dispatch = createEventDispatcher() diff --git a/packages/bbui/src/Form/Select.svelte b/packages/bbui/src/Form/Select.svelte index 260090c7b7..0e67a1f532 100644 --- a/packages/bbui/src/Form/Select.svelte +++ b/packages/bbui/src/Form/Select.svelte @@ -17,18 +17,18 @@ export let getOptionIcon = option => option?.icon export let getOptionColour = option => option?.colour export let useOptionIconImage = false - export let isOptionEnabled + export let isOptionEnabled = undefined export let quiet = false export let autoWidth = false export let sort = false export let tooltip = "" export let autocomplete = false - export let customPopoverHeight - export let align + export let customPopoverHeight = undefined + export let align = undefined export let footer = null export let tag = null export let helpText = null - export let compare + export let compare = undefined export let onOptionMouseenter = () => {} export let onOptionMouseleave = () => {} diff --git a/packages/bbui/src/Table/Table.svelte b/packages/bbui/src/Table/Table.svelte index 3916bec738..228838e477 100644 --- a/packages/bbui/src/Table/Table.svelte +++ b/packages/bbui/src/Table/Table.svelte @@ -43,7 +43,7 @@ export let showHeaderBorder = true export let placeholderText = "No rows found" export let snippets = [] - export let defaultSortColumn + export let defaultSortColumn = undefined export let defaultSortOrder = "Ascending" const dispatch = createEventDispatcher() diff --git a/packages/bbui/src/Typography/Heading.svelte b/packages/bbui/src/Typography/Heading.svelte index 50522fffc3..90d53fb208 100644 --- a/packages/bbui/src/Typography/Heading.svelte +++ b/packages/bbui/src/Typography/Heading.svelte @@ -1,9 +1,9 @@ - diff --git a/packages/builder/.gitignore b/packages/builder/.gitignore index abc5671984..e5c961d509 100644 --- a/packages/builder/.gitignore +++ b/packages/builder/.gitignore @@ -5,5 +5,4 @@ package-lock.json release/ dist/ routify -.routify/ -svelte.config.js \ No newline at end of file +.routify/ \ No newline at end of file diff --git a/packages/builder/package.json b/packages/builder/package.json index 46e454f5b9..fd819b8a9c 100644 --- a/packages/builder/package.json +++ b/packages/builder/package.json @@ -4,7 +4,8 @@ "license": "GPL-3.0", "private": true, "scripts": { - "build": "routify -b && vite build --emptyOutDir", + "svelte-check": "svelte-check --no-tsconfig", + "build": "yarn svelte-check && routify -b && vite build --emptyOutDir", "start": "routify -c rollup", "dev": "routify -c dev:vite", "dev:vite": "vite --host 0.0.0.0", @@ -97,6 +98,7 @@ "jest": "29.7.0", "jsdom": "^21.1.1", "resize-observer-polyfill": "^1.5.1", + "svelte-check": "^4.1.0", "svelte-jester": "^1.3.2", "vite": "^4.5.0", "vite-plugin-static-copy": "^0.17.0", diff --git a/packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte b/packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte index b16533c2ca..b901d21a6d 100644 --- a/packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte +++ b/packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte @@ -114,7 +114,7 @@ $: schemaFields = search.getFields( $tables.list, Object.values(schema || {}), - { allowLinks: true } + { allowLinks: false } ) $: queryLimit = tableId?.includes("datasource") ? "∞" : "1000" $: isTrigger = $memoBlock?.type === AutomationStepType.TRIGGER diff --git a/packages/builder/src/dataBinding.js b/packages/builder/src/dataBinding.js index 8adc27ee59..c89221f163 100644 --- a/packages/builder/src/dataBinding.js +++ b/packages/builder/src/dataBinding.js @@ -1141,10 +1141,11 @@ export const buildFormSchema = (component, asset) => { const fieldSetting = settings.find( setting => setting.key === "field" && setting.type.startsWith("field/") ) - if (fieldSetting && component.field) { + if (fieldSetting) { const type = fieldSetting.type.split("field/")[1] - if (type) { - schema[component.field] = { type } + const key = component.field || component._instanceName + if (type && key) { + schema[key] = { type } } } component._children?.forEach(child => { diff --git a/packages/builder/src/pages/builder/app/[application]/data/datasource/[datasourceId]/index.svelte b/packages/builder/src/pages/builder/app/[application]/data/datasource/[datasourceId]/index.svelte index b723788695..3757ed7dee 100644 --- a/packages/builder/src/pages/builder/app/[application]/data/datasource/[datasourceId]/index.svelte +++ b/packages/builder/src/pages/builder/app/[application]/data/datasource/[datasourceId]/index.svelte @@ -117,7 +117,4 @@ align-items: center; margin-bottom: 12px; } - - .tabs { - } diff --git a/packages/builder/src/pages/builder/app/[application]/design/[screenId]/_components/ComponentList/DNDPositionIndicator.svelte b/packages/builder/src/pages/builder/app/[application]/design/[screenId]/_components/ComponentList/DNDPositionIndicator.svelte index 4612440a2c..45185d8e2d 100644 --- a/packages/builder/src/pages/builder/app/[application]/design/[screenId]/_components/ComponentList/DNDPositionIndicator.svelte +++ b/packages/builder/src/pages/builder/app/[application]/design/[screenId]/_components/ComponentList/DNDPositionIndicator.svelte @@ -50,8 +50,6 @@ border-radius: 4px; pointer-events: none; } - .indicator.above { - } .indicator.below { margin-top: 32px; } diff --git a/packages/builder/src/pages/builder/portal/plugins/index.svelte b/packages/builder/src/pages/builder/portal/plugins/index.svelte index d8dd4eae11..e16da42e10 100644 --- a/packages/builder/src/pages/builder/portal/plugins/index.svelte +++ b/packages/builder/src/pages/builder/portal/plugins/index.svelte @@ -1,4 +1,4 @@ - -
- {#key $component.editing} - - {/key} -
- {#if !formContext} - - {:else if !fieldState} - - {:else if schemaType && schemaType !== type && !["options", "longform"].includes(type)} - - {:else} - - {#if fieldState.error} -
- - {fieldState.error} -
- {:else if helpText} -
- {helpText} -
- {/if} - {/if} -
-
+ + + + + {:else} +
+ {#key $component.editing} + + {/key} +
+ {#if !fieldState} + + {:else if schemaType && schemaType !== type && !["options", "longform"].includes(type)} + + {:else} + + {#if fieldState.error} +
+ + {fieldState.error} +
+ {:else if helpText} +
+ {helpText} +
+ {/if} + {/if} +
+
+ {/if} +