diff --git a/.github/workflows/budibase_ci.yml b/.github/workflows/budibase_ci.yml index 1a2e74f863..a9de0ba342 100644 --- a/.github/workflows/budibase_ci.yml +++ b/.github/workflows/budibase_ci.yml @@ -56,7 +56,6 @@ jobs: run: yarn install:pro $BRANCH $BASE_BRANCH - run: yarn - run: yarn bootstrap - - run: yarn build:client - run: yarn test - uses: codecov/codecov-action@v3 with: @@ -80,14 +79,6 @@ jobs: integration-test: runs-on: ubuntu-latest - services: - couchdb: - image: ibmcom/couchdb3 - env: - COUCHDB_PASSWORD: budibase - COUCHDB_USER: budibase - ports: - - 4567:5984 steps: - uses: actions/checkout@v2 - name: Use Node.js 14.x @@ -96,10 +87,11 @@ jobs: node-version: 14.x - name: Install Pro run: yarn install:pro $BRANCH $BASE_BRANCH - - run: yarn - - run: yarn bootstrap - - run: yarn build + - run: yarn && yarn bootstrap && yarn build - run: | cd qa-core - yarn - yarn api:test:ci + yarn setup + yarn test:ci + env: + BB_ADMIN_USER_EMAIL: admin + BB_ADMIN_USER_PASSWORD: admin \ No newline at end of file diff --git a/.nvmrc b/.nvmrc index e0bcfe01fb..436b34d932 100644 --- a/.nvmrc +++ b/.nvmrc @@ -1 +1 @@ -v14.19.3 +v14.20.1 \ No newline at end of file diff --git a/.nxignore b/.nxignore new file mode 100644 index 0000000000..0255e9d274 --- /dev/null +++ b/.nxignore @@ -0,0 +1,14 @@ +node_modules +**/node_modules + +**/dist/ +**/.routify/ + +**/coverage/ +**/yarn-error.log + +**/prebuilds/ +**/build/ + +packages/server/builder/* +packages/server/client/* \ No newline at end of file diff --git a/.python-version b/.python-version index 371cfe355d..e06d07afe1 100644 --- a/.python-version +++ b/.python-version @@ -1 +1 @@ -3.11.1 +3.10.0 \ No newline at end of file diff --git a/.tool-versions b/.tool-versions index 6ee8cc60be..094292d096 100644 --- a/.tool-versions +++ b/.tool-versions @@ -1,2 +1,2 @@ -nodejs 14.19.3 +nodejs 14.20.1 python 3.10.0 \ No newline at end of file diff --git a/charts/budibase/templates/app-service-deployment.yaml b/charts/budibase/templates/app-service-deployment.yaml index df4070806f..6ba9337f5b 100644 --- a/charts/budibase/templates/app-service-deployment.yaml +++ b/charts/budibase/templates/app-service-deployment.yaml @@ -64,6 +64,8 @@ spec: value: {{ .Values.globals.enableAnalytics | quote }} - name: API_ENCRYPTION_KEY value: {{ .Values.globals.apiEncryptionKey | quote }} + - name: HTTP_LOGGING + value: {{ .Values.services.apps.httpLogging | quote }} - name: INTERNAL_API_KEY valueFrom: secretKeyRef: @@ -119,7 +121,7 @@ spec: - name: MULTI_TENANCY value: {{ .Values.globals.multiTenancy | quote }} - name: LOG_LEVEL - value: {{ default "info" .Values.services.apps.logLevel | quote }} + value: {{ .Values.services.apps.logLevel | quote }} - name: REDIS_PASSWORD value: {{ .Values.services.redis.password }} - name: REDIS_URL @@ -180,18 +182,6 @@ spec: - name: DD_APM_DD_URL value: https://trace.agent.datadoghq.eu {{ end }} - {{ if .Values.globals.elasticApmEnabled }} - - name: ELASTIC_APM_ENABLED - value: {{ .Values.globals.elasticApmEnabled | quote }} - {{ end }} - {{ if .Values.globals.elasticApmSecretToken }} - - name: ELASTIC_APM_SECRET_TOKEN - value: {{ .Values.globals.elasticApmSecretToken | quote }} - {{ end }} - {{ if .Values.globals.elasticApmServerUrl }} - - name: ELASTIC_APM_SERVER_URL - value: {{ .Values.globals.elasticApmServerUrl | quote }} - {{ end }} {{ if .Values.globals.globalAgentHttpProxy }} - name: GLOBAL_AGENT_HTTP_PROXY value: {{ .Values.globals.globalAgentHttpProxy | quote }} diff --git a/charts/budibase/templates/worker-service-deployment.yaml b/charts/budibase/templates/worker-service-deployment.yaml index f48d19689b..0df295b5d6 100644 --- a/charts/budibase/templates/worker-service-deployment.yaml +++ b/charts/budibase/templates/worker-service-deployment.yaml @@ -64,6 +64,8 @@ spec: {{ end }} - name: API_ENCRYPTION_KEY value: {{ .Values.globals.apiEncryptionKey | quote }} + - name: HTTP_LOGGING + value: {{ .Values.services.worker.httpLogging | quote }} - name: INTERNAL_API_KEY valueFrom: secretKeyRef: @@ -115,7 +117,7 @@ spec: - name: MULTI_TENANCY value: {{ .Values.globals.multiTenancy | quote }} - name: LOG_LEVEL - value: {{ default "info" .Values.services.worker.logLevel | quote }} + value: {{ .Values.services.worker.logLevel | quote }} - name: REDIS_PASSWORD value: {{ .Values.services.redis.password | quote }} - name: REDIS_URL @@ -170,18 +172,6 @@ spec: - name: DD_APM_DD_URL value: https://trace.agent.datadoghq.eu {{ end }} - {{ if .Values.globals.elasticApmEnabled }} - - name: ELASTIC_APM_ENABLED - value: {{ .Values.globals.elasticApmEnabled | quote }} - {{ end }} - {{ if .Values.globals.elasticApmSecretToken }} - - name: ELASTIC_APM_SECRET_TOKEN - value: {{ .Values.globals.elasticApmSecretToken | quote }} - {{ end }} - {{ if .Values.globals.elasticApmServerUrl }} - - name: ELASTIC_APM_SERVER_URL - value: {{ .Values.globals.elasticApmServerUrl | quote }} - {{ end }} {{ if .Values.globals.globalAgentHttpProxy }} - name: GLOBAL_AGENT_HTTP_PROXY value: {{ .Values.globals.globalAgentHttpProxy | quote }} diff --git a/charts/budibase/values.yaml b/charts/budibase/values.yaml index ccbbf9878e..dbf6b6b46e 100644 --- a/charts/budibase/values.yaml +++ b/charts/budibase/values.yaml @@ -80,7 +80,6 @@ globals: enableAnalytics: "1" sentryDSN: "" posthogToken: "phc_bIjZL7oh2GEUd2vqvTBH8WvrX0fWTFQMs6H5KQxiUxU" - logLevel: info selfHosted: "1" # set to 0 for budibase cloud environment, set to 1 for self-hosted setup multiTenancy: "0" # set to 0 to disable multiple orgs, set to 1 to enable multiple orgs accountPortalUrl: "" @@ -107,9 +106,6 @@ globals: smtp: enabled: false -# elasticApmEnabled: -# elasticApmSecretToken: -# elasticApmServerUrl: # globalAgentHttpProxy: # globalAgentHttpsProxy: # globalAgentNoProxy: @@ -137,6 +133,7 @@ services: port: 4002 replicaCount: 1 logLevel: info + httpLogging: 1 resources: {} # nodeDebug: "" # set the value of NODE_DEBUG # annotations: @@ -147,6 +144,8 @@ services: worker: port: 4003 replicaCount: 1 + logLevel: info + httpLogging: 1 resources: {} # annotations: # co.elastic.logs/multiline.type: pattern diff --git a/lerna.json b/lerna.json index 7926e03ecb..29c24ab35e 100644 --- a/lerna.json +++ b/lerna.json @@ -1,9 +1,7 @@ { - "version": "2.4.42-alpha.5", + "version": "2.4.44-alpha.12", "npmClient": "yarn", - "packages": [ - "packages/*" - ], + "packages": ["packages/*"], "command": { "publish": { "ignoreChanges": [ @@ -13,6 +11,9 @@ "# We ignore every JSON file, except for built-in-modules, built-ins and plugins defined in babel-preset-env/data.", "@(!(built-in-modules|built-ins|plugins|package)).json" ] + }, + "run": { + "loadEnvFiles": false } } } diff --git a/nx.json b/nx.json new file mode 100644 index 0000000000..efb577a01b --- /dev/null +++ b/nx.json @@ -0,0 +1,10 @@ +{ + "tasksRunnerOptions": { + "default": { + "runner": "nx/tasks-runners/default", + "options": { + "cacheableOperations": ["build", "test"] + } + } + } +} diff --git a/package.json b/package.json index 592d389dec..e1a768b2be 100644 --- a/package.json +++ b/package.json @@ -12,7 +12,7 @@ "husky": "^7.0.1", "js-yaml": "^4.1.0", "kill-port": "^1.6.1", - "lerna": "3.14.1", + "lerna": "^6.6.1", "madge": "^6.0.0", "prettier": "^2.3.1", "prettier-plugin-svelte": "^2.3.0", @@ -23,12 +23,13 @@ }, "scripts": { "setup": "node ./hosting/scripts/setup.js && yarn && yarn bootstrap && yarn build && yarn dev", - "bootstrap": "lerna bootstrap && lerna link && ./scripts/link-dependencies.sh", - "build": "lerna run build", - "build:client": "lerna run build --ignore @budibase/backend-core --ignore @budibase/worker --ignore @budibase/server --ignore @budibase/builder --ignore @budibase/cli --ignore @budibase/sdk", - "build:dev": "lerna run prebuild && tsc --build --watch --preserveWatchOutput", - "build:backend": "lerna run build --ignore @budibase/client --ignore @budibase/bbui --ignore @budibase/builder --ignore @budibase/cli", - "build:sdk": "lerna run build:sdk", + "bootstrap": "lerna bootstrap", + "postbootstrap": "lerna link && ./scripts/link-dependencies.sh", + "build": "lerna run --stream build", + "build:dev": "lerna run --stream prebuild && tsc --build --watch --preserveWatchOutput", + "backend:bootstrap": "./scripts/scopeBackend.sh 'lerna bootstrap' && yarn run postbootstrap", + "backend:build": "./scripts/scopeBackend.sh 'lerna run --stream build'", + "build:sdk": "lerna run --stream build:sdk", "deps:circular": "madge packages/server/dist/index.js packages/worker/src/index.ts packages/backend-core/dist/src/index.js packages/cli/src/index.js --circular", "release": "lerna publish ${RELEASE_VERSION_TYPE:-patch} --yes --force-publish && yarn release:pro", "release:develop": "lerna publish prerelease --yes --force-publish --dist-tag develop --exact && yarn release:pro:develop", @@ -37,15 +38,16 @@ "restore": "yarn run clean && yarn run bootstrap && yarn run build", "nuke": "yarn run nuke:packages && yarn run nuke:docker", "nuke:packages": "yarn run restore", - "nuke:docker": "lerna run --parallel dev:stack:nuke", + "nuke:docker": "lerna run --stream --parallel dev:stack:nuke", "clean": "lerna clean", "kill-builder": "kill-port 3000", "kill-server": "kill-port 4001 4002", "kill-all": "yarn run kill-builder && yarn run kill-server", - "dev": "yarn run kill-all && lerna link && lerna run --parallel dev:builder --concurrency 1", - "dev:noserver": "yarn run kill-builder && lerna link && lerna run dev:stack:up && lerna run --parallel dev:builder --concurrency 1 --ignore @budibase/backend-core --ignore @budibase/server --ignore @budibase/worker", - "dev:server": "yarn run kill-server && lerna run --parallel dev:builder --concurrency 1 --scope @budibase/backend-core --scope @budibase/worker --scope @budibase/server", - "test": "lerna run test", + "dev": "yarn run kill-all && lerna link && lerna run --stream --parallel dev:builder --concurrency 1 --stream", + "dev:noserver": "yarn run kill-builder && lerna link && lerna run --stream dev:stack:up && lerna run --stream --parallel dev:builder --concurrency 1 --ignore @budibase/backend-core --ignore @budibase/server --ignore @budibase/worker", + "dev:server": "yarn run kill-server && lerna run --stream --parallel dev:builder --concurrency 1 --scope @budibase/backend-core --scope @budibase/worker --scope @budibase/server", + "dev:built": "cd packages/server && yarn dev:stack:up && cd ../../ && lerna run --stream --parallel dev:built", + "test": "lerna run --stream test --stream", "test:pro": "bash scripts/pro/test.sh", "lint:eslint": "eslint packages && eslint qa-core", "lint:prettier": "prettier --check \"packages/**/*.{js,ts,svelte}\" && prettier --write \"examples/**/*.{js,ts,svelte}\" && prettier --check \"qa-core/**/*.{js,ts,svelte}\"", @@ -53,12 +55,12 @@ "lint:fix:eslint": "eslint --fix packages qa-core", "lint:fix:prettier": "prettier --write \"packages/**/*.{js,ts,svelte}\" && prettier --write \"examples/**/*.{js,ts,svelte}\" && prettier --write \"qa-core/**/*.{js,ts,svelte}\"", "lint:fix": "yarn run lint:fix:prettier && yarn run lint:fix:eslint", - "build:specs": "lerna run specs", - "build:docker": "lerna run build:docker && npm run build:docker:proxy && cd hosting/scripts/linux/ && ./release-to-docker-hub.sh $BUDIBASE_RELEASE_VERSION && cd -", - "build:docker:pre": "lerna run build && lerna run predocker", + "build:specs": "lerna run --stream specs", + "build:docker": "lerna run --stream build:docker && npm run build:docker:proxy && cd hosting/scripts/linux/ && ./release-to-docker-hub.sh $BUDIBASE_RELEASE_VERSION && cd -", + "build:docker:pre": "lerna run --stream build && lerna run --stream predocker", "build:docker:proxy": "docker build hosting/proxy -t proxy-service", - "build:docker:selfhost": "lerna run build:docker && cd hosting/scripts/linux/ && ./release-to-docker-hub.sh latest && cd -", - "build:docker:develop": "node scripts/pinVersions && lerna run build:docker && npm run build:docker:proxy && cd hosting/scripts/linux/ && ./release-to-docker-hub.sh develop && cd -", + "build:docker:selfhost": "lerna run --stream build:docker && cd hosting/scripts/linux/ && ./release-to-docker-hub.sh latest && cd -", + "build:docker:develop": "node scripts/pinVersions && lerna run --stream build:docker && npm run build:docker:proxy && cd hosting/scripts/linux/ && ./release-to-docker-hub.sh develop && cd -", "build:docker:airgap": "node hosting/scripts/airgapped/airgappedDockerBuild", "build:digitalocean": "cd hosting/digitalocean && ./build.sh && cd -", "build:docker:single:multiarch": "docker buildx build --platform linux/arm64,linux/amd64 -f hosting/single/Dockerfile -t budibase:latest .", @@ -67,16 +69,16 @@ "build:docker:dependencies": "docker build -f hosting/dependencies/Dockerfile -t budibase/dependencies:latest ./hosting", "publish:docker:couch": "docker buildx build --platform linux/arm64,linux/amd64 -f hosting/couchdb/Dockerfile -t budibase/couchdb:latest -t budibase/couchdb:v3.2.1 --push ./hosting/couchdb", "publish:docker:dependencies": "docker buildx build --platform linux/arm64,linux/amd64 -f hosting/dependencies/Dockerfile -t budibase/dependencies:latest -t budibase/dependencies:v3.2.1 --push ./hosting", - "build:docs": "lerna run build:docs", + "build:docs": "lerna run --stream build:docs", "release:helm": "node scripts/releaseHelmChart", - "env:multi:enable": "lerna run env:multi:enable", - "env:multi:disable": "lerna run env:multi:disable", - "env:selfhost:enable": "lerna run env:selfhost:enable", - "env:selfhost:disable": "lerna run env:selfhost:disable", + "env:multi:enable": "lerna run --stream env:multi:enable", + "env:multi:disable": "lerna run --stream env:multi:disable", + "env:selfhost:enable": "lerna run --stream env:selfhost:enable", + "env:selfhost:disable": "lerna run --stream env:selfhost:disable", "env:localdomain:enable": "./scripts/localdomain.sh enable", "env:localdomain:disable": "./scripts/localdomain.sh disable", - "env:account:enable": "lerna run env:account:enable", - "env:account:disable": "lerna run env:account:disable", + "env:account:enable": "lerna run --stream env:account:enable", + "env:account:disable": "lerna run --stream env:account:disable", "mode:self": "yarn env:selfhost:enable && yarn env:multi:disable && yarn env:account:disable", "mode:cloud": "yarn env:selfhost:disable && yarn env:multi:enable && yarn env:account:disable", "mode:account": "yarn mode:cloud && yarn env:account:enable", diff --git a/packages/backend-core/package.json b/packages/backend-core/package.json index e7c6217cee..9d3f75c583 100644 --- a/packages/backend-core/package.json +++ b/packages/backend-core/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/backend-core", - "version": "2.4.42-alpha.5", + "version": "2.4.44-alpha.12", "description": "Budibase backend core libraries used in server and worker", "main": "dist/src/index.js", "types": "dist/src/index.d.ts", @@ -24,7 +24,7 @@ "dependencies": { "@budibase/nano": "10.1.2", "@budibase/pouchdb-replication-stream": "1.2.10", - "@budibase/types": "2.4.42-alpha.5", + "@budibase/types": "2.4.44-alpha.12", "@shopify/jest-koa-mocks": "5.0.1", "@techpass/passport-openidconnect": "0.3.2", "aws-cloudfront-sign": "2.2.0", @@ -39,6 +39,7 @@ "joi": "17.6.0", "jsonwebtoken": "9.0.0", "koa-passport": "4.1.4", + "koa-pino-logger": "4.0.0", "lodash": "4.17.21", "lodash.isarguments": "3.1.0", "node-fetch": "2.6.7", @@ -64,11 +65,9 @@ "@types/ioredis": "4.28.0", "@types/jest": "28.1.1", "@types/koa": "2.13.4", - "@types/koa-pino-logger": "3.0.0", "@types/lodash": "4.14.180", "@types/node": "14.18.20", "@types/node-fetch": "2.6.1", - "@types/pino-http": "5.8.1", "@types/pouchdb": "6.4.0", "@types/redlock": "4.0.3", "@types/semver": "7.3.7", @@ -80,6 +79,7 @@ "jest-serial-runner": "^1.2.1", "koa": "2.13.4", "nodemon": "2.0.16", + "pino-pretty": "10.0.0", "pouchdb-adapter-memory": "7.2.2", "timekeeper": "2.2.0", "ts-jest": "28.0.4", diff --git a/packages/backend-core/scripts/test.sh b/packages/backend-core/scripts/test.sh index 2965134399..3d8240e65a 100644 --- a/packages/backend-core/scripts/test.sh +++ b/packages/backend-core/scripts/test.sh @@ -1,4 +1,5 @@ #!/bin/bash +set -e if [[ -n $CI ]] then @@ -7,6 +8,6 @@ then jest --coverage --runInBand --forceExit else # --maxWorkers performs better in development - echo "jest --coverage" - jest --coverage -fi + echo "jest --coverage --forceExit" + jest --coverage --forceExit +fi \ No newline at end of file diff --git a/packages/backend-core/src/cache/tests/writethrough.spec.ts b/packages/backend-core/src/cache/tests/writethrough.spec.ts index a34f05e881..363344c84d 100644 --- a/packages/backend-core/src/cache/tests/writethrough.spec.ts +++ b/packages/backend-core/src/cache/tests/writethrough.spec.ts @@ -2,6 +2,7 @@ import { structures, DBTestConfiguration, expectFunctionWasCalledTimesWith, + mocks, } from "../../../tests" import { Writethrough } from "../writethrough" import { getDB } from "../../db" @@ -77,9 +78,9 @@ describe("writethrough", () => { expect.arrayContaining([current._rev, current._rev, newRev]) ) expectFunctionWasCalledTimesWith( - console.warn, + mocks.alerts.logWarn, 2, - "bb-warn: Ignoring redlock conflict in write-through cache" + "Ignoring redlock conflict in write-through cache" ) const output = await db.get(current._id) diff --git a/packages/backend-core/src/configs/configs.ts b/packages/backend-core/src/configs/configs.ts index b461497747..701b5bb329 100644 --- a/packages/backend-core/src/configs/configs.ts +++ b/packages/backend-core/src/configs/configs.ts @@ -5,6 +5,8 @@ import { GoogleInnerConfig, OIDCConfig, OIDCInnerConfig, + SCIMConfig, + SCIMInnerConfig, SettingsConfig, SettingsInnerConfig, SMTPConfig, @@ -32,8 +34,7 @@ export async function getConfig( const db = context.getGlobalDB() try { // await to catch error - const config = (await db.get(generateConfigID(type))) as T - return config + return (await db.get(generateConfigID(type))) as T } catch (e: any) { if (e.status === 404) { return @@ -242,3 +243,10 @@ export async function getSMTPConfig( } } } + +// SCIM + +export async function getSCIMConfig(): Promise { + const config = await getConfig(ConfigType.SCIM) + return config?.config +} diff --git a/packages/backend-core/src/constants/misc.ts b/packages/backend-core/src/constants/misc.ts index 15cec7a6b9..2bb8f815cf 100644 --- a/packages/backend-core/src/constants/misc.ts +++ b/packages/backend-core/src/constants/misc.ts @@ -22,6 +22,7 @@ export enum Header { TOKEN = "x-budibase-token", CSRF_TOKEN = "x-csrf-token", CORRELATION_ID = "x-budibase-correlation-id", + AUTHORIZATION = "authorization", } export enum GlobalRole { @@ -38,6 +39,7 @@ export enum Config { GOOGLE = "google", OIDC = "oidc", OIDC_LOGOS = "logos_oidc", + SCIM = "scim", } export const MIN_VALID_DATE = new Date(-2147483647000) diff --git a/packages/backend-core/src/context/mainContext.ts b/packages/backend-core/src/context/mainContext.ts index 02ba16aa8c..e1bd535b78 100644 --- a/packages/backend-core/src/context/mainContext.ts +++ b/packages/backend-core/src/context/mainContext.ts @@ -214,6 +214,13 @@ export function doInEnvironmentContext( return newContext(updates, task) } +export function doInScimContext(task: any) { + const updates: ContextMap = { + isScim: true, + } + return newContext(updates, task) +} + export function getEnvironmentVariables() { const context = Context.get() if (!context.environmentVariables) { @@ -270,3 +277,9 @@ export function getDevAppDB(opts?: any): Database { } return getDB(conversions.getDevelopmentAppID(appId), opts) } + +export function isScim(): boolean { + const context = Context.get() + const scimCall = context?.isScim + return !!scimCall +} diff --git a/packages/backend-core/src/context/tests/index.spec.ts b/packages/backend-core/src/context/tests/index.spec.ts index 5c8ce6fc19..3f760d4a49 100644 --- a/packages/backend-core/src/context/tests/index.spec.ts +++ b/packages/backend-core/src/context/tests/index.spec.ts @@ -1,6 +1,6 @@ import { testEnv } from "../../../tests" -const context = require("../") -const { DEFAULT_TENANT_ID } = require("../../constants") +import * as context from "../" +import { DEFAULT_TENANT_ID } from "../../constants" describe("context", () => { describe("doInTenant", () => { @@ -131,4 +131,17 @@ describe("context", () => { }) }) }) + + describe("doInScimContext", () => { + it("returns true when set", () => { + context.doInScimContext(() => { + const isScim = context.isScim() + expect(isScim).toBe(true) + }) + }) + it("returns false when not set", () => { + const isScim = context.isScim() + expect(isScim).toBe(false) + }) + }) }) diff --git a/packages/backend-core/src/context/types.ts b/packages/backend-core/src/context/types.ts index 78197ed528..727dad80bc 100644 --- a/packages/backend-core/src/context/types.ts +++ b/packages/backend-core/src/context/types.ts @@ -6,4 +6,5 @@ export type ContextMap = { appId?: string identity?: IdentityContext environmentVariables?: Record + isScim?: boolean } diff --git a/packages/backend-core/src/db/index.ts b/packages/backend-core/src/db/index.ts index a569b17b36..ea93b91d14 100644 --- a/packages/backend-core/src/db/index.ts +++ b/packages/backend-core/src/db/index.ts @@ -8,3 +8,4 @@ export { default as Replication } from "./Replication" export * from "../constants/db" export { getGlobalDBName, baseGlobalDBName } from "../context" export * from "./lucene" +export * as searchIndexes from "./searchIndexes" diff --git a/packages/backend-core/src/db/lucene.ts b/packages/backend-core/src/db/lucene.ts index 71ce4ba9ac..6f2f4fc991 100644 --- a/packages/backend-core/src/db/lucene.ts +++ b/packages/backend-core/src/db/lucene.ts @@ -1,12 +1,14 @@ import fetch from "node-fetch" import { getCouchInfo } from "./couch" import { SearchFilters, Row } from "@budibase/types" +import { createUserIndex } from "./searchIndexes/searchIndexes" const QUERY_START_REGEX = /\d[0-9]*:/g interface SearchResponse { rows: T[] | any[] - bookmark: string + bookmark?: string + totalRows: number } interface PaginatedSearchResponse extends SearchResponse { @@ -42,23 +44,26 @@ export function removeKeyNumbering(key: any): string { * Optionally takes a base lucene query object. */ export class QueryBuilder { - dbName: string - index: string - query: SearchFilters - limit: number - sort?: string - bookmark?: string - sortOrder: string - sortType: string - includeDocs: boolean - version?: string - indexBuilder?: () => Promise - noEscaping = false + #dbName: string + #index: string + #query: SearchFilters + #limit: number + #sort?: string + #bookmark?: string + #sortOrder: string + #sortType: string + #includeDocs: boolean + #version?: string + #indexBuilder?: () => Promise + #noEscaping = false + #skip?: number + + static readonly maxLimit = 200 constructor(dbName: string, index: string, base?: SearchFilters) { - this.dbName = dbName - this.index = index - this.query = { + this.#dbName = dbName + this.#index = index + this.#query = { allOr: false, string: {}, fuzzy: {}, @@ -73,86 +78,96 @@ export class QueryBuilder { containsAny: {}, ...base, } - this.limit = 50 - this.sortOrder = "ascending" - this.sortType = "string" - this.includeDocs = true + this.#limit = 50 + this.#sortOrder = "ascending" + this.#sortType = "string" + this.#includeDocs = true } disableEscaping() { - this.noEscaping = true + this.#noEscaping = true return this } setIndexBuilder(builderFn: () => Promise) { - this.indexBuilder = builderFn + this.#indexBuilder = builderFn return this } setVersion(version?: string) { if (version != null) { - this.version = version + this.#version = version } return this } setTable(tableId: string) { - this.query.equal!.tableId = tableId + this.#query.equal!.tableId = tableId return this } setLimit(limit?: number) { if (limit != null) { - this.limit = limit + this.#limit = limit } return this } setSort(sort?: string) { if (sort != null) { - this.sort = sort + this.#sort = sort } return this } setSortOrder(sortOrder?: string) { if (sortOrder != null) { - this.sortOrder = sortOrder + this.#sortOrder = sortOrder } return this } setSortType(sortType?: string) { if (sortType != null) { - this.sortType = sortType + this.#sortType = sortType } return this } setBookmark(bookmark?: string) { if (bookmark != null) { - this.bookmark = bookmark + this.#bookmark = bookmark } return this } + setSkip(skip: number | undefined) { + this.#skip = skip + return this + } + excludeDocs() { - this.includeDocs = false + this.#includeDocs = false + return this + } + + includeDocs() { + this.#includeDocs = true return this } addString(key: string, partial: string) { - this.query.string![key] = partial + this.#query.string![key] = partial return this } addFuzzy(key: string, fuzzy: string) { - this.query.fuzzy![key] = fuzzy + this.#query.fuzzy![key] = fuzzy return this } addRange(key: string, low: string | number, high: string | number) { - this.query.range![key] = { + this.#query.range![key] = { low, high, } @@ -160,51 +175,51 @@ export class QueryBuilder { } addEqual(key: string, value: any) { - this.query.equal![key] = value + this.#query.equal![key] = value return this } addNotEqual(key: string, value: any) { - this.query.notEqual![key] = value + this.#query.notEqual![key] = value return this } addEmpty(key: string, value: any) { - this.query.empty![key] = value + this.#query.empty![key] = value return this } addNotEmpty(key: string, value: any) { - this.query.notEmpty![key] = value + this.#query.notEmpty![key] = value return this } addOneOf(key: string, value: any) { - this.query.oneOf![key] = value + this.#query.oneOf![key] = value return this } addContains(key: string, value: any) { - this.query.contains![key] = value + this.#query.contains![key] = value return this } addNotContains(key: string, value: any) { - this.query.notContains![key] = value + this.#query.notContains![key] = value return this } addContainsAny(key: string, value: any) { - this.query.containsAny![key] = value + this.#query.containsAny![key] = value return this } setAllOr() { - this.query.allOr = true + this.#query.allOr = true } handleSpaces(input: string) { - if (this.noEscaping) { + if (this.#noEscaping) { return input } else { return input.replace(/ /g, "_") @@ -219,7 +234,7 @@ export class QueryBuilder { * @returns {string|*} */ preprocess(value: any, { escape, lowercase, wrap, type }: any = {}) { - const hasVersion = !!this.version + const hasVersion = !!this.#version // Determine if type needs wrapped const originalType = typeof value // Convert to lowercase @@ -227,7 +242,7 @@ export class QueryBuilder { value = value.toLowerCase ? value.toLowerCase() : value } // Escape characters - if (!this.noEscaping && escape && originalType === "string") { + if (!this.#noEscaping && escape && originalType === "string") { value = `${value}`.replace(/[ #+\-&|!(){}\]^"~*?:\\]/g, "\\$&") } @@ -242,7 +257,7 @@ export class QueryBuilder { isMultiCondition() { let count = 0 - for (let filters of Object.values(this.query)) { + for (let filters of Object.values(this.#query)) { // not contains is one massive filter in allOr mode if (typeof filters === "object") { count += Object.keys(filters).length @@ -272,13 +287,13 @@ export class QueryBuilder { buildSearchQuery() { const builder = this - let allOr = this.query && this.query.allOr + let allOr = this.#query && this.#query.allOr let query = allOr ? "" : "*:*" const allPreProcessingOpts = { escape: true, lowercase: true, wrap: true } let tableId - if (this.query.equal!.tableId) { - tableId = this.query.equal!.tableId - delete this.query.equal!.tableId + if (this.#query.equal!.tableId) { + tableId = this.#query.equal!.tableId + delete this.#query.equal!.tableId } const equal = (key: string, value: any) => { @@ -363,8 +378,8 @@ export class QueryBuilder { } // Construct the actual lucene search query string from JSON structure - if (this.query.string) { - build(this.query.string, (key: string, value: any) => { + if (this.#query.string) { + build(this.#query.string, (key: string, value: any) => { if (!value) { return null } @@ -376,8 +391,8 @@ export class QueryBuilder { return `${key}:${value}*` }) } - if (this.query.range) { - build(this.query.range, (key: string, value: any) => { + if (this.#query.range) { + build(this.#query.range, (key: string, value: any) => { if (!value) { return null } @@ -392,8 +407,8 @@ export class QueryBuilder { return `${key}:[${low} TO ${high}]` }) } - if (this.query.fuzzy) { - build(this.query.fuzzy, (key: string, value: any) => { + if (this.#query.fuzzy) { + build(this.#query.fuzzy, (key: string, value: any) => { if (!value) { return null } @@ -405,34 +420,34 @@ export class QueryBuilder { return `${key}:${value}~` }) } - if (this.query.equal) { - build(this.query.equal, equal) + if (this.#query.equal) { + build(this.#query.equal, equal) } - if (this.query.notEqual) { - build(this.query.notEqual, (key: string, value: any) => { + if (this.#query.notEqual) { + build(this.#query.notEqual, (key: string, value: any) => { if (!value) { return null } return `!${key}:${builder.preprocess(value, allPreProcessingOpts)}` }) } - if (this.query.empty) { - build(this.query.empty, (key: string) => `!${key}:["" TO *]`) + if (this.#query.empty) { + build(this.#query.empty, (key: string) => `!${key}:["" TO *]`) } - if (this.query.notEmpty) { - build(this.query.notEmpty, (key: string) => `${key}:["" TO *]`) + if (this.#query.notEmpty) { + build(this.#query.notEmpty, (key: string) => `${key}:["" TO *]`) } - if (this.query.oneOf) { - build(this.query.oneOf, oneOf) + if (this.#query.oneOf) { + build(this.#query.oneOf, oneOf) } - if (this.query.contains) { - build(this.query.contains, contains) + if (this.#query.contains) { + build(this.#query.contains, contains) } - if (this.query.notContains) { - build(this.compressFilters(this.query.notContains), notContains) + if (this.#query.notContains) { + build(this.compressFilters(this.#query.notContains), notContains) } - if (this.query.containsAny) { - build(this.query.containsAny, containsAny) + if (this.#query.containsAny) { + build(this.#query.containsAny, containsAny) } // make sure table ID is always added as an AND if (tableId) { @@ -446,29 +461,65 @@ export class QueryBuilder { buildSearchBody() { let body: any = { q: this.buildSearchQuery(), - limit: Math.min(this.limit, 200), - include_docs: this.includeDocs, + limit: Math.min(this.#limit, QueryBuilder.maxLimit), + include_docs: this.#includeDocs, } - if (this.bookmark) { - body.bookmark = this.bookmark + if (this.#bookmark) { + body.bookmark = this.#bookmark } - if (this.sort) { - const order = this.sortOrder === "descending" ? "-" : "" - const type = `<${this.sortType}>` - body.sort = `${order}${this.handleSpaces(this.sort)}${type}` + if (this.#sort) { + const order = this.#sortOrder === "descending" ? "-" : "" + const type = `<${this.#sortType}>` + body.sort = `${order}${this.handleSpaces(this.#sort)}${type}` } return body } async run() { + if (this.#skip) { + await this.#skipItems(this.#skip) + } + return await this.#execute() + } + + /** + * Lucene queries do not support pagination and use bookmarks instead. + * For the given builder, walk through pages using bookmarks until the desired + * page has been met. + */ + async #skipItems(skip: number) { + // Lucene does not support pagination. + // Handle pagination by finding the right bookmark + const prevIncludeDocs = this.#includeDocs + const prevLimit = this.#limit + + this.excludeDocs() + let skipRemaining = skip + let iterationFetched = 0 + do { + const toSkip = Math.min(QueryBuilder.maxLimit, skipRemaining) + this.setLimit(toSkip) + const { bookmark, rows } = await this.#execute() + this.setBookmark(bookmark) + iterationFetched = rows.length + skipRemaining -= rows.length + } while (skipRemaining > 0 && iterationFetched > 0) + + this.#includeDocs = prevIncludeDocs + this.#limit = prevLimit + } + + async #execute() { const { url, cookie } = getCouchInfo() - const fullPath = `${url}/${this.dbName}/_design/database/_search/${this.index}` + const fullPath = `${url}/${this.#dbName}/_design/database/_search/${ + this.#index + }` const body = this.buildSearchBody() try { return await runQuery(fullPath, body, cookie) } catch (err: any) { - if (err.status === 404 && this.indexBuilder) { - await this.indexBuilder() + if (err.status === 404 && this.#indexBuilder) { + await this.#indexBuilder() return await runQuery(fullPath, body, cookie) } else { throw err @@ -502,8 +553,9 @@ async function runQuery( } const json = await response.json() - let output: any = { + let output: SearchResponse = { rows: [], + totalRows: 0, } if (json.rows != null && json.rows.length > 0) { output.rows = json.rows.map((row: any) => row.doc) @@ -511,6 +563,9 @@ async function runQuery( if (json.bookmark) { output.bookmark = json.bookmark } + if (json.total_rows) { + output.totalRows = json.total_rows + } return output } @@ -543,8 +598,8 @@ async function recursiveSearch( if (rows.length >= params.limit) { return rows } - let pageSize = 200 - if (rows.length > params.limit - 200) { + let pageSize = QueryBuilder.maxLimit + if (rows.length > params.limit - QueryBuilder.maxLimit) { pageSize = params.limit - rows.length } const page = await new QueryBuilder(dbName, index, query) @@ -559,7 +614,7 @@ async function recursiveSearch( if (!page.rows.length) { return rows } - if (page.rows.length < 200) { + if (page.rows.length < QueryBuilder.maxLimit) { return [...rows, ...page.rows] } const newParams = { @@ -597,7 +652,7 @@ export async function paginatedSearch( if (limit == null || isNaN(limit) || limit < 0) { limit = 50 } - limit = Math.min(limit, 200) + limit = Math.min(limit, QueryBuilder.maxLimit) const search = new QueryBuilder(dbName, index, query) if (params.version) { search.setVersion(params.version) diff --git a/packages/backend-core/src/db/searchIndexes/index.ts b/packages/backend-core/src/db/searchIndexes/index.ts new file mode 100644 index 0000000000..d3054e90c0 --- /dev/null +++ b/packages/backend-core/src/db/searchIndexes/index.ts @@ -0,0 +1 @@ +export * from "./searchIndexes" diff --git a/packages/backend-core/src/db/searchIndexes/searchIndexes.ts b/packages/backend-core/src/db/searchIndexes/searchIndexes.ts new file mode 100644 index 0000000000..f03259b47f --- /dev/null +++ b/packages/backend-core/src/db/searchIndexes/searchIndexes.ts @@ -0,0 +1,62 @@ +import { User, SearchIndex } from "@budibase/types" +import { getGlobalDB } from "../../context" + +export async function createUserIndex() { + const db = getGlobalDB() + let designDoc + try { + designDoc = await db.get("_design/database") + } catch (err: any) { + if (err.status === 404) { + designDoc = { _id: "_design/database" } + } + } + + const fn = function (user: User) { + if (user._id && !user._id.startsWith("us_")) { + return + } + const ignoredFields = [ + "_id", + "_rev", + "password", + "account", + "license", + "budibaseAccess", + "accountPortalAccess", + "csrfToken", + ] + + function idx(input: Record, prev?: string) { + for (let key of Object.keys(input)) { + if (ignoredFields.includes(key)) { + continue + } + let idxKey = prev != null ? `${prev}.${key}` : key + if (typeof input[key] === "string") { + // eslint-disable-next-line no-undef + // @ts-ignore + index(idxKey, input[key].toLowerCase(), { facet: true }) + } else if (typeof input[key] !== "object") { + // eslint-disable-next-line no-undef + // @ts-ignore + index(idxKey, input[key], { facet: true }) + } else { + idx(input[key], idxKey) + } + } + } + idx(user) + } + + designDoc.indexes = { + [SearchIndex.USER]: { + index: fn.toString(), + analyzer: { + default: "keyword", + name: "perfield", + }, + }, + } + await db.put(designDoc) +} diff --git a/packages/backend-core/src/db/tests/lucene.spec.ts b/packages/backend-core/src/db/tests/lucene.spec.ts index 52017cc94c..26ce316a9d 100644 --- a/packages/backend-core/src/db/tests/lucene.spec.ts +++ b/packages/backend-core/src/db/tests/lucene.spec.ts @@ -136,6 +136,106 @@ describe("lucene", () => { const resp = await builder.run() expect(resp.rows.length).toBe(2) }) + + describe("skip", () => { + const skipDbName = `db-${newid()}` + let docs: { + _id: string + property: string + array: string[] + }[] + + beforeAll(async () => { + const db = getDB(skipDbName) + + docs = Array(QueryBuilder.maxLimit * 2.5) + .fill(0) + .map((_, i) => ({ + _id: i.toString().padStart(3, "0"), + property: `value_${i.toString().padStart(3, "0")}`, + array: [], + })) + await db.bulkDocs(docs) + + await db.put({ + _id: "_design/database", + indexes: { + [INDEX_NAME]: { + index: index, + analyzer: "standard", + }, + }, + }) + }) + + it("should be able to apply skip", async () => { + const builder = new QueryBuilder(skipDbName, INDEX_NAME) + const firstResponse = await builder.run() + builder.setSkip(40) + const secondResponse = await builder.run() + + // Return the default limit + expect(firstResponse.rows.length).toBe(50) + expect(secondResponse.rows.length).toBe(50) + + // Should have the expected overlap + expect(firstResponse.rows.slice(40)).toEqual( + secondResponse.rows.slice(0, 10) + ) + }) + + it("should handle limits", async () => { + const builder = new QueryBuilder(skipDbName, INDEX_NAME) + builder.setLimit(10) + builder.setSkip(50) + builder.setSort("_id") + + const resp = await builder.run() + expect(resp.rows.length).toBe(10) + expect(resp.rows).toEqual( + docs.slice(50, 60).map(expect.objectContaining) + ) + }) + + it("should be able to skip searching through multiple responses", async () => { + const builder = new QueryBuilder(skipDbName, INDEX_NAME) + // Skipping 2 max limits plus a little bit more + const skip = QueryBuilder.maxLimit * 2 + 37 + builder.setSkip(skip) + builder.setSort("_id") + const resp = await builder.run() + + expect(resp.rows.length).toBe(50) + expect(resp.rows).toEqual( + docs.slice(skip, skip + resp.rows.length).map(expect.objectContaining) + ) + }) + + it("should not return results if skipping all docs", async () => { + const builder = new QueryBuilder(skipDbName, INDEX_NAME) + // Skipping 2 max limits plus a little bit more + const skip = docs.length + 1 + builder.setSkip(skip) + + const resp = await builder.run() + + expect(resp.rows.length).toBe(0) + }) + + it("skip should respect with filters", async () => { + const builder = new QueryBuilder(skipDbName, INDEX_NAME) + builder.setLimit(10) + builder.setSkip(50) + builder.addString("property", "value_1") + builder.setSort("property") + + const resp = await builder.run() + expect(resp.rows.length).toBe(10) + expect(resp.rows).toEqual( + docs.slice(150, 160).map(expect.objectContaining) + ) + }) + }) }) describe("paginated search", () => { diff --git a/packages/backend-core/src/db/utils.ts b/packages/backend-core/src/db/utils.ts index 76c52d08ad..441c118235 100644 --- a/packages/backend-core/src/db/utils.ts +++ b/packages/backend-core/src/db/utils.ts @@ -434,8 +434,8 @@ export const getPluginParams = (pluginId?: string | null, otherProps = {}) => { return getDocParams(DocumentType.PLUGIN, pluginId, otherProps) } -export function pagination( - data: any[], +export function pagination( + data: T[], pageSize: number, { paginate, @@ -444,7 +444,7 @@ export function pagination( }: { paginate: boolean property: string - getKey?: (doc: any) => string | undefined + getKey?: (doc: T) => string | undefined } = { paginate: true, property: "_id", diff --git a/packages/backend-core/src/environment.ts b/packages/backend-core/src/environment.ts index 5718494fc4..68f056c80c 100644 --- a/packages/backend-core/src/environment.ts +++ b/packages/backend-core/src/environment.ts @@ -36,6 +36,15 @@ function getAPIEncryptionKey() { : process.env.JWT_SECRET // fallback to the JWT_SECRET used historically } +function httpLogging() { + if (process.env.HTTP_LOGGING === undefined) { + // on by default unless otherwise specified + return true + } + + return process.env.HTTP_LOGGING +} + const environment = { isTest, isJest, @@ -90,11 +99,11 @@ const environment = { USE_COUCH: process.env.USE_COUCH || true, DEFAULT_LICENSE: process.env.DEFAULT_LICENSE, SERVICE: process.env.SERVICE || "budibase", - LOG_LEVEL: process.env.LOG_LEVEL, + LOG_LEVEL: process.env.LOG_LEVEL || "info", SESSION_UPDATE_PERIOD: process.env.SESSION_UPDATE_PERIOD, DEPLOYMENT_ENVIRONMENT: process.env.DEPLOYMENT_ENVIRONMENT || "docker-compose", - ENABLE_4XX_HTTP_LOGGING: process.env.ENABLE_4XX_HTTP_LOGGING || true, + HTTP_LOGGING: httpLogging(), ENABLE_AUDIT_LOG_IP_ADDR: process.env.ENABLE_AUDIT_LOG_IP_ADDR, // smtp SMTP_FALLBACK_ENABLED: process.env.SMTP_FALLBACK_ENABLED, diff --git a/packages/backend-core/src/errors/errors.ts b/packages/backend-core/src/errors/errors.ts index 54ca8456ab..4e1f1abbb5 100644 --- a/packages/backend-core/src/errors/errors.ts +++ b/packages/backend-core/src/errors/errors.ts @@ -97,3 +97,11 @@ export class InvalidAPIKeyError extends BudibaseError { ) } } + +// USERS + +export class EmailUnavailableError extends Error { + constructor(email: string) { + super(`Email already in use: '${email}'`) + } +} diff --git a/packages/backend-core/src/events/processors/LoggingProcessor.ts b/packages/backend-core/src/events/processors/LoggingProcessor.ts index 6bb691a83a..0f4d02b99c 100644 --- a/packages/backend-core/src/events/processors/LoggingProcessor.ts +++ b/packages/backend-core/src/events/processors/LoggingProcessor.ts @@ -2,14 +2,6 @@ import { Event, Identity, Group } from "@budibase/types" import { EventProcessor } from "./types" import env from "../../environment" -const getTimestampString = (timestamp?: string | number) => { - let timestampString = "" - if (timestamp) { - timestampString = `[timestamp=${new Date(timestamp).toISOString()}]` - } - return timestampString -} - const skipLogging = env.SELF_HOSTED && !env.isDev() export default class LoggingProcessor implements EventProcessor { @@ -22,32 +14,21 @@ export default class LoggingProcessor implements EventProcessor { if (skipLogging) { return } - let timestampString = getTimestampString(timestamp) - let message = `[audit] [identityType=${identity.type}] ${timestampString} ${event} ` - if (env.isDev()) { - message = message + `[debug: [properties=${JSON.stringify(properties)}] ]` - } - console.log(message) + console.log(`[audit] [identityType=${identity.type}] ${event}`, properties) } async identify(identity: Identity, timestamp?: string | number) { if (skipLogging) { return } - let timestampString = getTimestampString(timestamp) - console.log( - `[audit] [${JSON.stringify(identity)}] ${timestampString} identified` - ) + console.log(`[audit] identified`, identity) } async identifyGroup(group: Group, timestamp?: string | number) { if (skipLogging) { return } - let timestampString = getTimestampString(timestamp) - console.log( - `[audit] [${JSON.stringify(group)}] ${timestampString} group identified` - ) + console.log(`[audit] group identified`, group) } shutdown(): void { diff --git a/packages/backend-core/src/events/publishers/group.ts b/packages/backend-core/src/events/publishers/group.ts index a000b880a2..6870ceb350 100644 --- a/packages/backend-core/src/events/publishers/group.ts +++ b/packages/backend-core/src/events/publishers/group.ts @@ -9,12 +9,13 @@ import { GroupUsersDeletedEvent, GroupAddedOnboardingEvent, GroupPermissionsEditedEvent, - UserGroupRoles, } from "@budibase/types" +import { isScim } from "../../context" async function created(group: UserGroup, timestamp?: number) { const properties: GroupCreatedEvent = { groupId: group._id as string, + viaScim: isScim(), audited: { name: group.name, }, @@ -25,6 +26,7 @@ async function created(group: UserGroup, timestamp?: number) { async function updated(group: UserGroup) { const properties: GroupUpdatedEvent = { groupId: group._id as string, + viaScim: isScim(), audited: { name: group.name, }, @@ -35,6 +37,7 @@ async function updated(group: UserGroup) { async function deleted(group: UserGroup) { const properties: GroupDeletedEvent = { groupId: group._id as string, + viaScim: isScim(), audited: { name: group.name, }, @@ -46,6 +49,7 @@ async function usersAdded(count: number, group: UserGroup) { const properties: GroupUsersAddedEvent = { count, groupId: group._id as string, + viaScim: isScim(), audited: { name: group.name, }, @@ -57,6 +61,7 @@ async function usersDeleted(count: number, group: UserGroup) { const properties: GroupUsersDeletedEvent = { count, groupId: group._id as string, + viaScim: isScim(), audited: { name: group.name, }, diff --git a/packages/backend-core/src/events/publishers/user.ts b/packages/backend-core/src/events/publishers/user.ts index 8dbc494d1e..0d08c0a759 100644 --- a/packages/backend-core/src/events/publishers/user.ts +++ b/packages/backend-core/src/events/publishers/user.ts @@ -15,10 +15,12 @@ import { UserUpdatedEvent, UserOnboardingEvent, } from "@budibase/types" +import { isScim } from "../../context" async function created(user: User, timestamp?: number) { const properties: UserCreatedEvent = { userId: user._id as string, + viaScim: isScim(), audited: { email: user.email, }, @@ -29,6 +31,7 @@ async function created(user: User, timestamp?: number) { async function updated(user: User) { const properties: UserUpdatedEvent = { userId: user._id as string, + viaScim: isScim(), audited: { email: user.email, }, @@ -39,6 +42,7 @@ async function updated(user: User) { async function deleted(user: User) { const properties: UserDeletedEvent = { userId: user._id as string, + viaScim: isScim(), audited: { email: user.email, }, diff --git a/packages/backend-core/src/logging.ts b/packages/backend-core/src/logging.ts deleted file mode 100644 index 8f2ae6e619..0000000000 --- a/packages/backend-core/src/logging.ts +++ /dev/null @@ -1,60 +0,0 @@ -import { Header } from "./constants" -import env from "./environment" -const correlator = require("correlation-id") -import { Options } from "pino-http" -import { IncomingMessage } from "http" - -const NonErrors = ["AccountError"] - -function isSuppressed(e?: any) { - return e && e["suppressAlert"] -} - -export function logAlert(message: string, e?: any) { - if (e && NonErrors.includes(e.name) && isSuppressed(e)) { - return - } - let errorJson = "" - if (e) { - errorJson = ": " + JSON.stringify(e, Object.getOwnPropertyNames(e)) - } - console.error(`bb-alert: ${message} ${errorJson}`) -} - -export function logAlertWithInfo( - message: string, - db: string, - id: string, - error: any -) { - message = `${message} - db: ${db} - doc: ${id} - error: ` - logAlert(message, error) -} - -export function logWarn(message: string) { - console.warn(`bb-warn: ${message}`) -} - -export function pinoSettings(): Options { - return { - prettyPrint: { - levelFirst: true, - }, - genReqId: correlator.getId, - level: env.LOG_LEVEL || "error", - autoLogging: { - ignore: (req: IncomingMessage) => !!req.url?.includes("/health"), - }, - } -} - -const setCorrelationHeader = (headers: any) => { - const correlationId = correlator.getId() - if (correlationId) { - headers[Header.CORRELATION_ID] = correlationId - } -} - -export const correlation = { - setHeader: setCorrelationHeader, -} diff --git a/packages/backend-core/src/logging/alerts.ts b/packages/backend-core/src/logging/alerts.ts new file mode 100644 index 0000000000..625ecfcf15 --- /dev/null +++ b/packages/backend-core/src/logging/alerts.ts @@ -0,0 +1,26 @@ +const NonErrors = ["AccountError"] + +function isSuppressed(e?: any) { + return e && e["suppressAlert"] +} + +export function logAlert(message: string, e?: any) { + if (e && NonErrors.includes(e.name) && isSuppressed(e)) { + return + } + console.error(`bb-alert: ${message}`, e) +} + +export function logAlertWithInfo( + message: string, + db: string, + id: string, + error: any +) { + message = `${message} - db: ${db} - doc: ${id} - error: ` + logAlert(message, error) +} + +export function logWarn(message: string) { + console.warn(`bb-warn: ${message}`) +} diff --git a/packages/backend-core/src/logging/correlation/correlation.ts b/packages/backend-core/src/logging/correlation/correlation.ts new file mode 100644 index 0000000000..b5b47df9c6 --- /dev/null +++ b/packages/backend-core/src/logging/correlation/correlation.ts @@ -0,0 +1,13 @@ +import { Header } from "../../constants" +const correlator = require("correlation-id") + +export const setHeader = (headers: any) => { + const correlationId = correlator.getId() + if (correlationId) { + headers[Header.CORRELATION_ID] = correlationId + } +} + +export function getId() { + return correlator.getId() +} diff --git a/packages/backend-core/src/logging/correlation/index.ts b/packages/backend-core/src/logging/correlation/index.ts new file mode 100644 index 0000000000..a6d8b920fb --- /dev/null +++ b/packages/backend-core/src/logging/correlation/index.ts @@ -0,0 +1 @@ +export * from "./correlation" diff --git a/packages/backend-core/src/logging/correlation/middleware.ts b/packages/backend-core/src/logging/correlation/middleware.ts new file mode 100644 index 0000000000..f77714a5ae --- /dev/null +++ b/packages/backend-core/src/logging/correlation/middleware.ts @@ -0,0 +1,17 @@ +import { Header } from "../../constants" +import { v4 as uuid } from "uuid" +const correlator = require("correlation-id") + +const correlation = (ctx: any, next: any) => { + // use the provided correlation id header if present + let correlationId = ctx.headers[Header.CORRELATION_ID] + if (!correlationId) { + correlationId = uuid() + } + + return correlator.withId(correlationId, () => { + return next() + }) +} + +export default correlation diff --git a/packages/backend-core/src/logging/index.ts b/packages/backend-core/src/logging/index.ts new file mode 100644 index 0000000000..276a8d627c --- /dev/null +++ b/packages/backend-core/src/logging/index.ts @@ -0,0 +1,6 @@ +export * as correlation from "./correlation/correlation" +export { default as logger } from "./pino/logger" +export * from "./alerts" + +// turn off or on context logging i.e. tenantId, appId etc +export let LOG_CONTEXT = true diff --git a/packages/backend-core/src/logging/pino/logger.ts b/packages/backend-core/src/logging/pino/logger.ts new file mode 100644 index 0000000000..c82876f49a --- /dev/null +++ b/packages/backend-core/src/logging/pino/logger.ts @@ -0,0 +1,170 @@ +import env from "../../environment" +import pino, { LoggerOptions } from "pino" +import * as context from "../../context" +import * as correlation from "../correlation" +import { IdentityType } from "@budibase/types" +import { LOG_CONTEXT } from "../index" + +// LOGGER + +const pinoOptions: LoggerOptions = { + level: env.LOG_LEVEL, + formatters: { + level: label => { + return { level: label.toUpperCase() } + }, + bindings: () => { + return {} + }, + }, + timestamp: () => `,"timestamp":"${new Date(Date.now()).toISOString()}"`, +} + +if (env.isDev()) { + pinoOptions.transport = { + target: "pino-pretty", + options: { + singleLine: true, + }, + } +} + +export const logger = pino(pinoOptions) + +// CONSOLE OVERRIDES + +interface MergingObject { + objects?: any[] + tenantId?: string + appId?: string + identityId?: string + identityType?: IdentityType + correlationId?: string + err?: Error +} + +function isPlainObject(obj: any) { + return typeof obj === "object" && obj !== null && !(obj instanceof Error) +} + +function isError(obj: any) { + return obj instanceof Error +} + +function isMessage(obj: any) { + return typeof obj === "string" +} + +/** + * Backwards compatibility between console logging statements + * and pino logging requirements. + */ +function getLogParams(args: any[]): [MergingObject, string] { + let error = undefined + let objects: any[] = [] + let message = "" + + args.forEach(arg => { + if (isMessage(arg)) { + message = `${message} ${arg}`.trimStart() + } + if (isPlainObject(arg)) { + objects.push(arg) + } + if (isError(arg)) { + error = arg + } + }) + + const identity = getIdentity() + + let contextObject = {} + + if (LOG_CONTEXT) { + contextObject = { + tenantId: getTenantId(), + appId: getAppId(), + identityId: identity?._id, + identityType: identity?.type, + correlationId: correlation.getId(), + } + } + + const mergingObject = { + objects: objects.length ? objects : undefined, + err: error, + ...contextObject, + } + + return [mergingObject, message] +} + +console.log = (...arg: any[]) => { + const [obj, msg] = getLogParams(arg) + logger.info(obj, msg) +} +console.info = (...arg: any[]) => { + const [obj, msg] = getLogParams(arg) + logger.info(obj, msg) +} +console.warn = (...arg: any[]) => { + const [obj, msg] = getLogParams(arg) + logger.warn(obj, msg) +} +console.error = (...arg: any[]) => { + const [obj, msg] = getLogParams(arg) + logger.error(obj, msg) +} + +/** + * custom trace impl - this resembles the node trace behaviour rather + * than traditional trace logging + * @param arg + */ +console.trace = (...arg: any[]) => { + const [obj, msg] = getLogParams(arg) + if (!obj.err) { + // to get stack trace + obj.err = new Error() + } + logger.trace(obj, msg) +} + +console.debug = (...arg: any) => { + const [obj, msg] = getLogParams(arg) + logger.debug(obj, msg) +} + +// CONTEXT + +const getTenantId = () => { + let tenantId + try { + tenantId = context.getTenantId() + } catch (e: any) { + // do nothing + } + return tenantId +} + +const getAppId = () => { + let appId + try { + appId = context.getAppId() + } catch (e) { + // do nothing + } + return appId +} + +const getIdentity = () => { + let identity + try { + identity = context.getIdentity() + } catch (e) { + // do nothing + } + return identity +} + +export default logger diff --git a/packages/backend-core/src/logging/pino/middleware.ts b/packages/backend-core/src/logging/pino/middleware.ts new file mode 100644 index 0000000000..e9d37ab692 --- /dev/null +++ b/packages/backend-core/src/logging/pino/middleware.ts @@ -0,0 +1,45 @@ +import env from "../../environment" +import logger from "./logger" +import { IncomingMessage } from "http" +const pino = require("koa-pino-logger") +import { Options } from "pino-http" +import { Ctx } from "@budibase/types" +const correlator = require("correlation-id") + +export function pinoSettings(): Options { + return { + logger, + genReqId: correlator.getId, + autoLogging: { + ignore: (req: IncomingMessage) => !!req.url?.includes("/health"), + }, + serializers: { + req: req => { + return { + method: req.method, + url: req.url, + correlationId: req.id, + } + }, + res: res => { + return { + status: res.statusCode, + } + }, + }, + } +} + +function getMiddleware() { + if (env.HTTP_LOGGING) { + return pino(pinoSettings()) + } else { + return (ctx: Ctx, next: any) => { + return next() + } + } +} + +const pinoMiddleware = getMiddleware() + +export default pinoMiddleware diff --git a/packages/backend-core/src/middleware/authenticated.ts b/packages/backend-core/src/middleware/authenticated.ts index 8a97319586..f877985ee0 100644 --- a/packages/backend-core/src/middleware/authenticated.ts +++ b/packages/backend-core/src/middleware/authenticated.ts @@ -96,9 +96,15 @@ export default function ( } try { // check the actual user is authenticated first, try header or cookie - const headerToken = ctx.request.headers[Header.TOKEN] + let headerToken = ctx.request.headers[Header.TOKEN] + const authCookie = getCookie(ctx, Cookie.Auth) || openJwt(headerToken) - const apiKey = ctx.request.headers[Header.API_KEY] + let apiKey = ctx.request.headers[Header.API_KEY] + + if (!apiKey && ctx.request.headers[Header.AUTHORIZATION]) { + apiKey = ctx.request.headers[Header.AUTHORIZATION].split(" ")[1] + } + const tenantId = ctx.request.headers[Header.TENANT_ID] let authenticated = false, user = null, diff --git a/packages/backend-core/src/middleware/errorHandling.ts b/packages/backend-core/src/middleware/errorHandling.ts index 36aff2cdbc..ebdd4107e9 100644 --- a/packages/backend-core/src/middleware/errorHandling.ts +++ b/packages/backend-core/src/middleware/errorHandling.ts @@ -1,6 +1,5 @@ import { APIError } from "@budibase/types" import * as errors from "../errors" -import env from "../environment" export async function errorHandling(ctx: any, next: any) { try { @@ -9,9 +8,10 @@ export async function errorHandling(ctx: any, next: any) { const status = err.status || err.statusCode || 500 ctx.status = status - if (status > 499 || env.ENABLE_4XX_HTTP_LOGGING) { - ctx.log.error(err) - console.trace(err) + if (status >= 400 && status < 500) { + console.warn(err) + } else { + console.error(err) } const error = errors.getPublicError(err) diff --git a/packages/backend-core/src/middleware/index.ts b/packages/backend-core/src/middleware/index.ts index dce07168d4..980bf06b00 100644 --- a/packages/backend-core/src/middleware/index.ts +++ b/packages/backend-core/src/middleware/index.ts @@ -14,7 +14,8 @@ export { default as csrf } from "./csrf" export { default as adminOnly } from "./adminOnly" export { default as builderOrAdmin } from "./builderOrAdmin" export { default as builderOnly } from "./builderOnly" -export { default as logging } from "./logging" +export { default as pino } from "../logging/pino/middleware" +export { default as correlation } from "../logging/correlation/middleware" export { default as errorHandling } from "./errorHandling" export { default as querystringToBody } from "./querystringToBody" export * as joiValidator from "./joi-validator" diff --git a/packages/backend-core/src/middleware/logging.ts b/packages/backend-core/src/middleware/logging.ts deleted file mode 100644 index db9b64b883..0000000000 --- a/packages/backend-core/src/middleware/logging.ts +++ /dev/null @@ -1,90 +0,0 @@ -const correlator = require("correlation-id") -import { Header } from "../constants" -import { v4 as uuid } from "uuid" -import * as context from "../context" - -const debug = console.warn -const trace = console.trace -const log = console.log -const info = console.info -const warn = console.warn -const error = console.error - -const getTenantId = () => { - let tenantId - try { - tenantId = context.getTenantId() - } catch (e: any) { - // do nothing - } - return tenantId -} - -const getAppId = () => { - let appId - try { - appId = context.getAppId() - } catch (e) { - // do nothing - } - return appId -} - -const getIdentityId = () => { - let identityId - try { - const identity = context.getIdentity() - identityId = identity?._id - } catch (e) { - // do nothing - } - return identityId -} - -const print = (fn: any, data: any[]) => { - let message = "" - - const correlationId = correlator.getId() - if (correlationId) { - message = message + `[correlationId=${correlator.getId()}]` - } - - const tenantId = getTenantId() - if (tenantId) { - message = message + ` [tenantId=${tenantId}]` - } - - const appId = getAppId() - if (appId) { - message = message + ` [appId=${appId}]` - } - - const identityId = getIdentityId() - if (identityId) { - message = message + ` [identityId=${identityId}]` - } - - if (!process.env.CI) { - fn(message, data) - } -} - -const logging = (ctx: any, next: any) => { - // use the provided correlation id header if present - let correlationId = ctx.headers[Header.CORRELATION_ID] - if (!correlationId) { - correlationId = uuid() - } - - return correlator.withId(correlationId, () => { - console.debug = data => print(debug, data) - console.trace = data => print(trace, data) - console.log = data => print(log, data) - console.info = data => print(info, data) - console.warn = data => print(warn, data) - console.error = data => print(error, data) - return next() - }) -} - -export default logging diff --git a/packages/backend-core/src/middleware/passport/datasource/google.ts b/packages/backend-core/src/middleware/passport/datasource/google.ts index 32451cb8d2..6fd4e9ff32 100644 --- a/packages/backend-core/src/middleware/passport/datasource/google.ts +++ b/packages/backend-core/src/middleware/passport/datasource/google.ts @@ -78,17 +78,23 @@ export async function postAuth( ), { successRedirect: "/", failureRedirect: "/error" }, async (err: any, tokens: string[]) => { + const baseUrl = `/builder/app/${authStateCookie.appId}/data` // update the DB for the datasource with all the user info await doWithDB(authStateCookie.appId, async (db: Database) => { - const datasource = await db.get(authStateCookie.datasourceId) + let datasource + try { + datasource = await db.get(authStateCookie.datasourceId) + } catch (err: any) { + if (err.status === 404) { + ctx.redirect(baseUrl) + } + } if (!datasource.config) { datasource.config = {} } datasource.config.auth = { type: "google", ...tokens } await db.put(datasource) - ctx.redirect( - `/builder/app/${authStateCookie.appId}/data/datasource/${authStateCookie.datasourceId}` - ) + ctx.redirect(`${baseUrl}/datasource/${authStateCookie.datasourceId}`) }) } )(ctx, next) diff --git a/packages/backend-core/src/middleware/passport/sso/oidc.ts b/packages/backend-core/src/middleware/passport/sso/oidc.ts index b6d5eb52e9..83bfde28b6 100644 --- a/packages/backend-core/src/middleware/passport/sso/oidc.ts +++ b/packages/backend-core/src/middleware/passport/sso/oidc.ts @@ -1,6 +1,7 @@ import fetch from "node-fetch" import * as sso from "./sso" import { ssoCallbackUrl } from "../utils" +import { validEmail } from "../../../utils" import { ConfigType, OIDCInnerConfig, @@ -11,6 +12,7 @@ import { JwtClaims, SaveSSOUserFunction, } from "@budibase/types" + const OIDCStrategy = require("@techpass/passport-openidconnect").Strategy export function buildVerifyFn(saveUserFn: SaveSSOUserFunction) { @@ -86,15 +88,6 @@ function getEmail(profile: SSOProfile, jwtClaims: JwtClaims) { ) } -function validEmail(value: string) { - return ( - value && - !!value.match( - /^(([^<>()[\]\\.,;:\s@"]+(\.[^<>()[\]\\.,;:\s@"]+)*)|(".+"))@((\[[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}])|(([a-zA-Z\-0-9]+\.)+[a-zA-Z]{2,}))$/ - ) - ) -} - /** * Create an instance of the oidc passport strategy. This wrapper fetches the configuration * from couchDB rather than environment variables, using this factory is necessary for dynamically configuring passport. diff --git a/packages/backend-core/src/migrations/migrations.ts b/packages/backend-core/src/migrations/migrations.ts index ab72091d56..c750bc4882 100644 --- a/packages/backend-core/src/migrations/migrations.ts +++ b/packages/backend-core/src/migrations/migrations.ts @@ -99,9 +99,7 @@ export const runMigration = async ( options.force[migrationType] && options.force[migrationType].includes(migrationName) ) { - log( - `[Tenant: ${tenantId}] [Migration: ${migrationName}] [DB: ${dbName}] Forcing` - ) + log(`[Migration: ${migrationName}] [DB: ${dbName}] Forcing`) } else { // no force, exit return @@ -111,7 +109,7 @@ export const runMigration = async ( // check if the migration is not a no-op if (!options.noOp) { log( - `[Tenant: ${tenantId}] [Migration: ${migrationName}] [DB: ${dbName}] Running ${lengthStatement}` + `[Migration: ${migrationName}] [DB: ${dbName}] Running ${lengthStatement}` ) if (migration.preventRetry) { @@ -131,9 +129,7 @@ export const runMigration = async ( await migration.fn(db) } - log( - `[Tenant: ${tenantId}] [Migration: ${migrationName}] [DB: ${dbName}] Complete` - ) + log(`[Migration: ${migrationName}] [DB: ${dbName}] Complete`) } // mark as complete @@ -141,7 +137,7 @@ export const runMigration = async ( await db.put(doc) } catch (err) { console.error( - `[Tenant: ${tenantId}] [Migration: ${migrationName}] [DB: ${dbName}] Error: `, + `[Migration: ${migrationName}] [DB: ${dbName}] Error: `, err ) throw err diff --git a/packages/backend-core/src/users.ts b/packages/backend-core/src/users.ts index dfc544c3ed..c7d8a94e95 100644 --- a/packages/backend-core/src/users.ts +++ b/packages/backend-core/src/users.ts @@ -8,8 +8,10 @@ import { DocumentType, SEPARATOR, directCouchFind, + getGlobalUserParams, + pagination, } from "./db" -import { BulkDocsResponse, User } from "@budibase/types" +import { BulkDocsResponse, SearchUsersRequest, User } from "@budibase/types" import { getGlobalDB } from "./context" import * as context from "./context" @@ -199,3 +201,41 @@ export const searchGlobalUsersByEmail = async ( } return users } + +const PAGE_LIMIT = 8 +export const paginatedUsers = async ({ + page, + email, + appId, +}: SearchUsersRequest = {}) => { + const db = getGlobalDB() + // get one extra document, to have the next page + const opts: any = { + include_docs: true, + limit: PAGE_LIMIT + 1, + } + // add a startkey if the page was specified (anchor) + if (page) { + opts.startkey = page + } + // property specifies what to use for the page/anchor + let userList: User[], + property = "_id", + getKey + if (appId) { + userList = await searchGlobalUsersByApp(appId, opts) + getKey = (doc: any) => getGlobalUserByAppPage(appId, doc) + } else if (email) { + userList = await searchGlobalUsersByEmail(email, opts) + property = "email" + } else { + // no search, query allDocs + const response = await db.allDocs(getGlobalUserParams(null, opts)) + userList = response.rows.map((row: any) => row.doc) + } + return pagination(userList, PAGE_LIMIT, { + paginate: true, + property, + getKey, + }) +} diff --git a/packages/backend-core/src/utils/index.ts b/packages/backend-core/src/utils/index.ts index 8e663bce52..318a7f13ba 100644 --- a/packages/backend-core/src/utils/index.ts +++ b/packages/backend-core/src/utils/index.ts @@ -1,2 +1,3 @@ export * from "./hashing" export * from "./utils" +export * from "./stringUtils" diff --git a/packages/backend-core/src/utils/stringUtils.ts b/packages/backend-core/src/utils/stringUtils.ts new file mode 100644 index 0000000000..fd3b4c6132 --- /dev/null +++ b/packages/backend-core/src/utils/stringUtils.ts @@ -0,0 +1,8 @@ +export function validEmail(value: string) { + return ( + value && + !!value.match( + /^(([^<>()[\]\\.,;:\s@"]+(\.[^<>()[\]\\.,;:\s@"]+)*)|(".+"))@((\[[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}])|(([a-zA-Z\-0-9]+\.)+[a-zA-Z]{2,}))$/ + ) + ) +} diff --git a/packages/backend-core/tests/jestEnv.ts b/packages/backend-core/tests/jestEnv.ts index 3555973928..c2047118ec 100644 --- a/packages/backend-core/tests/jestEnv.ts +++ b/packages/backend-core/tests/jestEnv.ts @@ -3,5 +3,4 @@ process.env.MULTI_TENANCY = "1" process.env.NODE_ENV = "jest" process.env.MOCK_REDIS = "1" process.env.LOG_LEVEL = process.env.LOG_LEVEL || "error" -process.env.ENABLE_4XX_HTTP_LOGGING = "0" process.env.REDIS_PASSWORD = "budibase" diff --git a/packages/backend-core/tests/jestSetup.ts b/packages/backend-core/tests/jestSetup.ts index be81fbff75..f5542e8fdd 100644 --- a/packages/backend-core/tests/jestSetup.ts +++ b/packages/backend-core/tests/jestSetup.ts @@ -1,4 +1,3 @@ -import "./logging" import env from "../src/environment" import { cleanup } from "../src/timers" import { mocks, testContainerUtils } from "./utilities" diff --git a/packages/backend-core/tests/logging.ts b/packages/backend-core/tests/logging.ts deleted file mode 100644 index 271f4d62ff..0000000000 --- a/packages/backend-core/tests/logging.ts +++ /dev/null @@ -1,34 +0,0 @@ -export enum LogLevel { - TRACE = "trace", - DEBUG = "debug", - INFO = "info", - WARN = "warn", - ERROR = "error", -} - -const LOG_INDEX: { [key in LogLevel]: number } = { - [LogLevel.TRACE]: 1, - [LogLevel.DEBUG]: 2, - [LogLevel.INFO]: 3, - [LogLevel.WARN]: 4, - [LogLevel.ERROR]: 5, -} - -const setIndex = LOG_INDEX[process.env.LOG_LEVEL as LogLevel] - -if (setIndex > LOG_INDEX.trace) { - global.console.trace = jest.fn() -} - -if (setIndex > LOG_INDEX.debug) { - global.console.debug = jest.fn() -} - -if (setIndex > LOG_INDEX.info) { - global.console.info = jest.fn() - global.console.log = jest.fn() -} - -if (setIndex > LOG_INDEX.warn) { - global.console.warn = jest.fn() -} diff --git a/packages/backend-core/tests/utilities/mocks/alerts.ts b/packages/backend-core/tests/utilities/mocks/alerts.ts new file mode 100644 index 0000000000..501c543671 --- /dev/null +++ b/packages/backend-core/tests/utilities/mocks/alerts.ts @@ -0,0 +1,3 @@ +jest.mock("../../../src/logging/alerts") +import * as _alerts from "../../../src/logging/alerts" +export const alerts = jest.mocked(_alerts) diff --git a/packages/backend-core/tests/utilities/mocks/index.ts b/packages/backend-core/tests/utilities/mocks/index.ts index f5f45c0342..915021cecf 100644 --- a/packages/backend-core/tests/utilities/mocks/index.ts +++ b/packages/backend-core/tests/utilities/mocks/index.ts @@ -5,5 +5,6 @@ export const accounts = jest.mocked(_accounts) export * as date from "./date" export * as licenses from "./licenses" export { default as fetch } from "./fetch" +export * from "./alerts" import "./posthog" import "./events" diff --git a/packages/backend-core/tests/utilities/mocks/licenses.ts b/packages/backend-core/tests/utilities/mocks/licenses.ts index 2ca41616e4..839b22e5f9 100644 --- a/packages/backend-core/tests/utilities/mocks/licenses.ts +++ b/packages/backend-core/tests/utilities/mocks/licenses.ts @@ -86,6 +86,10 @@ export const useAuditLogs = () => { return useFeature(Feature.AUDIT_LOGS) } +export const useScimIntegration = () => { + return useFeature(Feature.SCIM) +} + // QUOTAS export const setAutomationLogsQuota = (value: number) => { diff --git a/packages/backend-core/tests/utilities/structures/index.ts b/packages/backend-core/tests/utilities/structures/index.ts index ca77f476d0..5592a7e1f9 100644 --- a/packages/backend-core/tests/utilities/structures/index.ts +++ b/packages/backend-core/tests/utilities/structures/index.ts @@ -8,4 +8,6 @@ export * as plugins from "./plugins" export * as sso from "./sso" export * as tenant from "./tenants" export * as users from "./users" +export * as userGroups from "./userGroups" export { generator } from "./generator" +export * as scim from "./scim" diff --git a/packages/backend-core/tests/utilities/structures/scim.ts b/packages/backend-core/tests/utilities/structures/scim.ts new file mode 100644 index 0000000000..6657bb90b5 --- /dev/null +++ b/packages/backend-core/tests/utilities/structures/scim.ts @@ -0,0 +1,69 @@ +import { ScimCreateGroupRequest, ScimCreateUserRequest } from "@budibase/types" +import { uuid } from "./common" +import { generator } from "./generator" + +export function createUserRequest(userData?: { + externalId?: string + email?: string + firstName?: string + lastName?: string + username?: string +}) { + const { + externalId = uuid(), + email = generator.email(), + firstName = generator.first(), + lastName = generator.last(), + username = generator.name(), + } = userData || {} + + const user: ScimCreateUserRequest = { + schemas: [ + "urn:ietf:params:scim:schemas:core:2.0:User", + "urn:ietf:params:scim:schemas:extension:enterprise:2.0:User", + ], + externalId, + userName: username, + active: true, + emails: [ + { + primary: true, + type: "work", + value: email, + }, + ], + meta: { + resourceType: "User", + }, + name: { + formatted: generator.name(), + familyName: lastName, + givenName: firstName, + }, + roles: [], + } + return user +} + +export function createGroupRequest(groupData?: { + externalId?: string + displayName?: string +}) { + const { externalId = uuid(), displayName = generator.word() } = + groupData || {} + + const group: ScimCreateGroupRequest = { + schemas: [ + "urn:ietf:params:scim:schemas:core:2.0:Group", + "http://schemas.microsoft.com/2006/11/ResourceManagement/ADSCIM/2.0/Group", + ], + externalId: externalId, + displayName: displayName, + meta: { + resourceType: "Group", + created: new Date(), + lastModified: new Date(), + }, + } + return group +} diff --git a/packages/backend-core/tests/utilities/structures/userGroups.ts b/packages/backend-core/tests/utilities/structures/userGroups.ts new file mode 100644 index 0000000000..4dc870a00a --- /dev/null +++ b/packages/backend-core/tests/utilities/structures/userGroups.ts @@ -0,0 +1,10 @@ +import { UserGroup } from "@budibase/types" +import { generator } from "./generator" + +export function userGroup(): UserGroup { + return { + name: generator.word(), + icon: generator.word(), + color: generator.word(), + } +} diff --git a/packages/backend-core/yarn.lock b/packages/backend-core/yarn.lock index 91c5c6c9f3..b7ac2a859b 100644 --- a/packages/backend-core/yarn.lock +++ b/packages/backend-core/yarn.lock @@ -1217,15 +1217,6 @@ dependencies: "@types/koa" "*" -"@types/koa-pino-logger@3.0.0": - version "3.0.0" - resolved "https://registry.yarnpkg.com/@types/koa-pino-logger/-/koa-pino-logger-3.0.0.tgz#275d4b000abc14b1928dc2e9ab476c8296a64b6a" - integrity sha512-sP+12JNX01q+nHpCRqkVIuLjaRemQEfDoFg0evpTnjUEI3jUI2ZrOkhQ5coxn3yVm2tedui/2YhlaPn/XrYNWA== - dependencies: - "@types/koa" "*" - "@types/pino" "*" - "@types/pino-http" "*" - "@types/koa@*", "@types/koa@2.13.4": version "2.13.4" resolved "https://registry.yarnpkg.com/@types/koa/-/koa-2.13.4.tgz#10620b3f24a8027ef5cbae88b393d1b31205726b" @@ -1273,44 +1264,6 @@ resolved "https://registry.yarnpkg.com/@types/node/-/node-14.18.20.tgz#268f028b36eaf51181c3300252f605488c4f0650" integrity sha512-Q8KKwm9YqEmUBRsqJ2GWJDtXltBDxTdC4m5vTdXBolu2PeQh8LX+f6BTwU+OuXPu37fLxoN6gidqBmnky36FXA== -"@types/pino-http@*", "@types/pino-http@5.8.1": - version "5.8.1" - resolved "https://registry.yarnpkg.com/@types/pino-http/-/pino-http-5.8.1.tgz#ebb194750ad2f9245c3028b5d2c4e6d64f685ba9" - integrity sha512-A9MW6VCnx5ii7s+Fs5aFIw+aSZcBCpsZ/atpxamu8tTsvWFacxSf2Hrn1Ohn1jkVRB/LiPGOapRXcFawDBnDnA== - dependencies: - "@types/pino" "6.3" - -"@types/pino-pretty@*": - version "5.0.0" - resolved "https://registry.yarnpkg.com/@types/pino-pretty/-/pino-pretty-5.0.0.tgz#aa7a61cfd553b051764acfa0a49872f7a09a1722" - integrity sha512-N1uzqSzioqz8R3AkDbSJwcfDWeI3YMPNapSQQhnB2ISU4NYgUIcAh+hYT5ygqBM+klX4htpEhXMmoJv3J7GrdA== - dependencies: - pino-pretty "*" - -"@types/pino-std-serializers@*": - version "4.0.0" - resolved "https://registry.yarnpkg.com/@types/pino-std-serializers/-/pino-std-serializers-4.0.0.tgz#1e28b80b554c8222858e99a4e0fc77fd070e10e8" - integrity sha512-gXfUZx2xIBbFYozGms53fT0nvkacx/+62c8iTxrEqH5PkIGAQvDbXg2774VWOycMPbqn5YJBQ3BMsg4Li3dWbg== - dependencies: - pino-std-serializers "*" - -"@types/pino@*": - version "7.0.5" - resolved "https://registry.yarnpkg.com/@types/pino/-/pino-7.0.5.tgz#1c84a81b924a6a9e263dbb581dffdbad7a3c60c4" - integrity sha512-wKoab31pknvILkxAF8ss+v9iNyhw5Iu/0jLtRkUD74cNfOOLJNnqfFKAv0r7wVaTQxRZtWrMpGfShwwBjOcgcg== - dependencies: - pino "*" - -"@types/pino@6.3": - version "6.3.12" - resolved "https://registry.yarnpkg.com/@types/pino/-/pino-6.3.12.tgz#4425db6ced806109c3df957100cba9dfcd73c228" - integrity sha512-dsLRTq8/4UtVSpJgl9aeqHvbh6pzdmjYD3C092SYgLD2TyoCqHpTJk6vp8DvCTGGc7iowZ2MoiYiVUUCcu7muw== - dependencies: - "@types/node" "*" - "@types/pino-pretty" "*" - "@types/pino-std-serializers" "*" - sonic-boom "^2.1.0" - "@types/pouchdb-adapter-cordova-sqlite@*": version "1.0.1" resolved "https://registry.yarnpkg.com/@types/pouchdb-adapter-cordova-sqlite/-/pouchdb-adapter-cordova-sqlite-1.0.1.tgz#49e5ee6df7cc0c23196fcb340f43a560e74eb1d6" @@ -2539,6 +2492,16 @@ duplexer3@^0.1.4: resolved "https://registry.yarnpkg.com/duplexer3/-/duplexer3-0.1.4.tgz#ee01dd1cac0ed3cbc7fdbea37dc0a8f1ce002ce2" integrity sha512-CEj8FwwNA4cVH2uFCoHUrmojhYh1vmCdOaneKJXwkeY1i9jnlslVo9dx+hQ5Hl9GnH/Bwy/IjxAyOePyPKYnzA== +duplexify@^4.1.2: + version "4.1.2" + resolved "https://registry.yarnpkg.com/duplexify/-/duplexify-4.1.2.tgz#18b4f8d28289132fa0b9573c898d9f903f81c7b0" + integrity sha512-fz3OjcNCHmRP12MJoZMPglx8m4rrFP8rovnk4vT8Fs+aonZoCwGg10dSsQsfP/E62eZcPTMSMP6686fu9Qlqtw== + dependencies: + end-of-stream "^1.4.1" + inherits "^2.0.3" + readable-stream "^3.1.1" + stream-shift "^1.0.0" + ecc-jsbn@~0.1.1: version "0.1.2" resolved "https://registry.yarnpkg.com/ecc-jsbn/-/ecc-jsbn-0.1.2.tgz#3a83a904e54353287874c564b7549386849a98c9" @@ -2728,9 +2691,9 @@ extsprintf@^1.2.0: integrity sha512-Wrk35e8ydCKDj/ArClo1VrPVmN8zph5V4AtHwIuHhvMXsKf73UT3BOD+azBIW+3wOJ4FhEH7zyaJCFvChjYvMA== fast-copy@^3.0.0: - version "3.0.0" - resolved "https://registry.yarnpkg.com/fast-copy/-/fast-copy-3.0.0.tgz#875ebf33b13948ae012b6e51d33da5e6e7571ab8" - integrity sha512-4HzS+9pQ5Yxtv13Lhs1Z1unMXamBdn5nA4bEi1abYpDNSpSp7ODYQ1KPMF6nTatfEzgH6/zPvXKU1zvHiUjWlA== + version "3.0.1" + resolved "https://registry.yarnpkg.com/fast-copy/-/fast-copy-3.0.1.tgz#9e89ef498b8c04c1cd76b33b8e14271658a732aa" + integrity sha512-Knr7NOtK3HWRYGtHoJrjkaWepqT8thIVGAwt0p0aUs1zqkAzXZV4vo9fFNwyb5fcqK1GKYFYxldQdIDVKhUAfA== fast-deep-equal@^3.1.1: version "3.1.3" @@ -2742,7 +2705,7 @@ fast-json-stable-stringify@2.x, fast-json-stable-stringify@^2.0.0: resolved "https://registry.yarnpkg.com/fast-json-stable-stringify/-/fast-json-stable-stringify-2.1.0.tgz#874bf69c6f404c2b5d99c481341399fd55892633" integrity sha512-lhd/wF+Lk98HZoTCtlVraHtfh5XYijIjalXck7saUtuanSDyLMxnHhSXEDJqHxD7msR8D0uCmqlkwjCV8xvwHw== -fast-redact@^3.1.1: +fast-redact@^3.0.0: version "3.1.2" resolved "https://registry.yarnpkg.com/fast-redact/-/fast-redact-3.1.2.tgz#d58e69e9084ce9fa4c1a6fa98a3e1ecf5d7839aa" integrity sha512-+0em+Iya9fKGfEQGcd62Yv6onjBmmhV1uh86XVfOU8VwAe6kaFdQCWI9s0/Nnugx5Vd9tdbZ7e6gE2tR9dzXdw== @@ -2752,6 +2715,13 @@ fast-safe-stringify@^2.1.1: resolved "https://registry.yarnpkg.com/fast-safe-stringify/-/fast-safe-stringify-2.1.1.tgz#c406a83b6e70d9e35ce3b30a81141df30aeba884" integrity sha512-W+KJc2dmILlPplD/H4K9l9LcAHAfPtP6BY84uVLXQ6Evcz9Lcg33Y2z1IVblT6xdY54PXYVHEv+0Wpq8Io6zkA== +fast-url-parser@^1.1.3: + version "1.1.3" + resolved "https://registry.yarnpkg.com/fast-url-parser/-/fast-url-parser-1.1.3.tgz#f4af3ea9f34d8a271cf58ad2b3759f431f0b318d" + integrity sha512-5jOCVXADYNuRkKFzNJ0dCCewsZiYo0dz8QNYljkOpFC6r2U4OBmKtvm/Tsuh4w1YYdDqDb31a8TVhBJ2OJKdqQ== + dependencies: + punycode "^1.3.2" + fb-watchman@^2.0.0: version "2.0.1" resolved "https://registry.yarnpkg.com/fb-watchman/-/fb-watchman-2.0.1.tgz#fc84fb39d2709cf3ff6d743706157bb5708a8a85" @@ -4027,6 +3997,13 @@ koa-passport@4.1.4: dependencies: passport "^0.4.0" +koa-pino-logger@4.0.0: + version "4.0.0" + resolved "https://registry.yarnpkg.com/koa-pino-logger/-/koa-pino-logger-4.0.0.tgz#a45c9fd0a39e9cd84ac23da502055054ee51b80d" + integrity sha512-YI/LB9ajyLPpjvf6e+7Ewmn+OQkCJpu/Y9eI1n7fnipu5Y1NchuNlke0mqh3/2z+5oDYr7pijjOWruEDIfua2A== + dependencies: + pino-http "^6.5.0" + koa@2.13.4, koa@^2.13.4: version "2.13.4" resolved "https://registry.yarnpkg.com/koa/-/koa-2.13.4.tgz#ee5b0cb39e0b8069c38d115139c774833d32462e" @@ -4623,6 +4600,11 @@ object-inspect@^1.9.0: resolved "https://registry.yarnpkg.com/object-inspect/-/object-inspect-1.12.2.tgz#c0641f26394532f28ab8d796ab954e43c009a8ea" integrity sha512-z+cPxW0QGUp0mcqcsgQyLVRDoXFQbXOwBaqyF7VIgI4TWNQsDHrBpUQslRmIfAoYWdYzs6UlKJtB2XJpTaNSpQ== +on-exit-leak-free@^0.2.0: + version "0.2.0" + resolved "https://registry.yarnpkg.com/on-exit-leak-free/-/on-exit-leak-free-0.2.0.tgz#b39c9e3bf7690d890f4861558b0d7b90a442d209" + integrity sha512-dqaz3u44QbRXQooZLTUKU41ZrzYrcvLISVgbrzbyCMxpmSLJvZ3ZamIJIZ29P6OhZIkNIQKosdeM6t1LYbA9hg== + on-exit-leak-free@^2.1.0: version "2.1.0" resolved "https://registry.yarnpkg.com/on-exit-leak-free/-/on-exit-leak-free-2.1.0.tgz#5c703c968f7e7f851885f6459bf8a8a57edc9cc4" @@ -4857,7 +4839,7 @@ picomatch@^2.0.4, picomatch@^2.2.1, picomatch@^2.2.3, picomatch@^2.3.1: resolved "https://registry.yarnpkg.com/picomatch/-/picomatch-2.3.1.tgz#3ba3833733646d9d3e4995946c1365a67fb07a42" integrity sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA== -pino-abstract-transport@^1.0.0, pino-abstract-transport@v1.0.0: +pino-abstract-transport@^1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/pino-abstract-transport/-/pino-abstract-transport-1.0.0.tgz#cc0d6955fffcadb91b7b49ef220a6cc111d48bb3" integrity sha512-c7vo5OpW4wIS42hUVcT5REsL8ZljsUfBjqV/e2sFxmFEFZiq1XLUp5EYLtuDH6PEHq9W1egWqRbnLUP5FuZmOA== @@ -4865,10 +4847,28 @@ pino-abstract-transport@^1.0.0, pino-abstract-transport@v1.0.0: readable-stream "^4.0.0" split2 "^4.0.0" -pino-pretty@*: - version "9.1.1" - resolved "https://registry.yarnpkg.com/pino-pretty/-/pino-pretty-9.1.1.tgz#e7d64c1db98266ca428ab56567b844ba780cd0e1" - integrity sha512-iJrnjgR4FWQIXZkUF48oNgoRI9BpyMhaEmihonHeCnZ6F50ZHAS4YGfGBT/ZVNsPmd+hzkIPGzjKdY08+/yAXw== +pino-abstract-transport@v0.5.0: + version "0.5.0" + resolved "https://registry.yarnpkg.com/pino-abstract-transport/-/pino-abstract-transport-0.5.0.tgz#4b54348d8f73713bfd14e3dc44228739aa13d9c0" + integrity sha512-+KAgmVeqXYbTtU2FScx1XS3kNyfZ5TrXY07V96QnUSFqo2gAqlvmaxH67Lj7SWazqsMabf+58ctdTcBgnOLUOQ== + dependencies: + duplexify "^4.1.2" + split2 "^4.0.0" + +pino-http@^6.5.0: + version "6.6.0" + resolved "https://registry.yarnpkg.com/pino-http/-/pino-http-6.6.0.tgz#d0a1deacada8c93327fdaa48f5bdc94bc43d3407" + integrity sha512-PlItaK2MLpoIMLEcClhfb1VQk/o6fKppINl5s6sPE/4rvufkdO3kCSs/92EwrBsB1yssRCQqDV+w1xpYuPVnjg== + dependencies: + fast-url-parser "^1.1.3" + get-caller-file "^2.0.5" + pino "^7.5.0" + pino-std-serializers "^5.0.0" + +pino-pretty@10.0.0: + version "10.0.0" + resolved "https://registry.yarnpkg.com/pino-pretty/-/pino-pretty-10.0.0.tgz#fd2f307ee897289f63d09b0b804ac2ecc9a18516" + integrity sha512-zKFjYXBzLaLTEAN1ayKpHXtL5UeRQC7R3lvhKe7fWs7hIVEjKGG/qIXwQt9HmeUp71ogUd/YcW+LmMwRp4KT6Q== dependencies: colorette "^2.0.7" dateformat "^4.6.3" @@ -4885,27 +4885,32 @@ pino-pretty@*: sonic-boom "^3.0.0" strip-json-comments "^3.1.1" -pino-std-serializers@*, pino-std-serializers@^6.0.0: - version "6.1.0" - resolved "https://registry.yarnpkg.com/pino-std-serializers/-/pino-std-serializers-6.1.0.tgz#307490fd426eefc95e06067e85d8558603e8e844" - integrity sha512-KO0m2f1HkrPe9S0ldjx7za9BJjeHqBku5Ch8JyxETxT8dEFGz1PwgrHaOQupVYitpzbFSYm7nnljxD8dik2c+g== +pino-std-serializers@^4.0.0: + version "4.0.0" + resolved "https://registry.yarnpkg.com/pino-std-serializers/-/pino-std-serializers-4.0.0.tgz#1791ccd2539c091ae49ce9993205e2cd5dbba1e2" + integrity sha512-cK0pekc1Kjy5w9V2/n+8MkZwusa6EyyxfeQCB799CQRhRt/CqYKiWs5adeu8Shve2ZNffvfC/7J64A2PJo1W/Q== -pino@*: - version "8.8.0" - resolved "https://registry.yarnpkg.com/pino/-/pino-8.8.0.tgz#1f0d6695a224aa06afc7ad60f2ccc4772d3b9233" - integrity sha512-cF8iGYeu2ODg2gIwgAHcPrtR63ILJz3f7gkogaHC/TXVVXxZgInmNYiIpDYEwgEkxZti2Se6P2W2DxlBIZe6eQ== +pino-std-serializers@^5.0.0: + version "5.6.0" + resolved "https://registry.yarnpkg.com/pino-std-serializers/-/pino-std-serializers-5.6.0.tgz#31b141155d6520967c5ec72944d08fb45c490fd3" + integrity sha512-VdUXCw8gO+xhir7sFuoYSjTnzB+TMDGxhAC/ph3YS3sdHnXNdsK0wMtADNUltfeGkn2KDxEM21fnjF3RwXyC8A== + +pino@^7.5.0: + version "7.11.0" + resolved "https://registry.yarnpkg.com/pino/-/pino-7.11.0.tgz#0f0ea5c4683dc91388081d44bff10c83125066f6" + integrity sha512-dMACeu63HtRLmCG8VKdy4cShCPKaYDR4youZqoSWLxl5Gu99HUw8bw75thbPv9Nip+H+QYX8o3ZJbTdVZZ2TVg== dependencies: atomic-sleep "^1.0.0" - fast-redact "^3.1.1" - on-exit-leak-free "^2.1.0" - pino-abstract-transport v1.0.0 - pino-std-serializers "^6.0.0" - process-warning "^2.0.0" + fast-redact "^3.0.0" + on-exit-leak-free "^0.2.0" + pino-abstract-transport v0.5.0 + pino-std-serializers "^4.0.0" + process-warning "^1.0.0" quick-format-unescaped "^4.0.3" - real-require "^0.2.0" - safe-stable-stringify "^2.3.1" - sonic-boom "^3.1.0" - thread-stream "^2.0.0" + real-require "^0.1.0" + safe-stable-stringify "^2.1.0" + sonic-boom "^2.2.1" + thread-stream "^0.15.1" pirates@^4.0.4: version "4.0.5" @@ -5156,10 +5161,10 @@ process-nextick-args@~2.0.0: resolved "https://registry.yarnpkg.com/process-nextick-args/-/process-nextick-args-2.0.1.tgz#7820d9b16120cc55ca9ae7792680ae7dba6d7fe2" integrity sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag== -process-warning@^2.0.0: - version "2.1.0" - resolved "https://registry.yarnpkg.com/process-warning/-/process-warning-2.1.0.tgz#1e60e3bfe8183033bbc1e702c2da74f099422d1a" - integrity sha512-9C20RLxrZU/rFnxWncDkuF6O999NdIf3E1ws4B0ZeY3sRVPzWBMsYDE2lxjxhiXxg464cQTgKUGm8/i6y2YGXg== +process-warning@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/process-warning/-/process-warning-1.0.0.tgz#980a0b25dc38cd6034181be4b7726d89066b4616" + integrity sha512-du4wfLyj4yCZq1VupnVSZmRsPJsNuxoDQFdCFHLaYiEbFBD7QE0a+I4D7hOxrVnh78QE/YipFAj9lXHiXocV+Q== process@^0.11.10: version "0.11.10" @@ -5207,6 +5212,11 @@ punycode@1.3.2: resolved "https://registry.yarnpkg.com/punycode/-/punycode-1.3.2.tgz#9653a036fb7c1ee42342f2325cceefea3926c48d" integrity sha512-RofWgt/7fL5wP1Y7fxE7/EmTLzQVnB0ycyibJ0OOHIlJqTNzglYFxVwETOcIoJqJmpDXJ9xImDv+Fq34F/d4Dw== +punycode@^1.3.2: + version "1.4.1" + resolved "https://registry.yarnpkg.com/punycode/-/punycode-1.4.1.tgz#c0d5a63b2718800ad8e1eb0fa5269c84dd41845e" + integrity sha512-jmYNElW7yvO7TV33CjSmvSiE2yco3bV2czu/OzDKdMNVZQWfxCblURLhf+47syQRBntjfLdd/H0egrzIG+oaFQ== + punycode@^2.1.0, punycode@^2.1.1: version "2.1.1" resolved "https://registry.yarnpkg.com/punycode/-/punycode-2.1.1.tgz#b58b010ac40c22c5657616c8d2c2c02c7bf479ec" @@ -5330,10 +5340,10 @@ readline-sync@^1.4.9: resolved "https://registry.yarnpkg.com/readline-sync/-/readline-sync-1.4.10.tgz#41df7fbb4b6312d673011594145705bf56d8873b" integrity sha512-gNva8/6UAe8QYepIQH/jQ2qn91Qj0B9sYjMBBs3QOB8F2CXcKgLxQaJRP76sWVRQt+QU+8fAkCbCvjjMFu7Ycw== -real-require@^0.2.0: - version "0.2.0" - resolved "https://registry.yarnpkg.com/real-require/-/real-require-0.2.0.tgz#209632dea1810be2ae063a6ac084fee7e33fba78" - integrity sha512-57frrGM/OCTLqLOAh0mhVA9VBMHd+9U7Zb2THMGdBUoZVOtGbJzjxsYGDJ3A9AYYCP4hn6y1TVbaOfzWtm5GFg== +real-require@^0.1.0: + version "0.1.0" + resolved "https://registry.yarnpkg.com/real-require/-/real-require-0.1.0.tgz#736ac214caa20632847b7ca8c1056a0767df9381" + integrity sha512-r/H9MzAWtrv8aSVjPCMFpDMl5q66GqtmmRkRjpHTsp4zBAa+snZyiQNlMONiUmEJcsnaw0wCauJ2GWODr/aFkg== redis-commands@1.7.0: version "1.7.0" @@ -5477,7 +5487,7 @@ safe-buffer@~5.1.0, safe-buffer@~5.1.1: resolved "https://registry.yarnpkg.com/safe-buffer/-/safe-buffer-5.1.2.tgz#991ec69d296e0313747d59bdfd2b745c35f8828d" integrity sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g== -safe-stable-stringify@^2.3.1: +safe-stable-stringify@^2.1.0: version "2.4.2" resolved "https://registry.yarnpkg.com/safe-stable-stringify/-/safe-stable-stringify-2.4.2.tgz#ec7b037768098bf65310d1d64370de0dc02353aa" integrity sha512-gMxvPJYhP0O9n2pvcfYfIuYgbledAOJFcqRThtPRmjscaipiwcwPPKLytpVzMkG2HAN87Qmo2d4PtGiri1dSLA== @@ -5589,14 +5599,14 @@ slash@^3.0.0: resolved "https://registry.yarnpkg.com/slash/-/slash-3.0.0.tgz#6539be870c165adbd5240220dbe361f1bc4d4634" integrity sha512-g9Q1haeby36OSStwb4ntCGGGaKsaVSjQ68fBxoQcutl5fS1vuY18H3wSt3jFyFtrkx+Kz0V1G85A4MyAdDMi2Q== -sonic-boom@^2.1.0: +sonic-boom@^2.2.1: version "2.8.0" resolved "https://registry.yarnpkg.com/sonic-boom/-/sonic-boom-2.8.0.tgz#c1def62a77425090e6ad7516aad8eb402e047611" integrity sha512-kuonw1YOYYNOve5iHdSahXPOK49GqwA+LZhI6Wz/l0rP57iKyXXIHaRagOBHAPmGwJC6od2Z9zgvZ5loSgMlVg== dependencies: atomic-sleep "^1.0.0" -sonic-boom@^3.0.0, sonic-boom@^3.1.0: +sonic-boom@^3.0.0: version "3.2.1" resolved "https://registry.yarnpkg.com/sonic-boom/-/sonic-boom-3.2.1.tgz#972ceab831b5840a08a002fa95a672008bda1c38" integrity sha512-iITeTHxy3B9FGu8aVdiDXUVAcHMF9Ss0cCsAOo2HfCrmVGT3/DT5oYaeu0M/YKZDlKTvChEyPq0zI9Hf33EX6A== @@ -5701,6 +5711,11 @@ step@0.0.x: resolved "https://registry.yarnpkg.com/step/-/step-0.0.6.tgz#143e7849a5d7d3f4a088fe29af94915216eeede2" integrity sha512-qSSeQinUJk2w38vUFobjFoE307GqsozMC8VisOCkJLpklvKPT0ptPHwWOrENoag8rgLudvTkfP3bancwP93/Jw== +stream-shift@^1.0.0: + version "1.0.1" + resolved "https://registry.yarnpkg.com/stream-shift/-/stream-shift-1.0.1.tgz#d7088281559ab2778424279b0877da3c392d5a3d" + integrity sha512-AiisoFqQ0vbGcZgQPY1cdP2I76glaVA/RauYR4G4thNFgkTqr90yXTo4LYX60Jl+sIlPNHHdGSwo01AvbKUSVQ== + stream-to-array@^2.3.0: version "2.3.0" resolved "https://registry.yarnpkg.com/stream-to-array/-/stream-to-array-2.3.0.tgz#bbf6b39f5f43ec30bc71babcb37557acecf34353" @@ -5897,12 +5912,12 @@ testcontainers@4.7.0: stream-to-array "^2.3.0" tar-fs "^2.1.0" -thread-stream@^2.0.0: - version "2.3.0" - resolved "https://registry.yarnpkg.com/thread-stream/-/thread-stream-2.3.0.tgz#4fc07fb39eff32ae7bad803cb7dd9598349fed33" - integrity sha512-kaDqm1DET9pp3NXwR8382WHbnpXnRkN9xGN9dQt3B2+dmXiW8X1SOwmFOxAErEQ47ObhZ96J6yhZNXuyCOL7KA== +thread-stream@^0.15.1: + version "0.15.2" + resolved "https://registry.yarnpkg.com/thread-stream/-/thread-stream-0.15.2.tgz#fb95ad87d2f1e28f07116eb23d85aba3bc0425f4" + integrity sha512-UkEhKIg2pD+fjkHQKyJO3yoIvAP3N6RlNFt2dUhcS1FGvCD1cQa1M/PGknCLFIyZdtJOWQjejp7bdNqmN7zwdA== dependencies: - real-require "^0.2.0" + real-require "^0.1.0" through2@3.0.2: version "3.0.2" diff --git a/packages/bbui/package.json b/packages/bbui/package.json index ee88119d3c..a1a6229aa1 100644 --- a/packages/bbui/package.json +++ b/packages/bbui/package.json @@ -1,7 +1,7 @@ { "name": "@budibase/bbui", "description": "A UI solution used in the different Budibase projects.", - "version": "2.4.42-alpha.5", + "version": "2.4.44-alpha.12", "license": "MPL-2.0", "svelte": "src/index.js", "module": "dist/bbui.es.js", @@ -38,8 +38,8 @@ ], "dependencies": { "@adobe/spectrum-css-workflow-icons": "1.2.1", - "@budibase/shared-core": "2.4.42-alpha.5", - "@budibase/string-templates": "2.4.42-alpha.5", + "@budibase/shared-core": "2.4.44-alpha.12", + "@budibase/string-templates": "2.4.44-alpha.12", "@spectrum-css/accordion": "3.0.24", "@spectrum-css/actionbutton": "1.0.1", "@spectrum-css/actiongroup": "1.0.1", diff --git a/packages/builder/package.json b/packages/builder/package.json index 1e175eb757..97fac1e9a4 100644 --- a/packages/builder/package.json +++ b/packages/builder/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/builder", - "version": "2.4.42-alpha.5", + "version": "2.4.44-alpha.12", "license": "GPL-3.0", "private": true, "scripts": { @@ -58,11 +58,11 @@ } }, "dependencies": { - "@budibase/bbui": "2.4.42-alpha.5", - "@budibase/client": "2.4.42-alpha.5", - "@budibase/frontend-core": "2.4.42-alpha.5", - "@budibase/shared-core": "2.4.42-alpha.5", - "@budibase/string-templates": "2.4.42-alpha.5", + "@budibase/bbui": "2.4.44-alpha.12", + "@budibase/client": "2.4.44-alpha.12", + "@budibase/frontend-core": "2.4.44-alpha.12", + "@budibase/shared-core": "2.4.44-alpha.12", + "@budibase/string-templates": "2.4.44-alpha.12", "@fortawesome/fontawesome-svg-core": "^6.2.1", "@fortawesome/free-brands-svg-icons": "^6.2.1", "@fortawesome/free-solid-svg-icons": "^6.2.1", diff --git a/packages/builder/reporterConfig.json b/packages/builder/reporterConfig.json deleted file mode 100644 index 2c2ef7c138..0000000000 --- a/packages/builder/reporterConfig.json +++ /dev/null @@ -1,10 +0,0 @@ -{ - "reporterEnabled": "mochawesome", - "mochawesomeReporterOptions": { - "reportDir": "cypress/reports", - "quiet": true, - "overwrite": false, - "html": false, - "json": true - } -} \ No newline at end of file diff --git a/packages/builder/setup.js b/packages/builder/setup.js deleted file mode 100644 index b3fd96877b..0000000000 --- a/packages/builder/setup.js +++ /dev/null @@ -1,43 +0,0 @@ -const testConfig = require("./testConfig.json") - -// normal development system -const SERVER_PORT = testConfig.env.PORT -const WORKER_PORT = testConfig.env.WORKER_PORT - -if (!process.env.NODE_ENV) { - process.env.NODE_ENV = "cypress" -} -process.env.ENABLE_ANALYTICS = "0" -process.env.JWT_SECRET = testConfig.env.JWT_SECRET -process.env.SELF_HOSTED = 1 -process.env.WORKER_URL = `http://localhost:${WORKER_PORT}/` -process.env.APPS_URL = `http://localhost:${SERVER_PORT}/` -process.env.MINIO_URL = `http://localhost:4004` -process.env.MINIO_ACCESS_KEY = "budibase" -process.env.MINIO_SECRET_KEY = "budibase" -process.env.COUCH_DB_USER = "budibase" -process.env.COUCH_DB_PASSWORD = "budibase" -process.env.INTERNAL_API_KEY = "budibase" -process.env.ALLOW_DEV_AUTOMATIONS = 1 -process.env.MOCK_REDIS = 1 - -// Stop info logs polluting test outputs -process.env.LOG_LEVEL = "error" - -exports.run = (serverLoc = "../server/dist", workerLoc = "../worker/dist") => { - // require("dotenv").config({ path: resolve(dir, ".env") }) - // don't make this a variable or top level require - // it will cause environment module to be loaded prematurely - - // override the port with the worker port temporarily - process.env.PORT = WORKER_PORT - require(workerLoc) - - // override the port with the server port - process.env.PORT = SERVER_PORT - require(serverLoc) -} - -if (require.main === module) { - exports.run() -} diff --git a/packages/builder/src/components/automation/AutomationBuilder/TestDisplay.svelte b/packages/builder/src/components/automation/AutomationBuilder/TestDisplay.svelte index 043844b6d2..aa441cec40 100644 --- a/packages/builder/src/components/automation/AutomationBuilder/TestDisplay.svelte +++ b/packages/builder/src/components/automation/AutomationBuilder/TestDisplay.svelte @@ -58,7 +58,7 @@ /> {#if openBlocks[block.id]} - {#if filteredResults?.[idx]?.outputs.iterations} + {#if filteredResults?.[idx]?.outputs?.iterations}
diff --git a/packages/builder/src/components/backend/DatasourceNavigator/_components/GoogleButton.svelte b/packages/builder/src/components/backend/DatasourceNavigator/_components/GoogleButton.svelte index 4622bd7b19..b7d70d88b7 100644 --- a/packages/builder/src/components/backend/DatasourceNavigator/_components/GoogleButton.svelte +++ b/packages/builder/src/components/backend/DatasourceNavigator/_components/GoogleButton.svelte @@ -5,20 +5,33 @@ export let preAuthStep export let datasource + export let disabled + export let samePage $: tenantId = $auth.tenantId diff --git a/packages/builder/src/pages/builder/portal/users/_components/SCIMBanner.svelte b/packages/builder/src/pages/builder/portal/users/_components/SCIMBanner.svelte new file mode 100644 index 0000000000..2d02214800 --- /dev/null +++ b/packages/builder/src/pages/builder/portal/users/_components/SCIMBanner.svelte @@ -0,0 +1,15 @@ + + +
+ + Users are synced from your AD +
+ + diff --git a/packages/builder/src/pages/builder/portal/users/_layout.svelte b/packages/builder/src/pages/builder/portal/users/_layout.svelte index db598c4e43..d53da89613 100644 --- a/packages/builder/src/pages/builder/portal/users/_layout.svelte +++ b/packages/builder/src/pages/builder/portal/users/_layout.svelte @@ -1,12 +1,23 @@ diff --git a/packages/builder/src/pages/builder/portal/users/groups/[groupId].svelte b/packages/builder/src/pages/builder/portal/users/groups/[groupId].svelte index a45414d307..bfb60f1c72 100644 --- a/packages/builder/src/pages/builder/portal/users/groups/[groupId].svelte +++ b/packages/builder/src/pages/builder/portal/users/groups/[groupId].svelte @@ -14,7 +14,7 @@ } from "@budibase/bbui" import UserGroupPicker from "components/settings/UserGroupPicker.svelte" import { createPaginationStore } from "helpers/pagination" - import { users, apps, groups, auth } from "stores/portal" + import { users, apps, groups, auth, features } from "stores/portal" import { onMount, setContext } from "svelte" import { roles } from "stores/backend" import ConfirmDialog from "components/common/ConfirmDialog.svelte" @@ -24,18 +24,23 @@ import AppNameTableRenderer from "../users/_components/AppNameTableRenderer.svelte" import RemoveUserTableRenderer from "./_components/RemoveUserTableRenderer.svelte" import AppRoleTableRenderer from "../users/_components/AppRoleTableRenderer.svelte" + import ScimBanner from "../_components/SCIMBanner.svelte" export let groupId - const userSchema = { + $: userSchema = { email: { width: "1fr", }, - _id: { - displayName: "", - width: "auto", - borderLeft: true, - }, + ...(readonly + ? {} + : { + _id: { + displayName: "", + width: "auto", + borderLeft: true, + }, + }), } const appSchema = { name: { @@ -70,7 +75,9 @@ let loaded = false let editModal, deleteModal - $: readonly = !$auth.isAdmin + const scimEnabled = $features.isScimEnabled + + $: readonly = !$auth.isAdmin || scimEnabled $: page = $pageInfo.page $: fetchUsers(page, searchTerm) $: group = $groups.find(x => x._id === groupId) @@ -182,11 +189,15 @@
Users -
- -
+ {#if !scimEnabled} +
+ +
+ {:else} + + {/if} {#if $licensing.groupsEnabled} - - + {#if !$features.isScimEnabled} + + + {:else} + + {/if} {:else}