diff --git a/.github/workflows/budibase_ci.yml b/.github/workflows/budibase_ci.yml index 39a79193d3..d6bbf19940 100644 --- a/.github/workflows/budibase_ci.yml +++ b/.github/workflows/budibase_ci.yml @@ -38,10 +38,10 @@ jobs: submodules: ${{ env.IS_OSS_CONTRIBUTOR == 'false' }} token: ${{ secrets.PERSONAL_ACCESS_TOKEN || github.token }} - - name: Use Node.js 18.x + - name: Use Node.js 20.x uses: actions/setup-node@v3 with: - node-version: 18.x + node-version: 20.x cache: yarn - run: yarn --frozen-lockfile - run: yarn lint @@ -56,10 +56,10 @@ jobs: token: ${{ secrets.PERSONAL_ACCESS_TOKEN || github.token }} fetch-depth: 0 - - name: Use Node.js 18.x + - name: Use Node.js 20.x uses: actions/setup-node@v3 with: - node-version: 18.x + node-version: 20.x cache: yarn - run: yarn --frozen-lockfile @@ -84,7 +84,7 @@ jobs: with: fetch-depth: 0 - - name: Use Node.js 18.x + - name: Use Node.js 20.x uses: azure/setup-helm@v3 - run: cd charts/budibase && helm lint . @@ -98,10 +98,10 @@ jobs: token: ${{ secrets.PERSONAL_ACCESS_TOKEN || github.token }} fetch-depth: 0 - - name: Use Node.js 18.x + - name: Use Node.js 20.x uses: actions/setup-node@v3 with: - node-version: 18.x + node-version: 20.x cache: yarn - run: yarn --frozen-lockfile - name: Test @@ -122,10 +122,10 @@ jobs: token: ${{ secrets.PERSONAL_ACCESS_TOKEN || github.token }} fetch-depth: 0 - - name: Use Node.js 18.x + - name: Use Node.js 20.x uses: actions/setup-node@v3 with: - node-version: 18.x + node-version: 20.x cache: yarn - run: yarn --frozen-lockfile - name: Test worker @@ -146,10 +146,10 @@ jobs: token: ${{ secrets.PERSONAL_ACCESS_TOKEN || github.token }} fetch-depth: 0 - - name: Use Node.js 18.x + - name: Use Node.js 20.x uses: actions/setup-node@v3 with: - node-version: 18.x + node-version: 20.x cache: yarn - run: yarn --frozen-lockfile - name: Test server @@ -171,10 +171,10 @@ jobs: token: ${{ secrets.PERSONAL_ACCESS_TOKEN || github.token }} fetch-depth: 0 - - name: Use Node.js 18.x + - name: Use Node.js 20.x uses: actions/setup-node@v3 with: - node-version: 18.x + node-version: 20.x cache: yarn - run: yarn --frozen-lockfile - name: Test @@ -194,10 +194,10 @@ jobs: submodules: ${{ env.IS_OSS_CONTRIBUTOR == 'false' }} token: ${{ secrets.PERSONAL_ACCESS_TOKEN || github.token }} - - name: Use Node.js 18.x + - name: Use Node.js 20.x uses: actions/setup-node@v3 with: - node-version: 18.x + node-version: 20.x cache: yarn - run: yarn --frozen-lockfile - name: Build packages diff --git a/.github/workflows/stale_bot.yml b/.github/workflows/stale_bot.yml index 8f3ab9c74c..411a70a463 100644 --- a/.github/workflows/stale_bot.yml +++ b/.github/workflows/stale_bot.yml @@ -16,8 +16,8 @@ jobs: days-before-pr-stale: 7 stale-issue-label: stale exempt-pr-labels: pinned,security,roadmap - days-before-pr-close: 7 + days-before-issue-close: 30 - uses: actions/stale@v8 with: @@ -26,6 +26,7 @@ jobs: days-before-stale: 30 only-issue-labels: bug,High priority stale-issue-label: warn + days-before-close: 30 - uses: actions/stale@v8 with: @@ -34,6 +35,7 @@ jobs: days-before-stale: 90 only-issue-labels: bug,Medium priority stale-issue-label: warn + days-before-close: 30 - uses: actions/stale@v8 with: @@ -43,5 +45,4 @@ jobs: stale-issue-label: stale only-issue-labels: bug stale-issue-message: "This issue has been automatically marked as stale because it has not had any activity for six months." - days-before-close: 30 diff --git a/.nvmrc b/.nvmrc index 7950a44576..790e1105f2 100644 --- a/.nvmrc +++ b/.nvmrc @@ -1 +1 @@ -v18.17.0 +v20.10.0 diff --git a/.tool-versions b/.tool-versions index a909d60941..946d5198ce 100644 --- a/.tool-versions +++ b/.tool-versions @@ -1,3 +1,3 @@ -nodejs 18.17.0 +nodejs 20.10.0 python 3.10.0 yarn 1.22.19 diff --git a/charts/budibase/templates/app-service-deployment.yaml b/charts/budibase/templates/app-service-deployment.yaml index c6ded3cee2..c7c4481122 100644 --- a/charts/budibase/templates/app-service-deployment.yaml +++ b/charts/budibase/templates/app-service-deployment.yaml @@ -252,4 +252,10 @@ spec: {{ end }} restartPolicy: Always serviceAccountName: "" + {{ if .Values.services.apps.ndots }} + dnsConfig: + options: + - name: ndots + value: {{ .Values.services.apps.ndots | quote }} + {{ end }} status: {} diff --git a/charts/budibase/templates/automation-worker-service-deployment.yaml b/charts/budibase/templates/automation-worker-service-deployment.yaml index b7eece6b85..36c3a8ffbf 100644 --- a/charts/budibase/templates/automation-worker-service-deployment.yaml +++ b/charts/budibase/templates/automation-worker-service-deployment.yaml @@ -227,6 +227,7 @@ spec: resources: {{- toYaml . | nindent 10 }} {{ end }} + {{ if .Values.services.automationWorkers.command }} command: {{- toYaml .Values.services.automationWorkers.command | nindent 10 }} {{ end }} @@ -251,6 +252,11 @@ spec: {{ end }} restartPolicy: Always serviceAccountName: "" - {{ if .Values.services.automationWorkers.command }}} + {{ if .Values.services.automationWorkers.ndots }} + dnsConfig: + options: + - name: ndots + value: {{ .Values.services.automationWorkers.ndots | quote }} + {{ end }} status: {} {{- end }} \ No newline at end of file diff --git a/charts/budibase/templates/proxy-service-deployment.yaml b/charts/budibase/templates/proxy-service-deployment.yaml index 2e6217008b..233028cafe 100644 --- a/charts/budibase/templates/proxy-service-deployment.yaml +++ b/charts/budibase/templates/proxy-service-deployment.yaml @@ -109,4 +109,10 @@ spec: {{- toYaml .Values.services.proxy.args | nindent 8 }} {{ end }} volumes: + {{ if .Values.services.proxy.ndots }} + dnsConfig: + options: + - name: ndots + value: {{ .Values.services.proxy.ndots | quote }} + {{ end }} status: {} diff --git a/charts/budibase/templates/worker-service-deployment.yaml b/charts/budibase/templates/worker-service-deployment.yaml index 04791df869..2f97508ae3 100644 --- a/charts/budibase/templates/worker-service-deployment.yaml +++ b/charts/budibase/templates/worker-service-deployment.yaml @@ -238,4 +238,10 @@ spec: {{ end }} restartPolicy: Always serviceAccountName: "" + {{ if .Values.services.worker.ndots }} + dnsConfig: + options: + - name: ndots + value: {{ .Values.services.worker.ndots | quote }} + {{ end }} status: {} diff --git a/docs/CONTRIBUTING.md b/docs/CONTRIBUTING.md index 21e4b52a4b..311afbe706 100644 --- a/docs/CONTRIBUTING.md +++ b/docs/CONTRIBUTING.md @@ -90,7 +90,7 @@ Component libraries are collections of components as well as the definition of t #### 1. Prerequisites -- NodeJS version `18.x.x` +- NodeJS version `20.x.x` - Python version `3.x` ### Using asdf (recommended) diff --git a/hosting/couchdb/runner.sh b/hosting/couchdb/runner.sh index e56b8e0e7f..9f6a853ca7 100644 --- a/hosting/couchdb/runner.sh +++ b/hosting/couchdb/runner.sh @@ -76,6 +76,6 @@ done # CouchDB needs the `_users` and `_replicator` databases to exist before it will # function correctly, so we create them here. -curl -X PUT http://${COUCHDB_USER}:${COUCHDB_PASSWORD}@localhost:5984/_users -curl -X PUT http://${COUCHDB_USER}:${COUCHDB_PASSWORD}@localhost:5984/_replicator +curl -X PUT -u "${COUCHDB_USER}:${COUCHDB_PASSWORD}" http://localhost:5984/_users +curl -X PUT -u "${COUCHDB_USER}:${COUCHDB_PASSWORD}" http://localhost:5984/_replicator sleep infinity \ No newline at end of file diff --git a/hosting/docker-compose.yaml b/hosting/docker-compose.yaml index 7803916069..36b88466fe 100644 --- a/hosting/docker-compose.yaml +++ b/hosting/docker-compose.yaml @@ -26,7 +26,7 @@ services: BB_ADMIN_USER_EMAIL: ${BB_ADMIN_USER_EMAIL} BB_ADMIN_USER_PASSWORD: ${BB_ADMIN_USER_PASSWORD} PLUGINS_DIR: ${PLUGINS_DIR} - OFFLINE_MODE: ${OFFLINE_MODE} + OFFLINE_MODE: ${OFFLINE_MODE:-} depends_on: - worker-service - redis-service @@ -53,7 +53,7 @@ services: INTERNAL_API_KEY: ${INTERNAL_API_KEY} REDIS_URL: redis-service:6379 REDIS_PASSWORD: ${REDIS_PASSWORD} - OFFLINE_MODE: ${OFFLINE_MODE} + OFFLINE_MODE: ${OFFLINE_MODE:-} depends_on: - redis-service - minio-service @@ -109,7 +109,7 @@ services: redis-service: restart: unless-stopped image: redis - command: redis-server --requirepass ${REDIS_PASSWORD} + command: redis-server --requirepass "${REDIS_PASSWORD}" volumes: - redis_data:/data diff --git a/hosting/single/Dockerfile b/hosting/single/Dockerfile index e9ff6c6596..67ac677984 100644 --- a/hosting/single/Dockerfile +++ b/hosting/single/Dockerfile @@ -1,4 +1,4 @@ -FROM node:18-slim as build +FROM node:20-slim as build # install node-gyp dependencies RUN apt-get update && apt-get install -y --no-install-recommends g++ make python3 jq @@ -42,7 +42,7 @@ COPY packages/string-templates packages/string-templates FROM budibase/couchdb as runner ARG TARGETARCH ENV TARGETARCH $TARGETARCH -ENV NODE_MAJOR 18 +ENV NODE_MAJOR 20 #TARGETBUILD can be set to single (for single docker image) or aas (for azure app service) # e.g. docker build --build-arg TARGETBUILD=aas .... ARG TARGETBUILD=single diff --git a/lerna.json b/lerna.json index 671935c34b..d6fa262685 100644 --- a/lerna.json +++ b/lerna.json @@ -1,5 +1,5 @@ { - "version": "2.13.50", + "version": "2.14.3", "npmClient": "yarn", "packages": [ "packages/*", diff --git a/package.json b/package.json index 3aa3f7c15d..8c2b6b099c 100644 --- a/package.json +++ b/package.json @@ -6,6 +6,7 @@ "@babel/eslint-parser": "^7.22.5", "@babel/preset-env": "^7.22.5", "@esbuild-plugins/tsconfig-paths": "^0.1.2", + "@types/node": "20.10.0", "@typescript-eslint/parser": "6.9.0", "esbuild": "^0.18.17", "esbuild-node-externals": "^1.8.0", @@ -99,7 +100,7 @@ "@budibase/types": "0.0.0" }, "engines": { - "node": ">=18.0.0 <19.0.0" + "node": ">=20.0.0 <21.0.0" }, "dependencies": {} } diff --git a/packages/account-portal b/packages/account-portal index e46a352a63..8ee2734e77 160000 --- a/packages/account-portal +++ b/packages/account-portal @@ -1 +1 @@ -Subproject commit e46a352a6326a838faa00f912de069aee95d7300 +Subproject commit 8ee2734e77709438cbcaaabc024f677c7b24c883 diff --git a/packages/backend-core/package.json b/packages/backend-core/package.json index 36b8de0f56..343bc67449 100644 --- a/packages/backend-core/package.json +++ b/packages/backend-core/package.json @@ -65,7 +65,6 @@ "@types/cookies": "0.7.8", "@types/jest": "29.5.5", "@types/lodash": "4.14.200", - "@types/node": "18.17.0", "@types/node-fetch": "2.6.4", "@types/pouchdb": "6.4.0", "@types/redlock": "4.0.3", diff --git a/packages/backend-core/src/cache/generic.ts b/packages/backend-core/src/cache/generic.ts index 7a2be5a0f0..3ac323a8d4 100644 --- a/packages/backend-core/src/cache/generic.ts +++ b/packages/backend-core/src/cache/generic.ts @@ -18,14 +18,15 @@ export enum TTL { ONE_DAY = 86400, } -function performExport(funcName: string) { - // @ts-ignore - return (...args: any) => GENERIC[funcName](...args) -} - -export const keys = performExport("keys") -export const get = performExport("get") -export const store = performExport("store") -export const destroy = performExport("delete") -export const withCache = performExport("withCache") -export const bustCache = performExport("bustCache") +export const keys = (...args: Parameters) => + GENERIC.keys(...args) +export const get = (...args: Parameters) => + GENERIC.get(...args) +export const store = (...args: Parameters) => + GENERIC.store(...args) +export const destroy = (...args: Parameters) => + GENERIC.delete(...args) +export const withCache = (...args: Parameters) => + GENERIC.withCache(...args) +export const bustCache = (...args: Parameters) => + GENERIC.bustCache(...args) diff --git a/packages/backend-core/src/cache/passwordReset.ts b/packages/backend-core/src/cache/passwordReset.ts index 7f5a93f149..db32b520f7 100644 --- a/packages/backend-core/src/cache/passwordReset.ts +++ b/packages/backend-core/src/cache/passwordReset.ts @@ -1,6 +1,6 @@ import * as redis from "../redis/init" import * as utils from "../utils" -import { Duration, DurationType } from "../utils" +import { Duration } from "../utils" const TTL_SECONDS = Duration.fromHours(1).toSeconds() @@ -32,7 +32,18 @@ export async function getCode(code: string): Promise { const client = await redis.getPasswordResetClient() const value = (await client.get(code)) as PasswordReset | undefined if (!value) { - throw "Provided information is not valid, cannot reset password - please try again." + throw new Error( + "Provided information is not valid, cannot reset password - please try again." + ) } return value } + +/** + * Given a reset code this will invalidate it. + * @param code The code provided via the email link. + */ +export async function invalidateCode(code: string): Promise { + const client = await redis.getPasswordResetClient() + await client.delete(code) +} diff --git a/packages/backend-core/src/context/mainContext.ts b/packages/backend-core/src/context/mainContext.ts index e439eb5fd0..36fd5dcb48 100644 --- a/packages/backend-core/src/context/mainContext.ts +++ b/packages/backend-core/src/context/mainContext.ts @@ -134,7 +134,7 @@ export async function doInContext(appId: string, task: any): Promise { } export async function doInTenant( - tenantId: string | null, + tenantId: string | undefined, task: () => T ): Promise { // make sure default always selected in single tenancy diff --git a/packages/backend-core/src/db/couch/DatabaseImpl.ts b/packages/backend-core/src/db/couch/DatabaseImpl.ts index c2c0b6b21d..3fec573bb9 100644 --- a/packages/backend-core/src/db/couch/DatabaseImpl.ts +++ b/packages/backend-core/src/db/couch/DatabaseImpl.ts @@ -17,6 +17,7 @@ import { directCouchUrlCall } from "./utils" import { getPouchDB } from "./pouchDB" import { WriteStream, ReadStream } from "fs" import { newid } from "../../docIds/newid" +import { DDInstrumentedDatabase } from "../instrumentation" function buildNano(couchInfo: { url: string; cookie: string }) { return Nano({ @@ -35,7 +36,8 @@ export function DatabaseWithConnection( connection: string, opts?: DatabaseOpts ) { - return new DatabaseImpl(dbName, opts, connection) + const db = new DatabaseImpl(dbName, opts, connection) + return new DDInstrumentedDatabase(db) } export class DatabaseImpl implements Database { diff --git a/packages/backend-core/src/db/db.ts b/packages/backend-core/src/db/db.ts index 3e69d49f0e..197770298e 100644 --- a/packages/backend-core/src/db/db.ts +++ b/packages/backend-core/src/db/db.ts @@ -1,8 +1,9 @@ import { directCouchQuery, DatabaseImpl } from "./couch" import { CouchFindOptions, Database, DatabaseOpts } from "@budibase/types" +import { DDInstrumentedDatabase } from "./instrumentation" export function getDB(dbName: string, opts?: DatabaseOpts): Database { - return new DatabaseImpl(dbName, opts) + return new DDInstrumentedDatabase(new DatabaseImpl(dbName, opts)) } // we have to use a callback for this so that we can close diff --git a/packages/backend-core/src/db/instrumentation.ts b/packages/backend-core/src/db/instrumentation.ts new file mode 100644 index 0000000000..ba5febcba6 --- /dev/null +++ b/packages/backend-core/src/db/instrumentation.ts @@ -0,0 +1,156 @@ +import { + DocumentScope, + DocumentDestroyResponse, + DocumentInsertResponse, + DocumentBulkResponse, + OkResponse, +} from "@budibase/nano" +import { + AllDocsResponse, + AnyDocument, + Database, + DatabaseDumpOpts, + DatabasePutOpts, + DatabaseQueryOpts, + Document, +} from "@budibase/types" +import tracer from "dd-trace" +import { Writable } from "stream" + +export class DDInstrumentedDatabase implements Database { + constructor(private readonly db: Database) {} + + get name(): string { + return this.db.name + } + + exists(): Promise { + return tracer.trace("db.exists", span => { + span?.addTags({ db_name: this.name }) + return this.db.exists() + }) + } + + checkSetup(): Promise> { + return tracer.trace("db.checkSetup", span => { + span?.addTags({ db_name: this.name }) + return this.db.checkSetup() + }) + } + + get(id?: string | undefined): Promise { + return tracer.trace("db.get", span => { + span?.addTags({ db_name: this.name, doc_id: id }) + return this.db.get(id) + }) + } + + getMultiple( + ids: string[], + opts?: { allowMissing?: boolean | undefined } | undefined + ): Promise { + return tracer.trace("db.getMultiple", span => { + span?.addTags({ + db_name: this.name, + num_docs: ids.length, + allow_missing: opts?.allowMissing, + }) + return this.db.getMultiple(ids, opts) + }) + } + + remove( + id: string | Document, + rev?: string | undefined + ): Promise { + return tracer.trace("db.remove", span => { + span?.addTags({ db_name: this.name, doc_id: id }) + return this.db.remove(id, rev) + }) + } + + put( + document: AnyDocument, + opts?: DatabasePutOpts | undefined + ): Promise { + return tracer.trace("db.put", span => { + span?.addTags({ db_name: this.name, doc_id: document._id }) + return this.db.put(document, opts) + }) + } + + bulkDocs(documents: AnyDocument[]): Promise { + return tracer.trace("db.bulkDocs", span => { + span?.addTags({ db_name: this.name, num_docs: documents.length }) + return this.db.bulkDocs(documents) + }) + } + + allDocs( + params: DatabaseQueryOpts + ): Promise> { + return tracer.trace("db.allDocs", span => { + span?.addTags({ db_name: this.name }) + return this.db.allDocs(params) + }) + } + + query( + viewName: string, + params: DatabaseQueryOpts + ): Promise> { + return tracer.trace("db.query", span => { + span?.addTags({ db_name: this.name, view_name: viewName }) + return this.db.query(viewName, params) + }) + } + + destroy(): Promise { + return tracer.trace("db.destroy", span => { + span?.addTags({ db_name: this.name }) + return this.db.destroy() + }) + } + + compact(): Promise { + return tracer.trace("db.compact", span => { + span?.addTags({ db_name: this.name }) + return this.db.compact() + }) + } + + dump(stream: Writable, opts?: DatabaseDumpOpts | undefined): Promise { + return tracer.trace("db.dump", span => { + span?.addTags({ db_name: this.name }) + return this.db.dump(stream, opts) + }) + } + + load(...args: any[]): Promise { + return tracer.trace("db.load", span => { + span?.addTags({ db_name: this.name }) + return this.db.load(...args) + }) + } + + createIndex(...args: any[]): Promise { + return tracer.trace("db.createIndex", span => { + span?.addTags({ db_name: this.name }) + return this.db.createIndex(...args) + }) + } + + deleteIndex(...args: any[]): Promise { + return tracer.trace("db.deleteIndex", span => { + span?.addTags({ db_name: this.name }) + return this.db.deleteIndex(...args) + }) + } + + getIndexes(...args: any[]): Promise { + return tracer.trace("db.getIndexes", span => { + span?.addTags({ db_name: this.name }) + return this.db.getIndexes(...args) + }) + } +} diff --git a/packages/backend-core/src/environment.ts b/packages/backend-core/src/environment.ts index 66c91b19fb..e0a4f91c67 100644 --- a/packages/backend-core/src/environment.ts +++ b/packages/backend-core/src/environment.ts @@ -166,6 +166,8 @@ const environment = { DISABLE_JWT_WARNING: process.env.DISABLE_JWT_WARNING, BLACKLIST_IPS: process.env.BLACKLIST_IPS, SERVICE_TYPE: "unknown", + PASSWORD_MIN_LENGTH: process.env.PASSWORD_MIN_LENGTH, + PASSWORD_MAX_LENGTH: process.env.PASSWORD_MAX_LENGTH, /** * Enable to allow an admin user to login using a password. * This can be useful to prevent lockout when configuring SSO. diff --git a/packages/backend-core/src/index.ts b/packages/backend-core/src/index.ts index d04f48e5fc..7bf26f3688 100644 --- a/packages/backend-core/src/index.ts +++ b/packages/backend-core/src/index.ts @@ -33,6 +33,7 @@ export * as docUpdates from "./docUpdates" export * from "./utils/Duration" export { SearchParams } from "./db" export * as docIds from "./docIds" +export * as security from "./security" // Add context to tenancy for backwards compatibility // only do this for external usages to prevent internal // circular dependencies diff --git a/packages/backend-core/src/logging/pino/logger.ts b/packages/backend-core/src/logging/pino/logger.ts index ad68bd300d..7a051e7f12 100644 --- a/packages/backend-core/src/logging/pino/logger.ts +++ b/packages/backend-core/src/logging/pino/logger.ts @@ -5,6 +5,7 @@ import { IdentityType } from "@budibase/types" import env from "../../environment" import * as context from "../../context" import * as correlation from "../correlation" +import tracer from "dd-trace" import { formats } from "dd-trace/ext" import { localFileDestination } from "../system" @@ -116,6 +117,11 @@ if (!env.DISABLE_PINO_LOGGER) { correlationId: correlation.getId(), } + const span = tracer.scope().active() + if (span) { + tracer.inject(span.context(), formats.LOG, contextObject) + } + const mergingObject: any = { err: error, pid: process.pid, diff --git a/packages/backend-core/src/middleware/authenticated.ts b/packages/backend-core/src/middleware/authenticated.ts index 16f658b90a..d357dbdbdc 100644 --- a/packages/backend-core/src/middleware/authenticated.ts +++ b/packages/backend-core/src/middleware/authenticated.ts @@ -15,6 +15,7 @@ import * as identity from "../context/identity" import env from "../environment" import { Ctx, EndpointMatcher, SessionCookie } from "@budibase/types" import { InvalidAPIKeyError, ErrorCode } from "../errors" +import tracer from "dd-trace" const ONE_MINUTE = env.SESSION_UPDATE_PERIOD ? parseInt(env.SESSION_UPDATE_PERIOD) @@ -166,6 +167,16 @@ export default function ( if (!authenticated) { authenticated = false } + + if (user) { + tracer.setUser({ + id: user?._id, + tenantId: user?.tenantId, + budibaseAccess: user?.budibaseAccess, + status: user?.status, + }) + } + // isAuthenticated is a function, so use a variable to be able to check authed state finalise(ctx, { authenticated, user, internal, version, publicEndpoint }) diff --git a/packages/backend-core/src/queue/queue.ts b/packages/backend-core/src/queue/queue.ts index 0657437a3b..b95dace5b2 100644 --- a/packages/backend-core/src/queue/queue.ts +++ b/packages/backend-core/src/queue/queue.ts @@ -47,7 +47,7 @@ export function createQueue( cleanupInterval = timers.set(cleanup, CLEANUP_PERIOD_MS) // fire off an initial cleanup cleanup().catch(err => { - console.error(`Unable to cleanup automation queue initially - ${err}`) + console.error(`Unable to cleanup ${jobQueue} initially - ${err}`) }) } return queue diff --git a/packages/backend-core/src/redis/redis.ts b/packages/backend-core/src/redis/redis.ts index 701e262091..d15453ba62 100644 --- a/packages/backend-core/src/redis/redis.ts +++ b/packages/backend-core/src/redis/redis.ts @@ -18,6 +18,7 @@ import { SelectableDatabase, getRedisConnectionDetails, } from "./utils" +import { logAlert } from "../logging" import * as timers from "../timers" const RETRY_PERIOD_MS = 2000 @@ -39,21 +40,16 @@ function pickClient(selectDb: number): any { return CLIENTS[selectDb] } -function connectionError( - selectDb: number, - timeout: NodeJS.Timeout, - err: Error | string -) { +function connectionError(timeout: NodeJS.Timeout, err: Error | string) { // manually shut down, ignore errors if (CLOSED) { return } - pickClient(selectDb).disconnect() CLOSED = true // always clear this on error clearTimeout(timeout) CONNECTED = false - console.error("Redis connection failed - " + err) + logAlert("Redis connection failed", err) setTimeout(() => { init() }, RETRY_PERIOD_MS) @@ -79,11 +75,7 @@ function init(selectDb = DEFAULT_SELECT_DB) { // start the timer - only allowed 5 seconds to connect timeout = setTimeout(() => { if (!CONNECTED) { - connectionError( - selectDb, - timeout, - "Did not successfully connect in timeout" - ) + connectionError(timeout, "Did not successfully connect in timeout") } }, STARTUP_TIMEOUT_MS) @@ -106,12 +98,13 @@ function init(selectDb = DEFAULT_SELECT_DB) { // allow the process to exit return } - connectionError(selectDb, timeout, err) + connectionError(timeout, err) }) client.on("error", (err: Error) => { - connectionError(selectDb, timeout, err) + connectionError(timeout, err) }) client.on("connect", () => { + console.log(`Connected to Redis DB: ${selectDb}`) clearTimeout(timeout) CONNECTED = true }) diff --git a/packages/backend-core/src/redis/redlockImpl.ts b/packages/backend-core/src/redis/redlockImpl.ts index e57a3721b5..7009dc6f55 100644 --- a/packages/backend-core/src/redis/redlockImpl.ts +++ b/packages/backend-core/src/redis/redlockImpl.ts @@ -2,7 +2,6 @@ import Redlock from "redlock" import { getLockClient } from "./init" import { LockOptions, LockType } from "@budibase/types" import * as context from "../context" -import { logWarn } from "../logging" import { utils } from "@budibase/shared-core" import { Duration } from "../utils" diff --git a/packages/backend-core/src/security/auth.ts b/packages/backend-core/src/security/auth.ts new file mode 100644 index 0000000000..1cce35a0af --- /dev/null +++ b/packages/backend-core/src/security/auth.ts @@ -0,0 +1,24 @@ +import env from "../environment" + +export const PASSWORD_MIN_LENGTH = +(env.PASSWORD_MIN_LENGTH || 8) +export const PASSWORD_MAX_LENGTH = +(env.PASSWORD_MAX_LENGTH || 512) + +export function validatePassword( + password: string +): { valid: true } | { valid: false; error: string } { + if (!password || password.length < PASSWORD_MIN_LENGTH) { + return { + valid: false, + error: `Password invalid. Minimum ${PASSWORD_MIN_LENGTH} characters.`, + } + } + + if (password.length > PASSWORD_MAX_LENGTH) { + return { + valid: false, + error: `Password invalid. Maximum ${PASSWORD_MAX_LENGTH} characters.`, + } + } + + return { valid: true } +} diff --git a/packages/backend-core/src/security/index.ts b/packages/backend-core/src/security/index.ts new file mode 100644 index 0000000000..306751af96 --- /dev/null +++ b/packages/backend-core/src/security/index.ts @@ -0,0 +1 @@ +export * from "./auth" diff --git a/packages/backend-core/src/security/tests/auth.spec.ts b/packages/backend-core/src/security/tests/auth.spec.ts new file mode 100644 index 0000000000..b1835fdfb3 --- /dev/null +++ b/packages/backend-core/src/security/tests/auth.spec.ts @@ -0,0 +1,45 @@ +import { generator } from "../../../tests" +import { PASSWORD_MAX_LENGTH, validatePassword } from "../auth" + +describe("auth", () => { + describe("validatePassword", () => { + it("a valid password returns successful", () => { + expect(validatePassword("password")).toEqual({ valid: true }) + }) + + it.each([ + ["undefined", undefined], + ["null", null], + ["empty", ""], + ])("%s returns unsuccessful", (_, password) => { + expect(validatePassword(password as string)).toEqual({ + valid: false, + error: "Password invalid. Minimum 8 characters.", + }) + }) + + it.each([ + generator.word({ length: PASSWORD_MAX_LENGTH }), + generator.paragraph().substring(0, PASSWORD_MAX_LENGTH), + ])(`can use passwords up to 512 characters in length`, password => { + expect(validatePassword(password)).toEqual({ + valid: true, + }) + }) + + it.each([ + generator.word({ length: PASSWORD_MAX_LENGTH + 1 }), + generator + .paragraph({ sentences: 50 }) + .substring(0, PASSWORD_MAX_LENGTH + 1), + ])( + `passwords cannot have more than ${PASSWORD_MAX_LENGTH} characters`, + password => { + expect(validatePassword(password)).toEqual({ + valid: false, + error: "Password invalid. Maximum 512 characters.", + }) + } + ) + }) +}) diff --git a/packages/backend-core/src/tenancy/tenancy.ts b/packages/backend-core/src/tenancy/tenancy.ts index 3603ef3462..8835960ca5 100644 --- a/packages/backend-core/src/tenancy/tenancy.ts +++ b/packages/backend-core/src/tenancy/tenancy.ts @@ -39,7 +39,7 @@ const ALL_STRATEGIES = Object.values(TenantResolutionStrategy) export const getTenantIDFromCtx = ( ctx: BBContext, opts: GetTenantIdOptions -): string | null => { +): string | undefined => { // exit early if not multi-tenant if (!isMultiTenant()) { return DEFAULT_TENANT_ID @@ -144,5 +144,5 @@ export const getTenantIDFromCtx = ( ctx.throw(403, "Tenant id not set") } - return null + return undefined } diff --git a/packages/backend-core/src/tenancy/tests/tenancy.spec.ts b/packages/backend-core/src/tenancy/tests/tenancy.spec.ts index ebeaca074c..95dd76a6dd 100644 --- a/packages/backend-core/src/tenancy/tests/tenancy.spec.ts +++ b/packages/backend-core/src/tenancy/tests/tenancy.spec.ts @@ -157,12 +157,12 @@ describe("getTenantIDFromCtx", () => { TenantResolutionStrategy.PATH, ], } - expect(getTenantIDFromCtx(ctx, mockOpts)).toBeNull() + expect(getTenantIDFromCtx(ctx, mockOpts)).toBeUndefined() expect(ctx.throw).toBeCalledTimes(1) expect(ctx.throw).toBeCalledWith(403, "Tenant id not set") }) - it("returns null if allowNoTenant is true", () => { + it("returns undefined if allowNoTenant is true", () => { const ctx = createCtx({}) mockOpts = { allowNoTenant: true, @@ -172,7 +172,7 @@ describe("getTenantIDFromCtx", () => { TenantResolutionStrategy.PATH, ], } - expect(getTenantIDFromCtx(ctx, mockOpts)).toBeNull() + expect(getTenantIDFromCtx(ctx, mockOpts)).toBeUndefined() }) }) diff --git a/packages/backend-core/src/timers/timers.ts b/packages/backend-core/src/timers/timers.ts index 9de57af7f1..9121c576cd 100644 --- a/packages/backend-core/src/timers/timers.ts +++ b/packages/backend-core/src/timers/timers.ts @@ -50,7 +50,7 @@ export class ExecutionTimeTracker { return this.totalTimeMs } - private checkLimit() { + checkLimit() { if (this.totalTimeMs > this.limitMs) { throw new ExecutionTimeoutError( `Execution time limit of ${this.limitMs}ms exceeded: ${this.totalTimeMs}ms` diff --git a/packages/backend-core/src/users/db.ts b/packages/backend-core/src/users/db.ts index 01fa4899d1..4d0d216603 100644 --- a/packages/backend-core/src/users/db.ts +++ b/packages/backend-core/src/users/db.ts @@ -27,6 +27,7 @@ import { } from "./utils" import { searchExistingEmails } from "./lookup" import { hash } from "../utils" +import { validatePassword } from "../security" type QuotaUpdateFn = ( change: number, @@ -43,6 +44,12 @@ type GroupFns = { getBulk: GroupGetFn getGroupBuilderAppIds: GroupBuildersFn } +type CreateAdminUserOpts = { + ssoId?: string + hashPassword?: boolean + requirePassword?: boolean + skipPasswordValidation?: boolean +} type FeatureFns = { isSSOEnforced: FeatureFn; isAppBuildersEnabled: FeatureFn } const bulkDeleteProcessing = async (dbUser: User) => { @@ -110,6 +117,14 @@ export class UserDB { if (await UserDB.isPreventPasswordActions(user, account)) { throw new HTTPError("Password change is disabled for this user", 400) } + + if (!opts.skipPasswordValidation) { + const passwordValidation = validatePassword(password) + if (!passwordValidation.valid) { + throw new HTTPError(passwordValidation.error, 400) + } + } + hashedPassword = opts.hashPassword ? await hash(password) : password } else if (dbUser) { hashedPassword = dbUser.password @@ -482,7 +497,7 @@ export class UserDB { email: string, password: string, tenantId: string, - opts?: { ssoId?: string; hashPassword?: boolean; requirePassword?: boolean } + opts?: CreateAdminUserOpts ) { const user: User = { email: email, @@ -506,6 +521,7 @@ export class UserDB { return await UserDB.save(user, { hashPassword: opts?.hashPassword, requirePassword: opts?.requirePassword, + skipPasswordValidation: opts?.skipPasswordValidation, }) } diff --git a/packages/backend-core/src/utils/utils.ts b/packages/backend-core/src/utils/utils.ts index 0554737518..30cf55b149 100644 --- a/packages/backend-core/src/utils/utils.ts +++ b/packages/backend-core/src/utils/utils.ts @@ -31,8 +31,8 @@ export async function resolveAppUrl(ctx: Ctx) { const appUrl = ctx.path.split("/")[2] let possibleAppUrl = `/${appUrl.toLowerCase()}` - let tenantId: string | null = context.getTenantId() - if (env.MULTI_TENANCY) { + let tenantId: string | undefined = context.getTenantId() + if (!env.isDev() && env.MULTI_TENANCY) { // always use the tenant id from the subdomain in multi tenancy // this ensures the logged-in user tenant id doesn't overwrite // e.g. in the case of viewing a public app while already logged-in to another tenant @@ -41,7 +41,7 @@ export async function resolveAppUrl(ctx: Ctx) { }) } - // search prod apps for a url that matches + // search prod apps for an url that matches const apps: App[] = await context.doInTenant( tenantId, () => getAllApps({ dev: false }) as Promise diff --git a/packages/backend-core/tests/core/utilities/structures/users.ts b/packages/backend-core/tests/core/utilities/structures/users.ts index 68ee29686c..8f4096d401 100644 --- a/packages/backend-core/tests/core/utilities/structures/users.ts +++ b/packages/backend-core/tests/core/utilities/structures/users.ts @@ -21,7 +21,7 @@ export const user = (userProps?: Partial>): User => { _id: userId, userId, email: newEmail(), - password: "test", + password: "password", roles: { app_test: "admin" }, firstName: generator.first(), lastName: generator.last(), diff --git a/packages/bbui/src/ActionButton/ActionButton.svelte b/packages/bbui/src/ActionButton/ActionButton.svelte index 427a98f888..0e6ec3d155 100644 --- a/packages/bbui/src/ActionButton/ActionButton.svelte +++ b/packages/bbui/src/ActionButton/ActionButton.svelte @@ -130,5 +130,6 @@ max-width: 150px; transform: translateX(-50%); text-align: center; + z-index: 1; } diff --git a/packages/bbui/src/DetailSummary/DetailSummary.svelte b/packages/bbui/src/DetailSummary/DetailSummary.svelte index e5d6fda86b..2cbb6796f3 100644 --- a/packages/bbui/src/DetailSummary/DetailSummary.svelte +++ b/packages/bbui/src/DetailSummary/DetailSummary.svelte @@ -78,7 +78,7 @@ var(--spacing-xl); } .property-panel.no-title { - padding: var(--spacing-xl); + padding-top: var(--spacing-xl); } .show { diff --git a/packages/bbui/src/Form/Field.svelte b/packages/bbui/src/Form/Field.svelte index 0c031b0235..1770438c3c 100644 --- a/packages/bbui/src/Form/Field.svelte +++ b/packages/bbui/src/Form/Field.svelte @@ -51,15 +51,13 @@ margin-top: var(--spectrum-global-dimension-size-75); align-items: center; } - .helpText :global(svg) { - width: 14px; - color: var(--grey-5); + width: 13px; + color: var(--spectrum-global-color-gray-600); margin-right: 6px; } - .helpText span { - color: var(--grey-7); + color: var(--spectrum-global-color-gray-800); font-size: var(--spectrum-global-dimension-font-size-75); } diff --git a/packages/bbui/src/Markdown/MarkdownEditor.svelte b/packages/bbui/src/Markdown/MarkdownEditor.svelte index 888187c8da..2f18c9d634 100644 --- a/packages/bbui/src/Markdown/MarkdownEditor.svelte +++ b/packages/bbui/src/Markdown/MarkdownEditor.svelte @@ -19,7 +19,7 @@ // Ensure the value is updated if the value prop changes outside the editor's // control $: checkValue(value) - $: mde?.codemirror.on("change", debouncedUpdate) + $: mde?.codemirror.on("blur", update) $: if (readonly || disabled) { mde?.togglePreview() } @@ -30,21 +30,10 @@ } } - const debounce = (fn, interval) => { - let timeout - return () => { - clearTimeout(timeout) - timeout = setTimeout(fn, interval) - } - } - const update = () => { latestValue = mde.value() dispatch("change", latestValue) } - - // Debounce the update function to avoid spamming it constantly - const debouncedUpdate = debounce(update, 250) {#key height} diff --git a/packages/builder/src/builderStore/dataBinding.js b/packages/builder/src/builderStore/dataBinding.js index d86e94aba2..52368a0723 100644 --- a/packages/builder/src/builderStore/dataBinding.js +++ b/packages/builder/src/builderStore/dataBinding.js @@ -465,8 +465,8 @@ const filterCategoryByContext = (component, context) => { const { _component } = component if (_component.endsWith("formblock")) { if ( - (component.actionType == "Create" && context.type === "schema") || - (component.actionType == "View" && context.type === "form") + (component.actionType === "Create" && context.type === "schema") || + (component.actionType === "View" && context.type === "form") ) { return false } @@ -474,20 +474,21 @@ const filterCategoryByContext = (component, context) => { return true } +// Enrich binding category information for certain components const getComponentBindingCategory = (component, context, def) => { let icon = def.icon let category = component._instanceName if (component._component.endsWith("formblock")) { - let contextCategorySuffix = { - form: "Fields", - schema: "Row", + if (context.type === "form") { + category = `${component._instanceName} - Fields` + icon = "Form" + } else if (context.type === "schema") { + category = `${component._instanceName} - Row` + icon = "Data" } - category = `${component._instanceName} - ${ - contextCategorySuffix[context.type] - }` - icon = context.type === "form" ? "Form" : "Data" } + return { icon, category, diff --git a/packages/builder/src/builderStore/store/frontend.js b/packages/builder/src/builderStore/store/frontend.js index aaa0eb0184..2d62a0667e 100644 --- a/packages/builder/src/builderStore/store/frontend.js +++ b/packages/builder/src/builderStore/store/frontend.js @@ -610,12 +610,12 @@ export const getFrontendStore = () => { // Use default config if the 'buttons' prop has never been initialised if (!("buttons" in enrichedComponent)) { enrichedComponent["buttons"] = - Utils.buildDynamicButtonConfig(enrichedComponent) + Utils.buildFormBlockButtonConfig(enrichedComponent) migrated = true } else if (enrichedComponent["buttons"] == null) { // Ignore legacy config if 'buttons' has been reset by 'resetOn' const { _id, actionType, dataSource } = enrichedComponent - enrichedComponent["buttons"] = Utils.buildDynamicButtonConfig({ + enrichedComponent["buttons"] = Utils.buildFormBlockButtonConfig({ _id, actionType, dataSource, @@ -1289,15 +1289,14 @@ export const getFrontendStore = () => { const settings = getComponentSettings(component._component) const updatedSetting = settings.find(setting => setting.key === name) - // Can be a single string or array of strings - const resetFields = settings.filter(setting => { - return ( + // Reset dependent fields + settings.forEach(setting => { + const needsReset = name === setting.resetOn || (Array.isArray(setting.resetOn) && setting.resetOn.includes(name)) - ) - }) - resetFields?.forEach(setting => { - component[setting.key] = null + if (needsReset) { + component[setting.key] = setting.defaultValue || null + } }) if ( diff --git a/packages/builder/src/components/backend/Datasources/ConfigEditor/stores/validation.js b/packages/builder/src/components/backend/Datasources/ConfigEditor/stores/validation.js index 08331b840d..2ec9539824 100644 --- a/packages/builder/src/components/backend/Datasources/ConfigEditor/stores/validation.js +++ b/packages/builder/src/components/backend/Datasources/ConfigEditor/stores/validation.js @@ -1,4 +1,4 @@ -import { string, number } from "yup" +import { string, number, object } from "yup" const propertyValidator = type => { if (type === "number") { @@ -9,6 +9,10 @@ const propertyValidator = type => { return string().email().nullable() } + if (type === "object") { + return object().nullable() + } + return string().nullable() } diff --git a/packages/builder/src/components/common/CodeEditor/CodeEditor.svelte b/packages/builder/src/components/common/CodeEditor/CodeEditor.svelte index 76d7a58ef1..a39634f9a3 100644 --- a/packages/builder/src/components/common/CodeEditor/CodeEditor.svelte +++ b/packages/builder/src/components/common/CodeEditor/CodeEditor.svelte @@ -53,6 +53,7 @@ export let value = "" export let placeholder = null export let autocompleteEnabled = true + export let autofocus = false // Export a function to expose caret position export const getCaretPosition = () => { @@ -241,6 +242,12 @@ }) } + $: { + if (autofocus && isEditorInitialised) { + editor.focus() + } + } + $: editorHeight = typeof height === "number" ? `${height}px` : height // Init when all elements are ready diff --git a/packages/builder/src/components/common/bindings/BindingPanel.svelte b/packages/builder/src/components/common/bindings/BindingPanel.svelte index 4df26c5d03..548a71b483 100644 --- a/packages/builder/src/components/common/bindings/BindingPanel.svelte +++ b/packages/builder/src/components/common/bindings/BindingPanel.svelte @@ -45,6 +45,7 @@ export let valid export let allowJS = false export let allowHelpers = true + export let autofocusEditor = false const drawerActions = getContext("drawer-actions") const bindingDrawerActions = getContext("binding-drawer-actions") @@ -199,6 +200,7 @@ ]} placeholder="" height="100%" + autofocus={autofocusEditor} />