diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md index 6ace2303d9..854bc2e6dc 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.md +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -2,10 +2,11 @@ name: Bug report about: Create a report to help us improve title: '' -labels: bug +labels: bug, linear assignees: '' --- + **Checklist** - [ ] I have searched budibase discussions and github issues to check if my issue already exists diff --git a/.github/workflows/deploy-preprod.yml b/.github/workflows/deploy-preprod.yml index 3015a9ad27..a1eee2c465 100644 --- a/.github/workflows/deploy-preprod.yml +++ b/.github/workflows/deploy-preprod.yml @@ -22,7 +22,6 @@ jobs: release_version=${{ github.event.inputs.version }} fi echo "RELEASE_VERSION=$release_version" >> $GITHUB_ENV - - name: Configure AWS Credentials uses: aws-actions/configure-aws-credentials@v1 with: @@ -37,7 +36,6 @@ jobs: -o values.preprod.yaml \ -L https://api.github.com/repos/budibase/budibase-infra/contents/kubernetes/budibase-preprod/values.yaml wc -l values.preprod.yaml - - name: Deploy to Preprod Environment uses: budibase/helm@v1.8.0 with: @@ -64,4 +62,4 @@ jobs: with: webhook-url: ${{ secrets.PROD_DEPLOY_WEBHOOK_URL }} content: "Preprod Deployment Complete: ${{ env.RELEASE_VERSION }} deployed to Budibase Pre-prod." - embed-title: ${{ env.RELEASE_VERSION }} \ No newline at end of file + embed-title: ${{ env.RELEASE_VERSION }} diff --git a/charts/budibase/templates/app-service-deployment.yaml b/charts/budibase/templates/app-service-deployment.yaml index 89e7db7796..41b0dc48c9 100644 --- a/charts/budibase/templates/app-service-deployment.yaml +++ b/charts/budibase/templates/app-service-deployment.yaml @@ -62,16 +62,22 @@ spec: {{ end }} - name: ENABLE_ANALYTICS value: {{ .Values.globals.enableAnalytics | quote }} + - name: API_ENCRYPTION_KEY + value: {{ .Values.globals.apiEncryptionKey | quote }} - name: INTERNAL_API_KEY valueFrom: secretKeyRef: name: {{ template "budibase.fullname" . }} key: internalApiKey + - name: INTERNAL_API_KEY_FALLBACK + value: {{ .Values.globals.internalApiKeyFallback | quote }} - name: JWT_SECRET valueFrom: secretKeyRef: name: {{ template "budibase.fullname" . }} key: jwtSecret + - name: JWT_SECRET_FALLBACK + value: {{ .Values.globals.jwtSecretFallback | quote }} {{ if .Values.services.objectStore.region }} - name: AWS_REGION value: {{ .Values.services.objectStore.region }} diff --git a/charts/budibase/templates/worker-service-deployment.yaml b/charts/budibase/templates/worker-service-deployment.yaml index 0e053dfb5a..7886d55b28 100644 --- a/charts/budibase/templates/worker-service-deployment.yaml +++ b/charts/budibase/templates/worker-service-deployment.yaml @@ -62,16 +62,22 @@ spec: {{ else }} value: http://{{ .Release.Name }}-svc-couchdb:{{ .Values.services.couchdb.port }} {{ end }} + - name: API_ENCRYPTION_KEY + value: {{ .Values.globals.apiEncryptionKey | quote }} - name: INTERNAL_API_KEY valueFrom: secretKeyRef: name: {{ template "budibase.fullname" . }} key: internalApiKey + - name: INTERNAL_API_KEY_FALLBACK + value: {{ .Values.globals.internalApiKeyFallback | quote }} - name: JWT_SECRET valueFrom: secretKeyRef: name: {{ template "budibase.fullname" . }} key: jwtSecret + - name: JWT_SECRET_FALLBACK + value: {{ .Values.globals.jwtSecretFallback | quote }} {{ if .Values.services.objectStore.region }} - name: AWS_REGION value: {{ .Values.services.objectStore.region }} diff --git a/charts/budibase/values.yaml b/charts/budibase/values.yaml index 536af8560f..ed4ff014a9 100644 --- a/charts/budibase/values.yaml +++ b/charts/budibase/values.yaml @@ -96,9 +96,13 @@ globals: createSecrets: true # creates an internal API key, JWT secrets and redis password for you # if createSecrets is set to false, you can hard-code your secrets here + apiEncryptionKey: "" internalApiKey: "" jwtSecret: "" cdnUrl: "" + # fallback values used during live rotation + internalApiKeyFallback: "" + jwtSecretFallback: "" smtp: enabled: false diff --git a/hosting/.env b/hosting/.env index 07b506a6b2..c2b6d55eef 100644 --- a/hosting/.env +++ b/hosting/.env @@ -3,6 +3,7 @@ MAIN_PORT=10000 # This section contains all secrets pertaining to the system # These should be updated +API_ENCRYPTION_KEY=testsecret JWT_SECRET=testsecret MINIO_ACCESS_KEY=budibase MINIO_SECRET_KEY=budibase diff --git a/hosting/docker-compose.yaml b/hosting/docker-compose.yaml index d36937910f..bad34a20ea 100644 --- a/hosting/docker-compose.yaml +++ b/hosting/docker-compose.yaml @@ -17,6 +17,7 @@ services: INTERNAL_API_KEY: ${INTERNAL_API_KEY} BUDIBASE_ENVIRONMENT: ${BUDIBASE_ENVIRONMENT} PORT: 4002 + API_ENCRYPTION_KEY: ${API_ENCRYPTION_KEY} JWT_SECRET: ${JWT_SECRET} LOG_LEVEL: info SENTRY_DSN: https://a34ae347621946bf8acded18e5b7d4b8@o420233.ingest.sentry.io/5338131 @@ -40,6 +41,7 @@ services: SELF_HOSTED: 1 PORT: 4003 CLUSTER_PORT: ${MAIN_PORT} + API_ENCRYPTION_KEY: ${API_ENCRYPTION_KEY} JWT_SECRET: ${JWT_SECRET} MINIO_ACCESS_KEY: ${MINIO_ACCESS_KEY} MINIO_SECRET_KEY: ${MINIO_SECRET_KEY} diff --git a/hosting/hosting.properties b/hosting/hosting.properties index c5638a266f..6c1d9e5dbd 100644 --- a/hosting/hosting.properties +++ b/hosting/hosting.properties @@ -3,6 +3,7 @@ MAIN_PORT=10000 # This section contains all secrets pertaining to the system # These should be updated +API_ENCRYPTION_KEY=testsecret JWT_SECRET=testsecret MINIO_ACCESS_KEY=budibase MINIO_SECRET_KEY=budibase diff --git a/lerna.json b/lerna.json index 45ebb41470..b47cca4c23 100644 --- a/lerna.json +++ b/lerna.json @@ -1,5 +1,5 @@ { - "version": "2.4.20", + "version": "2.4.26", "npmClient": "yarn", "packages": [ "packages/*" diff --git a/packages/backend-core/package.json b/packages/backend-core/package.json index be57c3bc94..c27893051d 100644 --- a/packages/backend-core/package.json +++ b/packages/backend-core/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/backend-core", - "version": "2.4.20", + "version": "2.4.26", "description": "Budibase backend core libraries used in server and worker", "main": "dist/src/index.js", "types": "dist/src/index.d.ts", @@ -24,7 +24,7 @@ "dependencies": { "@budibase/nano": "10.1.2", "@budibase/pouchdb-replication-stream": "1.2.10", - "@budibase/types": "^2.4.20", + "@budibase/types": "^2.4.26", "@shopify/jest-koa-mocks": "5.0.1", "@techpass/passport-openidconnect": "0.3.2", "aws-cloudfront-sign": "2.2.0", diff --git a/packages/backend-core/src/auth/auth.ts b/packages/backend-core/src/auth/auth.ts index 7e6fe4bcee..26c7cd4e26 100644 --- a/packages/backend-core/src/auth/auth.ts +++ b/packages/backend-core/src/auth/auth.ts @@ -1,6 +1,5 @@ const _passport = require("koa-passport") const LocalStrategy = require("passport-local").Strategy -const JwtStrategy = require("passport-jwt").Strategy import { getGlobalDB } from "../context" import { Cookie } from "../constants" import { getSessionsForUser, invalidateSessions } from "../security/sessions" @@ -8,7 +7,6 @@ import { authenticated, csrf, google, - jwt as jwtPassport, local, oidc, tenancy, @@ -21,14 +19,11 @@ import { OIDCInnerConfig, PlatformLogoutOpts, SSOProviderType, - User, } from "@budibase/types" -import { logAlert } from "../logging" import * as events from "../events" import * as configs from "../configs" import { clearCookie, getCookie } from "../utils" import { ssoSaveUserNoOp } from "../middleware/passport/sso/sso" -import env from "../environment" const refresh = require("passport-oauth2-refresh") export { @@ -51,25 +46,6 @@ export const jwt = require("jsonwebtoken") // Strategies _passport.use(new LocalStrategy(local.options, local.authenticate)) -if (jwtPassport.options.secretOrKey) { - _passport.use(new JwtStrategy(jwtPassport.options, jwtPassport.authenticate)) -} else if (!env.DISABLE_JWT_WARNING) { - logAlert("No JWT Secret supplied, cannot configure JWT strategy") -} - -_passport.serializeUser((user: User, done: any) => done(null, user)) - -_passport.deserializeUser(async (user: User, done: any) => { - const db = getGlobalDB() - - try { - const dbUser = await db.get(user._id) - return done(null, dbUser) - } catch (err) { - console.error(`User not found`, err) - return done(null, false, { message: "User not found" }) - } -}) async function refreshOIDCAccessToken( chosenConfig: OIDCInnerConfig, diff --git a/packages/backend-core/src/cache/tests/writethrough.spec.ts b/packages/backend-core/src/cache/tests/writethrough.spec.ts index d346788121..a34f05e881 100644 --- a/packages/backend-core/src/cache/tests/writethrough.spec.ts +++ b/packages/backend-core/src/cache/tests/writethrough.spec.ts @@ -1,10 +1,13 @@ -import { structures, DBTestConfiguration } from "../../../tests" +import { + structures, + DBTestConfiguration, + expectFunctionWasCalledTimesWith, +} from "../../../tests" import { Writethrough } from "../writethrough" import { getDB } from "../../db" import tk from "timekeeper" -const START_DATE = Date.now() -tk.freeze(START_DATE) +tk.freeze(Date.now()) const DELAY = 5000 @@ -17,34 +20,99 @@ describe("writethrough", () => { const writethrough = new Writethrough(db, DELAY) const writethrough2 = new Writethrough(db2, DELAY) + const docId = structures.uuid() + + beforeEach(() => { + jest.clearAllMocks() + }) + describe("put", () => { - let first: any + let current: any it("should be able to store, will go to DB", async () => { await config.doInTenant(async () => { - const response = await writethrough.put({ _id: "test", value: 1 }) + const response = await writethrough.put({ + _id: docId, + value: 1, + }) const output = await db.get(response.id) - first = output + current = output expect(output.value).toBe(1) }) }) it("second put shouldn't update DB", async () => { await config.doInTenant(async () => { - const response = await writethrough.put({ ...first, value: 2 }) + const response = await writethrough.put({ ...current, value: 2 }) const output = await db.get(response.id) - expect(first._rev).toBe(output._rev) + expect(current._rev).toBe(output._rev) expect(output.value).toBe(1) }) }) it("should put it again after delay period", async () => { await config.doInTenant(async () => { - tk.freeze(START_DATE + DELAY + 1) - const response = await writethrough.put({ ...first, value: 3 }) + tk.freeze(Date.now() + DELAY + 1) + const response = await writethrough.put({ ...current, value: 3 }) const output = await db.get(response.id) - expect(response.rev).not.toBe(first._rev) + expect(response.rev).not.toBe(current._rev) expect(output.value).toBe(3) + + current = output + }) + }) + + it("should handle parallel DB updates ignoring conflicts", async () => { + await config.doInTenant(async () => { + tk.freeze(Date.now() + DELAY + 1) + const responses = await Promise.all([ + writethrough.put({ ...current, value: 4 }), + writethrough.put({ ...current, value: 4 }), + writethrough.put({ ...current, value: 4 }), + ]) + + const newRev = responses.map(x => x.rev).find(x => x !== current._rev) + expect(newRev).toBeDefined() + expect(responses.map(x => x.rev)).toEqual( + expect.arrayContaining([current._rev, current._rev, newRev]) + ) + expectFunctionWasCalledTimesWith( + console.warn, + 2, + "bb-warn: Ignoring redlock conflict in write-through cache" + ) + + const output = await db.get(current._id) + expect(output.value).toBe(4) + expect(output._rev).toBe(newRev) + + current = output + }) + }) + + it("should handle updates with documents falling behind", async () => { + await config.doInTenant(async () => { + tk.freeze(Date.now() + DELAY + 1) + + const id = structures.uuid() + await writethrough.put({ _id: id, value: 1 }) + const doc = await writethrough.get(id) + + // Updating document + tk.freeze(Date.now() + DELAY + 1) + await writethrough.put({ ...doc, value: 2 }) + + // Update with the old rev value + tk.freeze(Date.now() + DELAY + 1) + const res = await writethrough.put({ + ...doc, + value: 3, + }) + expect(res.ok).toBe(true) + + const output = await db.get(id) + expect(output.value).toBe(3) + expect(output._rev).toBe(res.rev) }) }) }) @@ -52,8 +120,8 @@ describe("writethrough", () => { describe("get", () => { it("should be able to retrieve", async () => { await config.doInTenant(async () => { - const response = await writethrough.get("test") - expect(response.value).toBe(3) + const response = await writethrough.get(docId) + expect(response.value).toBe(4) }) }) }) diff --git a/packages/backend-core/src/cache/writethrough.ts b/packages/backend-core/src/cache/writethrough.ts index dc889d5b18..a3b1ecc08d 100644 --- a/packages/backend-core/src/cache/writethrough.ts +++ b/packages/backend-core/src/cache/writethrough.ts @@ -1,7 +1,8 @@ import BaseCache from "./base" import { getWritethroughClient } from "../redis/init" import { logWarn } from "../logging" -import { Database } from "@budibase/types" +import { Database, Document, LockName, LockType } from "@budibase/types" +import * as locks from "../redis/redlockImpl" const DEFAULT_WRITE_RATE_MS = 10000 let CACHE: BaseCache | null = null @@ -27,44 +28,62 @@ function makeCacheItem(doc: any, lastWrite: number | null = null): CacheItem { return { doc, lastWrite: lastWrite || Date.now() } } -export async function put( +async function put( db: Database, - doc: any, + doc: Document, writeRateMs: number = DEFAULT_WRITE_RATE_MS ) { const cache = await getCache() const key = doc._id - let cacheItem: CacheItem | undefined = await cache.get(makeCacheKey(db, key)) + let cacheItem: CacheItem | undefined + if (key) { + cacheItem = await cache.get(makeCacheKey(db, key)) + } const updateDb = !cacheItem || cacheItem.lastWrite < Date.now() - writeRateMs let output = doc if (updateDb) { - const writeDb = async (toWrite: any) => { - // doc should contain the _id and _rev - const response = await db.put(toWrite) - output = { - ...doc, - _id: response.id, - _rev: response.rev, - } - } - try { - await writeDb(doc) - } catch (err: any) { - if (err.status !== 409) { - throw err - } else { - // Swallow 409s but log them - logWarn(`Ignoring conflict in write-through cache`) + const lockResponse = await locks.doWithLock( + { + type: LockType.TRY_ONCE, + name: LockName.PERSIST_WRITETHROUGH, + resource: key, + ttl: 1000, + }, + async () => { + const writeDb = async (toWrite: any) => { + // doc should contain the _id and _rev + const response = await db.put(toWrite, { force: true }) + output = { + ...doc, + _id: response.id, + _rev: response.rev, + } + } + try { + await writeDb(doc) + } catch (err: any) { + if (err.status !== 409) { + throw err + } else { + // Swallow 409s but log them + logWarn(`Ignoring conflict in write-through cache`) + } + } } + ) + if (!lockResponse.executed) { + logWarn(`Ignoring redlock conflict in write-through cache`) } } // if we are updating the DB then need to set the lastWrite to now cacheItem = makeCacheItem(output, updateDb ? null : cacheItem?.lastWrite) - await cache.store(makeCacheKey(db, key), cacheItem) + if (output._id) { + await cache.store(makeCacheKey(db, output._id), cacheItem) + } return { ok: true, id: output._id, rev: output._rev } } -export async function get(db: Database, id: string): Promise { +async function get(db: Database, id: string): Promise { const cache = await getCache() const cacheKey = makeCacheKey(db, id) let cacheItem: CacheItem = await cache.get(cacheKey) @@ -76,11 +95,7 @@ export async function get(db: Database, id: string): Promise { return cacheItem.doc } -export async function remove( - db: Database, - docOrId: any, - rev?: any -): Promise { +async function remove(db: Database, docOrId: any, rev?: any): Promise { const cache = await getCache() if (!docOrId) { throw new Error("No ID/Rev provided.") diff --git a/packages/backend-core/src/db/lucene.ts b/packages/backend-core/src/db/lucene.ts index cba2f0138a..71ce4ba9ac 100644 --- a/packages/backend-core/src/db/lucene.ts +++ b/packages/backend-core/src/db/lucene.ts @@ -199,6 +199,10 @@ export class QueryBuilder { return this } + setAllOr() { + this.query.allOr = true + } + handleSpaces(input: string) { if (this.noEscaping) { return input @@ -236,6 +240,36 @@ export class QueryBuilder { return value } + isMultiCondition() { + let count = 0 + for (let filters of Object.values(this.query)) { + // not contains is one massive filter in allOr mode + if (typeof filters === "object") { + count += Object.keys(filters).length + } + } + return count > 1 + } + + compressFilters(filters: Record) { + const compressed: typeof filters = {} + for (let key of Object.keys(filters)) { + const finalKey = removeKeyNumbering(key) + if (compressed[finalKey]) { + compressed[finalKey] = compressed[finalKey].concat(filters[key]) + } else { + compressed[finalKey] = filters[key] + } + } + // add prefixes back + const final: typeof filters = {} + let count = 1 + for (let [key, value] of Object.entries(compressed)) { + final[`${count++}:${key}`] = value + } + return final + } + buildSearchQuery() { const builder = this let allOr = this.query && this.query.allOr @@ -272,9 +306,9 @@ export class QueryBuilder { } const notContains = (key: string, value: any) => { - // @ts-ignore - const allPrefix = allOr === "" ? "*:* AND" : "" - return allPrefix + "NOT " + contains(key, value) + const allPrefix = allOr ? "*:* AND " : "" + const mode = allOr ? "AND" : undefined + return allPrefix + "NOT " + contains(key, value, mode) } const containsAny = (key: string, value: any) => { @@ -299,21 +333,32 @@ export class QueryBuilder { return `${key}:(${orStatement})` } - function build(structure: any, queryFn: any) { + function build( + structure: any, + queryFn: (key: string, value: any) => string | null, + opts?: { returnBuilt?: boolean; mode?: string } + ) { + let built = "" for (let [key, value] of Object.entries(structure)) { // check for new format - remove numbering if needed key = removeKeyNumbering(key) key = builder.preprocess(builder.handleSpaces(key), { escape: true, }) - const expression = queryFn(key, value) + let expression = queryFn(key, value) if (expression == null) { continue } - if (query.length > 0) { - query += ` ${allOr ? "OR" : "AND"} ` + if (built.length > 0 || query.length > 0) { + const mode = opts?.mode ? opts.mode : allOr ? "OR" : "AND" + built += ` ${mode} ` } - query += expression + built += expression + } + if (opts?.returnBuilt) { + return built + } else { + query += built } } @@ -384,14 +429,14 @@ export class QueryBuilder { build(this.query.contains, contains) } if (this.query.notContains) { - build(this.query.notContains, notContains) + build(this.compressFilters(this.query.notContains), notContains) } if (this.query.containsAny) { build(this.query.containsAny, containsAny) } // make sure table ID is always added as an AND if (tableId) { - query = `(${query})` + query = this.isMultiCondition() ? `(${query})` : query allOr = false build({ tableId }, equal) } diff --git a/packages/backend-core/src/db/tests/lucene.spec.ts b/packages/backend-core/src/db/tests/lucene.spec.ts index 23b01e18df..52017cc94c 100644 --- a/packages/backend-core/src/db/tests/lucene.spec.ts +++ b/packages/backend-core/src/db/tests/lucene.spec.ts @@ -6,9 +6,13 @@ import { QueryBuilder, paginatedSearch, fullSearch } from "../lucene" const INDEX_NAME = "main" const index = `function(doc) { - let props = ["property", "number"] + let props = ["property", "number", "array"] for (let key of props) { - if (doc[key]) { + if (Array.isArray(doc[key])) { + for (let val of doc[key]) { + index(key, val) + } + } else if (doc[key]) { index(key, doc[key]) } } @@ -21,9 +25,14 @@ describe("lucene", () => { dbName = `db-${newid()}` // create the DB for testing db = getDB(dbName) - await db.put({ _id: newid(), property: "word" }) - await db.put({ _id: newid(), property: "word2" }) - await db.put({ _id: newid(), property: "word3", number: 1 }) + await db.put({ _id: newid(), property: "word", array: ["1", "4"] }) + await db.put({ _id: newid(), property: "word2", array: ["3", "1"] }) + await db.put({ + _id: newid(), + property: "word3", + number: 1, + array: ["1", "2"], + }) }) it("should be able to create a lucene index", async () => { @@ -118,6 +127,15 @@ describe("lucene", () => { const resp = await builder.run() expect(resp.rows.length).toBe(2) }) + + it("should be able to perform an or not contains search", async () => { + const builder = new QueryBuilder(dbName, INDEX_NAME) + builder.addNotContains("array", ["1"]) + builder.addNotContains("array", ["2"]) + builder.setAllOr() + const resp = await builder.run() + expect(resp.rows.length).toBe(2) + }) }) describe("paginated search", () => { diff --git a/packages/backend-core/src/environment.ts b/packages/backend-core/src/environment.ts index 8dc2cce487..f1c96c7fec 100644 --- a/packages/backend-core/src/environment.ts +++ b/packages/backend-core/src/environment.ts @@ -30,6 +30,12 @@ const DefaultBucketName = { const selfHosted = !!parseInt(process.env.SELF_HOSTED || "") +function getAPIEncryptionKey() { + return process.env.API_ENCRYPTION_KEY + ? process.env.API_ENCRYPTION_KEY + : process.env.JWT_SECRET // fallback to the JWT_SECRET used historically +} + const environment = { isTest, isJest, @@ -39,7 +45,9 @@ const environment = { }, JS_BCRYPT: process.env.JS_BCRYPT, JWT_SECRET: process.env.JWT_SECRET, + JWT_SECRET_FALLBACK: process.env.JWT_SECRET_FALLBACK, ENCRYPTION_KEY: process.env.ENCRYPTION_KEY, + API_ENCRYPTION_KEY: getAPIEncryptionKey(), COUCH_DB_URL: process.env.COUCH_DB_URL || "http://localhost:4005", COUCH_DB_USERNAME: process.env.COUCH_DB_USER, COUCH_DB_PASSWORD: process.env.COUCH_DB_PASSWORD, @@ -55,6 +63,7 @@ const environment = { MINIO_URL: process.env.MINIO_URL, MINIO_ENABLED: process.env.MINIO_ENABLED || 1, INTERNAL_API_KEY: process.env.INTERNAL_API_KEY, + INTERNAL_API_KEY_FALLBACK: process.env.INTERNAL_API_KEY_FALLBACK, MULTI_TENANCY: process.env.MULTI_TENANCY, ACCOUNT_PORTAL_URL: process.env.ACCOUNT_PORTAL_URL || "https://account.budibase.app", diff --git a/packages/backend-core/src/middleware/authenticated.ts b/packages/backend-core/src/middleware/authenticated.ts index 0708581570..5e546b4c1c 100644 --- a/packages/backend-core/src/middleware/authenticated.ts +++ b/packages/backend-core/src/middleware/authenticated.ts @@ -1,5 +1,10 @@ import { Cookie, Header } from "../constants" -import { getCookie, clearCookie, openJwt } from "../utils" +import { + getCookie, + clearCookie, + openJwt, + isValidInternalAPIKey, +} from "../utils" import { getUser } from "../cache/user" import { getSession, updateSessionTTL } from "../security/sessions" import { buildMatcherRegex, matches } from "./matchers" @@ -35,7 +40,9 @@ function finalise(ctx: any, opts: FinaliseOpts = {}) { } async function checkApiKey(apiKey: string, populateUser?: Function) { - if (apiKey === env.INTERNAL_API_KEY) { + // check both the primary and the fallback internal api keys + // this allows for rotation + if (isValidInternalAPIKey(apiKey)) { return { valid: true } } const decrypted = decrypt(apiKey) diff --git a/packages/backend-core/src/middleware/index.ts b/packages/backend-core/src/middleware/index.ts index addeac6a1a..dce07168d4 100644 --- a/packages/backend-core/src/middleware/index.ts +++ b/packages/backend-core/src/middleware/index.ts @@ -1,4 +1,3 @@ -export * as jwt from "./passport/jwt" export * as local from "./passport/local" export * as google from "./passport/sso/google" export * as oidc from "./passport/sso/oidc" diff --git a/packages/backend-core/src/middleware/internalApi.ts b/packages/backend-core/src/middleware/internalApi.ts index fff761928b..dc73cd6b66 100644 --- a/packages/backend-core/src/middleware/internalApi.ts +++ b/packages/backend-core/src/middleware/internalApi.ts @@ -1,13 +1,21 @@ -import env from "../environment" import { Header } from "../constants" import { BBContext } from "@budibase/types" +import { isValidInternalAPIKey } from "../utils" /** * API Key only endpoint. */ export default async (ctx: BBContext, next: any) => { const apiKey = ctx.request.headers[Header.API_KEY] - if (apiKey !== env.INTERNAL_API_KEY) { + if (!apiKey) { + ctx.throw(403, "Unauthorized") + } + + if (Array.isArray(apiKey)) { + ctx.throw(403, "Unauthorized") + } + + if (!isValidInternalAPIKey(apiKey)) { ctx.throw(403, "Unauthorized") } diff --git a/packages/backend-core/src/middleware/passport/jwt.ts b/packages/backend-core/src/middleware/passport/jwt.ts deleted file mode 100644 index 95dc8f2656..0000000000 --- a/packages/backend-core/src/middleware/passport/jwt.ts +++ /dev/null @@ -1,19 +0,0 @@ -import { Cookie } from "../../constants" -import env from "../../environment" -import { authError } from "./utils" -import { BBContext } from "@budibase/types" - -export const options = { - secretOrKey: env.JWT_SECRET, - jwtFromRequest: function (ctx: BBContext) { - return ctx.cookies.get(Cookie.Auth) - }, -} - -export async function authenticate(jwt: Function, done: Function) { - try { - return done(null, jwt) - } catch (err) { - return authError(done, "JWT invalid", err) - } -} diff --git a/packages/backend-core/src/redis/redlockImpl.ts b/packages/backend-core/src/redis/redlockImpl.ts index 136d7f5d33..5e71488689 100644 --- a/packages/backend-core/src/redis/redlockImpl.ts +++ b/packages/backend-core/src/redis/redlockImpl.ts @@ -24,7 +24,7 @@ const getClient = async (type: LockType): Promise => { } } -export const OPTIONS = { +const OPTIONS = { TRY_ONCE: { // immediately throws an error if the lock is already held retryCount: 0, @@ -56,14 +56,29 @@ export const OPTIONS = { }, } -export const newRedlock = async (opts: Options = {}) => { +const newRedlock = async (opts: Options = {}) => { let options = { ...OPTIONS.DEFAULT, ...opts } const redisWrapper = await getLockClient() const client = redisWrapper.getClient() return new Redlock([client], options) } -export const doWithLock = async (opts: LockOptions, task: any) => { +type SuccessfulRedlockExecution = { + executed: true + result: T +} +type UnsuccessfulRedlockExecution = { + executed: false +} + +type RedlockExecution = + | SuccessfulRedlockExecution + | UnsuccessfulRedlockExecution + +export const doWithLock = async ( + opts: LockOptions, + task: () => Promise +): Promise> => { const redlock = await getClient(opts.type) let lock try { @@ -73,8 +88,8 @@ export const doWithLock = async (opts: LockOptions, task: any) => { let name: string = `lock:${prefix}_${opts.name}` // add additional unique name if required - if (opts.nameSuffix) { - name = name + `_${opts.nameSuffix}` + if (opts.resource) { + name = name + `_${opts.resource}` } // create the lock @@ -83,7 +98,7 @@ export const doWithLock = async (opts: LockOptions, task: any) => { // perform locked task // need to await to ensure completion before unlocking const result = await task() - return result + return { executed: true, result } } catch (e: any) { console.warn("lock error") // lock limit exceeded @@ -92,7 +107,7 @@ export const doWithLock = async (opts: LockOptions, task: any) => { // don't throw for try-once locks, they will always error // due to retry count (0) exceeded console.warn(e) - return + return { executed: false } } else { console.error(e) throw e diff --git a/packages/backend-core/src/security/encryption.ts b/packages/backend-core/src/security/encryption.ts index d0707cb850..d2c8f18f73 100644 --- a/packages/backend-core/src/security/encryption.ts +++ b/packages/backend-core/src/security/encryption.ts @@ -8,7 +8,7 @@ const RANDOM_BYTES = 16 const STRETCH_LENGTH = 32 export enum SecretOption { - JWT = "jwt", + API = "api", ENCRYPTION = "encryption", } @@ -19,10 +19,10 @@ function getSecret(secretOption: SecretOption): string { secret = env.ENCRYPTION_KEY secretName = "ENCRYPTION_KEY" break - case SecretOption.JWT: + case SecretOption.API: default: - secret = env.JWT_SECRET - secretName = "JWT_SECRET" + secret = env.API_ENCRYPTION_KEY + secretName = "API_ENCRYPTION_KEY" break } if (!secret) { @@ -37,7 +37,7 @@ function stretchString(string: string, salt: Buffer) { export function encrypt( input: string, - secretOption: SecretOption = SecretOption.JWT + secretOption: SecretOption = SecretOption.API ) { const salt = crypto.randomBytes(RANDOM_BYTES) const stretched = stretchString(getSecret(secretOption), salt) @@ -50,7 +50,7 @@ export function encrypt( export function decrypt( input: string, - secretOption: SecretOption = SecretOption.JWT + secretOption: SecretOption = SecretOption.API ) { const [salt, encrypted] = input.split(SEPARATOR) const saltBuffer = Buffer.from(salt, "hex") diff --git a/packages/backend-core/src/users.ts b/packages/backend-core/src/users.ts index 8963f7c141..dfc544c3ed 100644 --- a/packages/backend-core/src/users.ts +++ b/packages/backend-core/src/users.ts @@ -5,6 +5,8 @@ import { generateAppUserID, queryGlobalView, UNICODE_MAX, + DocumentType, + SEPARATOR, directCouchFind, } from "./db" import { BulkDocsResponse, User } from "@budibase/types" @@ -45,6 +47,16 @@ export const bulkGetGlobalUsersById = async ( return users } +export const getAllUserIds = async () => { + const db = getGlobalDB() + const startKey = `${DocumentType.USER}${SEPARATOR}` + const response = await db.allDocs({ + startkey: startKey, + endkey: `${startKey}${UNICODE_MAX}`, + }) + return response.rows.map(row => row.id) +} + export const bulkUpdateGlobalUsers = async (users: User[]) => { const db = getGlobalDB() return (await db.bulkDocs(users)) as BulkDocsResponse diff --git a/packages/backend-core/src/utils/utils.ts b/packages/backend-core/src/utils/utils.ts index 3efd40ca80..7c222a9831 100644 --- a/packages/backend-core/src/utils/utils.ts +++ b/packages/backend-core/src/utils/utils.ts @@ -1,5 +1,4 @@ import { getAllApps, queryGlobalView } from "../db" -import { options } from "../middleware/passport/jwt" import { Header, MAX_VALID_DATE, @@ -133,7 +132,30 @@ export function openJwt(token: string) { if (!token) { return token } - return jwt.verify(token, options.secretOrKey) + try { + return jwt.verify(token, env.JWT_SECRET) + } catch (e) { + if (env.JWT_SECRET_FALLBACK) { + // fallback to enable rotation + return jwt.verify(token, env.JWT_SECRET_FALLBACK) + } else { + throw e + } + } +} + +export function isValidInternalAPIKey(apiKey: string) { + if (env.INTERNAL_API_KEY && env.INTERNAL_API_KEY === apiKey) { + return true + } + // fallback to enable rotation + if ( + env.INTERNAL_API_KEY_FALLBACK && + env.INTERNAL_API_KEY_FALLBACK === apiKey + ) { + return true + } + return false } /** @@ -165,7 +187,7 @@ export function setCookie( opts = { sign: true } ) { if (value && opts && opts.sign) { - value = jwt.sign(value, options.secretOrKey) + value = jwt.sign(value, env.JWT_SECRET) } const config: SetOption = { diff --git a/packages/backend-core/tests/utilities/index.ts b/packages/backend-core/tests/utilities/index.ts index efe014908b..1c73216d76 100644 --- a/packages/backend-core/tests/utilities/index.ts +++ b/packages/backend-core/tests/utilities/index.ts @@ -4,4 +4,6 @@ export { generator } from "./structures" export * as testEnv from "./testEnv" export * as testContainerUtils from "./testContainerUtils" +export * from "./jestUtils" + export { default as DBTestConfiguration } from "./DBTestConfiguration" diff --git a/packages/backend-core/tests/utilities/jestUtils.ts b/packages/backend-core/tests/utilities/jestUtils.ts new file mode 100644 index 0000000000..d84eac548c --- /dev/null +++ b/packages/backend-core/tests/utilities/jestUtils.ts @@ -0,0 +1,9 @@ +export function expectFunctionWasCalledTimesWith( + jestFunction: any, + times: number, + argument: any +) { + expect( + jestFunction.mock.calls.filter((call: any) => call[0] === argument).length + ).toBe(times) +} diff --git a/packages/backend-core/tests/utilities/structures/db.ts b/packages/backend-core/tests/utilities/structures/db.ts index e25b707cb9..f4a677e777 100644 --- a/packages/backend-core/tests/utilities/structures/db.ts +++ b/packages/backend-core/tests/utilities/structures/db.ts @@ -1,5 +1,12 @@ +import { structures } from ".." import { newid } from "../../../src/newid" export function id() { return `db_${newid()}` } + +export function rev() { + return `${structures.generator.character({ + numeric: true, + })}-${structures.uuid().replace(/-/, "")}` +} diff --git a/packages/bbui/package.json b/packages/bbui/package.json index 1145483efe..6f37501ca6 100644 --- a/packages/bbui/package.json +++ b/packages/bbui/package.json @@ -1,7 +1,7 @@ { "name": "@budibase/bbui", "description": "A UI solution used in the different Budibase projects.", - "version": "2.4.20", + "version": "2.4.26", "license": "MPL-2.0", "svelte": "src/index.js", "module": "dist/bbui.es.js", @@ -38,8 +38,8 @@ ], "dependencies": { "@adobe/spectrum-css-workflow-icons": "1.2.1", - "@budibase/shared-core": "^2.4.20", - "@budibase/string-templates": "^2.4.20", + "@budibase/shared-core": "^2.4.26", + "@budibase/string-templates": "^2.4.26", "@spectrum-css/accordion": "3.0.24", "@spectrum-css/actionbutton": "1.0.1", "@spectrum-css/actiongroup": "1.0.1", diff --git a/packages/bbui/src/ActionButton/ActionButton.svelte b/packages/bbui/src/ActionButton/ActionButton.svelte index 01f5033e6c..60c8bec80b 100644 --- a/packages/bbui/src/ActionButton/ActionButton.svelte +++ b/packages/bbui/src/ActionButton/ActionButton.svelte @@ -113,6 +113,9 @@ .spectrum-ActionButton--quiet { padding: 0 8px; } + .spectrum-ActionButton--quiet.is-selected { + color: var(--spectrum-global-color-gray-900); + } .is-selected:not(.emphasized) .spectrum-Icon { color: var(--spectrum-global-color-gray-900); } diff --git a/packages/bbui/src/Actions/position_dropdown.js b/packages/bbui/src/Actions/position_dropdown.js index abc7188985..ecbb5747c4 100644 --- a/packages/bbui/src/Actions/position_dropdown.js +++ b/packages/bbui/src/Actions/position_dropdown.js @@ -31,6 +31,7 @@ export default function positionDropdown(element, opts) { styles.top = anchorBounds.top } else if (window.innerHeight - anchorBounds.bottom < 100) { styles.top = anchorBounds.top - elementBounds.height - offset + styles.maxHeight = 240 } else { styles.top = anchorBounds.bottom + offset styles.maxHeight = window.innerHeight - anchorBounds.bottom - 20 diff --git a/packages/builder/src/components/common/inputs/CopyInput.svelte b/packages/bbui/src/Input/CopyInput.svelte similarity index 91% rename from packages/builder/src/components/common/inputs/CopyInput.svelte rename to packages/bbui/src/Input/CopyInput.svelte index fe7746d1f9..b4d6e5107f 100644 --- a/packages/builder/src/components/common/inputs/CopyInput.svelte +++ b/packages/bbui/src/Input/CopyInput.svelte @@ -1,5 +1,7 @@ + + + +
+
+ + +
+
+ {#each categories as [name, results], catIdx} +
+ {name} +
+ {#each results as command, cmdIdx} +
runAction(command)} + class:selected={command.idx === selected} + > + + {command.type}:  +
+ {command.name} +
+
+ {/each} +
+
+ {/each} +
+
+ + + diff --git a/packages/builder/src/components/deploy/DeployModal.svelte b/packages/builder/src/components/deploy/DeployModal.svelte index 2b881e524e..05b620da71 100644 --- a/packages/builder/src/components/deploy/DeployModal.svelte +++ b/packages/builder/src/components/deploy/DeployModal.svelte @@ -5,12 +5,12 @@ notifications, ModalContent, Layout, + ProgressCircle, + CopyInput, } from "@budibase/bbui" import { API } from "api" import analytics, { Events, EventSource } from "analytics" import { store } from "builderStore" - import { ProgressCircle } from "@budibase/bbui" - import CopyInput from "components/common/inputs/CopyInput.svelte" import TourWrap from "../portal/onboarding/TourWrap.svelte" import { TOUR_STEP_KEYS } from "../portal/onboarding/tours.js" diff --git a/packages/builder/src/components/deploy/VersionModal.svelte b/packages/builder/src/components/deploy/VersionModal.svelte index 23d9fd83a0..f357cc7820 100644 --- a/packages/builder/src/components/deploy/VersionModal.svelte +++ b/packages/builder/src/components/deploy/VersionModal.svelte @@ -24,7 +24,10 @@ let updateModal $: appId = $store.appId - $: updateAvailable = clientPackage.version !== $store.version + $: updateAvailable = + clientPackage.version && + $store.version && + clientPackage.version !== $store.version $: revertAvailable = $store.revertableVersion != null const refreshAppPackage = async () => { diff --git a/packages/builder/src/components/design/Panel.svelte b/packages/builder/src/components/design/Panel.svelte index a1e3bd7eb7..6c8753a99e 100644 --- a/packages/builder/src/components/design/Panel.svelte +++ b/packages/builder/src/components/design/Panel.svelte @@ -14,10 +14,11 @@ export let borderRight = false let wide = false + $: customHeaderContent = $$slots["panel-header-content"]
-
+
{#if showBackButton} {/if} @@ -43,6 +44,13 @@ {/if}
+ + {#if customHeaderContent} + + + + {/if} +
@@ -116,4 +124,10 @@ justify-content: flex-start; align-items: stretch; } + .header.custom { + border: none; + } + .custom-content-wrap { + border-bottom: var(--border-light); + } diff --git a/packages/builder/src/components/portal/onboarding/TourPopover.svelte b/packages/builder/src/components/portal/onboarding/TourPopover.svelte index 37ed0c6350..68e2e68a49 100644 --- a/packages/builder/src/components/portal/onboarding/TourPopover.svelte +++ b/packages/builder/src/components/portal/onboarding/TourPopover.svelte @@ -15,20 +15,12 @@ $: tourKey = $store.tourKey $: tourStepKey = $store.tourStepKey - const initTour = targetKey => { - if (!targetKey) { + const updateTourStep = (targetStepKey, tourKey) => { + if (!tourKey) { return } - tourSteps = [...TOURS[targetKey]] - tourStepIdx = 0 - tourStep = { ...tourSteps[tourStepIdx] } - } - - $: initTour(tourKey) - - const updateTourStep = targetStepKey => { if (!tourSteps?.length) { - return + tourSteps = [...TOURS[tourKey]] } tourStepIdx = getCurrentStepIdx(tourSteps, targetStepKey) lastStep = tourStepIdx + 1 == tourSteps.length @@ -36,7 +28,7 @@ tourStep.onLoad() } - $: updateTourStep(tourStepKey) + $: updateTourStep(tourStepKey, tourKey) const showPopover = (tourStep, tourNodes, popover) => { if (!tourStep) { diff --git a/packages/builder/src/components/portal/onboarding/TourWrap.svelte b/packages/builder/src/components/portal/onboarding/TourWrap.svelte index 2fff8507c5..fd47d7b48e 100644 --- a/packages/builder/src/components/portal/onboarding/TourWrap.svelte +++ b/packages/builder/src/components/portal/onboarding/TourWrap.svelte @@ -8,20 +8,28 @@ let currentTourStep let ready = false + let registered = false let handler + const registerTourNode = (tourKey, stepKey) => { + if (ready && !registered && tourKey) { + currentTourStep = TOURS[tourKey].find(step => step.id === stepKey) + if (!currentTourStep) { + return + } + const elem = document.querySelector(currentTourStep.query) + handler = tourHandler(elem, stepKey) + registered = true + } + } + + $: tourKeyWatch = $store.tourKey + $: registerTourNode(tourKeyWatch, tourStepKey, ready) + onMount(() => { - if (!$store.tourKey) return - - currentTourStep = TOURS[$store.tourKey].find( - step => step.id === tourStepKey - ) - if (!currentTourStep) return - - const elem = document.querySelector(currentTourStep.query) - handler = tourHandler(elem, tourStepKey) ready = true }) + onDestroy(() => { if (handler) { handler.destroy() diff --git a/packages/builder/src/components/settings/APIKeyModal.svelte b/packages/builder/src/components/settings/APIKeyModal.svelte index 6fb2c88634..a252691ace 100644 --- a/packages/builder/src/components/settings/APIKeyModal.svelte +++ b/packages/builder/src/components/settings/APIKeyModal.svelte @@ -1,9 +1,7 @@ + +
{ query = e.target.value.trim() }} diff --git a/packages/builder/src/pages/builder/app/[application]/_layout.svelte b/packages/builder/src/pages/builder/app/[application]/_layout.svelte index a4b4982fca..ff728312c1 100644 --- a/packages/builder/src/pages/builder/app/[application]/_layout.svelte +++ b/packages/builder/src/pages/builder/app/[application]/_layout.svelte @@ -10,6 +10,7 @@ Tabs, Tab, Heading, + Modal, notifications, } from "@budibase/bbui" @@ -18,6 +19,7 @@ import { isActive, goto, layout, redirect } from "@roxi/routify" import { capitalise } from "helpers" import { onMount, onDestroy } from "svelte" + import CommandPalette from "components/commandPalette/CommandPalette.svelte" import TourWrap from "components/portal/onboarding/TourWrap.svelte" import TourPopover from "components/portal/onboarding/TourPopover.svelte" import BuilderSidePanel from "./_components/BuilderSidePanel.svelte" @@ -25,12 +27,9 @@ export let application - // Get Package and set store let promise = getPackage() - // let betaAccess = false - - // Sync once when you load the app let hasSynced = false + let commandPaletteModal $: selected = capitalise( $layout.children.find(layout => $isActive(layout.path))?.title ?? "data" @@ -50,7 +49,6 @@ $redirect("../../") } } - // Handles navigation between frontend, backend, automation. // This remembers your last place on each of the sections // e.g. if one of your screens is selected on front end, then @@ -67,6 +65,14 @@ }) } + // Event handler for the command palette + const handleKeyDown = e => { + if (e.key === "k" && (e.ctrlKey || e.metaKey)) { + e.preventDefault() + commandPaletteModal.toggle() + } + } + const initTour = async () => { // Check if onboarding is enabled. if (isEnabled(TENANT_FEATURE_FLAGS.ONBOARDING_TOUR)) { @@ -120,89 +126,91 @@ }) -{#await promise} - -
-{:then _} - + - {#if $store.builderSidePanel} - - {/if} +{#if $store.builderSidePanel} + +{/if} -
-
-
- -
- -
- $goto("../../portal/apps")}> - Exit to portal - - $goto(`../../portal/overview/${application}`)} - > - Overview - - - $goto(`../../portal/overview/${application}/access`)} - > - Access - - - $goto(`../../portal/overview/${application}/automation-history`)} - > - Automation history - - - $goto(`../../portal/overview/${application}/backups`)} - > - Backups - +
+
+
+ +
+ +
+ $goto("../../portal/apps")}> + Exit to portal + + $goto(`../../portal/overview/${application}`)} + > + Overview + + $goto(`../../portal/overview/${application}/access`)} + > + Access + + + $goto(`../../portal/overview/${application}/automation-history`)} + > + Automation history + + $goto(`../../portal/overview/${application}/backups`)} + > + Backups + - - $goto(`../../portal/overview/${application}/name-and-url`)} - > - Name and URL - - - $goto(`../../portal/overview/${application}/version`)} - > - Version - - - {$store.name || "App"} -
-
- - {#each $layout.children as { path, title }} - - - - {/each} - -
-
- -
+ + $goto(`../../portal/overview/${application}/name-and-url`)} + > + Name and URL + + $goto(`../../portal/overview/${application}/version`)} + > + Version + + + {$store.name} +
+
+ + {#each $layout.children as { path, title }} + + + + {/each} + +
+
+
-
-{:catch error} -

Something went wrong: {error.message}

-{/await} + {#await promise} + +
+ {:then _} + + {:catch error} +

Something went wrong: {error.message}

+ {/await} +
+ + + + + diff --git a/packages/builder/src/pages/builder/portal/_components/UpgradeButton.svelte b/packages/builder/src/pages/builder/portal/_components/UpgradeButton.svelte index d8481efc41..b67693a552 100644 --- a/packages/builder/src/pages/builder/portal/_components/UpgradeButton.svelte +++ b/packages/builder/src/pages/builder/portal/_components/UpgradeButton.svelte @@ -1,11 +1,11 @@ -{#if isEnabled(TENANT_FEATURE_FLAGS.LICENSING)} +{#if isEnabled(TENANT_FEATURE_FLAGS.LICENSING) && !$licensing.isEnterprisePlan} {#if $admin.cloud && $auth?.user?.accountPortalAccess}