diff --git a/packages/backend-core/src/context/identity.ts b/packages/backend-core/src/context/identity.ts index 648dd1b5fd..84de3b68c9 100644 --- a/packages/backend-core/src/context/identity.ts +++ b/packages/backend-core/src/context/identity.ts @@ -5,6 +5,8 @@ import { isCloudAccount, Account, AccountUserContext, + UserContext, + Ctx, } from "@budibase/types" import * as context from "." @@ -16,15 +18,22 @@ export function doInIdentityContext(identity: IdentityContext, task: any) { return context.doInIdentityContext(identity, task) } -export function doInUserContext(user: User, task: any) { - const userContext: any = { +// used in server/worker +export function doInUserContext(user: User, ctx: Ctx, task: any) { + const userContext: UserContext = { ...user, _id: user._id as string, type: IdentityType.USER, + hostInfo: { + ipAddress: ctx.request.ip, + // filled in by koa-useragent package + userAgent: ctx.userAgent._agent.source, + }, } return doInIdentityContext(userContext, task) } +// used in account portal export function doInAccountContext(account: Account, task: any) { const _id = getAccountUserId(account) const tenantId = account.tenantId diff --git a/packages/backend-core/src/events/events.ts b/packages/backend-core/src/events/events.ts index ba4b0a7448..6176fb6724 100644 --- a/packages/backend-core/src/events/events.ts +++ b/packages/backend-core/src/events/events.ts @@ -1,13 +1,37 @@ -import { Event, IdentityType, AuditLogFn } from "@budibase/types" +import { AuditLogFn, Event, IdentityType, HostInfo } from "@budibase/types" import { processors } from "./processors" import identification from "./identification" import { getAppId } from "../context" import * as backfill from "./backfill" +import { createQueue, JobQueue } from "../queue" +import BullQueue from "bull" -let writeAuditLogs: AuditLogFn | undefined +type AuditLogEvent = { + event: Event + properties: any + opts: { + timestamp?: string | number + userId?: string + appId?: string + hostInfo?: HostInfo + } +} + +let auditLogsEnabled = false +let auditLogQueue: BullQueue.Queue export const configure = (fn: AuditLogFn) => { - writeAuditLogs = fn + auditLogsEnabled = true + const writeAuditLogs = fn + auditLogQueue = createQueue(JobQueue.AUDIT_LOG) + return auditLogQueue.process(async job => { + await writeAuditLogs(job.data.event, job.data.properties, { + userId: job.data.opts.userId, + timestamp: job.data.opts.timestamp, + appId: job.data.opts.appId, + hostInfo: job.data.opts.hostInfo, + }) + }) } export const publishEvent = async ( @@ -21,16 +45,22 @@ export const publishEvent = async ( const backfilling = await backfill.isBackfillingEvent(event) // no backfill - send the event and exit if (!backfilling) { - // only audit log actual events, don't include backfills - const userId = identity.type === IdentityType.USER ? identity.id : undefined - if (writeAuditLogs) { - await writeAuditLogs(event, properties, { - userId, - timestamp, - appId: getAppId(), + await processors.processEvent(event, identity, properties, timestamp) + if (auditLogsEnabled) { + // only audit log actual events, don't include backfills + const userId = + identity.type === IdentityType.USER ? identity.id : undefined + // add to event queue, rather than just writing immediately + await auditLogQueue.add({ + event, + properties, + opts: { + userId, + timestamp, + appId: getAppId(), + }, }) } - await processors.processEvent(event, identity, properties, timestamp) return } diff --git a/packages/backend-core/src/events/identification.ts b/packages/backend-core/src/events/identification.ts index 7cade9e14b..e18d96bbe0 100644 --- a/packages/backend-core/src/events/identification.ts +++ b/packages/backend-core/src/events/identification.ts @@ -89,6 +89,7 @@ const getCurrentIdentity = async (): Promise => { installationId, tenantId, environment, + hostInfo: userContext.host, } } else { throw new Error("Unknown identity type") diff --git a/packages/backend-core/src/middleware/authenticated.ts b/packages/backend-core/src/middleware/authenticated.ts index 4bb2aaba76..cce27823c0 100644 --- a/packages/backend-core/src/middleware/authenticated.ts +++ b/packages/backend-core/src/middleware/authenticated.ts @@ -8,7 +8,7 @@ import { getGlobalDB, doInTenant } from "../context" import { decrypt } from "../security/encryption" import * as identity from "../context/identity" import env from "../environment" -import { BBContext, EndpointMatcher } from "@budibase/types" +import { Ctx, EndpointMatcher } from "@budibase/types" const ONE_MINUTE = env.SESSION_UPDATE_PERIOD ? parseInt(env.SESSION_UPDATE_PERIOD) @@ -73,7 +73,7 @@ export default function ( } ) { const noAuthOptions = noAuthPatterns ? buildMatcherRegex(noAuthPatterns) : [] - return async (ctx: BBContext | any, next: any) => { + return async (ctx: Ctx | any, next: any) => { let publicEndpoint = false const version = ctx.request.headers[Header.API_VER] // the path is not authenticated @@ -148,7 +148,7 @@ export default function ( finalise(ctx, { authenticated, user, internal, version, publicEndpoint }) if (user && user.email) { - return identity.doInUserContext(user, next) + return identity.doInUserContext(user, ctx, next) } else { return next() } diff --git a/packages/backend-core/src/queue/constants.ts b/packages/backend-core/src/queue/constants.ts index e8323dacb8..9261ed1176 100644 --- a/packages/backend-core/src/queue/constants.ts +++ b/packages/backend-core/src/queue/constants.ts @@ -1,4 +1,5 @@ export enum JobQueue { AUTOMATION = "automationQueue", APP_BACKUP = "appBackupQueue", + AUDIT_LOG = "auditLogQueue", } diff --git a/packages/frontend-core/src/api/auditLogs.js b/packages/frontend-core/src/api/auditLogs.js index 745bfbc742..ee8f1b5520 100644 --- a/packages/frontend-core/src/api/auditLogs.js +++ b/packages/frontend-core/src/api/auditLogs.js @@ -18,15 +18,15 @@ const buildOpts = ({ } if (events) { - opts.event = events + opts.events = events } if (userIds) { - opts.userId = userIds + opts.userIds = userIds } if (appIds) { - opts.appId = appIds + opts.appIds = appIds } return opts diff --git a/packages/server/package.json b/packages/server/package.json index 3eb133e272..98b69d4b61 100644 --- a/packages/server/package.json +++ b/packages/server/package.json @@ -87,6 +87,7 @@ "koa-send": "5.0.0", "koa-session": "5.12.0", "koa-static": "5.0.0", + "koa-useragent": "^4.1.0", "koa2-ratelimit": "1.1.1", "lodash": "4.17.21", "memorystream": "0.3.1", diff --git a/packages/server/src/app.ts b/packages/server/src/app.ts index 6779b35457..86b2783533 100644 --- a/packages/server/src/app.ts +++ b/packages/server/src/app.ts @@ -33,6 +33,7 @@ import { initialise as initialiseWebsockets } from "./websocket" import { startup } from "./startup" const Sentry = require("@sentry/node") const destroyable = require("server-destroy") +const { userAgent } = require("koa-useragent") // configure events to use the pro audit log write // can't integrate directly into backend-core due to cyclic issues @@ -58,6 +59,7 @@ app.use( ) app.use(middleware.logging) +app.use(userAgent) if (env.isProd()) { env._set("NODE_ENV", "production") diff --git a/packages/server/yarn.lock b/packages/server/yarn.lock index eb0a7c4def..383c3f1bff 100644 --- a/packages/server/yarn.lock +++ b/packages/server/yarn.lock @@ -6976,6 +6976,11 @@ expose-loader@^3.1.0: resolved "https://registry.yarnpkg.com/expose-loader/-/expose-loader-3.1.0.tgz#7a0bdecb345b921ca238a8c4715a4ea7e227213f" integrity sha512-2RExSo0yJiqP+xiUue13jQa2IHE8kLDzTI7b6kn+vUlBVvlzNSiLDzo4e5Pp5J039usvTUnxZ8sUOhv0Kg15NA== +express-useragent@^1.0.15: + version "1.0.15" + resolved "https://registry.yarnpkg.com/express-useragent/-/express-useragent-1.0.15.tgz#cefda5fa4904345d51d3368b117a8dd4124985d9" + integrity sha512-eq5xMiYCYwFPoekffMjvEIk+NWdlQY9Y38OsTyl13IvA728vKT+q/CSERYWzcw93HGBJcIqMIsZC5CZGARPVdg== + ext-list@^2.0.0: version "2.2.2" resolved "https://registry.yarnpkg.com/ext-list/-/ext-list-2.2.2.tgz#0b98e64ed82f5acf0f2931babf69212ef52ddd37" @@ -10236,6 +10241,13 @@ koa-static@5.0.0, koa-static@^5.0.0: debug "^3.1.0" koa-send "^5.0.0" +koa-useragent@^4.1.0: + version "4.1.0" + resolved "https://registry.yarnpkg.com/koa-useragent/-/koa-useragent-4.1.0.tgz#d3f128b552c6da3e5e9e9e9c887b2922b16e4468" + integrity sha512-x/HUDZ1zAmNNh5hA9hHbPm9p3UVg2prlpHzxCXQCzbibrNS0kmj7MkCResCbAbG7ZT6FVxNSMjR94ZGamdMwxA== + dependencies: + express-useragent "^1.0.15" + koa-views@^7.0.1: version "7.0.2" resolved "https://registry.yarnpkg.com/koa-views/-/koa-views-7.0.2.tgz#c96fd9e2143ef00c29dc5160c5ed639891aa723d" diff --git a/packages/types/src/sdk/auditLogs.ts b/packages/types/src/sdk/auditLogs.ts index 9f8eb3825c..dea36b0164 100644 --- a/packages/types/src/sdk/auditLogs.ts +++ b/packages/types/src/sdk/auditLogs.ts @@ -1,9 +1,10 @@ -import { Event } from "./events" +import { Event, HostInfo } from "./events" export type AuditWriteOpts = { appId?: string timestamp?: string | number userId?: string + hostInfo?: HostInfo } export type AuditLogFn = ( diff --git a/packages/types/src/sdk/context.ts b/packages/types/src/sdk/context.ts index b3403df8af..c8345de196 100644 --- a/packages/types/src/sdk/context.ts +++ b/packages/types/src/sdk/context.ts @@ -1,5 +1,5 @@ import { User, Account } from "../documents" -import { IdentityType } from "./events" +import { IdentityType, HostInfo } from "./events" export interface BaseContext { _id: string @@ -16,6 +16,7 @@ export interface UserContext extends BaseContext, User { _id: string tenantId: string account?: Account + hostInfo: HostInfo } export type IdentityContext = BaseContext | AccountUserContext | UserContext diff --git a/packages/types/src/sdk/events/identification.ts b/packages/types/src/sdk/events/identification.ts index 3f4e7ec9d4..bbbf74d33e 100644 --- a/packages/types/src/sdk/events/identification.ts +++ b/packages/types/src/sdk/events/identification.ts @@ -34,6 +34,11 @@ export enum IdentityType { INSTALLATION = "installation", } +export interface HostInfo { + ipAddress: string + userAgent: string +} + export interface Identity { id: string type: IdentityType @@ -41,6 +46,7 @@ export interface Identity { environment: string installationId?: string tenantId?: string + hostInfo: HostInfo } export interface UserIdentity extends Identity { diff --git a/packages/worker/package.json b/packages/worker/package.json index 0fb3abe53b..c1912054c4 100644 --- a/packages/worker/package.json +++ b/packages/worker/package.json @@ -60,6 +60,7 @@ "koa-send": "5.0.1", "koa-session": "5.13.1", "koa-static": "5.0.0", + "koa-useragent": "^4.1.0", "node-fetch": "2.6.7", "nodemailer": "6.7.2", "passport-google-oauth": "2.0.0", diff --git a/packages/worker/src/api/controllers/global/auth.ts b/packages/worker/src/api/controllers/global/auth.ts index 1e55d4de0d..318f69c7ad 100644 --- a/packages/worker/src/api/controllers/global/auth.ts +++ b/packages/worker/src/api/controllers/global/auth.ts @@ -69,7 +69,7 @@ export const login = async (ctx: Ctx, next: any) => { "local", async (err: any, user: User, info: any) => { await passportCallback(ctx, user, err, info) - await context.identity.doInUserContext(user, async () => { + await context.identity.doInUserContext(user, ctx, async () => { await events.auth.login("local", user.email) }) ctx.status = 200 @@ -211,7 +211,7 @@ export const googleCallback = async (ctx: any, next: any) => { { successRedirect: "/", failureRedirect: "/error" }, async (err: any, user: SSOUser, info: any) => { await passportCallback(ctx, user, err, info) - await context.identity.doInUserContext(user, async () => { + await context.identity.doInUserContext(user, ctx, async () => { await events.auth.login("google-internal", user.email) }) ctx.redirect("/") @@ -281,7 +281,7 @@ export const oidcCallback = async (ctx: any, next: any) => { { successRedirect: "/", failureRedirect: "/error" }, async (err: any, user: SSOUser, info: any) => { await passportCallback(ctx, user, err, info) - await context.identity.doInUserContext(user, async () => { + await context.identity.doInUserContext(user, ctx, async () => { await events.auth.login("oidc", user.email) }) ctx.redirect("/") diff --git a/packages/worker/src/index.ts b/packages/worker/src/index.ts index 69cd900637..750ae0055d 100644 --- a/packages/worker/src/index.ts +++ b/packages/worker/src/index.ts @@ -24,6 +24,8 @@ import * as redis from "./utilities/redis" const Sentry = require("@sentry/node") const koaSession = require("koa-session") const logger = require("koa-pino-logger") +const { userAgent } = require("koa-useragent") + import destroyable from "server-destroy" // configure events to use the pro audit log write @@ -48,6 +50,7 @@ app.use(koaBody({ multipart: true })) app.use(koaSession(app)) app.use(middleware.logging) app.use(logger(logging.pinoSettings())) +app.use(userAgent) // authentication app.use(auth.passport.initialize()) diff --git a/packages/worker/yarn.lock b/packages/worker/yarn.lock index 83417a2e84..35af60204e 100644 --- a/packages/worker/yarn.lock +++ b/packages/worker/yarn.lock @@ -3653,6 +3653,11 @@ expect@^28.1.3: jest-message-util "^28.1.3" jest-util "^28.1.3" +express-useragent@^1.0.15: + version "1.0.15" + resolved "https://registry.yarnpkg.com/express-useragent/-/express-useragent-1.0.15.tgz#cefda5fa4904345d51d3368b117a8dd4124985d9" + integrity sha512-eq5xMiYCYwFPoekffMjvEIk+NWdlQY9Y38OsTyl13IvA728vKT+q/CSERYWzcw93HGBJcIqMIsZC5CZGARPVdg== + extend@~3.0.2: version "3.0.2" resolved "https://registry.yarnpkg.com/extend/-/extend-3.0.2.tgz#f8b1136b4071fbd8eb140aff858b1019ec2915fa" @@ -5467,6 +5472,13 @@ koa-static@5.0.0: debug "^3.1.0" koa-send "^5.0.0" +koa-useragent@^4.1.0: + version "4.1.0" + resolved "https://registry.yarnpkg.com/koa-useragent/-/koa-useragent-4.1.0.tgz#d3f128b552c6da3e5e9e9e9c887b2922b16e4468" + integrity sha512-x/HUDZ1zAmNNh5hA9hHbPm9p3UVg2prlpHzxCXQCzbibrNS0kmj7MkCResCbAbG7ZT6FVxNSMjR94ZGamdMwxA== + dependencies: + express-useragent "^1.0.15" + koa@2.13.4, koa@^2.13.4: version "2.13.4" resolved "https://registry.yarnpkg.com/koa/-/koa-2.13.4.tgz#ee5b0cb39e0b8069c38d115139c774833d32462e"