Adding new mechanism to get the ipAddress and userAgent of call for audit logs.

packages/backend-core/src/context/identity.ts

@@ -5,6 +5,8 @@ import {
   isCloudAccount,
   Account,
   AccountUserContext,
+  UserContext,
+  Ctx,
 } from "@budibase/types"
 import * as context from "."
 
@@ -16,15 +18,22 @@ export function doInIdentityContext(identity: IdentityContext, task: any) {
   return context.doInIdentityContext(identity, task)
 }
 
-export function doInUserContext(user: User, task: any) {
+// used in server/worker
-  const userContext: any = {
+export function doInUserContext(user: User, ctx: Ctx, task: any) {
+  const userContext: UserContext = {
     ...user,
     _id: user._id as string,
     type: IdentityType.USER,
+    hostInfo: {
+      ipAddress: ctx.request.ip,
+      // filled in by koa-useragent package
+      userAgent: ctx.userAgent._agent.source,
+    },
   }
   return doInIdentityContext(userContext, task)
 }
 
+// used in account portal
 export function doInAccountContext(account: Account, task: any) {
   const _id = getAccountUserId(account)
   const tenantId = account.tenantId

packages/backend-core/src/events/events.ts

@@ -1,13 +1,37 @@
-import { Event, IdentityType, AuditLogFn } from "@budibase/types"
+import { AuditLogFn, Event, IdentityType, HostInfo } from "@budibase/types"
 import { processors } from "./processors"
 import identification from "./identification"
 import { getAppId } from "../context"
 import * as backfill from "./backfill"
+import { createQueue, JobQueue } from "../queue"
+import BullQueue from "bull"
 
-let writeAuditLogs: AuditLogFn | undefined
+type AuditLogEvent = {
+  event: Event
+  properties: any
+  opts: {
+    timestamp?: string | number
+    userId?: string
+    appId?: string
+    hostInfo?: HostInfo
+  }
+}
+
+let auditLogsEnabled = false
+let auditLogQueue: BullQueue.Queue<AuditLogEvent>
 
 export const configure = (fn: AuditLogFn) => {
-  writeAuditLogs = fn
+  auditLogsEnabled = true
+  const writeAuditLogs = fn
+  auditLogQueue = createQueue<AuditLogEvent>(JobQueue.AUDIT_LOG)
+  return auditLogQueue.process(async job => {
+    await writeAuditLogs(job.data.event, job.data.properties, {
+      userId: job.data.opts.userId,
+      timestamp: job.data.opts.timestamp,
+      appId: job.data.opts.appId,
+      hostInfo: job.data.opts.hostInfo,
+    })
+  })
 }
 
 export const publishEvent = async (
@@ -21,16 +45,22 @@ export const publishEvent = async (
   const backfilling = await backfill.isBackfillingEvent(event)
   // no backfill - send the event and exit
   if (!backfilling) {
-    // only audit log actual events, don't include backfills
+    await processors.processEvent(event, identity, properties, timestamp)
-    const userId = identity.type === IdentityType.USER ? identity.id : undefined
+    if (auditLogsEnabled) {
-    if (writeAuditLogs) {
+      // only audit log actual events, don't include backfills
-      await writeAuditLogs(event, properties, {
+      const userId =
-        userId,
+        identity.type === IdentityType.USER ? identity.id : undefined
-        timestamp,
+      // add to event queue, rather than just writing immediately
-        appId: getAppId(),
+      await auditLogQueue.add({
+        event,
+        properties,
+        opts: {
+          userId,
+          timestamp,
+          appId: getAppId(),
+        },
       })
     }
-    await processors.processEvent(event, identity, properties, timestamp)
     return
   }
 

packages/backend-core/src/events/identification.ts

@@ -89,6 +89,7 @@ const getCurrentIdentity = async (): Promise<Identity> => {
       installationId,
       tenantId,
       environment,
+      hostInfo: userContext.host,
     }
   } else {
     throw new Error("Unknown identity type")

packages/backend-core/src/middleware/authenticated.ts

@@ -8,7 +8,7 @@ import { getGlobalDB, doInTenant } from "../context"
 import { decrypt } from "../security/encryption"
 import * as identity from "../context/identity"
 import env from "../environment"
-import { BBContext, EndpointMatcher } from "@budibase/types"
+import { Ctx, EndpointMatcher } from "@budibase/types"
 
 const ONE_MINUTE = env.SESSION_UPDATE_PERIOD
   ? parseInt(env.SESSION_UPDATE_PERIOD)
@@ -73,7 +73,7 @@ export default function (
   }
 ) {
   const noAuthOptions = noAuthPatterns ? buildMatcherRegex(noAuthPatterns) : []
-  return async (ctx: BBContext | any, next: any) => {
+  return async (ctx: Ctx | any, next: any) => {
     let publicEndpoint = false
     const version = ctx.request.headers[Header.API_VER]
     // the path is not authenticated
@@ -148,7 +148,7 @@ export default function (
       finalise(ctx, { authenticated, user, internal, version, publicEndpoint })
 
       if (user && user.email) {
-        return identity.doInUserContext(user, next)
+        return identity.doInUserContext(user, ctx, next)
       } else {
         return next()
       }

packages/backend-core/src/queue/constants.ts

@@ -1,4 +1,5 @@
 export enum JobQueue {
   AUTOMATION = "automationQueue",
   APP_BACKUP = "appBackupQueue",
+  AUDIT_LOG = "auditLogQueue",
 }

packages/frontend-core/src/api/auditLogs.js

@@ -18,15 +18,15 @@ const buildOpts = ({
   }
 
   if (events) {
-    opts.event = events
+    opts.events = events
   }
 
   if (userIds) {
-    opts.userId = userIds
+    opts.userIds = userIds
   }
 
   if (appIds) {
-    opts.appId = appIds
+    opts.appIds = appIds
   }
 
   return opts

packages/server/package.json

@@ -87,6 +87,7 @@
     "koa-send": "5.0.0",
     "koa-session": "5.12.0",
     "koa-static": "5.0.0",
+    "koa-useragent": "^4.1.0",
     "koa2-ratelimit": "1.1.1",
     "lodash": "4.17.21",
     "memorystream": "0.3.1",

packages/server/src/app.ts

@@ -33,6 +33,7 @@ import { initialise as initialiseWebsockets } from "./websocket"
 import { startup } from "./startup"
 const Sentry = require("@sentry/node")
 const destroyable = require("server-destroy")
+const { userAgent } = require("koa-useragent")
 
 // configure events to use the pro audit log write
 // can't integrate directly into backend-core due to cyclic issues
@@ -58,6 +59,7 @@ app.use(
 )
 
 app.use(middleware.logging)
+app.use(userAgent)
 
 if (env.isProd()) {
   env._set("NODE_ENV", "production")

packages/server/yarn.lock

@@ -6976,6 +6976,11 @@ expose-loader@^3.1.0:
   resolved "https://registry.yarnpkg.com/expose-loader/-/expose-loader-3.1.0.tgz#7a0bdecb345b921ca238a8c4715a4ea7e227213f"
   integrity sha512-2RExSo0yJiqP+xiUue13jQa2IHE8kLDzTI7b6kn+vUlBVvlzNSiLDzo4e5Pp5J039usvTUnxZ8sUOhv0Kg15NA==
 
+express-useragent@^1.0.15:
+  version "1.0.15"
+  resolved "https://registry.yarnpkg.com/express-useragent/-/express-useragent-1.0.15.tgz#cefda5fa4904345d51d3368b117a8dd4124985d9"
+  integrity sha512-eq5xMiYCYwFPoekffMjvEIk+NWdlQY9Y38OsTyl13IvA728vKT+q/CSERYWzcw93HGBJcIqMIsZC5CZGARPVdg==
+
 ext-list@^2.0.0:
   version "2.2.2"
   resolved "https://registry.yarnpkg.com/ext-list/-/ext-list-2.2.2.tgz#0b98e64ed82f5acf0f2931babf69212ef52ddd37"
@@ -10236,6 +10241,13 @@ koa-static@5.0.0, koa-static@^5.0.0:
     debug "^3.1.0"
     koa-send "^5.0.0"
 
+koa-useragent@^4.1.0:
+  version "4.1.0"
+  resolved "https://registry.yarnpkg.com/koa-useragent/-/koa-useragent-4.1.0.tgz#d3f128b552c6da3e5e9e9e9c887b2922b16e4468"
+  integrity sha512-x/HUDZ1zAmNNh5hA9hHbPm9p3UVg2prlpHzxCXQCzbibrNS0kmj7MkCResCbAbG7ZT6FVxNSMjR94ZGamdMwxA==
+  dependencies:
+    express-useragent "^1.0.15"
+
 koa-views@^7.0.1:
   version "7.0.2"
   resolved "https://registry.yarnpkg.com/koa-views/-/koa-views-7.0.2.tgz#c96fd9e2143ef00c29dc5160c5ed639891aa723d"

packages/types/src/sdk/auditLogs.ts

@@ -1,9 +1,10 @@
-import { Event } from "./events"
+import { Event, HostInfo } from "./events"
 
 export type AuditWriteOpts = {
   appId?: string
   timestamp?: string | number
   userId?: string
+  hostInfo?: HostInfo
 }
 
 export type AuditLogFn = (

packages/types/src/sdk/context.ts

@@ -1,5 +1,5 @@
 import { User, Account } from "../documents"
-import { IdentityType } from "./events"
+import { IdentityType, HostInfo } from "./events"
 
 export interface BaseContext {
   _id: string
@@ -16,6 +16,7 @@ export interface UserContext extends BaseContext, User {
   _id: string
   tenantId: string
   account?: Account
+  hostInfo: HostInfo
 }
 
 export type IdentityContext = BaseContext | AccountUserContext | UserContext

packages/types/src/sdk/events/identification.ts

@@ -34,6 +34,11 @@ export enum IdentityType {
   INSTALLATION = "installation",
 }
 
+export interface HostInfo {
+  ipAddress: string
+  userAgent: string
+}
+
 export interface Identity {
   id: string
   type: IdentityType
@@ -41,6 +46,7 @@ export interface Identity {
   environment: string
   installationId?: string
   tenantId?: string
+  hostInfo: HostInfo
 }
 
 export interface UserIdentity extends Identity {

packages/worker/package.json

@@ -60,6 +60,7 @@
     "koa-send": "5.0.1",
     "koa-session": "5.13.1",
     "koa-static": "5.0.0",
+    "koa-useragent": "^4.1.0",
     "node-fetch": "2.6.7",
     "nodemailer": "6.7.2",
     "passport-google-oauth": "2.0.0",

packages/worker/src/api/controllers/global/auth.ts

@@ -69,7 +69,7 @@ export const login = async (ctx: Ctx<LoginRequest>, next: any) => {
     "local",
     async (err: any, user: User, info: any) => {
       await passportCallback(ctx, user, err, info)
-      await context.identity.doInUserContext(user, async () => {
+      await context.identity.doInUserContext(user, ctx, async () => {
         await events.auth.login("local", user.email)
       })
       ctx.status = 200
@@ -211,7 +211,7 @@ export const googleCallback = async (ctx: any, next: any) => {
     { successRedirect: "/", failureRedirect: "/error" },
     async (err: any, user: SSOUser, info: any) => {
       await passportCallback(ctx, user, err, info)
-      await context.identity.doInUserContext(user, async () => {
+      await context.identity.doInUserContext(user, ctx, async () => {
         await events.auth.login("google-internal", user.email)
       })
       ctx.redirect("/")
@@ -281,7 +281,7 @@ export const oidcCallback = async (ctx: any, next: any) => {
     { successRedirect: "/", failureRedirect: "/error" },
     async (err: any, user: SSOUser, info: any) => {
       await passportCallback(ctx, user, err, info)
-      await context.identity.doInUserContext(user, async () => {
+      await context.identity.doInUserContext(user, ctx, async () => {
         await events.auth.login("oidc", user.email)
       })
       ctx.redirect("/")

packages/worker/src/index.ts

@@ -24,6 +24,8 @@ import * as redis from "./utilities/redis"
 const Sentry = require("@sentry/node")
 const koaSession = require("koa-session")
 const logger = require("koa-pino-logger")
+const { userAgent } = require("koa-useragent")
+
 import destroyable from "server-destroy"
 
 // configure events to use the pro audit log write
@@ -48,6 +50,7 @@ app.use(koaBody({ multipart: true }))
 app.use(koaSession(app))
 app.use(middleware.logging)
 app.use(logger(logging.pinoSettings()))
+app.use(userAgent)
 
 // authentication
 app.use(auth.passport.initialize())

packages/worker/yarn.lock

@@ -3653,6 +3653,11 @@ expect@^28.1.3:
     jest-message-util "^28.1.3"
     jest-util "^28.1.3"
 
+express-useragent@^1.0.15:
+  version "1.0.15"
+  resolved "https://registry.yarnpkg.com/express-useragent/-/express-useragent-1.0.15.tgz#cefda5fa4904345d51d3368b117a8dd4124985d9"
+  integrity sha512-eq5xMiYCYwFPoekffMjvEIk+NWdlQY9Y38OsTyl13IvA728vKT+q/CSERYWzcw93HGBJcIqMIsZC5CZGARPVdg==
+
 extend@~3.0.2:
   version "3.0.2"
   resolved "https://registry.yarnpkg.com/extend/-/extend-3.0.2.tgz#f8b1136b4071fbd8eb140aff858b1019ec2915fa"
@@ -5467,6 +5472,13 @@ koa-static@5.0.0:
     debug "^3.1.0"
     koa-send "^5.0.0"
 
+koa-useragent@^4.1.0:
+  version "4.1.0"
+  resolved "https://registry.yarnpkg.com/koa-useragent/-/koa-useragent-4.1.0.tgz#d3f128b552c6da3e5e9e9e9c887b2922b16e4468"
+  integrity sha512-x/HUDZ1zAmNNh5hA9hHbPm9p3UVg2prlpHzxCXQCzbibrNS0kmj7MkCResCbAbG7ZT6FVxNSMjR94ZGamdMwxA==
+  dependencies:
+    express-useragent "^1.0.15"
+
 koa@2.13.4, koa@^2.13.4:
   version "2.13.4"
   resolved "https://registry.yarnpkg.com/koa/-/koa-2.13.4.tgz#ee5b0cb39e0b8069c38d115139c774833d32462e"