Merge pull request #9795 from Budibase/feature/audit-logs

Audit Logs

hosting/docker-compose.test.yaml

@@ -8,8 +8,8 @@ services:
     # Last version that supports the "fs" backend
     image: minio/minio:RELEASE.2022-10-24T18-35-07Z
     ports:
-      - 9000
+      - "9000"
-      - 9001
+      - "9001"
     environment:
       MINIO_ACCESS_KEY: ${MINIO_ACCESS_KEY}
       MINIO_SECRET_KEY: ${MINIO_SECRET_KEY}
@@ -28,9 +28,9 @@ services:
       - COUCHDB_PASSWORD=${COUCH_DB_PASSWORD}
       - COUCHDB_USER=${COUCH_DB_USER}
     ports:
-      - 5984
+      - "5984"
-      - 4369
+      - "4369"
-      - 9100
+      - "9100"
     healthcheck:
       test: ["CMD", "curl", "-f", "http://localhost:5984/_up"]
       interval: 30s
@@ -42,6 +42,6 @@ services:
     image: redis
     command: redis-server --requirepass ${REDIS_PASSWORD}
     ports:
-      - 6379
+      - "6379"
     healthcheck:
-      test: ["CMD", "redis-cli", "ping"]
+      test: ["CMD", "redis-cli", "ping"]

package.json

@@ -13,7 +13,7 @@
     "js-yaml": "^4.1.0",
     "kill-port": "^1.6.1",
     "lerna": "3.14.1",
-    "madge": "^5.0.1",
+    "madge": "^6.0.0",
     "prettier": "^2.3.1",
     "prettier-plugin-svelte": "^2.3.0",
     "rimraf": "^3.0.2",
@@ -84,4 +84,4 @@
     "install:pro": "bash scripts/pro/install.sh",
     "dep:clean": "yarn clean && yarn bootstrap"
   }
-}
+}

packages/backend-core/src/auth/auth.ts

@@ -227,6 +227,6 @@ export async function platformLogout(opts: PlatformLogoutOpts) {
 
   const sessionIds = sessions.map(({ sessionId }) => sessionId)
   await invalidateSessions(userId, { sessionIds, reason: "logout" })
-  await events.auth.logout()
+  await events.auth.logout(ctx.user?.email)
   await userCache.invalidateUser(userId)
 }

packages/backend-core/src/constants/db.ts

@@ -68,6 +68,7 @@ export enum DocumentType {
   MEM_VIEW = "view",
   USER_FLAG = "flag",
   AUTOMATION_METADATA = "meta_au",
+  AUDIT_LOG = "al",
 }
 
 export const StaticDatabases = {
@@ -88,6 +89,9 @@ export const StaticDatabases = {
       install: "install",
     },
   },
+  AUDIT_LOGS: {
+    name: "audit-logs",
+  },
 }
 
 export const APP_PREFIX = DocumentType.APP + SEPARATOR

packages/backend-core/src/constants/misc.ts

@@ -41,5 +41,6 @@ export enum Config {
   OIDC_LOGOS = "logos_oidc",
 }
 
+export const MIN_VALID_DATE = new Date(-2147483647000)
 export const MAX_VALID_DATE = new Date(2147483647000)
 export const DEFAULT_TENANT_ID = "default"

packages/backend-core/src/context/Context.ts

@@ -1,5 +1,5 @@
 import { AsyncLocalStorage } from "async_hooks"
-import { ContextMap } from "./mainContext"
+import { ContextMap } from "./types"
 
 export default class Context {
   static storage = new AsyncLocalStorage<ContextMap>()

packages/backend-core/src/context/identity.ts

@@ -5,6 +5,8 @@ import {
   isCloudAccount,
   Account,
   AccountUserContext,
+  UserContext,
+  Ctx,
 } from "@budibase/types"
 import * as context from "."
 
@@ -16,15 +18,22 @@ export function doInIdentityContext(identity: IdentityContext, task: any) {
   return context.doInIdentityContext(identity, task)
 }
 
-export function doInUserContext(user: User, task: any) {
+// used in server/worker
-  const userContext: any = {
+export function doInUserContext(user: User, ctx: Ctx, task: any) {
+  const userContext: UserContext = {
     ...user,
     _id: user._id as string,
     type: IdentityType.USER,
+    hostInfo: {
+      ipAddress: ctx.request.ip,
+      // filled in by koa-useragent package
+      userAgent: ctx.userAgent._agent.source,
+    },
   }
   return doInIdentityContext(userContext, task)
 }
 
+// used in account portal
 export function doInAccountContext(account: Account, task: any) {
   const _id = getAccountUserId(account)
   const tenantId = account.tenantId

packages/backend-core/src/context/mainContext.ts

@@ -11,13 +11,7 @@ import {
   DEFAULT_TENANT_ID,
 } from "../constants"
 import { Database, IdentityContext } from "@budibase/types"
-
+import { ContextMap } from "./types"
-export type ContextMap = {
-  tenantId?: string
-  appId?: string
-  identity?: IdentityContext
-  environmentVariables?: Record<string, string>
-}
 
 let TEST_APP_ID: string | null = null
 
@@ -30,14 +24,23 @@ export function getGlobalDBName(tenantId?: string) {
   return baseGlobalDBName(tenantId)
 }
 
-export function baseGlobalDBName(tenantId: string | undefined | null) {
+export function getAuditLogDBName(tenantId?: string) {
-  let dbName
+  if (!tenantId) {
-  if (!tenantId || tenantId === DEFAULT_TENANT_ID) {
+    tenantId = getTenantId()
-    dbName = StaticDatabases.GLOBAL.name
+  }
-  } else {
+  if (tenantId === DEFAULT_TENANT_ID) {
-    dbName = `${tenantId}${SEPARATOR}${StaticDatabases.GLOBAL.name}`
+    return StaticDatabases.AUDIT_LOGS.name
+  } else {
+    return `${tenantId}${SEPARATOR}${StaticDatabases.AUDIT_LOGS.name}`
+  }
+}
+
+export function baseGlobalDBName(tenantId: string | undefined | null) {
+  if (!tenantId || tenantId === DEFAULT_TENANT_ID) {
+    return StaticDatabases.GLOBAL.name
+  } else {
+    return `${tenantId}${SEPARATOR}${StaticDatabases.GLOBAL.name}`
   }
-  return dbName
 }
 
 export function isMultiTenant() {
@@ -228,6 +231,13 @@ export function getGlobalDB(): Database {
   return getDB(baseGlobalDBName(context?.tenantId))
 }
 
+export function getAuditLogsDB(): Database {
+  if (!getTenantId()) {
+    throw new Error("No tenant ID found - cannot open audit log DB")
+  }
+  return getDB(getAuditLogDBName())
+}
+
 /**
  * Gets the app database based on whatever the request
  * contained, dev or prod.

packages/backend-core/src/context/types.ts

@@ -0,0 +1,9 @@
+import { IdentityContext } from "@budibase/types"
+
+// keep this out of Budibase types, don't want to expose context info
+export type ContextMap = {
+  tenantId?: string
+  appId?: string
+  identity?: IdentityContext
+  environmentVariables?: Record<string, string>
+}

packages/backend-core/src/db/index.ts

@@ -7,3 +7,4 @@ export { default as Replication } from "./Replication"
 // exports to support old export structure
 export * from "../constants/db"
 export { getGlobalDBName, baseGlobalDBName } from "../context"
+export * from "./lucene"

packages/backend-core/src/db/lucene.ts

@@ -0,0 +1,624 @@
+import fetch from "node-fetch"
+import { getCouchInfo } from "./couch"
+import { SearchFilters, Row } from "@budibase/types"
+
+const QUERY_START_REGEX = /\d[0-9]*:/g
+
+interface SearchResponse<T> {
+  rows: T[] | any[]
+  bookmark: string
+}
+
+interface PaginatedSearchResponse<T> extends SearchResponse<T> {
+  hasNextPage: boolean
+}
+
+export type SearchParams<T> = {
+  tableId?: string
+  sort?: string
+  sortOrder?: string
+  sortType?: string
+  limit?: number
+  bookmark?: string
+  version?: string
+  indexer?: () => Promise<any>
+  disableEscaping?: boolean
+  rows?: T | Row[]
+}
+
+export function removeKeyNumbering(key: any): string {
+  if (typeof key === "string" && key.match(QUERY_START_REGEX) != null) {
+    const parts = key.split(":")
+    // remove the number
+    parts.shift()
+    return parts.join(":")
+  } else {
+    return key
+  }
+}
+
+/**
+ * Class to build lucene query URLs.
+ * Optionally takes a base lucene query object.
+ */
+export class QueryBuilder<T> {
+  dbName: string
+  index: string
+  query: SearchFilters
+  limit: number
+  sort?: string
+  bookmark?: string
+  sortOrder: string
+  sortType: string
+  includeDocs: boolean
+  version?: string
+  indexBuilder?: () => Promise<any>
+  noEscaping = false
+
+  constructor(dbName: string, index: string, base?: SearchFilters) {
+    this.dbName = dbName
+    this.index = index
+    this.query = {
+      allOr: false,
+      string: {},
+      fuzzy: {},
+      range: {},
+      equal: {},
+      notEqual: {},
+      empty: {},
+      notEmpty: {},
+      oneOf: {},
+      contains: {},
+      notContains: {},
+      containsAny: {},
+      ...base,
+    }
+    this.limit = 50
+    this.sortOrder = "ascending"
+    this.sortType = "string"
+    this.includeDocs = true
+  }
+
+  disableEscaping() {
+    this.noEscaping = true
+    return this
+  }
+
+  setIndexBuilder(builderFn: () => Promise<any>) {
+    this.indexBuilder = builderFn
+    return this
+  }
+
+  setVersion(version?: string) {
+    if (version != null) {
+      this.version = version
+    }
+    return this
+  }
+
+  setTable(tableId: string) {
+    this.query.equal!.tableId = tableId
+    return this
+  }
+
+  setLimit(limit?: number) {
+    if (limit != null) {
+      this.limit = limit
+    }
+    return this
+  }
+
+  setSort(sort?: string) {
+    if (sort != null) {
+      this.sort = sort
+    }
+    return this
+  }
+
+  setSortOrder(sortOrder?: string) {
+    if (sortOrder != null) {
+      this.sortOrder = sortOrder
+    }
+    return this
+  }
+
+  setSortType(sortType?: string) {
+    if (sortType != null) {
+      this.sortType = sortType
+    }
+    return this
+  }
+
+  setBookmark(bookmark?: string) {
+    if (bookmark != null) {
+      this.bookmark = bookmark
+    }
+    return this
+  }
+
+  excludeDocs() {
+    this.includeDocs = false
+    return this
+  }
+
+  addString(key: string, partial: string) {
+    this.query.string![key] = partial
+    return this
+  }
+
+  addFuzzy(key: string, fuzzy: string) {
+    this.query.fuzzy![key] = fuzzy
+    return this
+  }
+
+  addRange(key: string, low: string | number, high: string | number) {
+    this.query.range![key] = {
+      low,
+      high,
+    }
+    return this
+  }
+
+  addEqual(key: string, value: any) {
+    this.query.equal![key] = value
+    return this
+  }
+
+  addNotEqual(key: string, value: any) {
+    this.query.notEqual![key] = value
+    return this
+  }
+
+  addEmpty(key: string, value: any) {
+    this.query.empty![key] = value
+    return this
+  }
+
+  addNotEmpty(key: string, value: any) {
+    this.query.notEmpty![key] = value
+    return this
+  }
+
+  addOneOf(key: string, value: any) {
+    this.query.oneOf![key] = value
+    return this
+  }
+
+  addContains(key: string, value: any) {
+    this.query.contains![key] = value
+    return this
+  }
+
+  addNotContains(key: string, value: any) {
+    this.query.notContains![key] = value
+    return this
+  }
+
+  addContainsAny(key: string, value: any) {
+    this.query.containsAny![key] = value
+    return this
+  }
+
+  handleSpaces(input: string) {
+    if (this.noEscaping) {
+      return input
+    } else {
+      return input.replace(/ /g, "_")
+    }
+  }
+
+  /**
+   * Preprocesses a value before going into a lucene search.
+   * Transforms strings to lowercase and wraps strings and bools in quotes.
+   * @param value The value to process
+   * @param options The preprocess options
+   * @returns {string|*}
+   */
+  preprocess(value: any, { escape, lowercase, wrap, type }: any = {}) {
+    const hasVersion = !!this.version
+    // Determine if type needs wrapped
+    const originalType = typeof value
+    // Convert to lowercase
+    if (value && lowercase) {
+      value = value.toLowerCase ? value.toLowerCase() : value
+    }
+    // Escape characters
+    if (!this.noEscaping && escape && originalType === "string") {
+      value = `${value}`.replace(/[ #+\-&|!(){}\]^"~*?:\\]/g, "\\$&")
+    }
+
+    // Wrap in quotes
+    if (originalType === "string" && !isNaN(value) && !type) {
+      value = `"${value}"`
+    } else if (hasVersion && wrap) {
+      value = originalType === "number" ? value : `"${value}"`
+    }
+    return value
+  }
+
+  buildSearchQuery() {
+    const builder = this
+    let allOr = this.query && this.query.allOr
+    let query = allOr ? "" : "*:*"
+    const allPreProcessingOpts = { escape: true, lowercase: true, wrap: true }
+    let tableId
+    if (this.query.equal!.tableId) {
+      tableId = this.query.equal!.tableId
+      delete this.query.equal!.tableId
+    }
+
+    const equal = (key: string, value: any) => {
+      // 0 evaluates to false, which means we would return all rows if we don't check it
+      if (!value && value !== 0) {
+        return null
+      }
+      return `${key}:${builder.preprocess(value, allPreProcessingOpts)}`
+    }
+
+    const contains = (key: string, value: any, mode = "AND") => {
+      if (Array.isArray(value) && value.length === 0) {
+        return null
+      }
+      if (!Array.isArray(value)) {
+        return `${key}:${value}`
+      }
+      let statement = `${builder.preprocess(value[0], { escape: true })}`
+      for (let i = 1; i < value.length; i++) {
+        statement += ` ${mode} ${builder.preprocess(value[i], {
+          escape: true,
+        })}`
+      }
+      return `${key}:(${statement})`
+    }
+
+    const notContains = (key: string, value: any) => {
+      // @ts-ignore
+      const allPrefix = allOr === "" ? "*:* AND" : ""
+      return allPrefix + "NOT " + contains(key, value)
+    }
+
+    const containsAny = (key: string, value: any) => {
+      return contains(key, value, "OR")
+    }
+
+    const oneOf = (key: string, value: any) => {
+      if (!Array.isArray(value)) {
+        if (typeof value === "string") {
+          value = value.split(",")
+        } else {
+          return ""
+        }
+      }
+      let orStatement = `${builder.preprocess(value[0], allPreProcessingOpts)}`
+      for (let i = 1; i < value.length; i++) {
+        orStatement += ` OR ${builder.preprocess(
+          value[i],
+          allPreProcessingOpts
+        )}`
+      }
+      return `${key}:(${orStatement})`
+    }
+
+    function build(structure: any, queryFn: any) {
+      for (let [key, value] of Object.entries(structure)) {
+        // check for new format - remove numbering if needed
+        key = removeKeyNumbering(key)
+        key = builder.preprocess(builder.handleSpaces(key), {
+          escape: true,
+        })
+        const expression = queryFn(key, value)
+        if (expression == null) {
+          continue
+        }
+        if (query.length > 0) {
+          query += ` ${allOr ? "OR" : "AND"} `
+        }
+        query += expression
+      }
+    }
+
+    // Construct the actual lucene search query string from JSON structure
+    if (this.query.string) {
+      build(this.query.string, (key: string, value: any) => {
+        if (!value) {
+          return null
+        }
+        value = builder.preprocess(value, {
+          escape: true,
+          lowercase: true,
+          type: "string",
+        })
+        return `${key}:${value}*`
+      })
+    }
+    if (this.query.range) {
+      build(this.query.range, (key: string, value: any) => {
+        if (!value) {
+          return null
+        }
+        if (value.low == null || value.low === "") {
+          return null
+        }
+        if (value.high == null || value.high === "") {
+          return null
+        }
+        const low = builder.preprocess(value.low, allPreProcessingOpts)
+        const high = builder.preprocess(value.high, allPreProcessingOpts)
+        return `${key}:[${low} TO ${high}]`
+      })
+    }
+    if (this.query.fuzzy) {
+      build(this.query.fuzzy, (key: string, value: any) => {
+        if (!value) {
+          return null
+        }
+        value = builder.preprocess(value, {
+          escape: true,
+          lowercase: true,
+          type: "fuzzy",
+        })
+        return `${key}:${value}~`
+      })
+    }
+    if (this.query.equal) {
+      build(this.query.equal, equal)
+    }
+    if (this.query.notEqual) {
+      build(this.query.notEqual, (key: string, value: any) => {
+        if (!value) {
+          return null
+        }
+        return `!${key}:${builder.preprocess(value, allPreProcessingOpts)}`
+      })
+    }
+    if (this.query.empty) {
+      build(this.query.empty, (key: string) => `!${key}:["" TO *]`)
+    }
+    if (this.query.notEmpty) {
+      build(this.query.notEmpty, (key: string) => `${key}:["" TO *]`)
+    }
+    if (this.query.oneOf) {
+      build(this.query.oneOf, oneOf)
+    }
+    if (this.query.contains) {
+      build(this.query.contains, contains)
+    }
+    if (this.query.notContains) {
+      build(this.query.notContains, notContains)
+    }
+    if (this.query.containsAny) {
+      build(this.query.containsAny, containsAny)
+    }
+    // make sure table ID is always added as an AND
+    if (tableId) {
+      query = `(${query})`
+      allOr = false
+      build({ tableId }, equal)
+    }
+    return query
+  }
+
+  buildSearchBody() {
+    let body: any = {
+      q: this.buildSearchQuery(),
+      limit: Math.min(this.limit, 200),
+      include_docs: this.includeDocs,
+    }
+    if (this.bookmark) {
+      body.bookmark = this.bookmark
+    }
+    if (this.sort) {
+      const order = this.sortOrder === "descending" ? "-" : ""
+      const type = `<${this.sortType}>`
+      body.sort = `${order}${this.handleSpaces(this.sort)}${type}`
+    }
+    return body
+  }
+
+  async run() {
+    const { url, cookie } = getCouchInfo()
+    const fullPath = `${url}/${this.dbName}/_design/database/_search/${this.index}`
+    const body = this.buildSearchBody()
+    try {
+      return await runQuery<T>(fullPath, body, cookie)
+    } catch (err: any) {
+      if (err.status === 404 && this.indexBuilder) {
+        await this.indexBuilder()
+        return await runQuery<T>(fullPath, body, cookie)
+      } else {
+        throw err
+      }
+    }
+  }
+}
+
+/**
+ * Executes a lucene search query.
+ * @param url The query URL
+ * @param body The request body defining search criteria
+ * @param cookie The auth cookie for CouchDB
+ * @returns {Promise<{rows: []}>}
+ */
+async function runQuery<T>(
+  url: string,
+  body: any,
+  cookie: string
+): Promise<SearchResponse<T>> {
+  const response = await fetch(url, {
+    body: JSON.stringify(body),
+    method: "POST",
+    headers: {
+      Authorization: cookie,
+    },
+  })
+
+  if (response.status === 404) {
+    throw response
+  }
+  const json = await response.json()
+
+  let output: any = {
+    rows: [],
+  }
+  if (json.rows != null && json.rows.length > 0) {
+    output.rows = json.rows.map((row: any) => row.doc)
+  }
+  if (json.bookmark) {
+    output.bookmark = json.bookmark
+  }
+  return output
+}
+
+/**
+ * Gets round the fixed limit of 200 results from a query by fetching as many
+ * pages as required and concatenating the results. This recursively operates
+ * until enough results have been found.
+ * @param dbName {string} Which database to run a lucene query on
+ * @param index {string} Which search index to utilise
+ * @param query {object} The JSON query structure
+ * @param params {object} The search params including:
+ *   tableId {string} The table ID to search
+ *   sort {string} The sort column
+ *   sortOrder {string} The sort order ("ascending" or "descending")
+ *   sortType {string} Whether to treat sortable values as strings or
+ *     numbers. ("string" or "number")
+ *   limit {number} The number of results to fetch
+ *   bookmark {string|null} Current bookmark in the recursive search
+ *   rows {array|null} Current results in the recursive search
+ * @returns {Promise<*[]|*>}
+ */
+async function recursiveSearch<T>(
+  dbName: string,
+  index: string,
+  query: any,
+  params: any
+): Promise<any> {
+  const bookmark = params.bookmark
+  const rows = params.rows || []
+  if (rows.length >= params.limit) {
+    return rows
+  }
+  let pageSize = 200
+  if (rows.length > params.limit - 200) {
+    pageSize = params.limit - rows.length
+  }
+  const page = await new QueryBuilder<T>(dbName, index, query)
+    .setVersion(params.version)
+    .setTable(params.tableId)
+    .setBookmark(bookmark)
+    .setLimit(pageSize)
+    .setSort(params.sort)
+    .setSortOrder(params.sortOrder)
+    .setSortType(params.sortType)
+    .run()
+  if (!page.rows.length) {
+    return rows
+  }
+  if (page.rows.length < 200) {
+    return [...rows, ...page.rows]
+  }
+  const newParams = {
+    ...params,
+    bookmark: page.bookmark,
+    rows: [...rows, ...page.rows],
+  }
+  return await recursiveSearch(dbName, index, query, newParams)
+}
+
+/**
+ * Performs a paginated search. A bookmark will be returned to allow the next
+ * page to be fetched. There is a max limit off 200 results per page in a
+ * paginated search.
+ * @param dbName {string} Which database to run a lucene query on
+ * @param index {string} Which search index to utilise
+ * @param query {object} The JSON query structure
+ * @param params {object} The search params including:
+ *   tableId {string} The table ID to search
+ *   sort {string} The sort column
+ *   sortOrder {string} The sort order ("ascending" or "descending")
+ *   sortType {string} Whether to treat sortable values as strings or
+ *     numbers. ("string" or "number")
+ *   limit {number} The desired page size
+ *   bookmark {string} The bookmark to resume from
+ * @returns {Promise<{hasNextPage: boolean, rows: *[]}>}
+ */
+export async function paginatedSearch<T>(
+  dbName: string,
+  index: string,
+  query: SearchFilters,
+  params: SearchParams<T>
+) {
+  let limit = params.limit
+  if (limit == null || isNaN(limit) || limit < 0) {
+    limit = 50
+  }
+  limit = Math.min(limit, 200)
+  const search = new QueryBuilder<T>(dbName, index, query)
+  if (params.version) {
+    search.setVersion(params.version)
+  }
+  if (params.tableId) {
+    search.setTable(params.tableId)
+  }
+  if (params.sort) {
+    search
+      .setSort(params.sort)
+      .setSortOrder(params.sortOrder)
+      .setSortType(params.sortType)
+  }
+  if (params.indexer) {
+    search.setIndexBuilder(params.indexer)
+  }
+  if (params.disableEscaping) {
+    search.disableEscaping()
+  }
+  const searchResults = await search
+    .setBookmark(params.bookmark)
+    .setLimit(limit)
+    .run()
+
+  // Try fetching 1 row in the next page to see if another page of results
+  // exists or not
+  search.setBookmark(searchResults.bookmark).setLimit(1)
+  if (params.tableId) {
+    search.setTable(params.tableId)
+  }
+  const nextResults = await search.run()
+
+  return {
+    ...searchResults,
+    hasNextPage: nextResults.rows && nextResults.rows.length > 0,
+  }
+}
+
+/**
+ * Performs a full search, fetching multiple pages if required to return the
+ * desired amount of results. There is a limit of 1000 results to avoid
+ * heavy performance hits, and to avoid client components breaking from
+ * handling too much data.
+ * @param dbName {string} Which database to run a lucene query on
+ * @param index {string} Which search index to utilise
+ * @param query {object} The JSON query structure
+ * @param params {object} The search params including:
+ *   tableId {string} The table ID to search
+ *   sort {string} The sort column
+ *   sortOrder {string} The sort order ("ascending" or "descending")
+ *   sortType {string} Whether to treat sortable values as strings or
+ *     numbers. ("string" or "number")
+ *   limit {number} The desired number of results
+ * @returns {Promise<{rows: *}>}
+ */
+export async function fullSearch<T>(
+  dbName: string,
+  index: string,
+  query: SearchFilters,
+  params: SearchParams<T>
+) {
+  let limit = params.limit
+  if (limit == null || isNaN(limit) || limit < 0) {
+    limit = 1000
+  }
+  params.limit = Math.min(limit, 1000)
+  const rows = await recursiveSearch<T>(dbName, index, query, params)
+  return { rows }
+}

packages/backend-core/src/db/tests/lucene.spec.ts

@@ -0,0 +1,161 @@
+import { newid } from "../../newid"
+import { getDB } from "../db"
+import { Database } from "@budibase/types"
+import { QueryBuilder, paginatedSearch, fullSearch } from "../lucene"
+
+const INDEX_NAME = "main"
+
+const index = `function(doc) {
+  let props = ["property", "number"]
+  for (let key of props) {
+    if (doc[key]) {
+      index(key, doc[key])
+    }
+  }
+}`
+
+describe("lucene", () => {
+  let db: Database, dbName: string
+
+  beforeAll(async () => {
+    dbName = `db-${newid()}`
+    // create the DB for testing
+    db = getDB(dbName)
+    await db.put({ _id: newid(), property: "word" })
+    await db.put({ _id: newid(), property: "word2" })
+    await db.put({ _id: newid(), property: "word3", number: 1 })
+  })
+
+  it("should be able to create a lucene index", async () => {
+    const response = await db.put({
+      _id: "_design/database",
+      indexes: {
+        [INDEX_NAME]: {
+          index: index,
+          analyzer: "standard",
+        },
+      },
+    })
+    expect(response.ok).toBe(true)
+  })
+
+  describe("query builder", () => {
+    it("should be able to perform a basic query", async () => {
+      const builder = new QueryBuilder(dbName, INDEX_NAME)
+      builder.setSort("property")
+      builder.setSortOrder("desc")
+      builder.setSortType("string")
+      const resp = await builder.run()
+      expect(resp.rows.length).toBe(3)
+    })
+
+    it("should handle limits", async () => {
+      const builder = new QueryBuilder(dbName, INDEX_NAME)
+      builder.setLimit(1)
+      const resp = await builder.run()
+      expect(resp.rows.length).toBe(1)
+    })
+
+    it("should be able to perform a string search", async () => {
+      const builder = new QueryBuilder(dbName, INDEX_NAME)
+      builder.addString("property", "wo")
+      const resp = await builder.run()
+      expect(resp.rows.length).toBe(3)
+    })
+
+    it("should be able to perform a range search", async () => {
+      const builder = new QueryBuilder(dbName, INDEX_NAME)
+      builder.addRange("number", 0, 1)
+      const resp = await builder.run()
+      expect(resp.rows.length).toBe(1)
+    })
+
+    it("should be able to perform an equal search", async () => {
+      const builder = new QueryBuilder(dbName, INDEX_NAME)
+      builder.addEqual("property", "word2")
+      const resp = await builder.run()
+      expect(resp.rows.length).toBe(1)
+    })
+
+    it("should be able to perform a not equal search", async () => {
+      const builder = new QueryBuilder(dbName, INDEX_NAME)
+      builder.addNotEqual("property", "word2")
+      const resp = await builder.run()
+      expect(resp.rows.length).toBe(2)
+    })
+
+    it("should be able to perform an empty search", async () => {
+      const builder = new QueryBuilder(dbName, INDEX_NAME)
+      builder.addEmpty("number", true)
+      const resp = await builder.run()
+      expect(resp.rows.length).toBe(2)
+    })
+
+    it("should be able to perform a not empty search", async () => {
+      const builder = new QueryBuilder(dbName, INDEX_NAME)
+      builder.addNotEmpty("number", true)
+      const resp = await builder.run()
+      expect(resp.rows.length).toBe(1)
+    })
+
+    it("should be able to perform a one of search", async () => {
+      const builder = new QueryBuilder(dbName, INDEX_NAME)
+      builder.addOneOf("property", ["word", "word2"])
+      const resp = await builder.run()
+      expect(resp.rows.length).toBe(2)
+    })
+
+    it("should be able to perform a contains search", async () => {
+      const builder = new QueryBuilder(dbName, INDEX_NAME)
+      builder.addContains("property", ["word"])
+      const resp = await builder.run()
+      expect(resp.rows.length).toBe(1)
+    })
+
+    it("should be able to perform a not contains search", async () => {
+      const builder = new QueryBuilder(dbName, INDEX_NAME)
+      builder.addNotContains("property", ["word2"])
+      const resp = await builder.run()
+      expect(resp.rows.length).toBe(2)
+    })
+  })
+
+  describe("paginated search", () => {
+    it("should be able to perform a paginated search", async () => {
+      const page = await paginatedSearch(
+        dbName,
+        INDEX_NAME,
+        {
+          string: {
+            property: "wo",
+          },
+        },
+        {
+          limit: 1,
+          sort: "property",
+          sortType: "string",
+          sortOrder: "desc",
+        }
+      )
+      expect(page.rows.length).toBe(1)
+      expect(page.hasNextPage).toBe(true)
+      expect(page.bookmark).toBeDefined()
+    })
+  })
+
+  describe("full search", () => {
+    it("should be able to perform a full search", async () => {
+      const page = await fullSearch(
+        dbName,
+        INDEX_NAME,
+        {
+          string: {
+            property: "wo",
+          },
+        },
+        {}
+      )
+      expect(page.rows.length).toBe(3)
+    })
+  })
+})

packages/backend-core/src/db/utils.ts

@@ -365,6 +365,16 @@ export async function getAllApps({
   }
 }
 
+export async function getAppsByIDs(appIds: string[]) {
+  const settled = await Promise.allSettled(
+    appIds.map(appId => getAppMetadata(appId))
+  )
+  // have to list the apps which exist, some may have been deleted
+  return settled
+    .filter(promise => promise.status === "fulfilled")
+    .map(promise => (promise as PromiseFulfilledResult<App>).value)
+}
+
 /**
  * Utility function for getAllApps but filters to production apps only.
  */
@@ -381,6 +391,16 @@ export async function getDevAppIDs() {
   return apps.filter((id: any) => isDevAppID(id))
 }
 
+export function isSameAppID(
+  appId1: string | undefined,
+  appId2: string | undefined
+) {
+  if (appId1 == undefined || appId2 == undefined) {
+    return false
+  }
+  return getProdAppID(appId1) === getProdAppID(appId2)
+}
+
 export async function dbExists(dbName: any) {
   return doWithDB(
     dbName,

packages/backend-core/src/environment.ts

@@ -86,6 +86,7 @@ const environment = {
   DEPLOYMENT_ENVIRONMENT:
     process.env.DEPLOYMENT_ENVIRONMENT || "docker-compose",
   ENABLE_4XX_HTTP_LOGGING: process.env.ENABLE_4XX_HTTP_LOGGING || true,
+  ENABLE_AUDIT_LOG_IP_ADDR: process.env.ENABLE_AUDIT_LOG_IP_ADDR,
   // smtp
   SMTP_FALLBACK_ENABLED: process.env.SMTP_FALLBACK_ENABLED,
   SMTP_USER: process.env.SMTP_USER,

packages/backend-core/src/events/events.ts

@@ -1,4 +1,4 @@
-import { Event } from "@budibase/types"
+import { Event, AuditedEventFriendlyName } from "@budibase/types"
 import { processors } from "./processors"
 import identification from "./identification"
 import * as backfill from "./backfill"

packages/backend-core/src/events/identification.ts

@@ -87,6 +87,7 @@ const getCurrentIdentity = async (): Promise<Identity> => {
       installationId,
       tenantId,
       environment,
+      hostInfo: userContext.hostInfo,
     }
   } else {
     throw new Error("Unknown identity type")

packages/backend-core/src/events/processors/AuditLogsProcessor.ts

@@ -0,0 +1,90 @@
+import {
+  Event,
+  Identity,
+  Group,
+  IdentityType,
+  AuditLogQueueEvent,
+  AuditLogFn,
+  HostInfo,
+} from "@budibase/types"
+import { EventProcessor } from "./types"
+import { getAppId } from "../../context"
+import BullQueue from "bull"
+import { createQueue, JobQueue } from "../../queue"
+import { isAudited } from "../../utils"
+import env from "../../environment"
+
+export default class AuditLogsProcessor implements EventProcessor {
+  static auditLogsEnabled = false
+  static auditLogQueue: BullQueue.Queue<AuditLogQueueEvent>
+
+  // can't use constructor as need to return promise
+  static init(fn: AuditLogFn) {
+    AuditLogsProcessor.auditLogsEnabled = true
+    const writeAuditLogs = fn
+    AuditLogsProcessor.auditLogQueue = createQueue<AuditLogQueueEvent>(
+      JobQueue.AUDIT_LOG
+    )
+    return AuditLogsProcessor.auditLogQueue.process(async job => {
+      let properties = job.data.properties
+      if (properties.audited) {
+        properties = {
+          ...properties,
+          ...properties.audited,
+        }
+        delete properties.audited
+      }
+
+      // this feature is disabled by default due to privacy requirements
+      // in some countries - available as env var in-case it is desired
+      // in self host deployments
+      let hostInfo: HostInfo | undefined = {}
+      if (env.ENABLE_AUDIT_LOG_IP_ADDR) {
+        hostInfo = job.data.opts.hostInfo
+      }
+
+      await writeAuditLogs(job.data.event, properties, {
+        userId: job.data.opts.userId,
+        timestamp: job.data.opts.timestamp,
+        appId: job.data.opts.appId,
+        hostInfo,
+      })
+    })
+  }
+
+  async processEvent(
+    event: Event,
+    identity: Identity,
+    properties: any,
+    timestamp?: string
+  ): Promise<void> {
+    if (AuditLogsProcessor.auditLogsEnabled && isAudited(event)) {
+      // only audit log actual events, don't include backfills
+      const userId =
+        identity.type === IdentityType.USER ? identity.id : undefined
+      // add to the event queue, rather than just writing immediately
+      await AuditLogsProcessor.auditLogQueue.add({
+        event,
+        properties,
+        opts: {
+          userId,
+          timestamp,
+          appId: getAppId(),
+          hostInfo: identity.hostInfo,
+        },
+      })
+    }
+  }
+
+  async identify(identity: Identity, timestamp?: string | number) {
+    // no-op
+  }
+
+  async identifyGroup(group: Group, timestamp?: string | number) {
+    // no-op
+  }
+
+  shutdown(): void {
+    AuditLogsProcessor.auditLogQueue?.close()
+  }
+}

packages/backend-core/src/events/processors/index.ts

@@ -1,8 +1,19 @@
 import AnalyticsProcessor from "./AnalyticsProcessor"
 import LoggingProcessor from "./LoggingProcessor"
+import AuditLogsProcessor from "./AuditLogsProcessor"
 import Processors from "./Processors"
+import { AuditLogFn } from "@budibase/types"
 
 export const analyticsProcessor = new AnalyticsProcessor()
 const loggingProcessor = new LoggingProcessor()
+const auditLogsProcessor = new AuditLogsProcessor()
 
-export const processors = new Processors([analyticsProcessor, loggingProcessor])
+export function init(auditingFn: AuditLogFn) {
+  return AuditLogsProcessor.init(auditingFn)
+}
+
+export const processors = new Processors([
+  analyticsProcessor,
+  loggingProcessor,
+  auditLogsProcessor,
+])

packages/backend-core/src/events/processors/posthog/PosthogProcessor.ts

@@ -47,6 +47,8 @@ export default class PosthogProcessor implements EventProcessor {
       return
     }
 
+    properties = this.clearPIIProperties(properties)
+
     properties.version = pkg.version
     properties.service = env.SERVICE
     properties.environment = identity.environment
@@ -79,6 +81,16 @@ export default class PosthogProcessor implements EventProcessor {
     this.posthog.capture(payload)
   }
 
+  clearPIIProperties(properties: any) {
+    if (properties.email) {
+      delete properties.email
+    }
+    if (properties.audited) {
+      delete properties.audited
+    }
+    return properties
+  }
+
   async identify(identity: Identity, timestamp?: string | number) {
     const payload: any = { distinctId: identity.id, properties: identity }
     if (timestamp) {

packages/backend-core/src/events/processors/posthog/tests/PosthogProcessor.spec.ts

@@ -49,6 +49,25 @@ describe("PosthogProcessor", () => {
       expect(processor.posthog.capture).toHaveBeenCalledTimes(0)
     })
 
+    it("removes audited information", async () => {
+      const processor = new PosthogProcessor("test")
+
+      const identity = newIdentity()
+      const properties = {
+        email: "test",
+        audited: {
+          name: "test",
+        },
+      }
+
+      await processor.processEvent(Event.USER_CREATED, identity, properties)
+      expect(processor.posthog.capture).toHaveBeenCalled()
+      // @ts-ignore
+      const call = processor.posthog.capture.mock.calls[0][0]
+      expect(call.properties.audited).toBeUndefined()
+      expect(call.properties.email).toBeUndefined()
+    })
+
     describe("rate limiting", () => {
       it("sends daily event once in same day", async () => {
         const processor = new PosthogProcessor("test")

packages/backend-core/src/events/publishers/app.ts

@@ -19,6 +19,9 @@ const created = async (app: App, timestamp?: string | number) => {
   const properties: AppCreatedEvent = {
     appId: app.appId,
     version: app.version,
+    audited: {
+      name: app.name,
+    },
   }
   await publishEvent(Event.APP_CREATED, properties, timestamp)
 }
@@ -27,6 +30,9 @@ async function updated(app: App) {
   const properties: AppUpdatedEvent = {
     appId: app.appId,
     version: app.version,
+    audited: {
+      name: app.name,
+    },
   }
   await publishEvent(Event.APP_UPDATED, properties)
 }
@@ -34,6 +40,9 @@ async function updated(app: App) {
 async function deleted(app: App) {
   const properties: AppDeletedEvent = {
     appId: app.appId,
+    audited: {
+      name: app.name,
+    },
   }
   await publishEvent(Event.APP_DELETED, properties)
 }
@@ -41,6 +50,9 @@ async function deleted(app: App) {
 async function published(app: App, timestamp?: string | number) {
   const properties: AppPublishedEvent = {
     appId: app.appId,
+    audited: {
+      name: app.name,
+    },
   }
   await publishEvent(Event.APP_PUBLISHED, properties, timestamp)
 }
@@ -48,6 +60,9 @@ async function published(app: App, timestamp?: string | number) {
 async function unpublished(app: App) {
   const properties: AppUnpublishedEvent = {
     appId: app.appId,
+    audited: {
+      name: app.name,
+    },
   }
   await publishEvent(Event.APP_UNPUBLISHED, properties)
 }
@@ -55,6 +70,9 @@ async function unpublished(app: App) {
 async function fileImported(app: App) {
   const properties: AppFileImportedEvent = {
     appId: app.appId,
+    audited: {
+      name: app.name,
+    },
   }
   await publishEvent(Event.APP_FILE_IMPORTED, properties)
 }
@@ -63,6 +81,9 @@ async function templateImported(app: App, templateKey: string) {
   const properties: AppTemplateImportedEvent = {
     appId: app.appId,
     templateKey,
+    audited: {
+      name: app.name,
+    },
   }
   await publishEvent(Event.APP_TEMPLATE_IMPORTED, properties)
 }
@@ -76,6 +97,9 @@ async function versionUpdated(
     appId: app.appId,
     currentVersion,
     updatedToVersion,
+    audited: {
+      name: app.name,
+    },
   }
   await publishEvent(Event.APP_VERSION_UPDATED, properties)
 }
@@ -89,6 +113,9 @@ async function versionReverted(
     appId: app.appId,
     currentVersion,
     revertedToVersion,
+    audited: {
+      name: app.name,
+    },
   }
   await publishEvent(Event.APP_VERSION_REVERTED, properties)
 }
@@ -96,6 +123,9 @@ async function versionReverted(
 async function reverted(app: App) {
   const properties: AppRevertedEvent = {
     appId: app.appId,
+    audited: {
+      name: app.name,
+    },
   }
   await publishEvent(Event.APP_REVERTED, properties)
 }
@@ -103,6 +133,9 @@ async function reverted(app: App) {
 async function exported(app: App) {
   const properties: AppExportedEvent = {
     appId: app.appId,
+    audited: {
+      name: app.name,
+    },
   }
   await publishEvent(Event.APP_EXPORTED, properties)
 }

packages/backend-core/src/events/publishers/auditLog.ts

@@ -0,0 +1,26 @@
+import {
+  Event,
+  AuditLogSearchParams,
+  AuditLogFilteredEvent,
+  AuditLogDownloadedEvent,
+} from "@budibase/types"
+import { publishEvent } from "../events"
+
+async function filtered(search: AuditLogSearchParams) {
+  const properties: AuditLogFilteredEvent = {
+    filters: search,
+  }
+  await publishEvent(Event.AUDIT_LOGS_FILTERED, properties)
+}
+
+async function downloaded(search: AuditLogSearchParams) {
+  const properties: AuditLogDownloadedEvent = {
+    filters: search,
+  }
+  await publishEvent(Event.AUDIT_LOGS_DOWNLOADED, properties)
+}
+
+export default {
+  filtered,
+  downloaded,
+}

packages/backend-core/src/events/publishers/auth.ts

@@ -12,19 +12,25 @@ import {
 } from "@budibase/types"
 import { identification } from ".."
 
-async function login(source: LoginSource) {
+async function login(source: LoginSource, email: string) {
   const identity = await identification.getCurrentIdentity()
   const properties: LoginEvent = {
     userId: identity.id,
     source,
+    audited: {
+      email,
+    },
   }
   await publishEvent(Event.AUTH_LOGIN, properties)
 }
 
-async function logout() {
+async function logout(email?: string) {
   const identity = await identification.getCurrentIdentity()
   const properties: LogoutEvent = {
     userId: identity.id,
+    audited: {
+      email,
+    },
   }
   await publishEvent(Event.AUTH_LOGOUT, properties)
 }

packages/backend-core/src/events/publishers/automation.ts

@@ -18,6 +18,9 @@ async function created(automation: Automation, timestamp?: string | number) {
     automationId: automation._id as string,
     triggerId: automation.definition?.trigger?.id,
     triggerType: automation.definition?.trigger?.stepId,
+    audited: {
+      name: automation.name,
+    },
   }
   await publishEvent(Event.AUTOMATION_CREATED, properties, timestamp)
 }
@@ -38,6 +41,9 @@ async function deleted(automation: Automation) {
     automationId: automation._id as string,
     triggerId: automation.definition?.trigger?.id,
     triggerType: automation.definition?.trigger?.stepId,
+    audited: {
+      name: automation.name,
+    },
   }
   await publishEvent(Event.AUTOMATION_DELETED, properties)
 }
@@ -71,6 +77,9 @@ async function stepCreated(
     triggerType: automation.definition?.trigger?.stepId,
     stepId: step.id!,
     stepType: step.stepId,
+    audited: {
+      name: automation.name,
+    },
   }
   await publishEvent(Event.AUTOMATION_STEP_CREATED, properties, timestamp)
 }
@@ -83,6 +92,9 @@ async function stepDeleted(automation: Automation, step: AutomationStep) {
     triggerType: automation.definition?.trigger?.stepId,
     stepId: step.id!,
     stepType: step.stepId,
+    audited: {
+      name: automation.name,
+    },
   }
   await publishEvent(Event.AUTOMATION_STEP_DELETED, properties)
 }

packages/backend-core/src/events/publishers/backup.ts

@@ -13,6 +13,7 @@ async function appBackupRestored(backup: AppBackup) {
     appId: backup.appId,
     restoreId: backup._id!,
     backupCreatedAt: backup.timestamp,
+    name: backup.name as string,
   }
 
   await publishEvent(Event.APP_BACKUP_RESTORED, properties)
@@ -22,13 +23,15 @@ async function appBackupTriggered(
   appId: string,
   backupId: string,
   type: AppBackupType,
-  trigger: AppBackupTrigger
+  trigger: AppBackupTrigger,
+  name: string
 ) {
   const properties: AppBackupTriggeredEvent = {
     appId: appId,
     backupId,
     type,
     trigger,
+    name,
   }
   await publishEvent(Event.APP_BACKUP_TRIGGERED, properties)
 }

packages/backend-core/src/events/publishers/group.ts

@@ -8,12 +8,16 @@ import {
   GroupUsersAddedEvent,
   GroupUsersDeletedEvent,
   GroupAddedOnboardingEvent,
+  GroupPermissionsEditedEvent,
   UserGroupRoles,
 } from "@budibase/types"
 
 async function created(group: UserGroup, timestamp?: number) {
   const properties: GroupCreatedEvent = {
     groupId: group._id as string,
+    audited: {
+      name: group.name,
+    },
   }
   await publishEvent(Event.USER_GROUP_CREATED, properties, timestamp)
 }
@@ -21,6 +25,9 @@ async function created(group: UserGroup, timestamp?: number) {
 async function updated(group: UserGroup) {
   const properties: GroupUpdatedEvent = {
     groupId: group._id as string,
+    audited: {
+      name: group.name,
+    },
   }
   await publishEvent(Event.USER_GROUP_UPDATED, properties)
 }
@@ -28,6 +35,9 @@ async function updated(group: UserGroup) {
 async function deleted(group: UserGroup) {
   const properties: GroupDeletedEvent = {
     groupId: group._id as string,
+    audited: {
+      name: group.name,
+    },
   }
   await publishEvent(Event.USER_GROUP_DELETED, properties)
 }
@@ -36,6 +46,9 @@ async function usersAdded(count: number, group: UserGroup) {
   const properties: GroupUsersAddedEvent = {
     count,
     groupId: group._id as string,
+    audited: {
+      name: group.name,
+    },
   }
   await publishEvent(Event.USER_GROUP_USERS_ADDED, properties)
 }
@@ -44,6 +57,9 @@ async function usersDeleted(count: number, group: UserGroup) {
   const properties: GroupUsersDeletedEvent = {
     count,
     groupId: group._id as string,
+    audited: {
+      name: group.name,
+    },
   }
   await publishEvent(Event.USER_GROUP_USERS_REMOVED, properties)
 }
@@ -56,9 +72,13 @@ async function createdOnboarding(groupId: string) {
   await publishEvent(Event.USER_GROUP_ONBOARDING, properties)
 }
 
-async function permissionsEdited(roles: UserGroupRoles) {
+async function permissionsEdited(group: UserGroup) {
-  const properties: UserGroupRoles = {
+  const properties: GroupPermissionsEditedEvent = {
-    ...roles,
+    permissions: group.roles!,
+    groupId: group._id as string,
+    audited: {
+      name: group.name,
+    },
   }
   await publishEvent(Event.USER_GROUP_PERMISSIONS_EDITED, properties)
 }

packages/backend-core/src/events/publishers/index.ts

@@ -21,3 +21,4 @@ export { default as group } from "./group"
 export { default as plugin } from "./plugin"
 export { default as backup } from "./backup"
 export { default as environmentVariable } from "./environmentVariable"
+export { default as auditLog } from "./auditLog"

packages/backend-core/src/events/publishers/screen.ts

@@ -11,6 +11,9 @@ async function created(screen: Screen, timestamp?: string | number) {
     layoutId: screen.layoutId,
     screenId: screen._id as string,
     roleId: screen.routing.roleId,
+    audited: {
+      name: screen.routing?.route,
+    },
   }
   await publishEvent(Event.SCREEN_CREATED, properties, timestamp)
 }
@@ -20,6 +23,9 @@ async function deleted(screen: Screen) {
     layoutId: screen.layoutId,
     screenId: screen._id as string,
     roleId: screen.routing.roleId,
+    audited: {
+      name: screen.routing?.route,
+    },
   }
   await publishEvent(Event.SCREEN_DELETED, properties)
 }

packages/backend-core/src/events/publishers/table.ts

@@ -13,6 +13,9 @@ import {
 async function created(table: Table, timestamp?: string | number) {
   const properties: TableCreatedEvent = {
     tableId: table._id as string,
+    audited: {
+      name: table.name,
+    },
   }
   await publishEvent(Event.TABLE_CREATED, properties, timestamp)
 }
@@ -20,6 +23,9 @@ async function created(table: Table, timestamp?: string | number) {
 async function updated(table: Table) {
   const properties: TableUpdatedEvent = {
     tableId: table._id as string,
+    audited: {
+      name: table.name,
+    },
   }
   await publishEvent(Event.TABLE_UPDATED, properties)
 }
@@ -27,6 +33,9 @@ async function updated(table: Table) {
 async function deleted(table: Table) {
   const properties: TableDeletedEvent = {
     tableId: table._id as string,
+    audited: {
+      name: table.name,
+    },
   }
   await publishEvent(Event.TABLE_DELETED, properties)
 }
@@ -35,6 +44,9 @@ async function exported(table: Table, format: TableExportFormat) {
   const properties: TableExportedEvent = {
     tableId: table._id as string,
     format,
+    audited: {
+      name: table.name,
+    },
   }
   await publishEvent(Event.TABLE_EXPORTED, properties)
 }
@@ -42,6 +54,9 @@ async function exported(table: Table, format: TableExportFormat) {
 async function imported(table: Table) {
   const properties: TableImportedEvent = {
     tableId: table._id as string,
+    audited: {
+      name: table.name,
+    },
   }
   await publishEvent(Event.TABLE_IMPORTED, properties)
 }

packages/backend-core/src/events/publishers/user.ts

@@ -19,6 +19,9 @@ import {
 async function created(user: User, timestamp?: number) {
   const properties: UserCreatedEvent = {
     userId: user._id as string,
+    audited: {
+      email: user.email,
+    },
   }
   await publishEvent(Event.USER_CREATED, properties, timestamp)
 }
@@ -26,6 +29,9 @@ async function created(user: User, timestamp?: number) {
 async function updated(user: User) {
   const properties: UserUpdatedEvent = {
     userId: user._id as string,
+    audited: {
+      email: user.email,
+    },
   }
   await publishEvent(Event.USER_UPDATED, properties)
 }
@@ -33,6 +39,9 @@ async function updated(user: User) {
 async function deleted(user: User) {
   const properties: UserDeletedEvent = {
     userId: user._id as string,
+    audited: {
+      email: user.email,
+    },
   }
   await publishEvent(Event.USER_DELETED, properties)
 }
@@ -40,6 +49,9 @@ async function deleted(user: User) {
 export async function onboardingComplete(user: User) {
   const properties: UserOnboardingEvent = {
     userId: user._id as string,
+    audited: {
+      email: user.email,
+    },
   }
   await publishEvent(Event.USER_ONBOARDING_COMPLETE, properties)
 }
@@ -49,6 +61,9 @@ export async function onboardingComplete(user: User) {
 async function permissionAdminAssigned(user: User, timestamp?: number) {
   const properties: UserPermissionAssignedEvent = {
     userId: user._id as string,
+    audited: {
+      email: user.email,
+    },
   }
   await publishEvent(
     Event.USER_PERMISSION_ADMIN_ASSIGNED,
@@ -60,6 +75,9 @@ async function permissionAdminAssigned(user: User, timestamp?: number) {
 async function permissionAdminRemoved(user: User) {
   const properties: UserPermissionRemovedEvent = {
     userId: user._id as string,
+    audited: {
+      email: user.email,
+    },
   }
   await publishEvent(Event.USER_PERMISSION_ADMIN_REMOVED, properties)
 }
@@ -67,6 +85,9 @@ async function permissionAdminRemoved(user: User) {
 async function permissionBuilderAssigned(user: User, timestamp?: number) {
   const properties: UserPermissionAssignedEvent = {
     userId: user._id as string,
+    audited: {
+      email: user.email,
+    },
   }
   await publishEvent(
     Event.USER_PERMISSION_BUILDER_ASSIGNED,
@@ -78,20 +99,30 @@ async function permissionBuilderAssigned(user: User, timestamp?: number) {
 async function permissionBuilderRemoved(user: User) {
   const properties: UserPermissionRemovedEvent = {
     userId: user._id as string,
+    audited: {
+      email: user.email,
+    },
   }
   await publishEvent(Event.USER_PERMISSION_BUILDER_REMOVED, properties)
 }
 
 // INVITE
 
-async function invited() {
+async function invited(email: string) {
-  const properties: UserInvitedEvent = {}
+  const properties: UserInvitedEvent = {
+    audited: {
+      email,
+    },
+  }
   await publishEvent(Event.USER_INVITED, properties)
 }
 
 async function inviteAccepted(user: User) {
   const properties: UserInviteAcceptedEvent = {
     userId: user._id as string,
+    audited: {
+      email: user.email,
+    },
   }
   await publishEvent(Event.USER_INVITED_ACCEPTED, properties)
 }
@@ -101,6 +132,9 @@ async function inviteAccepted(user: User) {
 async function passwordForceReset(user: User) {
   const properties: UserPasswordForceResetEvent = {
     userId: user._id as string,
+    audited: {
+      email: user.email,
+    },
   }
   await publishEvent(Event.USER_PASSWORD_FORCE_RESET, properties)
 }
@@ -108,6 +142,9 @@ async function passwordForceReset(user: User) {
 async function passwordUpdated(user: User) {
   const properties: UserPasswordUpdatedEvent = {
     userId: user._id as string,
+    audited: {
+      email: user.email,
+    },
   }
   await publishEvent(Event.USER_PASSWORD_UPDATED, properties)
 }
@@ -115,6 +152,9 @@ async function passwordUpdated(user: User) {
 async function passwordResetRequested(user: User) {
   const properties: UserPasswordResetRequestedEvent = {
     userId: user._id as string,
+    audited: {
+      email: user.email,
+    },
   }
   await publishEvent(Event.USER_PASSWORD_RESET_REQUESTED, properties)
 }
@@ -122,6 +162,9 @@ async function passwordResetRequested(user: User) {
 async function passwordReset(user: User) {
   const properties: UserPasswordResetEvent = {
     userId: user._id as string,
+    audited: {
+      email: user.email,
+    },
   }
   await publishEvent(Event.USER_PASSWORD_RESET, properties)
 }

packages/backend-core/src/index.ts

@@ -21,11 +21,11 @@ export * as context from "./context"
 export * as cache from "./cache"
 export * as objectStore from "./objectStore"
 export * as redis from "./redis"
-export * as locks from "./redis/redlock"
+export * as locks from "./redis/redlockImpl"
 export * as utils from "./utils"
 export * as errors from "./errors"
 export { default as env } from "./environment"
-
+export { SearchParams } from "./db"
 // Add context to tenancy for backwards compatibility
 // only do this for external usages to prevent internal
 // circular dependencies

packages/backend-core/src/middleware/authenticated.ts

@@ -8,7 +8,7 @@ import { getGlobalDB, doInTenant } from "../context"
 import { decrypt } from "../security/encryption"
 import * as identity from "../context/identity"
 import env from "../environment"
-import { BBContext, EndpointMatcher } from "@budibase/types"
+import { Ctx, EndpointMatcher } from "@budibase/types"
 
 const ONE_MINUTE = env.SESSION_UPDATE_PERIOD
   ? parseInt(env.SESSION_UPDATE_PERIOD)
@@ -73,7 +73,7 @@ export default function (
   }
 ) {
   const noAuthOptions = noAuthPatterns ? buildMatcherRegex(noAuthPatterns) : []
-  return async (ctx: BBContext | any, next: any) => {
+  return async (ctx: Ctx | any, next: any) => {
     let publicEndpoint = false
     const version = ctx.request.headers[Header.API_VER]
     // the path is not authenticated
@@ -149,7 +149,7 @@ export default function (
       finalise(ctx, { authenticated, user, internal, version, publicEndpoint })
 
       if (user && user.email) {
-        return identity.doInUserContext(user, next)
+        return identity.doInUserContext(user, ctx, next)
       } else {
         return next()
       }

packages/backend-core/src/middleware/index.ts

@@ -17,4 +17,5 @@ export { default as builderOrAdmin } from "./builderOrAdmin"
 export { default as builderOnly } from "./builderOnly"
 export { default as logging } from "./logging"
 export { default as errorHandling } from "./errorHandling"
+export { default as querystringToBody } from "./querystringToBody"
 export * as joiValidator from "./joi-validator"

packages/backend-core/src/middleware/querystringToBody.ts

@@ -0,0 +1,28 @@
+import { Ctx } from "@budibase/types"
+
+/**
+ * Expects a standard "query" query string property which is the JSON body
+ * of the request, which has to be sent via query string due to the requirement
+ * of making an endpoint a GET request e.g. downloading a file stream.
+ */
+export default function (ctx: Ctx, next: any) {
+  const queryString = ctx.request.query?.query as string | undefined
+  if (ctx.request.method.toLowerCase() !== "get") {
+    ctx.throw(
+      500,
+      "Query to download middleware can only be used for get requests."
+    )
+  }
+  if (!queryString) {
+    return next()
+  }
+  const decoded = decodeURIComponent(queryString)
+  let json
+  try {
+    json = JSON.parse(decoded)
+  } catch (err) {
+    return next()
+  }
+  ctx.request.body = json
+  return next()
+}

packages/backend-core/src/platform/tenants.ts

@@ -1,7 +1,7 @@
 import { StaticDatabases } from "../constants"
 import { getPlatformDB } from "./platformDb"
 import { LockName, LockOptions, LockType, Tenants } from "@budibase/types"
-import * as locks from "../redis/redlock"
+import * as locks from "../redis/redlockImpl"
 
 const TENANT_DOC = StaticDatabases.PLATFORM_INFO.docs.tenants
 

packages/backend-core/src/queue/constants.ts

@@ -1,4 +1,5 @@
 export enum JobQueue {
   AUTOMATION = "automationQueue",
   APP_BACKUP = "appBackupQueue",
+  AUDIT_LOG = "auditLogQueue",
 }

packages/backend-core/src/queue/queue.ts

@@ -40,8 +40,10 @@ export function createQueue<T>(
 }
 
 export async function shutdown() {
-  if (QUEUES.length) {
+  if (cleanupInterval) {
     clearInterval(cleanupInterval)
+  }
+  if (QUEUES.length) {
     for (let queue of QUEUES) {
       await queue.close()
     }

packages/backend-core/src/redis/index.ts

@@ -3,4 +3,4 @@
 export { default as Client } from "./redis"
 export * as utils from "./utils"
 export * as clients from "./init"
-export * as locks from "./redlock"
+export * as locks from "./redlockImpl"

packages/backend-core/src/users.ts

@@ -10,14 +10,38 @@ import { BulkDocsResponse, User } from "@budibase/types"
 import { getGlobalDB } from "./context"
 import * as context from "./context"
 
-export const bulkGetGlobalUsersById = async (userIds: string[]) => {
+type GetOpts = { cleanup?: boolean }
+
+function removeUserPassword(users: User | User[]) {
+  if (Array.isArray(users)) {
+    return users.map(user => {
+      if (user) {
+        delete user.password
+        return user
+      }
+    })
+  } else if (users) {
+    delete users.password
+    return users
+  }
+  return users
+}
+
+export const bulkGetGlobalUsersById = async (
+  userIds: string[],
+  opts?: GetOpts
+) => {
   const db = getGlobalDB()
-  return (
+  let users = (
     await db.allDocs({
       keys: userIds,
       include_docs: true,
     })
   ).rows.map(row => row.doc) as User[]
+  if (opts?.cleanup) {
+    users = removeUserPassword(users) as User[]
+  }
+  return users
 }
 
 export const bulkUpdateGlobalUsers = async (users: User[]) => {
@@ -25,18 +49,22 @@ export const bulkUpdateGlobalUsers = async (users: User[]) => {
   return (await db.bulkDocs(users)) as BulkDocsResponse
 }
 
-export async function getById(id: string): Promise<User> {
+export async function getById(id: string, opts?: GetOpts): Promise<User> {
   const db = context.getGlobalDB()
-  return db.get(id)
+  let user = await db.get(id)
+  if (opts?.cleanup) {
+    user = removeUserPassword(user)
+  }
+  return user
 }
 
 /**
  * Given an email address this will use a view to search through
  * all the users to find one with this email address.
- * @param {string} email the email to lookup the user by.
  */
 export const getGlobalUserByEmail = async (
-  email: String
+  email: String,
+  opts?: GetOpts
 ): Promise<User | undefined> => {
   if (email == null) {
     throw "Must supply an email address to view"
@@ -52,10 +80,19 @@ export const getGlobalUserByEmail = async (
     throw new Error(`Multiple users found with email address: ${email}`)
   }
 
-  return response
+  let user = response as User
+  if (opts?.cleanup) {
+    user = removeUserPassword(user) as User
+  }
+
+  return user
 }
 
-export const searchGlobalUsersByApp = async (appId: any, opts: any) => {
+export const searchGlobalUsersByApp = async (
+  appId: any,
+  opts: any,
+  getOpts?: GetOpts
+) => {
   if (typeof appId !== "string") {
     throw new Error("Must provide a string based app ID")
   }
@@ -67,7 +104,11 @@ export const searchGlobalUsersByApp = async (appId: any, opts: any) => {
   if (!response) {
     response = []
   }
-  return Array.isArray(response) ? response : [response]
+  let users: User[] = Array.isArray(response) ? response : [response]
+  if (getOpts?.cleanup) {
+    users = removeUserPassword(users) as User[]
+  }
+  return users
 }
 
 export const getGlobalUserByAppPage = (appId: string, user: User) => {
@@ -80,7 +121,11 @@ export const getGlobalUserByAppPage = (appId: string, user: User) => {
 /**
  * Performs a starts with search on the global email view.
  */
-export const searchGlobalUsersByEmail = async (email: string, opts: any) => {
+export const searchGlobalUsersByEmail = async (
+  email: string,
+  opts: any,
+  getOpts?: GetOpts
+) => {
   if (typeof email !== "string") {
     throw new Error("Must provide a string to search by")
   }
@@ -95,5 +140,9 @@ export const searchGlobalUsersByEmail = async (email: string, opts: any) => {
   if (!response) {
     response = []
   }
-  return Array.isArray(response) ? response : [response]
+  let users: User[] = Array.isArray(response) ? response : [response]
+  if (getOpts?.cleanup) {
+    users = removeUserPassword(users) as User[]
+  }
+  return users
 }

packages/backend-core/src/utils/utils.ts

@@ -10,7 +10,13 @@ import {
 import env from "../environment"
 import * as tenancy from "../tenancy"
 import * as context from "../context"
-import { App, Ctx, TenantResolutionStrategy } from "@budibase/types"
+import {
+  App,
+  AuditedEventFriendlyName,
+  Ctx,
+  Event,
+  TenantResolutionStrategy,
+} from "@budibase/types"
 import { SetOption } from "cookies"
 const jwt = require("jsonwebtoken")
 
@@ -217,3 +223,7 @@ export async function getBuildersCount() {
 export function timeout(timeMs: number) {
   return new Promise(resolve => setTimeout(resolve, timeMs))
 }
+
+export function isAudited(event: Event) {
+  return !!AuditedEventFriendlyName[event]
+}

packages/backend-core/tests/utilities/mocks/licenses.ts

@@ -82,6 +82,10 @@ export const useEnvironmentVariables = () => {
   return useFeature(Feature.ENVIRONMENT_VARIABLES)
 }
 
+export const useAuditLogs = () => {
+  return useFeature(Feature.AUDIT_LOGS)
+}
+
 // QUOTAS
 
 export const setAutomationLogsQuota = (value: number) => {

packages/backend-core/tests/utilities/structures/generator.ts

@@ -0,0 +1,2 @@
+import Chance from "chance"
+export const generator = new Chance()

packages/backend-core/tests/utilities/structures/index.ts

@@ -1,8 +1,4 @@
 export * from "./common"
-
-import Chance from "chance"
-export const generator = new Chance()
-
 export * as accounts from "./accounts"
 export * as apps from "./apps"
 export * as db from "./db"
@@ -12,3 +8,4 @@ export * as plugins from "./plugins"
 export * as sso from "./sso"
 export * as tenant from "./tenants"
 export * as users from "./users"
+export { generator } from "./generator"

packages/backend-core/tests/utilities/structures/shared.ts

@@ -0,0 +1,19 @@
+import { User } from "@budibase/types"
+import { generator } from "./generator"
+import { uuid } from "./common"
+
+export const newEmail = () => {
+  return `${uuid()}@test.com`
+}
+
+export const user = (userProps?: any): User => {
+  return {
+    email: newEmail(),
+    password: "test",
+    roles: { app_test: "admin" },
+    firstName: generator.first(),
+    lastName: generator.last(),
+    pictureUrl: "http://test.com",
+    ...userProps,
+  }
+}

packages/backend-core/tests/utilities/structures/sso.ts

@@ -8,8 +8,36 @@ import {
   SSOProviderType,
   User,
 } from "@budibase/types"
-import { uuid, generator, users, email } from "./index"
+import { generator } from "./generator"
+import { uuid, email } from "./common"
+import * as shared from "./shared"
 import _ from "lodash"
+import { user } from "./shared"
+
+export function authDetails(userDoc?: User): SSOAuthDetails {
+  if (!userDoc) {
+    userDoc = user()
+  }
+
+  const userId = userDoc._id || uuid()
+  const provider = generator.string()
+
+  const profile = ssoProfile(userDoc)
+  profile.provider = provider
+  profile.id = userId
+
+  return {
+    email: userDoc.email,
+    oauth2: {
+      refreshToken: generator.string(),
+      accessToken: generator.string(),
+    },
+    profile,
+    provider,
+    providerType: providerType(),
+    userId,
+  }
+}
 
 export function providerType(): SSOProviderType {
   return _.sample(Object.values(SSOProviderType)) as SSOProviderType
@@ -17,7 +45,7 @@ export function providerType(): SSOProviderType {
 
 export function ssoProfile(user?: User): SSOProfile {
   if (!user) {
-    user = users.user()
+    user = shared.user()
   }
   return {
     id: user._id!,
@@ -33,31 +61,6 @@ export function ssoProfile(user?: User): SSOProfile {
   }
 }
 
-export function authDetails(user?: User): SSOAuthDetails {
-  if (!user) {
-    user = users.user()
-  }
-
-  const userId = user._id || uuid()
-  const provider = generator.string()
-
-  const profile = ssoProfile(user)
-  profile.provider = provider
-  profile.id = userId
-
-  return {
-    email: user.email,
-    oauth2: {
-      refreshToken: generator.string(),
-      accessToken: generator.string(),
-    },
-    profile,
-    provider,
-    providerType: providerType(),
-    userId,
-  }
-}
-
 // OIDC
 
 export function oidcConfig(): OIDCInnerConfig {

packages/backend-core/tests/utilities/structures/users.ts

@@ -1,29 +1,13 @@
-import { generator } from "../"
 import {
   AdminUser,
   BuilderUser,
   SSOAuthDetails,
   SSOUser,
-  User,
 } from "@budibase/types"
-import { v4 as uuid } from "uuid"
+import { user } from "./shared"
-import * as sso from "./sso"
+import { authDetails } from "./sso"
 
-export const newEmail = () => {
+export { user, newEmail } from "./shared"
-  return `${uuid()}@test.com`
-}
-
-export const user = (userProps?: any): User => {
-  return {
-    email: newEmail(),
-    password: "test",
-    roles: { app_test: "admin" },
-    firstName: generator.first(),
-    lastName: generator.last(),
-    pictureUrl: "http://test.com",
-    ...userProps,
-  }
-}
 
 export const adminUser = (userProps?: any): AdminUser => {
   return {
@@ -53,7 +37,7 @@ export function ssoUser(
   delete base.password
 
   if (!opts.details) {
-    opts.details = sso.authDetails(base)
+    opts.details = authDetails(base)
   }
 
   return {

packages/backend-core/tests/utilities/testContainerUtils.ts

@@ -1,3 +1,31 @@
+import { execSync } from "child_process"
+
+let dockerPsResult: string | undefined
+
+function formatDockerPsResult(serverName: string, port: number) {
+  const lines = dockerPsResult?.split("\n")
+  let first = true
+  if (!lines) {
+    return null
+  }
+  for (let line of lines) {
+    if (first) {
+      first = false
+      continue
+    }
+    let toLookFor = serverName.split("-service")[0]
+    if (!line.includes(toLookFor)) {
+      continue
+    }
+    const regex = new RegExp(`0.0.0.0:([0-9]*)->${port}`, "g")
+    const found = line.match(regex)
+    if (found) {
+      return found[0].split(":")[1].split("->")[0]
+    }
+  }
+  return null
+}
+
 function getTestContainerSettings(
   serverName: string,
   key: string
@@ -14,10 +42,22 @@ function getTestContainerSettings(
 }
 
 function getContainerInfo(containerName: string, port: number) {
-  const assignedPort = getTestContainerSettings(
+  let assignedPort = getTestContainerSettings(
     containerName.toUpperCase(),
     `PORT_${port}`
   )
+  if (!dockerPsResult) {
+    try {
+      const outputBuffer = execSync("docker ps")
+      dockerPsResult = outputBuffer.toString("utf8")
+    } catch (err) {
+      //no-op
+    }
+  }
+  const possiblePort = formatDockerPsResult(containerName, port)
+  if (possiblePort) {
+    assignedPort = possiblePort
+  }
   const host = getTestContainerSettings(containerName.toUpperCase(), "IP")
   return {
     port: assignedPort,
@@ -39,12 +79,15 @@ function getRedisConfig() {
 }
 
 export function setupEnv(...envs: any[]) {
+  const couch = getCouchConfig(),
+    minio = getCouchConfig(),
+    redis = getRedisConfig()
   const configs = [
-    { key: "COUCH_DB_PORT", value: getCouchConfig().port },
+    { key: "COUCH_DB_PORT", value: couch.port },
-    { key: "COUCH_DB_URL", value: getCouchConfig().url },
+    { key: "COUCH_DB_URL", value: couch.url },
-    { key: "MINIO_PORT", value: getMinioConfig().port },
+    { key: "MINIO_PORT", value: minio.port },
-    { key: "MINIO_URL", value: getMinioConfig().url },
+    { key: "MINIO_URL", value: minio.url },
-    { key: "REDIS_URL", value: getRedisConfig().url },
+    { key: "REDIS_URL", value: redis.url },
   ]
 
   for (const config of configs.filter(x => !!x.value)) {

packages/bbui/src/Form/Core/Multiselect.svelte

@@ -14,6 +14,8 @@
   export let autocomplete = false
   export let sort = false
   export let autoWidth = false
+  export let fetchTerm = null
+  export let customPopoverHeight
 
   const dispatch = createEventDispatcher()
 
@@ -83,10 +85,12 @@
   {options}
   isPlaceholder={!value?.length}
   {autocomplete}
+  bind:fetchTerm
   {isOptionSelected}
   {getOptionLabel}
   {getOptionValue}
   onSelectOption={toggleOption}
   {sort}
   {autoWidth}
+  {customPopoverHeight}
 />

packages/bbui/src/Form/Core/Picker.svelte

@@ -31,7 +31,8 @@
   export let autoWidth = false
   export let autocomplete = false
   export let sort = false
-
+  export let fetchTerm = null
+  export let customPopoverHeight
   const dispatch = createEventDispatcher()
 
   let searchTerm = null
@@ -71,7 +72,7 @@
   }
 
   const getFilteredOptions = (options, term, getLabel) => {
-    if (autocomplete && term) {
+    if (autocomplete && term && !fetchTerm) {
       const lowerCaseTerm = term.toLowerCase()
       return options.filter(option => {
         return `${getLabel(option)}`.toLowerCase().includes(lowerCaseTerm)
@@ -136,6 +137,7 @@
   on:close={() => (open = false)}
   useAnchorWidth={!autoWidth}
   maxWidth={autoWidth ? 400 : null}
+  customHeight={customPopoverHeight}
 >
   <div
     class="popover-content"
@@ -144,8 +146,9 @@
   >
     {#if autocomplete}
       <Search
-        value={searchTerm}
+        value={fetchTerm ? fetchTerm : searchTerm}
-        on:change={event => (searchTerm = event.detail)}
+        on:change={event =>
+          fetchTerm ? (fetchTerm = event.detail) : (searchTerm = event.detail)}
         {disabled}
         placeholder="Search"
       />
@@ -247,7 +250,7 @@
   }
   .popover-content.auto-width .spectrum-Menu-itemLabel {
     white-space: nowrap;
-    overflow: hidden;
+    overflow: none;
     text-overflow: ellipsis;
   }
   .popover-content:not(.auto-width) .spectrum-Menu-itemLabel {

packages/bbui/src/Form/Multiselect.svelte

@@ -15,6 +15,10 @@
   export let getOptionValue = option => option
   export let sort = false
   export let autoWidth = false
+  export let autocomplete = false
+  export let fetchTerm = null
+  export let customPopoverHeight
+
   const dispatch = createEventDispatcher()
   const onChange = e => {
     value = e.detail
@@ -34,6 +38,9 @@
     {getOptionLabel}
     {getOptionValue}
     {autoWidth}
+    {autocomplete}
+    {customPopoverHeight}
+    bind:fetchTerm
     on:change={onChange}
     on:click
   />

packages/bbui/src/Form/Select.svelte

@@ -20,6 +20,8 @@
   export let autoWidth = false
   export let sort = false
   export let tooltip = ""
+  export let autocomplete = false
+  export let customPopoverHeight
 
   const dispatch = createEventDispatcher()
   const onChange = e => {
@@ -51,6 +53,8 @@
     {getOptionIcon}
     {getOptionColour}
     {isOptionEnabled}
+    {autocomplete}
+    {customPopoverHeight}
     on:change={onChange}
     on:click
   />

packages/bbui/src/Popover/Popover.svelte

@@ -18,6 +18,7 @@
   export let useAnchorWidth = false
   export let dismissible = true
   export let offset = 5
+  export let customHeight
 
   $: target = portalTarget || getContext(Context.PopoverRoot) || ".spectrum"
 
@@ -74,6 +75,7 @@
       on:keydown={handleEscape}
       class="spectrum-Popover is-open"
       role="presentation"
+      style="height: {customHeight}"
       transition:fly|local={{ y: -20, duration: 200 }}
     >
       <slot />

packages/builder/src/pages/builder/portal/_components/LockedFeature.svelte

@@ -0,0 +1,76 @@
+<script>
+  import {
+    Layout,
+    Heading,
+    Body,
+    Button,
+    Divider,
+    Tags,
+    Tag,
+  } from "@budibase/bbui"
+  import { auth, admin } from "stores/portal"
+
+  export let title
+  export let planType
+  export let description
+  export let enabled
+  export let upgradeButtonClick
+</script>
+
+<Layout noPadding>
+  <Layout gap="XS" noPadding>
+    <div class="title">
+      <Heading size="M">{title}</Heading>
+      {#if !enabled}
+        <Tags>
+          <Tag icon="LockClosed">{planType}</Tag>
+        </Tags>
+      {/if}
+    </div>
+    <Body>{description}</Body>
+  </Layout>
+  <Divider size="S" />
+
+  {#if enabled}
+    <slot />
+  {:else}
+    <div class="buttons">
+      <Button
+        primary
+        disabled={!$auth.accountPortalAccess && $admin.cloud}
+        on:click={async () => upgradeButtonClick()}
+      >
+        Upgrade
+      </Button>
+      <!--Show the view plans button-->
+      <Button
+        secondary
+        on:click={() => {
+          window.open("https://budibase.com/pricing/", "_blank")
+        }}
+      >
+        View Plans
+      </Button>
+    </div>
+  {/if}
+</Layout>
+
+<style>
+  .buttons {
+    display: flex;
+    gap: var(--spacing-l);
+  }
+  .title {
+    display: flex;
+    flex-direction: row;
+    align-items: center;
+    justify-content: flex-start;
+    gap: var(--spacing-m);
+  }
+
+  .buttons {
+    display: flex;
+    flex-direction: row;
+    gap: var(--spacing-m);
+  }
+</style>

packages/builder/src/pages/builder/portal/account/_layout.svelte

@@ -4,12 +4,13 @@
   import { Content, SideNav, SideNavItem } from "components/portal/page"
   import { menu } from "stores/portal"
 
+  $: wide = $isActive("./auditLogs")
   $: pages = $menu.find(x => x.title === "Account")?.subPages || []
   $: !pages.length && $goto("../")
 </script>
 
 <Page>
-  <Content narrow>
+  <Content narrow={!wide}>
     <div slot="side-nav">
       <SideNav>
         {#each pages as { title, href }}

packages/builder/src/pages/builder/portal/account/auditLogs/_components/AppColumnRenderer.svelte

@@ -0,0 +1,5 @@
+<script>
+  export let value
+</script>
+
+<div>{value?.name || ""}</div>

packages/builder/src/pages/builder/portal/account/auditLogs/_components/TimeRenderer.svelte

@@ -0,0 +1,11 @@
+<script>
+  import dayjs from "dayjs"
+  import relativeTime from "dayjs/plugin/relativeTime"
+  dayjs.extend(relativeTime)
+
+  export let row
+</script>
+
+<div>
+  {dayjs(row.timestamp).fromNow()}
+</div>

packages/builder/src/pages/builder/portal/account/auditLogs/_components/UserRenderer.svelte

@@ -0,0 +1,45 @@
+<script>
+  import { Avatar, Tooltip } from "@budibase/bbui"
+  export let row
+
+  let showTooltip
+  const getInitials = user => {
+    let initials = ""
+    initials += user.firstName ? user.firstName[0] : ""
+    initials += user.lastName ? user.lastName[0] : ""
+
+    return initials === "" ? user.email[0] : initials
+  }
+</script>
+
+<div
+  class="container"
+  on:mouseover={() => (showTooltip = true)}
+  on:focus={() => (showTooltip = true)}
+  on:mouseleave={() => (showTooltip = false)}
+>
+  <Avatar size="M" initials={getInitials(row?.user)} />
+</div>
+{#if showTooltip}
+  <div class="tooltip">
+    <Tooltip textWrapping text={row?.user.email} direction="bottom" />
+  </div>
+{/if}
+
+<style>
+  .container {
+    position: relative;
+  }
+  .tooltip {
+    z-index: 1;
+    position: absolute;
+    top: 75%;
+    left: 120%;
+    transform: translateX(-100%) translateY(-50%);
+    display: flex;
+    flex-direction: row;
+    justify-content: flex-end;
+    width: 130px;
+    pointer-events: none;
+  }
+</style>

packages/builder/src/pages/builder/portal/account/auditLogs/_components/ViewDetailsRenderer.svelte

@@ -0,0 +1,13 @@
+<script>
+  import { ActionButton } from "@budibase/bbui"
+  import { getContext } from "svelte"
+
+  export let row
+  const auditLogs = getContext("auditLogs")
+  const onClick = e => {
+    e.stopPropagation()
+    auditLogs.viewDetails(row)
+  }
+</script>
+
+<ActionButton size="S" on:click={onClick}>Details</ActionButton>

packages/builder/src/pages/builder/portal/account/auditLogs/index.svelte

@@ -0,0 +1,501 @@
+<!-- If working on this file, you may notice that if you click the download button in the UI
+     hot reload will stop working due to the use of window.location. You'll need to reload the pag
+     to get it working again.
+-->
+<script>
+  import {
+    Layout,
+    Table,
+    Search,
+    Multiselect,
+    notifications,
+    Icon,
+    clickOutside,
+    CoreTextArea,
+    DatePicker,
+    Pagination,
+    Helpers,
+    Divider,
+    ActionButton,
+  } from "@budibase/bbui"
+  import { licensing, users, apps, auditLogs } from "stores/portal"
+  import LockedFeature from "../../_components/LockedFeature.svelte"
+  import { createPaginationStore } from "helpers/pagination"
+  import { onMount, setContext } from "svelte"
+  import ViewDetailsRenderer from "./_components/ViewDetailsRenderer.svelte"
+  import UserRenderer from "./_components/UserRenderer.svelte"
+  import TimeRenderer from "./_components/TimeRenderer.svelte"
+  import AppColumnRenderer from "./_components/AppColumnRenderer.svelte"
+  import { cloneDeep } from "lodash"
+
+  const schema = {
+    date: { width: "0.8fr" },
+    user: { width: "0.5fr" },
+    name: { width: "2fr", displayName: "Event" },
+    app: { width: "1.5fr" },
+    view: { width: "0.1fr", borderLeft: true, displayName: "" },
+  }
+
+  const customRenderers = [
+    {
+      column: "view",
+      component: ViewDetailsRenderer,
+    },
+    {
+      column: "user",
+      component: UserRenderer,
+    },
+    {
+      column: "date",
+      component: TimeRenderer,
+    },
+    {
+      column: "app",
+      component: AppColumnRenderer,
+    },
+  ]
+
+  let userSearchTerm = ""
+  let logSearchTerm = ""
+  let userPageInfo = createPaginationStore()
+  let logsPageInfo = createPaginationStore()
+
+  let prevUserSearch = undefined
+  let prevLogSearch = undefined
+  let selectedUsers = []
+  let selectedApps = []
+  let selectedEvents = []
+  let selectedLog
+  let sidePanelVisible = false
+  let wideSidePanel = false
+  let timer
+  let startDate = new Date()
+  startDate.setDate(startDate.getDate() - 30)
+  let endDate = new Date()
+
+  $: fetchUsers(userPage, userSearchTerm)
+  $: fetchLogs({
+    logsPage,
+    logSearchTerm,
+    startDate,
+    endDate,
+    selectedUsers,
+    selectedApps,
+    selectedEvents,
+  })
+  $: userPage = $userPageInfo.page
+  $: logsPage = $logsPageInfo.page
+
+  $: sortedUsers = sort(
+    enrich($users.data || [], selectedUsers, "_id"),
+    "email"
+  )
+  $: sortedEvents = sort(
+    enrich(parseEventObject($auditLogs.events), selectedEvents, "id"),
+    "id"
+  )
+  $: sortedApps = sort(enrich($apps, selectedApps, "appId"), "name")
+
+  const debounce = value => {
+    clearTimeout(timer)
+    timer = setTimeout(() => {
+      logSearchTerm = value
+    }, 400)
+  }
+
+  const fetchUsers = async (userPage, search) => {
+    if ($userPageInfo.loading) {
+      return
+    }
+    // need to remove the page if they've started searching
+    if (search && !prevUserSearch) {
+      userPageInfo.reset()
+      userPage = undefined
+    }
+    prevUserSearch = search
+    try {
+      userPageInfo.loading()
+      await users.search({ userPage, email: search })
+      userPageInfo.fetched($users.hasNextPage, $users.nextPage)
+    } catch (error) {
+      notifications.error("Error getting user list")
+    }
+  }
+
+  const fetchLogs = async ({
+    logsPage,
+    logSearchTerm,
+    startDate,
+    endDate,
+    selectedUsers,
+    selectedApps,
+    selectedEvents,
+  }) => {
+    if ($logsPageInfo.loading) {
+      return
+    }
+    // need to remove the page if they've started searching
+    if (logSearchTerm && !prevLogSearch) {
+      logsPageInfo.reset()
+      logsPage = undefined
+    }
+    prevLogSearch = logSearchTerm
+    try {
+      logsPageInfo.loading()
+      await auditLogs.search({
+        bookmark: logsPage,
+        startDate,
+        endDate,
+        fullSearch: logSearchTerm,
+        userIds: selectedUsers,
+        appIds: selectedApps,
+        events: selectedEvents,
+      })
+      logsPageInfo.fetched(
+        $auditLogs.logs.hasNextPage,
+        $auditLogs.logs.bookmark
+      )
+    } catch (error) {
+      notifications.error(`Error getting audit logs - ${error}`)
+    }
+  }
+
+  const enrich = (list, selected, key) => {
+    return list.map(item => {
+      return {
+        ...item,
+        selected:
+          selected.find(x => x === item[key] || x.includes(item[key])) != null,
+      }
+    })
+  }
+
+  const sort = (list, key) => {
+    let sortedList = list.slice()
+    sortedList?.sort((a, b) => {
+      if (a.selected === b.selected) {
+        return a[key] < b[key] ? -1 : 1
+      } else if (a.selected) {
+        return -1
+      } else if (b.selected) {
+        return 1
+      }
+      return 0
+    })
+    return sortedList
+  }
+
+  const parseEventObject = obj => {
+    // convert obj which is an object of key value pairs to an array of objects
+    // with the key as the id and the value as the name
+    if (obj) {
+      return Object.entries(obj).map(([id, label]) => {
+        return { id, label }
+      })
+    }
+  }
+
+  const viewDetails = detail => {
+    selectedLog = detail
+    sidePanelVisible = true
+  }
+
+  const downloadLogs = async () => {
+    try {
+      window.location = auditLogs.getDownloadUrl({
+        startDate,
+        endDate,
+        fullSearch: logSearchTerm,
+        userIds: selectedUsers,
+        appIds: selectedApps,
+        events: selectedEvents,
+      })
+    } catch (error) {
+      notifications.error(`Error downloading logs: ` + error.message)
+    }
+  }
+
+  setContext("auditLogs", {
+    viewDetails,
+  })
+
+  const copyToClipboard = async value => {
+    await Helpers.copyToClipboard(value)
+    notifications.success("Copied")
+  }
+
+  function cleanupMetadata(log) {
+    const cloned = cloneDeep(log)
+    cloned.userId = cloned.user._id
+    if (cloned.app) {
+      cloned.appId = cloned.app.appId
+    }
+    // remove props that are confused/not returned in download
+    delete cloned._id
+    delete cloned._rev
+    delete cloned.app
+    delete cloned.user
+    return cloned
+  }
+
+  onMount(async () => {
+    await auditLogs.getEventDefinitions()
+    await licensing.init()
+  })
+</script>
+
+<LockedFeature
+  title={"Audit Logs"}
+  planType={"Business plan"}
+  description={"View all events that have occurred in your Budibase installation"}
+  enabled={$licensing.auditLogsEnabled}
+  upgradeButtonClick={async () => {
+    $licensing.goToUpgradePage()
+  }}
+>
+  <div class="controls">
+    <div class="select">
+      <Multiselect
+        bind:fetchTerm={userSearchTerm}
+        placeholder="All users"
+        label="Users"
+        autocomplete
+        bind:value={selectedUsers}
+        getOptionValue={user => user._id}
+        getOptionLabel={user => user.email}
+        options={sortedUsers}
+      />
+    </div>
+    <div class="select">
+      <Multiselect
+        autocomplete
+        placeholder="All apps"
+        label="Apps"
+        getOptionValue={app => app.instance._id}
+        getOptionLabel={app => app.name}
+        options={sortedApps}
+        bind:value={selectedApps}
+      />
+    </div>
+    <div class="select">
+      <Multiselect
+        customPopoverHeight="500px"
+        autocomplete
+        getOptionValue={event => event.id}
+        getOptionLabel={event => event.label}
+        options={sortedEvents}
+        placeholder="All events"
+        label="Events"
+        bind:value={selectedEvents}
+      />
+    </div>
+
+    <div class="date-picker">
+      <DatePicker
+        value={[startDate, endDate]}
+        placeholder="Choose date range"
+        range={true}
+        on:change={e => {
+          if (e.detail[0]?.length === 1) {
+            startDate = e.detail[0][0].toISOString()
+            endDate = ""
+          } else if (e.detail[0]?.length > 1) {
+            startDate = e.detail[0][0].toISOString()
+            endDate = e.detail[0][1].toISOString()
+          } else {
+            startDate = ""
+            endDate = ""
+          }
+        }}
+      />
+    </div>
+    <div class="freeSearch">
+      <Search placeholder="Search" on:change={e => debounce(e.detail)} />
+    </div>
+
+    <div class="">
+      <ActionButton size="M" icon="Download" on:click={() => downloadLogs()} />
+    </div>
+  </div>
+  <Layout noPadding>
+    <Table
+      on:click={({ detail }) => viewDetails(detail)}
+      {customRenderers}
+      data={$auditLogs.logs.data}
+      allowEditColumns={false}
+      allowEditRows={false}
+      allowSelectRows={false}
+      {schema}
+    />
+    <div class="pagination">
+      <Pagination
+        page={$logsPageInfo.pageNumber}
+        hasPrevPage={$logsPageInfo.loading ? false : $logsPageInfo.hasPrevPage}
+        hasNextPage={$logsPageInfo.loading ? false : $logsPageInfo.hasNextPage}
+        goToPrevPage={logsPageInfo.prevPage}
+        goToNextPage={logsPageInfo.nextPage}
+      />
+    </div>
+  </Layout>
+</LockedFeature>
+
+{#if selectedLog}
+  <div
+    id="side-panel"
+    class:wide={wideSidePanel}
+    class:visible={sidePanelVisible}
+    use:clickOutside={() => {
+      sidePanelVisible = false
+    }}
+  >
+    <div class="side-panel-header">
+      Audit Log
+      <div class="side-panel-icons">
+        <Icon
+          size="S"
+          hoverable
+          name={wideSidePanel ? "Minimize" : "Maximize"}
+          on:click={() => {
+            wideSidePanel = !wideSidePanel
+          }}
+        />
+        <Icon
+          hoverable
+          name="Close"
+          on:click={() => {
+            sidePanelVisible = false
+          }}
+        />
+      </div>
+    </div>
+    <Divider />
+
+    <div class="side-panel-body">
+      <div
+        on:click={() => copyToClipboard(JSON.stringify(selectedLog.metadata))}
+        class="copy-icon"
+      >
+        <Icon name="Copy" size="S" />
+      </div>
+      <CoreTextArea
+        disabled
+        minHeight={"300px"}
+        height={"100%"}
+        value={JSON.stringify(cleanupMetadata(selectedLog), null, 2)}
+      />
+    </div>
+  </div>
+{/if}
+
+<style>
+  .copy-icon {
+    right: 16px;
+    top: 80px;
+    z-index: 10;
+    justify-content: center;
+    align-items: center;
+    display: flex;
+    flex-direction: row;
+    box-sizing: border-box;
+
+    border: 1px solid var(--spectrum-alias-border-color);
+    border-radius: var(--spectrum-alias-border-radius-regular);
+    width: 31px;
+    color: var(--spectrum-alias-text-color);
+    background-color: var(--spectrum-global-color-gray-75);
+    transition: background-color
+        var(--spectrum-global-animation-duration-100, 130ms),
+      box-shadow var(--spectrum-global-animation-duration-100, 130ms),
+      border-color var(--spectrum-global-animation-duration-100, 130ms);
+    height: calc(var(--spectrum-alias-item-height-m) - 2px);
+    position: absolute;
+  }
+  .copy-icon:hover {
+    cursor: pointer;
+    color: var(--spectrum-alias-text-color-hover);
+    background-color: var(--spectrum-global-color-gray-50);
+    border-color: var(--spectrum-alias-border-color-hover);
+  }
+  .side-panel-header {
+    display: flex;
+    padding: 20px 10px 10px 10px;
+    gap: var(--spacing-s);
+    justify-content: space-between;
+    align-items: center;
+  }
+
+  .pagination {
+    display: flex;
+    flex-direction: row;
+    justify-content: flex-end;
+    margin-top: var(--spacing-xl);
+  }
+
+  .side-panel-body {
+    padding: 10px;
+    height: calc(100% - 67px);
+  }
+  #side-panel {
+    position: absolute;
+    right: 0;
+    top: 0;
+    padding-bottom: 24px;
+    background: var(--background);
+    border-left: var(--border-light);
+    width: 320px;
+    max-width: calc(100vw - 48px - 48px);
+    transform: translateX(100%);
+    transition: transform 130ms ease-in-out;
+    height: calc(100% - 24px);
+    overflow-y: hidden;
+    overflow-x: hidden;
+    z-index: 2;
+  }
+  #side-panel.visible {
+    transform: translateX(0);
+  }
+
+  #side-panel.wide {
+    width: 500px;
+  }
+
+  #side-panel :global(textarea) {
+    min-height: 100% !important;
+    background-color: var(
+      --spectrum-textfield-m-background-color,
+      var(--spectrum-global-color-gray-50)
+    );
+    padding-top: var(--spacing-l);
+    padding-left: var(--spacing-l);
+    font-size: 13px;
+  }
+
+  .controls {
+    display: flex;
+    flex-direction: row;
+    gap: var(--spacing-l);
+    flex-wrap: wrap;
+    align-items: center;
+  }
+
+  .side-panel-icons {
+    display: flex;
+    gap: var(--spacing-l);
+  }
+
+  .select {
+    flex-basis: calc(33.33% - 10px);
+    width: 0;
+    min-width: 100px;
+  }
+
+  .date-picker {
+    flex-basis: calc(70% - 32px);
+    min-width: 100px;
+  }
+
+  .freeSearch {
+    flex-basis: 25%;
+    min-width: 100px;
+  }
+</style>

packages/builder/src/pages/builder/portal/overview/[appId]/backups/index.svelte

@@ -185,7 +185,7 @@
   <Divider />
 
   {#if !$licensing.backupsEnabled}
-    {#if !$auth.accountPortalAccess && !$licensing.groupsEnabled && $admin.cloud}
+    {#if !$auth.accountPortalAccess && $admin.cloud}
       <Body>Contact your account holder to upgrade your plan.</Body>
     {/if}
     <div class="pro-buttons">

packages/builder/src/pages/builder/portal/settings/environment/index.svelte

@@ -1,21 +1,17 @@
 <script>
   import {
     Layout,
-    Heading,
-    Body,
     Button,
-    Divider,
     Modal,
     Table,
-    Tags,
-    Tag,
     InlineAlert,
     notifications,
   } from "@budibase/bbui"
-  import { environment, licensing, auth, admin } from "stores/portal"
+  import { environment, licensing } from "stores/portal"
   import { onMount } from "svelte"
   import CreateEditVariableModal from "components/portal/environment/CreateEditVariableModal.svelte"
   import EditVariableColumn from "./_components/EditVariableColumn.svelte"
+  import LockedFeature from "../../_components/LockedFeature.svelte"
 
   let modal
 
@@ -61,91 +57,43 @@
   }
 </script>
 
-<Layout noPadding>
+<LockedFeature
-  <Layout gap="XS" noPadding>
+  title={"Environment Variables"}
-    <div class="title">
+  planType={"Business plan"}
-      <Heading size="M">Environment Variables</Heading>
+  description={"Add and manage environment variables for development and production"}
-      {#if !$licensing.environmentVariablesEnabled}
+  enabled={$licensing.environmentVariablesEnabled}
-        <Tags>
+  upgradeButtonClick={async () => {
-          <Tag icon="LockClosed">Business plan</Tag>
+    await environment.upgradePanelOpened()
-        </Tags>
+    $licensing.goToUpgradePage()
-      {/if}
+  }}
-    </div>
+>
-    <Body
+  {#if noEncryptionKey}
-      >Add and manage environment variables for development and production</Body
+    <InlineAlert
-    >
+      message="Your Budibase installation does not have a key for encryption, please update your app service's environment variables to contain an 'ENCRYPTION_KEY' value."
-  </Layout>
+      header="No encryption key found"
-  <Divider size="S" />
+      type="error"
-
+    />
-  {#if $licensing.environmentVariablesEnabled}
-    {#if noEncryptionKey}
-      <InlineAlert
-        message="Your Budibase installation does not have a key for encryption, please update your app service's environment variables to contain an 'ENCRYPTION_KEY' value."
-        header="No encryption key found"
-        type="error"
-      />
-    {/if}
-    <div>
-      <Button on:click={modal.show} cta disabled={noEncryptionKey}
-        >Add Variable</Button
-      >
-    </div>
-
-    <Layout noPadding>
-      <Table
-        {schema}
-        data={$environment.variables}
-        allowEditColumns={false}
-        allowEditRows={false}
-        allowSelectRows={false}
-        {customRenderers}
-      />
-    </Layout>
-  {:else}
-    <div class="buttons">
-      <Button
-        primary
-        disabled={!$auth.accountPortalAccess && $admin.cloud}
-        on:click={async () => {
-          await environment.upgradePanelOpened()
-          $licensing.goToUpgradePage()
-        }}
-      >
-        Upgrade
-      </Button>
-      <!--Show the view plans button-->
-      <Button
-        secondary
-        on:click={() => {
-          window.open("https://budibase.com/pricing/", "_blank")
-        }}
-      >
-        View Plans
-      </Button>
-    </div>
   {/if}
-</Layout>
+  <div>
+    <Button on:click={modal.show} cta disabled={noEncryptionKey}
+      >Add Variable</Button
+    >
+  </div>
+  <Layout noPadding>
+    <Table
+      {schema}
+      data={$environment.variables}
+      allowEditColumns={false}
+      allowEditRows={false}
+      allowSelectRows={false}
+      {customRenderers}
+    />
+  </Layout>
+</LockedFeature>
 
 <Modal bind:this={modal}>
   <CreateEditVariableModal {save} />
 </Modal>
 
 <style>
-  .buttons {
-    display: flex;
-    gap: var(--spacing-l);
-  }
-  .title {
-    display: flex;
-    flex-direction: row;
-    align-items: center;
-    justify-content: flex-start;
-    gap: var(--spacing-m);
-  }
-
-  .buttons {
-    display: flex;
-    flex-direction: row;
-    gap: var(--spacing-m);
-  }
 </style>

packages/builder/src/stores/portal/auditLogs.js

@@ -0,0 +1,43 @@
+import { writable, get } from "svelte/store"
+import { API } from "api"
+import { licensing } from "stores/portal"
+
+export function createAuditLogsStore() {
+  const { subscribe, update } = writable({
+    events: {},
+    logs: {},
+  })
+
+  async function search(opts = {}) {
+    if (get(licensing).auditLogsEnabled) {
+      const paged = await API.searchAuditLogs(opts)
+
+      update(state => {
+        return { ...state, logs: { ...paged, opts } }
+      })
+
+      return paged
+    }
+  }
+
+  async function getEventDefinitions() {
+    const events = await API.getEventDefinitions()
+
+    update(state => {
+      return { ...state, ...events }
+    })
+  }
+
+  function getDownloadUrl(opts = {}) {
+    return API.getDownloadUrl(opts)
+  }
+
+  return {
+    subscribe,
+    search,
+    getEventDefinitions,
+    getDownloadUrl,
+  }
+}
+
+export const auditLogs = createAuditLogsStore()

packages/builder/src/stores/portal/index.js

@@ -13,3 +13,4 @@ export { backups } from "./backups"
 export { overview } from "./overview"
 export { environment } from "./environment"
 export { menu } from "./menu"
+export { auditLogs } from "./auditLogs"

packages/builder/src/stores/portal/licensing.js

@@ -67,6 +67,9 @@ export const createLicensingStore = () => {
         Constants.Features.ENFORCEABLE_SSO
       )
 
+      const auditLogsEnabled = license.features.includes(
+        Constants.Features.AUDIT_LOGS
+      )
       store.update(state => {
         return {
           ...state,
@@ -75,6 +78,7 @@ export const createLicensingStore = () => {
           groupsEnabled,
           backupsEnabled,
           environmentVariablesEnabled,
+          auditLogsEnabled,
           enforceableSSO,
         }
       })

packages/builder/src/stores/portal/menu.js

@@ -75,6 +75,10 @@ export const menu = derived([admin, auth], ([$admin, $auth]) => {
         title: "Usage",
         href: "/builder/portal/account/usage",
       },
+      {
+        title: "Audit Logs",
+        href: "/builder/portal/account/auditLogs",
+      },
     ]
     if ($admin.cloud && $auth?.user?.accountPortalAccess) {
       accountSubPages.push({
@@ -87,6 +91,7 @@ export const menu = derived([admin, auth], ([$admin, $auth]) => {
         href: "/builder/portal/account/upgrade",
       })
     }
+    // add license check here
     if (
       $auth?.user?.accountPortalAccess &&
       $auth.user.account.stripeCustomerId

packages/frontend-core/src/api/auditLogs.js

@@ -0,0 +1,63 @@
+const buildOpts = ({
+  bookmark,
+  userIds,
+  appIds,
+  startDate,
+  endDate,
+  fullSearch,
+  events,
+}) => {
+  const opts = {}
+
+  if (bookmark) {
+    opts.bookmark = bookmark
+  }
+
+  if (startDate && endDate) {
+    opts.startDate = startDate
+    opts.endDate = endDate
+  } else if (startDate && !endDate) {
+    opts.startDate = startDate
+  }
+
+  if (fullSearch) {
+    opts.fullSearch = fullSearch
+  }
+
+  if (events.length) {
+    opts.events = events
+  }
+
+  if (userIds.length) {
+    opts.userIds = userIds
+  }
+
+  if (appIds.length) {
+    opts.appIds = appIds
+  }
+
+  return opts
+}
+
+export const buildAuditLogsEndpoints = API => ({
+  /**
+   * Gets a list of users in the current tenant.
+   */
+  searchAuditLogs: async opts => {
+    return await API.post({
+      url: `/api/global/auditlogs/search`,
+      body: buildOpts(opts),
+    })
+  },
+
+  getEventDefinitions: async () => {
+    return await API.get({
+      url: `/api/global/auditlogs/definitions`,
+    })
+  },
+
+  getDownloadUrl: opts => {
+    const query = encodeURIComponent(JSON.stringify(opts))
+    return `/api/global/auditlogs/download?query=${query}`
+  },
+})

packages/frontend-core/src/api/index.js

@@ -28,6 +28,8 @@ import { buildPluginEndpoints } from "./plugins"
 import { buildBackupsEndpoints } from "./backups"
 import { buildEnvironmentVariableEndpoints } from "./environmentVariables"
 import { buildEventEndpoints } from "./events"
+import { buildAuditLogsEndpoints } from "./auditLogs"
+
 const defaultAPIClientConfig = {
   /**
    * Certain definitions can't change at runtime for client apps, such as the
@@ -250,5 +252,6 @@ export const createAPIClient = config => {
     ...buildBackupsEndpoints(API),
     ...buildEnvironmentVariableEndpoints(API),
     ...buildEventEndpoints(API),
+    ...buildAuditLogsEndpoints(API),
   }
 }

packages/frontend-core/src/constants.js

@@ -115,6 +115,7 @@ export const Features = {
   USER_GROUPS: "userGroups",
   BACKUPS: "appBackups",
   ENVIRONMENT_VARIABLES: "environmentVariables",
+  AUDIT_LOGS: "auditLogs",
   ENFORCEABLE_SSO: "enforceableSSO",
 }
 

packages/server/jest.config.ts

@@ -39,6 +39,7 @@ const config: Config.InitialOptions = {
   ],
   collectCoverageFrom: [
     "src/**/*.{js,ts}",
+    "../backend-core/src/**/*.{js,ts}",
     // The use of coverage with couchdb view functions breaks tests
     "!src/db/views/staticViews.*",
   ],

packages/server/package.json

@@ -87,6 +87,7 @@
     "koa-send": "5.0.0",
     "koa-session": "5.12.0",
     "koa-static": "5.0.0",
+    "koa-useragent": "^4.1.0",
     "koa2-ratelimit": "1.1.1",
     "lodash": "4.17.21",
     "memorystream": "0.3.1",

packages/server/src/api/controllers/row/ExternalRequest.ts

@@ -24,8 +24,7 @@ import { breakExternalTableId, isSQL } from "../../../integrations/utils"
 import { processObjectSync } from "@budibase/string-templates"
 import { cloneDeep } from "lodash/fp"
 import { processFormulas, processDates } from "../../../utilities/rowProcessor"
-import { context } from "@budibase/backend-core"
+import { db as dbCore } from "@budibase/backend-core"
-import { removeKeyNumbering } from "./utils"
 import sdk from "../../../sdk"
 
 export interface ManyRelationship {
@@ -61,7 +60,7 @@ function buildFilters(
     let prefix = 1
     for (let operator of Object.values(filters)) {
       for (let field of Object.keys(operator || {})) {
-        if (removeKeyNumbering(field) === "_id") {
+        if (dbCore.removeKeyNumbering(field) === "_id") {
           if (primary) {
             const parts = breakRowIdField(operator[field])
             for (let field of primary) {

packages/server/src/api/controllers/row/internalSearch.ts

@@ -1,531 +1,18 @@
-import { SearchIndexes } from "../../../db/utils"
+import { db as dbCore, context, SearchParams } from "@budibase/backend-core"
-import { removeKeyNumbering } from "./utils"
+import { SearchFilters, Row, SearchIndex } from "@budibase/types"
-import fetch from "node-fetch"
-import { db as dbCore, context } from "@budibase/backend-core"
-import { SearchFilters, Row } from "@budibase/types"
 
-type SearchParams = {
-  tableId: string
-  sort?: string
-  sortOrder?: string
-  sortType?: string
-  limit?: number
-  bookmark?: string
-  version?: string
-  rows?: Row[]
-}
-
-/**
- * Class to build lucene query URLs.
- * Optionally takes a base lucene query object.
- */
-export class QueryBuilder {
-  query: SearchFilters
-  limit: number
-  sort?: string
-  bookmark?: string
-  sortOrder: string
-  sortType: string
-  includeDocs: boolean
-  version?: string
-
-  constructor(base?: SearchFilters) {
-    this.query = {
-      allOr: false,
-      string: {},
-      fuzzy: {},
-      range: {},
-      equal: {},
-      notEqual: {},
-      empty: {},
-      notEmpty: {},
-      oneOf: {},
-      contains: {},
-      notContains: {},
-      containsAny: {},
-      ...base,
-    }
-    this.limit = 50
-    this.sortOrder = "ascending"
-    this.sortType = "string"
-    this.includeDocs = true
-  }
-
-  setVersion(version?: string) {
-    if (version != null) {
-      this.version = version
-    }
-    return this
-  }
-
-  setTable(tableId: string) {
-    this.query.equal!.tableId = tableId
-    return this
-  }
-
-  setLimit(limit?: number) {
-    if (limit != null) {
-      this.limit = limit
-    }
-    return this
-  }
-
-  setSort(sort?: string) {
-    if (sort != null) {
-      this.sort = sort
-    }
-    return this
-  }
-
-  setSortOrder(sortOrder?: string) {
-    if (sortOrder != null) {
-      this.sortOrder = sortOrder
-    }
-    return this
-  }
-
-  setSortType(sortType?: string) {
-    if (sortType != null) {
-      this.sortType = sortType
-    }
-    return this
-  }
-
-  setBookmark(bookmark?: string) {
-    if (bookmark != null) {
-      this.bookmark = bookmark
-    }
-    return this
-  }
-
-  excludeDocs() {
-    this.includeDocs = false
-    return this
-  }
-
-  addString(key: string, partial: string) {
-    this.query.string![key] = partial
-    return this
-  }
-
-  addFuzzy(key: string, fuzzy: string) {
-    this.query.fuzzy![key] = fuzzy
-    return this
-  }
-
-  addRange(key: string, low: string | number, high: string | number) {
-    this.query.range![key] = {
-      low,
-      high,
-    }
-    return this
-  }
-
-  addEqual(key: string, value: any) {
-    this.query.equal![key] = value
-    return this
-  }
-
-  addNotEqual(key: string, value: any) {
-    this.query.notEqual![key] = value
-    return this
-  }
-
-  addEmpty(key: string, value: any) {
-    this.query.empty![key] = value
-    return this
-  }
-
-  addNotEmpty(key: string, value: any) {
-    this.query.notEmpty![key] = value
-    return this
-  }
-
-  addOneOf(key: string, value: any) {
-    this.query.oneOf![key] = value
-    return this
-  }
-
-  addContains(key: string, value: any) {
-    this.query.contains![key] = value
-    return this
-  }
-
-  addNotContains(key: string, value: any) {
-    this.query.notContains![key] = value
-    return this
-  }
-
-  addContainsAny(key: string, value: any) {
-    this.query.containsAny![key] = value
-    return this
-  }
-
-  /**
-   * Preprocesses a value before going into a lucene search.
-   * Transforms strings to lowercase and wraps strings and bools in quotes.
-   * @param value The value to process
-   * @param options The preprocess options
-   * @returns {string|*}
-   */
-  preprocess(value: any, { escape, lowercase, wrap, type }: any = {}) {
-    const hasVersion = !!this.version
-    // Determine if type needs wrapped
-    const originalType = typeof value
-    // Convert to lowercase
-    if (value && lowercase) {
-      value = value.toLowerCase ? value.toLowerCase() : value
-    }
-    // Escape characters
-    if (escape && originalType === "string") {
-      value = `${value}`.replace(/[ #+\-&|!(){}\]^"~*?:\\]/g, "\\$&")
-    }
-
-    // Wrap in quotes
-    if (originalType === "string" && !isNaN(value) && !type) {
-      value = `"${value}"`
-    } else if (hasVersion && wrap) {
-      value = originalType === "number" ? value : `"${value}"`
-    }
-    return value
-  }
-
-  buildSearchQuery() {
-    const builder = this
-    let allOr = this.query && this.query.allOr
-    let query = allOr ? "" : "*:*"
-    const allPreProcessingOpts = { escape: true, lowercase: true, wrap: true }
-    let tableId
-    if (this.query.equal!.tableId) {
-      tableId = this.query.equal!.tableId
-      delete this.query.equal!.tableId
-    }
-
-    const equal = (key: string, value: any) => {
-      // 0 evaluates to false, which means we would return all rows if we don't check it
-      if (!value && value !== 0) {
-        return null
-      }
-      return `${key}:${builder.preprocess(value, allPreProcessingOpts)}`
-    }
-
-    const contains = (key: string, value: any, mode = "AND") => {
-      if (Array.isArray(value) && value.length === 0) {
-        return null
-      }
-      if (!Array.isArray(value)) {
-        return `${key}:${value}`
-      }
-      let statement = `${builder.preprocess(value[0], { escape: true })}`
-      for (let i = 1; i < value.length; i++) {
-        statement += ` ${mode} ${builder.preprocess(value[i], {
-          escape: true,
-        })}`
-      }
-      return `${key}:(${statement})`
-    }
-
-    const notContains = (key: string, value: any) => {
-      // @ts-ignore
-      const allPrefix = allOr === "" ? "*:* AND" : ""
-      return allPrefix + "NOT " + contains(key, value)
-    }
-
-    const containsAny = (key: string, value: any) => {
-      return contains(key, value, "OR")
-    }
-
-    const oneOf = (key: string, value: any) => {
-      if (!Array.isArray(value)) {
-        if (typeof value === "string") {
-          value = value.split(",")
-        } else {
-          return ""
-        }
-      }
-      let orStatement = `${builder.preprocess(value[0], allPreProcessingOpts)}`
-      for (let i = 1; i < value.length; i++) {
-        orStatement += ` OR ${builder.preprocess(
-          value[i],
-          allPreProcessingOpts
-        )}`
-      }
-      return `${key}:(${orStatement})`
-    }
-
-    function build(structure: any, queryFn: any) {
-      for (let [key, value] of Object.entries(structure)) {
-        // check for new format - remove numbering if needed
-        key = removeKeyNumbering(key)
-        key = builder.preprocess(key.replace(/ /g, "_"), {
-          escape: true,
-        })
-        const expression = queryFn(key, value)
-        if (expression == null) {
-          continue
-        }
-        if (query.length > 0) {
-          query += ` ${allOr ? "OR" : "AND"} `
-        }
-        query += expression
-      }
-    }
-
-    // Construct the actual lucene search query string from JSON structure
-    if (this.query.string) {
-      build(this.query.string, (key: string, value: any) => {
-        if (!value) {
-          return null
-        }
-        value = builder.preprocess(value, {
-          escape: true,
-          lowercase: true,
-          type: "string",
-        })
-        return `${key}:${value}*`
-      })
-    }
-    if (this.query.range) {
-      build(this.query.range, (key: string, value: any) => {
-        if (!value) {
-          return null
-        }
-        if (value.low == null || value.low === "") {
-          return null
-        }
-        if (value.high == null || value.high === "") {
-          return null
-        }
-        const low = builder.preprocess(value.low, allPreProcessingOpts)
-        const high = builder.preprocess(value.high, allPreProcessingOpts)
-        return `${key}:[${low} TO ${high}]`
-      })
-    }
-    if (this.query.fuzzy) {
-      build(this.query.fuzzy, (key: string, value: any) => {
-        if (!value) {
-          return null
-        }
-        value = builder.preprocess(value, {
-          escape: true,
-          lowercase: true,
-          type: "fuzzy",
-        })
-        return `${key}:${value}~`
-      })
-    }
-    if (this.query.equal) {
-      build(this.query.equal, equal)
-    }
-    if (this.query.notEqual) {
-      build(this.query.notEqual, (key: string, value: any) => {
-        if (!value) {
-          return null
-        }
-        return `!${key}:${builder.preprocess(value, allPreProcessingOpts)}`
-      })
-    }
-    if (this.query.empty) {
-      build(this.query.empty, (key: string) => `!${key}:["" TO *]`)
-    }
-    if (this.query.notEmpty) {
-      build(this.query.notEmpty, (key: string) => `${key}:["" TO *]`)
-    }
-    if (this.query.oneOf) {
-      build(this.query.oneOf, oneOf)
-    }
-    if (this.query.contains) {
-      build(this.query.contains, contains)
-    }
-    if (this.query.notContains) {
-      build(this.query.notContains, notContains)
-    }
-    if (this.query.containsAny) {
-      build(this.query.containsAny, containsAny)
-    }
-    // make sure table ID is always added as an AND
-    if (tableId) {
-      query = `(${query})`
-      allOr = false
-      build({ tableId }, equal)
-    }
-    return query
-  }
-
-  buildSearchBody() {
-    let body: any = {
-      q: this.buildSearchQuery(),
-      limit: Math.min(this.limit, 200),
-      include_docs: this.includeDocs,
-    }
-    if (this.bookmark) {
-      body.bookmark = this.bookmark
-    }
-    if (this.sort) {
-      const order = this.sortOrder === "descending" ? "-" : ""
-      const type = `<${this.sortType}>`
-      body.sort = `${order}${this.sort.replace(/ /g, "_")}${type}`
-    }
-    return body
-  }
-
-  async run() {
-    const appId = context.getAppId()
-    const { url, cookie } = dbCore.getCouchInfo()
-    const fullPath = `${url}/${appId}/_design/database/_search/${SearchIndexes.ROWS}`
-    const body = this.buildSearchBody()
-    return await runQuery(fullPath, body, cookie)
-  }
-}
-
-/**
- * Executes a lucene search query.
- * @param url The query URL
- * @param body The request body defining search criteria
- * @param cookie The auth cookie for CouchDB
- * @returns {Promise<{rows: []}>}
- */
-const runQuery = async (url: string, body: any, cookie: string) => {
-  const response = await fetch(url, {
-    body: JSON.stringify(body),
-    method: "POST",
-    headers: {
-      Authorization: cookie,
-    },
-  })
-  const json = await response.json()
-
-  let output: any = {
-    rows: [],
-  }
-  if (json.rows != null && json.rows.length > 0) {
-    output.rows = json.rows.map((row: any) => row.doc)
-  }
-  if (json.bookmark) {
-    output.bookmark = json.bookmark
-  }
-  return output
-}
-
-/**
- * Gets round the fixed limit of 200 results from a query by fetching as many
- * pages as required and concatenating the results. This recursively operates
- * until enough results have been found.
- * @param query {object} The JSON query structure
- * @param params {object} The search params including:
- *   tableId {string} The table ID to search
- *   sort {string} The sort column
- *   sortOrder {string} The sort order ("ascending" or "descending")
- *   sortType {string} Whether to treat sortable values as strings or
- *     numbers. ("string" or "number")
- *   limit {number} The number of results to fetch
- *   bookmark {string|null} Current bookmark in the recursive search
- *   rows {array|null} Current results in the recursive search
- * @returns {Promise<*[]|*>}
- */
-async function recursiveSearch(query: any, params: any): Promise<any> {
-  const bookmark = params.bookmark
-  const rows = params.rows || []
-  if (rows.length >= params.limit) {
-    return rows
-  }
-  let pageSize = 200
-  if (rows.length > params.limit - 200) {
-    pageSize = params.limit - rows.length
-  }
-  const page = await new QueryBuilder(query)
-    .setVersion(params.version)
-    .setTable(params.tableId)
-    .setBookmark(bookmark)
-    .setLimit(pageSize)
-    .setSort(params.sort)
-    .setSortOrder(params.sortOrder)
-    .setSortType(params.sortType)
-    .run()
-  if (!page.rows.length) {
-    return rows
-  }
-  if (page.rows.length < 200) {
-    return [...rows, ...page.rows]
-  }
-  const newParams = {
-    ...params,
-    bookmark: page.bookmark,
-    rows: [...rows, ...page.rows],
-  }
-  return await recursiveSearch(query, newParams)
-}
-
-/**
- * Performs a paginated search. A bookmark will be returned to allow the next
- * page to be fetched. There is a max limit off 200 results per page in a
- * paginated search.
- * @param query {object} The JSON query structure
- * @param params {object} The search params including:
- *   tableId {string} The table ID to search
- *   sort {string} The sort column
- *   sortOrder {string} The sort order ("ascending" or "descending")
- *   sortType {string} Whether to treat sortable values as strings or
- *     numbers. ("string" or "number")
- *   limit {number} The desired page size
- *   bookmark {string} The bookmark to resume from
- * @returns {Promise<{hasNextPage: boolean, rows: *[]}>}
- */
 export async function paginatedSearch(
   query: SearchFilters,
-  params: SearchParams
+  params: SearchParams<Row>
 ) {
-  let limit = params.limit
+  const appId = context.getAppId()
-  if (limit == null || isNaN(limit) || limit < 0) {
+  return dbCore.paginatedSearch(appId!, SearchIndex.ROWS, query, params)
-    limit = 50
-  }
-  limit = Math.min(limit, 200)
-  const search = new QueryBuilder(query)
-    .setVersion(params.version)
-    .setTable(params.tableId)
-    .setSort(params.sort)
-    .setSortOrder(params.sortOrder)
-    .setSortType(params.sortType)
-  const searchResults = await search
-    .setBookmark(params.bookmark)
-    .setLimit(limit)
-    .run()
-
-  // Try fetching 1 row in the next page to see if another page of results
-  // exists or not
-  const nextResults = await search
-    .setTable(params.tableId)
-    .setBookmark(searchResults.bookmark)
-    .setLimit(1)
-    .run()
-
-  return {
-    ...searchResults,
-    hasNextPage: nextResults.rows && nextResults.rows.length > 0,
-  }
 }
 
-/**
+export async function fullSearch(
- * Performs a full search, fetching multiple pages if required to return the
+  query: SearchFilters,
- * desired amount of results. There is a limit of 1000 results to avoid
+  params: SearchParams<Row>
- * heavy performance hits, and to avoid client components breaking from
+) {
- * handling too much data.
+  const appId = context.getAppId()
- * @param query {object} The JSON query structure
+  return dbCore.fullSearch(appId!, SearchIndex.ROWS, query, params)
- * @param params {object} The search params including:
- *   tableId {string} The table ID to search
- *   sort {string} The sort column
- *   sortOrder {string} The sort order ("ascending" or "descending")
- *   sortType {string} Whether to treat sortable values as strings or
- *     numbers. ("string" or "number")
- *   limit {number} The desired number of results
- * @returns {Promise<{rows: *}>}
- */
-export async function fullSearch(query: SearchFilters, params: SearchParams) {
-  let limit = params.limit
-  if (limit == null || isNaN(limit) || limit < 0) {
-    limit = 1000
-  }
-  params.limit = Math.min(limit, 1000)
-  const rows = await recursiveSearch(query, params)
-  return { rows }
 }

packages/server/src/api/controllers/row/utils.ts

@@ -3,8 +3,7 @@ import * as userController from "../user"
 import { FieldTypes } from "../../../constants"
 import { context } from "@budibase/backend-core"
 import { makeExternalQuery } from "../../../integrations/base/query"
-import { BBContext, Row, Table } from "@budibase/types"
+import { Row, Table } from "@budibase/types"
-export { removeKeyNumbering } from "../../../integrations/base/utils"
 const validateJs = require("validate.js")
 const { cloneDeep } = require("lodash/fp")
 import { Format } from "../view/exporters"

packages/server/src/app.ts

@@ -32,6 +32,7 @@ import { initialise as initialiseWebsockets } from "./websocket"
 import { startup } from "./startup"
 const Sentry = require("@sentry/node")
 const destroyable = require("server-destroy")
+const { userAgent } = require("koa-useragent")
 
 const app = new Koa()
 
@@ -53,6 +54,7 @@ app.use(
 )
 
 app.use(middleware.logging)
+app.use(userAgent)
 
 if (env.isProd()) {
   env._set("NODE_ENV", "production")

packages/server/src/db/index.ts

@@ -1,4 +1,4 @@
-import { init as coreInit } from "@budibase/backend-core"
+import * as core from "@budibase/backend-core"
 import env from "../environment"
 
 export function init() {
@@ -12,5 +12,5 @@ export function init() {
     dbConfig.allDbs = true
   }
 
-  coreInit({ db: dbConfig })
+  core.init({ db: dbConfig })
 }

packages/server/src/db/utils.ts

@@ -9,10 +9,6 @@ export const AppStatus = {
   DEPLOYED: "published",
 }
 
-export const SearchIndexes = {
-  ROWS: "rows",
-}
-
 export const BudibaseInternalDB = {
   _id: "bb_internal",
   type: dbCore.BUDIBASE_DATASOURCE_TYPE,

packages/server/src/db/views/staticViews.ts

@@ -1,6 +1,6 @@
 import { context } from "@budibase/backend-core"
-import { DocumentType, SEPARATOR, ViewName, SearchIndexes } from "../utils"
+import { DocumentType, SEPARATOR, ViewName } from "../utils"
-import { LinkDocument, Row } from "@budibase/types"
+import { LinkDocument, Row, SearchIndex } from "@budibase/types"
 const SCREEN_PREFIX = DocumentType.SCREEN + SEPARATOR
 
 /**************************************************
@@ -91,7 +91,7 @@ async function searchIndex(indexName: string, fnString: string) {
 
 export async function createAllSearchIndex() {
   await searchIndex(
-    SearchIndexes.ROWS,
+    SearchIndex.ROWS,
     function (doc: Row) {
       function idx(input: Row, prev?: string) {
         for (let key of Object.keys(input)) {

packages/server/src/integrations/base/sql.ts

@@ -6,11 +6,11 @@ import {
   SearchFilters,
   SortDirection,
 } from "@budibase/types"
+import { db as dbCore } from "@budibase/backend-core"
 import { QueryOptions } from "../../definitions/datasource"
 import { isIsoDateString, SqlClient } from "../utils"
 import SqlTableQueryBuilder from "./sqlTable"
 import environment from "../../environment"
-import { removeKeyNumbering } from "./utils"
 
 const envLimit = environment.SQL_MAX_ROWS
   ? parseInt(environment.SQL_MAX_ROWS)
@@ -136,7 +136,7 @@ class InternalBuilder {
       fn: (key: string, value: any) => void
     ) {
       for (let [key, value] of Object.entries(structure)) {
-        const updatedKey = removeKeyNumbering(key)
+        const updatedKey = dbCore.removeKeyNumbering(key)
         const isRelationshipField = updatedKey.includes(".")
         if (!opts.relationship && !isRelationshipField) {
           fn(`${opts.tableName}.${updatedKey}`, value)

packages/server/src/integrations/base/utils.ts

@@ -1,12 +0,0 @@
-const QUERY_START_REGEX = /\d[0-9]*:/g
-
-export function removeKeyNumbering(key: any): string {
-  if (typeof key === "string" && key.match(QUERY_START_REGEX) != null) {
-    const parts = key.split(":")
-    // remove the number
-    parts.shift()
-    return parts.join(":")
-  } else {
-    return key
-  }
-}

packages/server/src/startup.ts

@@ -5,7 +5,7 @@ import {
   generateApiKey,
   getChecklist,
 } from "./utilities/workerRequests"
-import { installation, tenancy, logging } from "@budibase/backend-core"
+import { installation, tenancy, logging, events } from "@budibase/backend-core"
 import fs from "fs"
 import { watch } from "./watch"
 import * as automations from "./automations"
@@ -124,6 +124,9 @@ export async function startup(app?: any, server?: any) {
   // get the references to the queue promises, don't await as
   // they will never end, unless the processing stops
   let queuePromises = []
+  // configure events to use the pro audit log write
+  // can't integrate directly into backend-core due to cyclic issues
+  queuePromises.push(events.processors.init(pro.sdk.auditLogs.write))
   queuePromises.push(automations.init())
   queuePromises.push(initPro())
   if (app) {

packages/server/yarn.lock

@@ -6122,20 +6122,13 @@ dir-glob@^3.0.1:
   dependencies:
     path-type "^4.0.0"
 
-docker-compose@0.23.17:
+docker-compose@0.23.17, docker-compose@^0.23.5:
   version "0.23.17"
   resolved "https://registry.yarnpkg.com/docker-compose/-/docker-compose-0.23.17.tgz#8816bef82562d9417dc8c790aa4871350f93a2ba"
   integrity sha512-YJV18YoYIcxOdJKeFcCFihE6F4M2NExWM/d4S1ITcS9samHKnNUihz9kjggr0dNtsrbpFNc7/Yzd19DWs+m1xg==
   dependencies:
     yaml "^1.10.2"
 
-docker-compose@^0.23.5:
-  version "0.23.19"
-  resolved "https://registry.yarnpkg.com/docker-compose/-/docker-compose-0.23.19.tgz#9947726e2fe67bdfa9e8efe1ff15aa0de2e10eb8"
-  integrity sha512-v5vNLIdUqwj4my80wxFDkNH+4S85zsRuH29SO7dCWVWPCMt/ohZBsGN6g6KXWifT0pzQ7uOxqEKCYCDPJ8Vz4g==
-  dependencies:
-    yaml "^1.10.2"
-
 docker-modem@^3.0.0:
   version "3.0.6"
   resolved "https://registry.yarnpkg.com/docker-modem/-/docker-modem-3.0.6.tgz#8c76338641679e28ec2323abb65b3276fb1ce597"
@@ -6983,6 +6976,11 @@ expose-loader@^3.1.0:
   resolved "https://registry.yarnpkg.com/expose-loader/-/expose-loader-3.1.0.tgz#7a0bdecb345b921ca238a8c4715a4ea7e227213f"
   integrity sha512-2RExSo0yJiqP+xiUue13jQa2IHE8kLDzTI7b6kn+vUlBVvlzNSiLDzo4e5Pp5J039usvTUnxZ8sUOhv0Kg15NA==
 
+express-useragent@^1.0.15:
+  version "1.0.15"
+  resolved "https://registry.yarnpkg.com/express-useragent/-/express-useragent-1.0.15.tgz#cefda5fa4904345d51d3368b117a8dd4124985d9"
+  integrity sha512-eq5xMiYCYwFPoekffMjvEIk+NWdlQY9Y38OsTyl13IvA728vKT+q/CSERYWzcw93HGBJcIqMIsZC5CZGARPVdg==
+
 ext-list@^2.0.0:
   version "2.2.2"
   resolved "https://registry.yarnpkg.com/ext-list/-/ext-list-2.2.2.tgz#0b98e64ed82f5acf0f2931babf69212ef52ddd37"
@@ -10243,6 +10241,13 @@ koa-static@5.0.0, koa-static@^5.0.0:
     debug "^3.1.0"
     koa-send "^5.0.0"
 
+koa-useragent@^4.1.0:
+  version "4.1.0"
+  resolved "https://registry.yarnpkg.com/koa-useragent/-/koa-useragent-4.1.0.tgz#d3f128b552c6da3e5e9e9e9c887b2922b16e4468"
+  integrity sha512-x/HUDZ1zAmNNh5hA9hHbPm9p3UVg2prlpHzxCXQCzbibrNS0kmj7MkCResCbAbG7ZT6FVxNSMjR94ZGamdMwxA==
+  dependencies:
+    express-useragent "^1.0.15"
+
 koa-views@^7.0.1:
   version "7.0.2"
   resolved "https://registry.yarnpkg.com/koa-views/-/koa-views-7.0.2.tgz#c96fd9e2143ef00c29dc5160c5ed639891aa723d"
@@ -10706,7 +10711,12 @@ lru-cache@^6.0.0:
   dependencies:
     yallist "^4.0.0"
 
-lru-cache@^7.14.0, lru-cache@^7.14.1:
+lru-cache@^7.14.0:
+  version "7.16.1"
+  resolved "https://registry.yarnpkg.com/lru-cache/-/lru-cache-7.16.1.tgz#7acea16fecd9ed11430e78443c2bb81a06d3dea9"
+  integrity sha512-9kkuMZHnLH/8qXARvYSjNvq8S1GYFFzynQTAfKeaJ0sIrR3PUPuu37Z+EiIANiZBvpfTf2B5y8ecDLSMWlLv+w==
+
+lru-cache@^7.14.1:
   version "7.14.1"
   resolved "https://registry.yarnpkg.com/lru-cache/-/lru-cache-7.14.1.tgz#8da8d2f5f59827edb388e63e459ac23d6d408fea"
   integrity sha512-ysxwsnTKdAx96aTRdhDOCQfDgbHnt8SK0KY8SEjO0wHinhWOFTESbjVCMPbU1uGXg/ch4lifqx0wfjOawU2+WA==

packages/types/src/api/web/global/auditLogs.ts

@@ -0,0 +1,44 @@
+import { Event, AuditedEventFriendlyName } from "../../../sdk"
+import {
+  PaginationResponse,
+  PaginationRequest,
+  BasicPaginationRequest,
+} from "../"
+import { User, App } from "../../../"
+
+export interface AuditLogSearchParams {
+  userIds?: string[]
+  appIds?: string[]
+  events?: Event[]
+  startDate?: string
+  endDate?: string
+  fullSearch?: string
+  bookmark?: string
+}
+
+export interface DownloadAuditLogsRequest extends AuditLogSearchParams {}
+
+export interface SearchAuditLogsRequest
+  extends BasicPaginationRequest,
+    AuditLogSearchParams {}
+
+export enum AuditLogResourceStatus {
+  DELETED = "deleted",
+}
+
+export interface AuditLogEnriched {
+  app?: App | { _id: string; status: AuditLogResourceStatus }
+  user: User | { _id: string; status: AuditLogResourceStatus }
+  event: Event
+  timestamp: string
+  name: string
+  metadata: any
+}
+
+export interface SearchAuditLogsResponse extends PaginationResponse {
+  data: AuditLogEnriched[]
+}
+
+export interface DefinitionsAuditLogsResponse {
+  events: Record<string, string>
+}

packages/types/src/api/web/global/index.ts

@@ -1,3 +1,4 @@
 export * from "./environmentVariables"
+export * from "./auditLogs"
 export * from "./events"
 export * from "./configs"

packages/types/src/api/web/index.ts

@@ -5,3 +5,4 @@ export * from "./errors"
 export * from "./schedule"
 export * from "./app"
 export * from "./global"
+export * from "./pagination"

packages/types/src/api/web/pagination.ts

@@ -0,0 +1,27 @@
+export enum SortOrder {
+  ASCENDING = "ascending",
+  DESCENDING = "descending",
+}
+
+export enum SortType {
+  STRING = "string",
+  number = "number",
+}
+
+export interface BasicPaginationRequest {
+  bookmark?: string
+}
+
+export interface PaginationRequest extends BasicPaginationRequest {
+  limit?: number
+  sort?: {
+    order: SortOrder
+    column: string
+    type: SortType
+  }
+}
+
+export interface PaginationResponse {
+  bookmark: string
+  hasNextPage: boolean
+}

packages/types/src/documents/global/auditLogs.ts

@@ -0,0 +1,13 @@
+import { Document } from "../document"
+import { Event } from "../../sdk"
+
+export const AuditLogSystemUser = "SYSTEM"
+
+export interface AuditLogDoc extends Document {
+  appId?: string
+  event: Event
+  userId: string
+  timestamp: string
+  metadata: any
+  name: string
+}

packages/types/src/documents/global/index.ts

@@ -6,3 +6,4 @@ export * from "./quotas"
 export * from "./schedule"
 export * from "./templates"
 export * from "./environmentVariables"
+export * from "./auditLogs"

packages/types/src/sdk/auditLogs.ts

@@ -0,0 +1,21 @@
+import { Event, HostInfo } from "./events"
+import { AuditLogDoc } from "../documents"
+
+export type AuditWriteOpts = {
+  appId?: string
+  timestamp?: string | number
+  userId?: string
+  hostInfo?: HostInfo
+}
+
+export type AuditLogFn = (
+  event: Event,
+  metadata: any,
+  opts: AuditWriteOpts
+) => Promise<AuditLogDoc | undefined>
+
+export type AuditLogQueueEvent = {
+  event: Event
+  properties: any
+  opts: AuditWriteOpts
+}

packages/types/src/sdk/context.ts

@@ -1,5 +1,5 @@
 import { User, Account } from "../documents"
-import { IdentityType } from "./events"
+import { IdentityType, HostInfo } from "./events"
 
 export interface BaseContext {
   _id: string
@@ -16,6 +16,7 @@ export interface UserContext extends BaseContext, User {
   _id: string
   tenantId: string
   account?: Account
+  hostInfo: HostInfo
 }
 
 export type IdentityContext = BaseContext | AccountUserContext | UserContext

packages/types/src/sdk/db.ts

@@ -1,5 +1,11 @@
 import Nano from "@budibase/nano"
 import { AllDocsResponse, AnyDocument, Document } from "../"
+import { Writable } from "stream"
+
+export enum SearchIndex {
+  ROWS = "rows",
+  AUDIT = "audit",
+}
 
 export type PouchOptions = {
   inMemory?: boolean
@@ -63,6 +69,18 @@ export const isDocument = (doc: any): doc is Document => {
   return typeof doc === "object" && doc._id && doc._rev
 }
 
+export interface DatabaseDumpOpts {
+  filter?: (doc: AnyDocument) => boolean
+  batch_size?: number
+  batch_limit?: number
+  style?: "main_only" | "all_docs"
+  timeout?: number
+  doc_ids?: string[]
+  query_params?: any
+  view?: string
+  selector?: any
+}
+
 export interface Database {
   name: string
 
@@ -87,7 +105,7 @@ export interface Database {
   compact(): Promise<Nano.OkResponse | void>
   // these are all PouchDB related functions that are rarely used - in future
   // should be replaced by better typed/non-pouch implemented methods
-  dump(...args: any[]): Promise<any>
+  dump(stream: Writable, opts?: DatabaseDumpOpts): Promise<any>
   load(...args: any[]): Promise<any>
   createIndex(...args: any[]): Promise<any>
   deleteIndex(...args: any[]): Promise<any>

packages/types/src/sdk/events/app.ts

@@ -3,50 +3,83 @@ import { BaseEvent } from "./event"
 export interface AppCreatedEvent extends BaseEvent {
   appId: string
   version: string
+  audited: {
+    name: string
+  }
 }
 
 export interface AppUpdatedEvent extends BaseEvent {
   appId: string
   version: string
+  audited: {
+    name: string
+  }
 }
 
 export interface AppDeletedEvent extends BaseEvent {
   appId: string
+  audited: {
+    name: string
+  }
 }
 
 export interface AppPublishedEvent extends BaseEvent {
   appId: string
+  audited: {
+    name: string
+  }
 }
 
 export interface AppUnpublishedEvent extends BaseEvent {
   appId: string
+  audited: {
+    name: string
+  }
 }
 
 export interface AppFileImportedEvent extends BaseEvent {
   appId: string
+  audited: {
+    name: string
+  }
 }
 
 export interface AppTemplateImportedEvent extends BaseEvent {
   appId: string
   templateKey: string
+  audited: {
+    name: string
+  }
 }
 
 export interface AppVersionUpdatedEvent extends BaseEvent {
   appId: string
   currentVersion: string
   updatedToVersion: string
+  audited: {
+    name: string
+  }
 }
 
 export interface AppVersionRevertedEvent extends BaseEvent {
   appId: string
   currentVersion: string
   revertedToVersion: string
+  audited: {
+    name: string
+  }
 }
 
 export interface AppRevertedEvent extends BaseEvent {
   appId: string
+  audited: {
+    name: string
+  }
 }
 
 export interface AppExportedEvent extends BaseEvent {
   appId: string
+  audited: {
+    name: string
+  }
 }

packages/types/src/sdk/events/auditLog.ts

@@ -0,0 +1,10 @@
+import { BaseEvent } from "./event"
+import { AuditLogSearchParams } from "../../api"
+
+export interface AuditLogFilteredEvent extends BaseEvent {
+  filters: AuditLogSearchParams
+}
+
+export interface AuditLogDownloadedEvent extends BaseEvent {
+  filters: AuditLogSearchParams
+}

packages/types/src/sdk/events/auth.ts

@@ -7,10 +7,16 @@ export type SSOType = ConfigType.OIDC | ConfigType.GOOGLE
 export interface LoginEvent extends BaseEvent {
   userId: string
   source: LoginSource
+  audited: {
+    email: string
+  }
 }
 
 export interface LogoutEvent extends BaseEvent {
   userId: string
+  audited: {
+    email?: string
+  }
 }
 
 export interface SSOCreatedEvent extends BaseEvent {

packages/types/src/sdk/events/automation.ts

@@ -5,6 +5,9 @@ export interface AutomationCreatedEvent extends BaseEvent {
   automationId: string
   triggerId: string
   triggerType: string
+  audited: {
+    name: string
+  }
 }
 
 export interface AutomationTriggerUpdatedEvent extends BaseEvent {
@@ -19,6 +22,9 @@ export interface AutomationDeletedEvent extends BaseEvent {
   automationId: string
   triggerId: string
   triggerType: string
+  audited: {
+    name: string
+  }
 }
 
 export interface AutomationTestedEvent extends BaseEvent {
@@ -35,6 +41,9 @@ export interface AutomationStepCreatedEvent extends BaseEvent {
   triggerType: string
   stepId: string
   stepType: string
+  audited: {
+    name: string
+  }
 }
 
 export interface AutomationStepDeletedEvent extends BaseEvent {
@@ -44,6 +53,9 @@ export interface AutomationStepDeletedEvent extends BaseEvent {
   triggerType: string
   stepId: string
   stepType: string
+  audited: {
+    name: string
+  }
 }
 
 export interface AutomationsRunEvent extends BaseEvent {

packages/types/src/sdk/events/backup.ts

@@ -5,6 +5,7 @@ export interface AppBackupRestoreEvent extends BaseEvent {
   appId: string
   restoreId: string
   backupCreatedAt: string
+  name: string
 }
 
 export interface AppBackupTriggeredEvent extends BaseEvent {
@@ -12,4 +13,5 @@ export interface AppBackupTriggeredEvent extends BaseEvent {
   appId: string
   trigger: AppBackupTrigger
   type: AppBackupType
+  name: string
 }

packages/types/src/sdk/events/event.ts

@@ -180,6 +180,190 @@ export enum Event {
   ENVIRONMENT_VARIABLE_CREATED = "environment_variable:created",
   ENVIRONMENT_VARIABLE_DELETED = "environment_variable:deleted",
   ENVIRONMENT_VARIABLE_UPGRADE_PANEL_OPENED = "environment_variable:upgrade_panel_opened",
+
+  // AUDIT LOG
+  AUDIT_LOGS_FILTERED = "audit_log:filtered",
+  AUDIT_LOGS_DOWNLOADED = "audit_log:downloaded",
+}
+
+// all events that are not audited have been added to this record as undefined, this means
+// that Typescript can protect us against new events being added and auditing of those
+// events not being considered. This might be a little ugly, but provides a level of
+// Typescript build protection for the audit log feature, any new event also needs to be
+// added to this map, during which the developer will need to consider if it should be
+// a user facing event or not.
+export const AuditedEventFriendlyName: Record<Event, string | undefined> = {
+  // USER
+  [Event.USER_CREATED]: `User "{{ email }}" created`,
+  [Event.USER_UPDATED]: `User "{{ email }}" updated`,
+  [Event.USER_DELETED]: `User "{{ email }}" deleted`,
+  [Event.USER_PERMISSION_ADMIN_ASSIGNED]: `User "{{ email }}" admin role assigned`,
+  [Event.USER_PERMISSION_ADMIN_REMOVED]: `User "{{ email }}" admin role removed`,
+  [Event.USER_PERMISSION_BUILDER_ASSIGNED]: `User "{{ email }}" builder role assigned`,
+  [Event.USER_PERMISSION_BUILDER_REMOVED]: `User "{{ email }}" builder role removed`,
+  [Event.USER_INVITED]: `User "{{ email }}" invited`,
+  [Event.USER_INVITED_ACCEPTED]: `User "{{ email }}" accepted invite`,
+  [Event.USER_PASSWORD_UPDATED]: `User "{{ email }}" password updated`,
+  [Event.USER_PASSWORD_RESET_REQUESTED]: `User "{{ email }}" password reset requested`,
+  [Event.USER_PASSWORD_RESET]: `User "{{ email }}" password reset`,
+  [Event.USER_GROUP_CREATED]: `User group "{{ name }}" created`,
+  [Event.USER_GROUP_UPDATED]: `User group "{{ name }}" updated`,
+  [Event.USER_GROUP_DELETED]: `User group "{{ name }}" deleted`,
+  [Event.USER_GROUP_USERS_ADDED]: `User group "{{ name }}" {{ count }} users added`,
+  [Event.USER_GROUP_USERS_REMOVED]: `User group "{{ name }}" {{ count }} users removed`,
+  [Event.USER_GROUP_PERMISSIONS_EDITED]: `User group "{{ name }}" permissions edited`,
+  [Event.USER_PASSWORD_FORCE_RESET]: undefined,
+  [Event.USER_GROUP_ONBOARDING]: undefined,
+  [Event.USER_ONBOARDING_COMPLETE]: undefined,
+
+  // EMAIL
+  [Event.EMAIL_SMTP_CREATED]: `Email configuration created`,
+  [Event.EMAIL_SMTP_UPDATED]: `Email configuration updated`,
+
+  // AUTH
+  [Event.AUTH_SSO_CREATED]: `SSO configuration created`,
+  [Event.AUTH_SSO_UPDATED]: `SSO configuration updated`,
+  [Event.AUTH_SSO_ACTIVATED]: `SSO configuration activated`,
+  [Event.AUTH_SSO_DEACTIVATED]: `SSO configuration deactivated`,
+  [Event.AUTH_LOGIN]: `User "{{ email }}" logged in`,
+  [Event.AUTH_LOGOUT]: `User "{{ email }}" logged out`,
+
+  // ORG
+  [Event.ORG_NAME_UPDATED]: `Organisation name updated`,
+  [Event.ORG_LOGO_UPDATED]: `Organisation logo updated`,
+  [Event.ORG_PLATFORM_URL_UPDATED]: `Organisation platform URL updated`,
+
+  // APP
+  [Event.APP_CREATED]: `App "{{ name }}" created`,
+  [Event.APP_UPDATED]: `App "{{ name }}" updated`,
+  [Event.APP_DELETED]: `App "{{ name }}" deleted`,
+  [Event.APP_PUBLISHED]: `App "{{ name }}" published`,
+  [Event.APP_UNPUBLISHED]: `App "{{ name }}" unpublished`,
+  [Event.APP_TEMPLATE_IMPORTED]: `App "{{ name }}" template imported`,
+  [Event.APP_FILE_IMPORTED]: `App "{{ name }}" file imported`,
+  [Event.APP_VERSION_UPDATED]: `App "{{ name }}" version updated`,
+  [Event.APP_VERSION_REVERTED]: `App "{{ name }}" version reverted`,
+  [Event.APP_REVERTED]: `App "{{ name }}" reverted`,
+  [Event.APP_EXPORTED]: `App "{{ name }}" exported`,
+  [Event.APP_BACKUP_RESTORED]: `App backup "{{ name }}" restored`,
+  [Event.APP_BACKUP_TRIGGERED]: `App backup "{{ name }}" triggered`,
+
+  // DATASOURCE
+  [Event.DATASOURCE_CREATED]: `Datasource created`,
+  [Event.DATASOURCE_UPDATED]: `Datasource updated`,
+  [Event.DATASOURCE_DELETED]: `Datasource deleted`,
+
+  // QUERY
+  [Event.QUERY_CREATED]: `Query created`,
+  [Event.QUERY_UPDATED]: `Query updated`,
+  [Event.QUERY_DELETED]: `Query deleted`,
+  [Event.QUERY_IMPORT]: `Query import`,
+  [Event.QUERIES_RUN]: undefined,
+  [Event.QUERY_PREVIEWED]: undefined,
+
+  // TABLE
+  [Event.TABLE_CREATED]: `Table "{{ name }}" created`,
+  [Event.TABLE_UPDATED]: `Table "{{ name }}" updated`,
+  [Event.TABLE_DELETED]: `Table "{{ name }}" deleted`,
+  [Event.TABLE_EXPORTED]: `Table "{{ name }}" exported`,
+  [Event.TABLE_IMPORTED]: `Table "{{ name }}" imported`,
+  [Event.TABLE_DATA_IMPORTED]: `Data imported to table`,
+
+  // ROWS
+  [Event.ROWS_CREATED]: `Rows created`,
+  [Event.ROWS_IMPORTED]: `Rows imported`,
+
+  // AUTOMATION
+  [Event.AUTOMATION_CREATED]: `Automation "{{ name }}" created`,
+  [Event.AUTOMATION_DELETED]: `Automation "{{ name }}" deleted`,
+  [Event.AUTOMATION_STEP_CREATED]: `Automation "{{ name }}" step added`,
+  [Event.AUTOMATION_STEP_DELETED]: `Automation "{{ name }}" step removed`,
+  [Event.AUTOMATION_TESTED]: undefined,
+  [Event.AUTOMATIONS_RUN]: undefined,
+  [Event.AUTOMATION_TRIGGER_UPDATED]: undefined,
+
+  // SCREEN
+  [Event.SCREEN_CREATED]: `Screen "{{ name }}" created`,
+  [Event.SCREEN_DELETED]: `Screen "{{ name }}" deleted`,
+
+  // COMPONENT
+  [Event.COMPONENT_CREATED]: `Component created`,
+  [Event.COMPONENT_DELETED]: `Component deleted`,
+
+  // ENVIRONMENT VARIABLE
+  [Event.ENVIRONMENT_VARIABLE_CREATED]: `Environment variable created`,
+  [Event.ENVIRONMENT_VARIABLE_DELETED]: `Environment variable deleted`,
+  [Event.ENVIRONMENT_VARIABLE_UPGRADE_PANEL_OPENED]: undefined,
+
+  // PLUGIN
+  [Event.PLUGIN_IMPORTED]: `Plugin imported`,
+  [Event.PLUGIN_DELETED]: `Plugin deleted`,
+  [Event.PLUGIN_INIT]: undefined,
+
+  // ROLE - NOT AUDITED
+  [Event.ROLE_CREATED]: undefined,
+  [Event.ROLE_UPDATED]: undefined,
+  [Event.ROLE_DELETED]: undefined,
+  [Event.ROLE_ASSIGNED]: undefined,
+  [Event.ROLE_UNASSIGNED]: undefined,
+
+  // LICENSE - NOT AUDITED
+  [Event.LICENSE_PLAN_CHANGED]: undefined,
+  [Event.LICENSE_TIER_CHANGED]: undefined,
+  [Event.LICENSE_ACTIVATED]: undefined,
+  [Event.LICENSE_PAYMENT_FAILED]: undefined,
+  [Event.LICENSE_PAYMENT_RECOVERED]: undefined,
+  [Event.LICENSE_CHECKOUT_OPENED]: undefined,
+  [Event.LICENSE_CHECKOUT_SUCCESS]: undefined,
+  [Event.LICENSE_PORTAL_OPENED]: undefined,
+
+  // ACCOUNT - NOT AUDITED
+  [Event.ACCOUNT_CREATED]: undefined,
+  [Event.ACCOUNT_DELETED]: undefined,
+  [Event.ACCOUNT_VERIFIED]: undefined,
+
+  // BACKFILL - NOT AUDITED
+  [Event.APP_BACKFILL_SUCCEEDED]: undefined,
+  [Event.APP_BACKFILL_FAILED]: undefined,
+  [Event.TENANT_BACKFILL_SUCCEEDED]: undefined,
+  [Event.TENANT_BACKFILL_FAILED]: undefined,
+  [Event.INSTALLATION_BACKFILL_SUCCEEDED]: undefined,
+  [Event.INSTALLATION_BACKFILL_FAILED]: undefined,
+
+  // LAYOUT - NOT AUDITED
+  [Event.LAYOUT_CREATED]: undefined,
+  [Event.LAYOUT_DELETED]: undefined,
+
+  // VIEW - NOT AUDITED
+  [Event.VIEW_CREATED]: undefined,
+  [Event.VIEW_UPDATED]: undefined,
+  [Event.VIEW_DELETED]: undefined,
+  [Event.VIEW_EXPORTED]: undefined,
+  [Event.VIEW_FILTER_CREATED]: undefined,
+  [Event.VIEW_FILTER_UPDATED]: undefined,
+  [Event.VIEW_FILTER_DELETED]: undefined,
+  [Event.VIEW_CALCULATION_CREATED]: undefined,
+  [Event.VIEW_CALCULATION_UPDATED]: undefined,
+  [Event.VIEW_CALCULATION_DELETED]: undefined,
+
+  // SERVED - NOT AUDITED
+  [Event.SERVED_BUILDER]: undefined,
+  [Event.SERVED_APP]: undefined,
+  [Event.SERVED_APP_PREVIEW]: undefined,
+
+  // ANALYTICS - NOT AUDITED
+  [Event.ANALYTICS_OPT_OUT]: undefined,
+  [Event.ANALYTICS_OPT_IN]: undefined,
+
+  // INSTALLATION - NOT AUDITED
+  [Event.INSTALLATION_VERSION_CHECKED]: undefined,
+  [Event.INSTALLATION_VERSION_UPGRADED]: undefined,
+  [Event.INSTALLATION_VERSION_DOWNGRADED]: undefined,
+  [Event.INSTALLATION_FIRST_STARTUP]: undefined,
+
+  // AUDIT LOG - NOT AUDITED
+  [Event.AUDIT_LOGS_FILTERED]: undefined,
+  [Event.AUDIT_LOGS_DOWNLOADED]: undefined,
 }
 
 // properties added at the final stage of the event pipeline
@@ -191,6 +375,11 @@ export interface BaseEvent {
   installationId?: string
   tenantId?: string
   hosting?: Hosting
+  // any props in the audited section will be removed before passing events
+  // up out of system (purely for use with auditing)
+  audited?: {
+    [key: string]: any
+  }
 }
 
 export type TableExportFormat = "json" | "csv"

packages/types/src/sdk/events/identification.ts

@@ -34,6 +34,11 @@ export enum IdentityType {
   INSTALLATION = "installation",
 }
 
+export interface HostInfo {
+  ipAddress?: string
+  userAgent?: string
+}
+
 export interface Identity {
   id: string
   type: IdentityType
@@ -41,6 +46,7 @@ export interface Identity {
   environment: string
   installationId?: string
   tenantId?: string
+  hostInfo?: HostInfo
 }
 
 export interface UserIdentity extends Identity {

packages/types/src/sdk/events/index.ts

@@ -22,3 +22,4 @@ export * from "./userGroup"
 export * from "./plugin"
 export * from "./backup"
 export * from "./environmentVariable"
+export * from "./auditLog"

packages/types/src/sdk/events/screen.ts

@@ -4,10 +4,16 @@ export interface ScreenCreatedEvent extends BaseEvent {
   screenId: string
   layoutId?: string
   roleId: string
+  audited: {
+    name: string
+  }
 }
 
 export interface ScreenDeletedEvent extends BaseEvent {
   screenId: string
   layoutId?: string
   roleId: string
+  audited: {
+    name: string
+  }
 }