diff --git a/hosting/docker-compose.test.yaml b/hosting/docker-compose.test.yaml index dfd78621c5..f059173d2d 100644 --- a/hosting/docker-compose.test.yaml +++ b/hosting/docker-compose.test.yaml @@ -8,8 +8,8 @@ services: # Last version that supports the "fs" backend image: minio/minio:RELEASE.2022-10-24T18-35-07Z ports: - - 9000 - - 9001 + - "9000" + - "9001" environment: MINIO_ACCESS_KEY: ${MINIO_ACCESS_KEY} MINIO_SECRET_KEY: ${MINIO_SECRET_KEY} @@ -28,9 +28,9 @@ services: - COUCHDB_PASSWORD=${COUCH_DB_PASSWORD} - COUCHDB_USER=${COUCH_DB_USER} ports: - - 5984 - - 4369 - - 9100 + - "5984" + - "4369" + - "9100" healthcheck: test: ["CMD", "curl", "-f", "http://localhost:5984/_up"] interval: 30s @@ -42,6 +42,6 @@ services: image: redis command: redis-server --requirepass ${REDIS_PASSWORD} ports: - - 6379 + - "6379" healthcheck: - test: ["CMD", "redis-cli", "ping"] + test: ["CMD", "redis-cli", "ping"] \ No newline at end of file diff --git a/package.json b/package.json index 3958917380..815e470916 100644 --- a/package.json +++ b/package.json @@ -13,7 +13,7 @@ "js-yaml": "^4.1.0", "kill-port": "^1.6.1", "lerna": "3.14.1", - "madge": "^5.0.1", + "madge": "^6.0.0", "prettier": "^2.3.1", "prettier-plugin-svelte": "^2.3.0", "rimraf": "^3.0.2", @@ -84,4 +84,4 @@ "install:pro": "bash scripts/pro/install.sh", "dep:clean": "yarn clean && yarn bootstrap" } -} \ No newline at end of file +} diff --git a/packages/backend-core/src/auth/auth.ts b/packages/backend-core/src/auth/auth.ts index dd28344b1f..936d06ddff 100644 --- a/packages/backend-core/src/auth/auth.ts +++ b/packages/backend-core/src/auth/auth.ts @@ -227,6 +227,6 @@ export async function platformLogout(opts: PlatformLogoutOpts) { const sessionIds = sessions.map(({ sessionId }) => sessionId) await invalidateSessions(userId, { sessionIds, reason: "logout" }) - await events.auth.logout() + await events.auth.logout(ctx.user?.email) await userCache.invalidateUser(userId) } diff --git a/packages/backend-core/src/constants/db.ts b/packages/backend-core/src/constants/db.ts index f7d15b3880..d41098c405 100644 --- a/packages/backend-core/src/constants/db.ts +++ b/packages/backend-core/src/constants/db.ts @@ -68,6 +68,7 @@ export enum DocumentType { MEM_VIEW = "view", USER_FLAG = "flag", AUTOMATION_METADATA = "meta_au", + AUDIT_LOG = "al", } export const StaticDatabases = { @@ -88,6 +89,9 @@ export const StaticDatabases = { install: "install", }, }, + AUDIT_LOGS: { + name: "audit-logs", + }, } export const APP_PREFIX = DocumentType.APP + SEPARATOR diff --git a/packages/backend-core/src/constants/misc.ts b/packages/backend-core/src/constants/misc.ts index 0bf3df4094..e25c90575f 100644 --- a/packages/backend-core/src/constants/misc.ts +++ b/packages/backend-core/src/constants/misc.ts @@ -41,5 +41,6 @@ export enum Config { OIDC_LOGOS = "logos_oidc", } +export const MIN_VALID_DATE = new Date(-2147483647000) export const MAX_VALID_DATE = new Date(2147483647000) export const DEFAULT_TENANT_ID = "default" diff --git a/packages/backend-core/src/context/Context.ts b/packages/backend-core/src/context/Context.ts index 02b7713764..d29b6935a8 100644 --- a/packages/backend-core/src/context/Context.ts +++ b/packages/backend-core/src/context/Context.ts @@ -1,5 +1,5 @@ import { AsyncLocalStorage } from "async_hooks" -import { ContextMap } from "./mainContext" +import { ContextMap } from "./types" export default class Context { static storage = new AsyncLocalStorage() diff --git a/packages/backend-core/src/context/identity.ts b/packages/backend-core/src/context/identity.ts index 648dd1b5fd..84de3b68c9 100644 --- a/packages/backend-core/src/context/identity.ts +++ b/packages/backend-core/src/context/identity.ts @@ -5,6 +5,8 @@ import { isCloudAccount, Account, AccountUserContext, + UserContext, + Ctx, } from "@budibase/types" import * as context from "." @@ -16,15 +18,22 @@ export function doInIdentityContext(identity: IdentityContext, task: any) { return context.doInIdentityContext(identity, task) } -export function doInUserContext(user: User, task: any) { - const userContext: any = { +// used in server/worker +export function doInUserContext(user: User, ctx: Ctx, task: any) { + const userContext: UserContext = { ...user, _id: user._id as string, type: IdentityType.USER, + hostInfo: { + ipAddress: ctx.request.ip, + // filled in by koa-useragent package + userAgent: ctx.userAgent._agent.source, + }, } return doInIdentityContext(userContext, task) } +// used in account portal export function doInAccountContext(account: Account, task: any) { const _id = getAccountUserId(account) const tenantId = account.tenantId diff --git a/packages/backend-core/src/context/mainContext.ts b/packages/backend-core/src/context/mainContext.ts index 9884d25d5a..02ba16aa8c 100644 --- a/packages/backend-core/src/context/mainContext.ts +++ b/packages/backend-core/src/context/mainContext.ts @@ -11,13 +11,7 @@ import { DEFAULT_TENANT_ID, } from "../constants" import { Database, IdentityContext } from "@budibase/types" - -export type ContextMap = { - tenantId?: string - appId?: string - identity?: IdentityContext - environmentVariables?: Record -} +import { ContextMap } from "./types" let TEST_APP_ID: string | null = null @@ -30,14 +24,23 @@ export function getGlobalDBName(tenantId?: string) { return baseGlobalDBName(tenantId) } -export function baseGlobalDBName(tenantId: string | undefined | null) { - let dbName - if (!tenantId || tenantId === DEFAULT_TENANT_ID) { - dbName = StaticDatabases.GLOBAL.name - } else { - dbName = `${tenantId}${SEPARATOR}${StaticDatabases.GLOBAL.name}` +export function getAuditLogDBName(tenantId?: string) { + if (!tenantId) { + tenantId = getTenantId() + } + if (tenantId === DEFAULT_TENANT_ID) { + return StaticDatabases.AUDIT_LOGS.name + } else { + return `${tenantId}${SEPARATOR}${StaticDatabases.AUDIT_LOGS.name}` + } +} + +export function baseGlobalDBName(tenantId: string | undefined | null) { + if (!tenantId || tenantId === DEFAULT_TENANT_ID) { + return StaticDatabases.GLOBAL.name + } else { + return `${tenantId}${SEPARATOR}${StaticDatabases.GLOBAL.name}` } - return dbName } export function isMultiTenant() { @@ -228,6 +231,13 @@ export function getGlobalDB(): Database { return getDB(baseGlobalDBName(context?.tenantId)) } +export function getAuditLogsDB(): Database { + if (!getTenantId()) { + throw new Error("No tenant ID found - cannot open audit log DB") + } + return getDB(getAuditLogDBName()) +} + /** * Gets the app database based on whatever the request * contained, dev or prod. diff --git a/packages/backend-core/src/context/types.ts b/packages/backend-core/src/context/types.ts new file mode 100644 index 0000000000..78197ed528 --- /dev/null +++ b/packages/backend-core/src/context/types.ts @@ -0,0 +1,9 @@ +import { IdentityContext } from "@budibase/types" + +// keep this out of Budibase types, don't want to expose context info +export type ContextMap = { + tenantId?: string + appId?: string + identity?: IdentityContext + environmentVariables?: Record +} diff --git a/packages/backend-core/src/db/index.ts b/packages/backend-core/src/db/index.ts index 0d9f75fa18..a569b17b36 100644 --- a/packages/backend-core/src/db/index.ts +++ b/packages/backend-core/src/db/index.ts @@ -7,3 +7,4 @@ export { default as Replication } from "./Replication" // exports to support old export structure export * from "../constants/db" export { getGlobalDBName, baseGlobalDBName } from "../context" +export * from "./lucene" diff --git a/packages/backend-core/src/db/lucene.ts b/packages/backend-core/src/db/lucene.ts new file mode 100644 index 0000000000..cba2f0138a --- /dev/null +++ b/packages/backend-core/src/db/lucene.ts @@ -0,0 +1,624 @@ +import fetch from "node-fetch" +import { getCouchInfo } from "./couch" +import { SearchFilters, Row } from "@budibase/types" + +const QUERY_START_REGEX = /\d[0-9]*:/g + +interface SearchResponse { + rows: T[] | any[] + bookmark: string +} + +interface PaginatedSearchResponse extends SearchResponse { + hasNextPage: boolean +} + +export type SearchParams = { + tableId?: string + sort?: string + sortOrder?: string + sortType?: string + limit?: number + bookmark?: string + version?: string + indexer?: () => Promise + disableEscaping?: boolean + rows?: T | Row[] +} + +export function removeKeyNumbering(key: any): string { + if (typeof key === "string" && key.match(QUERY_START_REGEX) != null) { + const parts = key.split(":") + // remove the number + parts.shift() + return parts.join(":") + } else { + return key + } +} + +/** + * Class to build lucene query URLs. + * Optionally takes a base lucene query object. + */ +export class QueryBuilder { + dbName: string + index: string + query: SearchFilters + limit: number + sort?: string + bookmark?: string + sortOrder: string + sortType: string + includeDocs: boolean + version?: string + indexBuilder?: () => Promise + noEscaping = false + + constructor(dbName: string, index: string, base?: SearchFilters) { + this.dbName = dbName + this.index = index + this.query = { + allOr: false, + string: {}, + fuzzy: {}, + range: {}, + equal: {}, + notEqual: {}, + empty: {}, + notEmpty: {}, + oneOf: {}, + contains: {}, + notContains: {}, + containsAny: {}, + ...base, + } + this.limit = 50 + this.sortOrder = "ascending" + this.sortType = "string" + this.includeDocs = true + } + + disableEscaping() { + this.noEscaping = true + return this + } + + setIndexBuilder(builderFn: () => Promise) { + this.indexBuilder = builderFn + return this + } + + setVersion(version?: string) { + if (version != null) { + this.version = version + } + return this + } + + setTable(tableId: string) { + this.query.equal!.tableId = tableId + return this + } + + setLimit(limit?: number) { + if (limit != null) { + this.limit = limit + } + return this + } + + setSort(sort?: string) { + if (sort != null) { + this.sort = sort + } + return this + } + + setSortOrder(sortOrder?: string) { + if (sortOrder != null) { + this.sortOrder = sortOrder + } + return this + } + + setSortType(sortType?: string) { + if (sortType != null) { + this.sortType = sortType + } + return this + } + + setBookmark(bookmark?: string) { + if (bookmark != null) { + this.bookmark = bookmark + } + return this + } + + excludeDocs() { + this.includeDocs = false + return this + } + + addString(key: string, partial: string) { + this.query.string![key] = partial + return this + } + + addFuzzy(key: string, fuzzy: string) { + this.query.fuzzy![key] = fuzzy + return this + } + + addRange(key: string, low: string | number, high: string | number) { + this.query.range![key] = { + low, + high, + } + return this + } + + addEqual(key: string, value: any) { + this.query.equal![key] = value + return this + } + + addNotEqual(key: string, value: any) { + this.query.notEqual![key] = value + return this + } + + addEmpty(key: string, value: any) { + this.query.empty![key] = value + return this + } + + addNotEmpty(key: string, value: any) { + this.query.notEmpty![key] = value + return this + } + + addOneOf(key: string, value: any) { + this.query.oneOf![key] = value + return this + } + + addContains(key: string, value: any) { + this.query.contains![key] = value + return this + } + + addNotContains(key: string, value: any) { + this.query.notContains![key] = value + return this + } + + addContainsAny(key: string, value: any) { + this.query.containsAny![key] = value + return this + } + + handleSpaces(input: string) { + if (this.noEscaping) { + return input + } else { + return input.replace(/ /g, "_") + } + } + + /** + * Preprocesses a value before going into a lucene search. + * Transforms strings to lowercase and wraps strings and bools in quotes. + * @param value The value to process + * @param options The preprocess options + * @returns {string|*} + */ + preprocess(value: any, { escape, lowercase, wrap, type }: any = {}) { + const hasVersion = !!this.version + // Determine if type needs wrapped + const originalType = typeof value + // Convert to lowercase + if (value && lowercase) { + value = value.toLowerCase ? value.toLowerCase() : value + } + // Escape characters + if (!this.noEscaping && escape && originalType === "string") { + value = `${value}`.replace(/[ #+\-&|!(){}\]^"~*?:\\]/g, "\\$&") + } + + // Wrap in quotes + if (originalType === "string" && !isNaN(value) && !type) { + value = `"${value}"` + } else if (hasVersion && wrap) { + value = originalType === "number" ? value : `"${value}"` + } + return value + } + + buildSearchQuery() { + const builder = this + let allOr = this.query && this.query.allOr + let query = allOr ? "" : "*:*" + const allPreProcessingOpts = { escape: true, lowercase: true, wrap: true } + let tableId + if (this.query.equal!.tableId) { + tableId = this.query.equal!.tableId + delete this.query.equal!.tableId + } + + const equal = (key: string, value: any) => { + // 0 evaluates to false, which means we would return all rows if we don't check it + if (!value && value !== 0) { + return null + } + return `${key}:${builder.preprocess(value, allPreProcessingOpts)}` + } + + const contains = (key: string, value: any, mode = "AND") => { + if (Array.isArray(value) && value.length === 0) { + return null + } + if (!Array.isArray(value)) { + return `${key}:${value}` + } + let statement = `${builder.preprocess(value[0], { escape: true })}` + for (let i = 1; i < value.length; i++) { + statement += ` ${mode} ${builder.preprocess(value[i], { + escape: true, + })}` + } + return `${key}:(${statement})` + } + + const notContains = (key: string, value: any) => { + // @ts-ignore + const allPrefix = allOr === "" ? "*:* AND" : "" + return allPrefix + "NOT " + contains(key, value) + } + + const containsAny = (key: string, value: any) => { + return contains(key, value, "OR") + } + + const oneOf = (key: string, value: any) => { + if (!Array.isArray(value)) { + if (typeof value === "string") { + value = value.split(",") + } else { + return "" + } + } + let orStatement = `${builder.preprocess(value[0], allPreProcessingOpts)}` + for (let i = 1; i < value.length; i++) { + orStatement += ` OR ${builder.preprocess( + value[i], + allPreProcessingOpts + )}` + } + return `${key}:(${orStatement})` + } + + function build(structure: any, queryFn: any) { + for (let [key, value] of Object.entries(structure)) { + // check for new format - remove numbering if needed + key = removeKeyNumbering(key) + key = builder.preprocess(builder.handleSpaces(key), { + escape: true, + }) + const expression = queryFn(key, value) + if (expression == null) { + continue + } + if (query.length > 0) { + query += ` ${allOr ? "OR" : "AND"} ` + } + query += expression + } + } + + // Construct the actual lucene search query string from JSON structure + if (this.query.string) { + build(this.query.string, (key: string, value: any) => { + if (!value) { + return null + } + value = builder.preprocess(value, { + escape: true, + lowercase: true, + type: "string", + }) + return `${key}:${value}*` + }) + } + if (this.query.range) { + build(this.query.range, (key: string, value: any) => { + if (!value) { + return null + } + if (value.low == null || value.low === "") { + return null + } + if (value.high == null || value.high === "") { + return null + } + const low = builder.preprocess(value.low, allPreProcessingOpts) + const high = builder.preprocess(value.high, allPreProcessingOpts) + return `${key}:[${low} TO ${high}]` + }) + } + if (this.query.fuzzy) { + build(this.query.fuzzy, (key: string, value: any) => { + if (!value) { + return null + } + value = builder.preprocess(value, { + escape: true, + lowercase: true, + type: "fuzzy", + }) + return `${key}:${value}~` + }) + } + if (this.query.equal) { + build(this.query.equal, equal) + } + if (this.query.notEqual) { + build(this.query.notEqual, (key: string, value: any) => { + if (!value) { + return null + } + return `!${key}:${builder.preprocess(value, allPreProcessingOpts)}` + }) + } + if (this.query.empty) { + build(this.query.empty, (key: string) => `!${key}:["" TO *]`) + } + if (this.query.notEmpty) { + build(this.query.notEmpty, (key: string) => `${key}:["" TO *]`) + } + if (this.query.oneOf) { + build(this.query.oneOf, oneOf) + } + if (this.query.contains) { + build(this.query.contains, contains) + } + if (this.query.notContains) { + build(this.query.notContains, notContains) + } + if (this.query.containsAny) { + build(this.query.containsAny, containsAny) + } + // make sure table ID is always added as an AND + if (tableId) { + query = `(${query})` + allOr = false + build({ tableId }, equal) + } + return query + } + + buildSearchBody() { + let body: any = { + q: this.buildSearchQuery(), + limit: Math.min(this.limit, 200), + include_docs: this.includeDocs, + } + if (this.bookmark) { + body.bookmark = this.bookmark + } + if (this.sort) { + const order = this.sortOrder === "descending" ? "-" : "" + const type = `<${this.sortType}>` + body.sort = `${order}${this.handleSpaces(this.sort)}${type}` + } + return body + } + + async run() { + const { url, cookie } = getCouchInfo() + const fullPath = `${url}/${this.dbName}/_design/database/_search/${this.index}` + const body = this.buildSearchBody() + try { + return await runQuery(fullPath, body, cookie) + } catch (err: any) { + if (err.status === 404 && this.indexBuilder) { + await this.indexBuilder() + return await runQuery(fullPath, body, cookie) + } else { + throw err + } + } + } +} + +/** + * Executes a lucene search query. + * @param url The query URL + * @param body The request body defining search criteria + * @param cookie The auth cookie for CouchDB + * @returns {Promise<{rows: []}>} + */ +async function runQuery( + url: string, + body: any, + cookie: string +): Promise> { + const response = await fetch(url, { + body: JSON.stringify(body), + method: "POST", + headers: { + Authorization: cookie, + }, + }) + + if (response.status === 404) { + throw response + } + const json = await response.json() + + let output: any = { + rows: [], + } + if (json.rows != null && json.rows.length > 0) { + output.rows = json.rows.map((row: any) => row.doc) + } + if (json.bookmark) { + output.bookmark = json.bookmark + } + return output +} + +/** + * Gets round the fixed limit of 200 results from a query by fetching as many + * pages as required and concatenating the results. This recursively operates + * until enough results have been found. + * @param dbName {string} Which database to run a lucene query on + * @param index {string} Which search index to utilise + * @param query {object} The JSON query structure + * @param params {object} The search params including: + * tableId {string} The table ID to search + * sort {string} The sort column + * sortOrder {string} The sort order ("ascending" or "descending") + * sortType {string} Whether to treat sortable values as strings or + * numbers. ("string" or "number") + * limit {number} The number of results to fetch + * bookmark {string|null} Current bookmark in the recursive search + * rows {array|null} Current results in the recursive search + * @returns {Promise<*[]|*>} + */ +async function recursiveSearch( + dbName: string, + index: string, + query: any, + params: any +): Promise { + const bookmark = params.bookmark + const rows = params.rows || [] + if (rows.length >= params.limit) { + return rows + } + let pageSize = 200 + if (rows.length > params.limit - 200) { + pageSize = params.limit - rows.length + } + const page = await new QueryBuilder(dbName, index, query) + .setVersion(params.version) + .setTable(params.tableId) + .setBookmark(bookmark) + .setLimit(pageSize) + .setSort(params.sort) + .setSortOrder(params.sortOrder) + .setSortType(params.sortType) + .run() + if (!page.rows.length) { + return rows + } + if (page.rows.length < 200) { + return [...rows, ...page.rows] + } + const newParams = { + ...params, + bookmark: page.bookmark, + rows: [...rows, ...page.rows], + } + return await recursiveSearch(dbName, index, query, newParams) +} + +/** + * Performs a paginated search. A bookmark will be returned to allow the next + * page to be fetched. There is a max limit off 200 results per page in a + * paginated search. + * @param dbName {string} Which database to run a lucene query on + * @param index {string} Which search index to utilise + * @param query {object} The JSON query structure + * @param params {object} The search params including: + * tableId {string} The table ID to search + * sort {string} The sort column + * sortOrder {string} The sort order ("ascending" or "descending") + * sortType {string} Whether to treat sortable values as strings or + * numbers. ("string" or "number") + * limit {number} The desired page size + * bookmark {string} The bookmark to resume from + * @returns {Promise<{hasNextPage: boolean, rows: *[]}>} + */ +export async function paginatedSearch( + dbName: string, + index: string, + query: SearchFilters, + params: SearchParams +) { + let limit = params.limit + if (limit == null || isNaN(limit) || limit < 0) { + limit = 50 + } + limit = Math.min(limit, 200) + const search = new QueryBuilder(dbName, index, query) + if (params.version) { + search.setVersion(params.version) + } + if (params.tableId) { + search.setTable(params.tableId) + } + if (params.sort) { + search + .setSort(params.sort) + .setSortOrder(params.sortOrder) + .setSortType(params.sortType) + } + if (params.indexer) { + search.setIndexBuilder(params.indexer) + } + if (params.disableEscaping) { + search.disableEscaping() + } + const searchResults = await search + .setBookmark(params.bookmark) + .setLimit(limit) + .run() + + // Try fetching 1 row in the next page to see if another page of results + // exists or not + search.setBookmark(searchResults.bookmark).setLimit(1) + if (params.tableId) { + search.setTable(params.tableId) + } + const nextResults = await search.run() + + return { + ...searchResults, + hasNextPage: nextResults.rows && nextResults.rows.length > 0, + } +} + +/** + * Performs a full search, fetching multiple pages if required to return the + * desired amount of results. There is a limit of 1000 results to avoid + * heavy performance hits, and to avoid client components breaking from + * handling too much data. + * @param dbName {string} Which database to run a lucene query on + * @param index {string} Which search index to utilise + * @param query {object} The JSON query structure + * @param params {object} The search params including: + * tableId {string} The table ID to search + * sort {string} The sort column + * sortOrder {string} The sort order ("ascending" or "descending") + * sortType {string} Whether to treat sortable values as strings or + * numbers. ("string" or "number") + * limit {number} The desired number of results + * @returns {Promise<{rows: *}>} + */ +export async function fullSearch( + dbName: string, + index: string, + query: SearchFilters, + params: SearchParams +) { + let limit = params.limit + if (limit == null || isNaN(limit) || limit < 0) { + limit = 1000 + } + params.limit = Math.min(limit, 1000) + const rows = await recursiveSearch(dbName, index, query, params) + return { rows } +} diff --git a/packages/backend-core/src/db/tests/lucene.spec.ts b/packages/backend-core/src/db/tests/lucene.spec.ts new file mode 100644 index 0000000000..23b01e18df --- /dev/null +++ b/packages/backend-core/src/db/tests/lucene.spec.ts @@ -0,0 +1,161 @@ +import { newid } from "../../newid" +import { getDB } from "../db" +import { Database } from "@budibase/types" +import { QueryBuilder, paginatedSearch, fullSearch } from "../lucene" + +const INDEX_NAME = "main" + +const index = `function(doc) { + let props = ["property", "number"] + for (let key of props) { + if (doc[key]) { + index(key, doc[key]) + } + } +}` + +describe("lucene", () => { + let db: Database, dbName: string + + beforeAll(async () => { + dbName = `db-${newid()}` + // create the DB for testing + db = getDB(dbName) + await db.put({ _id: newid(), property: "word" }) + await db.put({ _id: newid(), property: "word2" }) + await db.put({ _id: newid(), property: "word3", number: 1 }) + }) + + it("should be able to create a lucene index", async () => { + const response = await db.put({ + _id: "_design/database", + indexes: { + [INDEX_NAME]: { + index: index, + analyzer: "standard", + }, + }, + }) + expect(response.ok).toBe(true) + }) + + describe("query builder", () => { + it("should be able to perform a basic query", async () => { + const builder = new QueryBuilder(dbName, INDEX_NAME) + builder.setSort("property") + builder.setSortOrder("desc") + builder.setSortType("string") + const resp = await builder.run() + expect(resp.rows.length).toBe(3) + }) + + it("should handle limits", async () => { + const builder = new QueryBuilder(dbName, INDEX_NAME) + builder.setLimit(1) + const resp = await builder.run() + expect(resp.rows.length).toBe(1) + }) + + it("should be able to perform a string search", async () => { + const builder = new QueryBuilder(dbName, INDEX_NAME) + builder.addString("property", "wo") + const resp = await builder.run() + expect(resp.rows.length).toBe(3) + }) + + it("should be able to perform a range search", async () => { + const builder = new QueryBuilder(dbName, INDEX_NAME) + builder.addRange("number", 0, 1) + const resp = await builder.run() + expect(resp.rows.length).toBe(1) + }) + + it("should be able to perform an equal search", async () => { + const builder = new QueryBuilder(dbName, INDEX_NAME) + builder.addEqual("property", "word2") + const resp = await builder.run() + expect(resp.rows.length).toBe(1) + }) + + it("should be able to perform a not equal search", async () => { + const builder = new QueryBuilder(dbName, INDEX_NAME) + builder.addNotEqual("property", "word2") + const resp = await builder.run() + expect(resp.rows.length).toBe(2) + }) + + it("should be able to perform an empty search", async () => { + const builder = new QueryBuilder(dbName, INDEX_NAME) + builder.addEmpty("number", true) + const resp = await builder.run() + expect(resp.rows.length).toBe(2) + }) + + it("should be able to perform a not empty search", async () => { + const builder = new QueryBuilder(dbName, INDEX_NAME) + builder.addNotEmpty("number", true) + const resp = await builder.run() + expect(resp.rows.length).toBe(1) + }) + + it("should be able to perform a one of search", async () => { + const builder = new QueryBuilder(dbName, INDEX_NAME) + builder.addOneOf("property", ["word", "word2"]) + const resp = await builder.run() + expect(resp.rows.length).toBe(2) + }) + + it("should be able to perform a contains search", async () => { + const builder = new QueryBuilder(dbName, INDEX_NAME) + builder.addContains("property", ["word"]) + const resp = await builder.run() + expect(resp.rows.length).toBe(1) + }) + + it("should be able to perform a not contains search", async () => { + const builder = new QueryBuilder(dbName, INDEX_NAME) + builder.addNotContains("property", ["word2"]) + const resp = await builder.run() + expect(resp.rows.length).toBe(2) + }) + }) + + describe("paginated search", () => { + it("should be able to perform a paginated search", async () => { + const page = await paginatedSearch( + dbName, + INDEX_NAME, + { + string: { + property: "wo", + }, + }, + { + limit: 1, + sort: "property", + sortType: "string", + sortOrder: "desc", + } + ) + expect(page.rows.length).toBe(1) + expect(page.hasNextPage).toBe(true) + expect(page.bookmark).toBeDefined() + }) + }) + + describe("full search", () => { + it("should be able to perform a full search", async () => { + const page = await fullSearch( + dbName, + INDEX_NAME, + { + string: { + property: "wo", + }, + }, + {} + ) + expect(page.rows.length).toBe(3) + }) + }) +}) diff --git a/packages/backend-core/src/db/utils.ts b/packages/backend-core/src/db/utils.ts index 948b59b13a..76c52d08ad 100644 --- a/packages/backend-core/src/db/utils.ts +++ b/packages/backend-core/src/db/utils.ts @@ -365,6 +365,16 @@ export async function getAllApps({ } } +export async function getAppsByIDs(appIds: string[]) { + const settled = await Promise.allSettled( + appIds.map(appId => getAppMetadata(appId)) + ) + // have to list the apps which exist, some may have been deleted + return settled + .filter(promise => promise.status === "fulfilled") + .map(promise => (promise as PromiseFulfilledResult).value) +} + /** * Utility function for getAllApps but filters to production apps only. */ @@ -381,6 +391,16 @@ export async function getDevAppIDs() { return apps.filter((id: any) => isDevAppID(id)) } +export function isSameAppID( + appId1: string | undefined, + appId2: string | undefined +) { + if (appId1 == undefined || appId2 == undefined) { + return false + } + return getProdAppID(appId1) === getProdAppID(appId2) +} + export async function dbExists(dbName: any) { return doWithDB( dbName, diff --git a/packages/backend-core/src/environment.ts b/packages/backend-core/src/environment.ts index 0579d709d7..cd7bcca11d 100644 --- a/packages/backend-core/src/environment.ts +++ b/packages/backend-core/src/environment.ts @@ -86,6 +86,7 @@ const environment = { DEPLOYMENT_ENVIRONMENT: process.env.DEPLOYMENT_ENVIRONMENT || "docker-compose", ENABLE_4XX_HTTP_LOGGING: process.env.ENABLE_4XX_HTTP_LOGGING || true, + ENABLE_AUDIT_LOG_IP_ADDR: process.env.ENABLE_AUDIT_LOG_IP_ADDR, // smtp SMTP_FALLBACK_ENABLED: process.env.SMTP_FALLBACK_ENABLED, SMTP_USER: process.env.SMTP_USER, diff --git a/packages/backend-core/src/events/events.ts b/packages/backend-core/src/events/events.ts index 01928221a0..c2f7cf66ec 100644 --- a/packages/backend-core/src/events/events.ts +++ b/packages/backend-core/src/events/events.ts @@ -1,4 +1,4 @@ -import { Event } from "@budibase/types" +import { Event, AuditedEventFriendlyName } from "@budibase/types" import { processors } from "./processors" import identification from "./identification" import * as backfill from "./backfill" diff --git a/packages/backend-core/src/events/identification.ts b/packages/backend-core/src/events/identification.ts index 08ed9c54b2..9534fb293d 100644 --- a/packages/backend-core/src/events/identification.ts +++ b/packages/backend-core/src/events/identification.ts @@ -87,6 +87,7 @@ const getCurrentIdentity = async (): Promise => { installationId, tenantId, environment, + hostInfo: userContext.hostInfo, } } else { throw new Error("Unknown identity type") diff --git a/packages/backend-core/src/events/processors/AuditLogsProcessor.ts b/packages/backend-core/src/events/processors/AuditLogsProcessor.ts new file mode 100644 index 0000000000..fd68b66871 --- /dev/null +++ b/packages/backend-core/src/events/processors/AuditLogsProcessor.ts @@ -0,0 +1,90 @@ +import { + Event, + Identity, + Group, + IdentityType, + AuditLogQueueEvent, + AuditLogFn, + HostInfo, +} from "@budibase/types" +import { EventProcessor } from "./types" +import { getAppId } from "../../context" +import BullQueue from "bull" +import { createQueue, JobQueue } from "../../queue" +import { isAudited } from "../../utils" +import env from "../../environment" + +export default class AuditLogsProcessor implements EventProcessor { + static auditLogsEnabled = false + static auditLogQueue: BullQueue.Queue + + // can't use constructor as need to return promise + static init(fn: AuditLogFn) { + AuditLogsProcessor.auditLogsEnabled = true + const writeAuditLogs = fn + AuditLogsProcessor.auditLogQueue = createQueue( + JobQueue.AUDIT_LOG + ) + return AuditLogsProcessor.auditLogQueue.process(async job => { + let properties = job.data.properties + if (properties.audited) { + properties = { + ...properties, + ...properties.audited, + } + delete properties.audited + } + + // this feature is disabled by default due to privacy requirements + // in some countries - available as env var in-case it is desired + // in self host deployments + let hostInfo: HostInfo | undefined = {} + if (env.ENABLE_AUDIT_LOG_IP_ADDR) { + hostInfo = job.data.opts.hostInfo + } + + await writeAuditLogs(job.data.event, properties, { + userId: job.data.opts.userId, + timestamp: job.data.opts.timestamp, + appId: job.data.opts.appId, + hostInfo, + }) + }) + } + + async processEvent( + event: Event, + identity: Identity, + properties: any, + timestamp?: string + ): Promise { + if (AuditLogsProcessor.auditLogsEnabled && isAudited(event)) { + // only audit log actual events, don't include backfills + const userId = + identity.type === IdentityType.USER ? identity.id : undefined + // add to the event queue, rather than just writing immediately + await AuditLogsProcessor.auditLogQueue.add({ + event, + properties, + opts: { + userId, + timestamp, + appId: getAppId(), + hostInfo: identity.hostInfo, + }, + }) + } + } + + async identify(identity: Identity, timestamp?: string | number) { + // no-op + } + + async identifyGroup(group: Group, timestamp?: string | number) { + // no-op + } + + shutdown(): void { + AuditLogsProcessor.auditLogQueue?.close() + } +} diff --git a/packages/backend-core/src/events/processors/index.ts b/packages/backend-core/src/events/processors/index.ts index 0e75f050db..6646764e47 100644 --- a/packages/backend-core/src/events/processors/index.ts +++ b/packages/backend-core/src/events/processors/index.ts @@ -1,8 +1,19 @@ import AnalyticsProcessor from "./AnalyticsProcessor" import LoggingProcessor from "./LoggingProcessor" +import AuditLogsProcessor from "./AuditLogsProcessor" import Processors from "./Processors" +import { AuditLogFn } from "@budibase/types" export const analyticsProcessor = new AnalyticsProcessor() const loggingProcessor = new LoggingProcessor() +const auditLogsProcessor = new AuditLogsProcessor() -export const processors = new Processors([analyticsProcessor, loggingProcessor]) +export function init(auditingFn: AuditLogFn) { + return AuditLogsProcessor.init(auditingFn) +} + +export const processors = new Processors([ + analyticsProcessor, + loggingProcessor, + auditLogsProcessor, +]) diff --git a/packages/backend-core/src/events/processors/posthog/PosthogProcessor.ts b/packages/backend-core/src/events/processors/posthog/PosthogProcessor.ts index 593e5ff082..0dbe70d543 100644 --- a/packages/backend-core/src/events/processors/posthog/PosthogProcessor.ts +++ b/packages/backend-core/src/events/processors/posthog/PosthogProcessor.ts @@ -47,6 +47,8 @@ export default class PosthogProcessor implements EventProcessor { return } + properties = this.clearPIIProperties(properties) + properties.version = pkg.version properties.service = env.SERVICE properties.environment = identity.environment @@ -79,6 +81,16 @@ export default class PosthogProcessor implements EventProcessor { this.posthog.capture(payload) } + clearPIIProperties(properties: any) { + if (properties.email) { + delete properties.email + } + if (properties.audited) { + delete properties.audited + } + return properties + } + async identify(identity: Identity, timestamp?: string | number) { const payload: any = { distinctId: identity.id, properties: identity } if (timestamp) { diff --git a/packages/backend-core/src/events/processors/posthog/tests/PosthogProcessor.spec.ts b/packages/backend-core/src/events/processors/posthog/tests/PosthogProcessor.spec.ts index 2c1340d36e..8df4e40bcf 100644 --- a/packages/backend-core/src/events/processors/posthog/tests/PosthogProcessor.spec.ts +++ b/packages/backend-core/src/events/processors/posthog/tests/PosthogProcessor.spec.ts @@ -49,6 +49,25 @@ describe("PosthogProcessor", () => { expect(processor.posthog.capture).toHaveBeenCalledTimes(0) }) + it("removes audited information", async () => { + const processor = new PosthogProcessor("test") + + const identity = newIdentity() + const properties = { + email: "test", + audited: { + name: "test", + }, + } + + await processor.processEvent(Event.USER_CREATED, identity, properties) + expect(processor.posthog.capture).toHaveBeenCalled() + // @ts-ignore + const call = processor.posthog.capture.mock.calls[0][0] + expect(call.properties.audited).toBeUndefined() + expect(call.properties.email).toBeUndefined() + }) + describe("rate limiting", () => { it("sends daily event once in same day", async () => { const processor = new PosthogProcessor("test") diff --git a/packages/backend-core/src/events/publishers/app.ts b/packages/backend-core/src/events/publishers/app.ts index 90da21f3f5..d08d59b5f1 100644 --- a/packages/backend-core/src/events/publishers/app.ts +++ b/packages/backend-core/src/events/publishers/app.ts @@ -19,6 +19,9 @@ const created = async (app: App, timestamp?: string | number) => { const properties: AppCreatedEvent = { appId: app.appId, version: app.version, + audited: { + name: app.name, + }, } await publishEvent(Event.APP_CREATED, properties, timestamp) } @@ -27,6 +30,9 @@ async function updated(app: App) { const properties: AppUpdatedEvent = { appId: app.appId, version: app.version, + audited: { + name: app.name, + }, } await publishEvent(Event.APP_UPDATED, properties) } @@ -34,6 +40,9 @@ async function updated(app: App) { async function deleted(app: App) { const properties: AppDeletedEvent = { appId: app.appId, + audited: { + name: app.name, + }, } await publishEvent(Event.APP_DELETED, properties) } @@ -41,6 +50,9 @@ async function deleted(app: App) { async function published(app: App, timestamp?: string | number) { const properties: AppPublishedEvent = { appId: app.appId, + audited: { + name: app.name, + }, } await publishEvent(Event.APP_PUBLISHED, properties, timestamp) } @@ -48,6 +60,9 @@ async function published(app: App, timestamp?: string | number) { async function unpublished(app: App) { const properties: AppUnpublishedEvent = { appId: app.appId, + audited: { + name: app.name, + }, } await publishEvent(Event.APP_UNPUBLISHED, properties) } @@ -55,6 +70,9 @@ async function unpublished(app: App) { async function fileImported(app: App) { const properties: AppFileImportedEvent = { appId: app.appId, + audited: { + name: app.name, + }, } await publishEvent(Event.APP_FILE_IMPORTED, properties) } @@ -63,6 +81,9 @@ async function templateImported(app: App, templateKey: string) { const properties: AppTemplateImportedEvent = { appId: app.appId, templateKey, + audited: { + name: app.name, + }, } await publishEvent(Event.APP_TEMPLATE_IMPORTED, properties) } @@ -76,6 +97,9 @@ async function versionUpdated( appId: app.appId, currentVersion, updatedToVersion, + audited: { + name: app.name, + }, } await publishEvent(Event.APP_VERSION_UPDATED, properties) } @@ -89,6 +113,9 @@ async function versionReverted( appId: app.appId, currentVersion, revertedToVersion, + audited: { + name: app.name, + }, } await publishEvent(Event.APP_VERSION_REVERTED, properties) } @@ -96,6 +123,9 @@ async function versionReverted( async function reverted(app: App) { const properties: AppRevertedEvent = { appId: app.appId, + audited: { + name: app.name, + }, } await publishEvent(Event.APP_REVERTED, properties) } @@ -103,6 +133,9 @@ async function reverted(app: App) { async function exported(app: App) { const properties: AppExportedEvent = { appId: app.appId, + audited: { + name: app.name, + }, } await publishEvent(Event.APP_EXPORTED, properties) } diff --git a/packages/backend-core/src/events/publishers/auditLog.ts b/packages/backend-core/src/events/publishers/auditLog.ts new file mode 100644 index 0000000000..7cfb76147a --- /dev/null +++ b/packages/backend-core/src/events/publishers/auditLog.ts @@ -0,0 +1,26 @@ +import { + Event, + AuditLogSearchParams, + AuditLogFilteredEvent, + AuditLogDownloadedEvent, +} from "@budibase/types" +import { publishEvent } from "../events" + +async function filtered(search: AuditLogSearchParams) { + const properties: AuditLogFilteredEvent = { + filters: search, + } + await publishEvent(Event.AUDIT_LOGS_FILTERED, properties) +} + +async function downloaded(search: AuditLogSearchParams) { + const properties: AuditLogDownloadedEvent = { + filters: search, + } + await publishEvent(Event.AUDIT_LOGS_DOWNLOADED, properties) +} + +export default { + filtered, + downloaded, +} diff --git a/packages/backend-core/src/events/publishers/auth.ts b/packages/backend-core/src/events/publishers/auth.ts index 4436045599..e275d2dbb0 100644 --- a/packages/backend-core/src/events/publishers/auth.ts +++ b/packages/backend-core/src/events/publishers/auth.ts @@ -12,19 +12,25 @@ import { } from "@budibase/types" import { identification } from ".." -async function login(source: LoginSource) { +async function login(source: LoginSource, email: string) { const identity = await identification.getCurrentIdentity() const properties: LoginEvent = { userId: identity.id, source, + audited: { + email, + }, } await publishEvent(Event.AUTH_LOGIN, properties) } -async function logout() { +async function logout(email?: string) { const identity = await identification.getCurrentIdentity() const properties: LogoutEvent = { userId: identity.id, + audited: { + email, + }, } await publishEvent(Event.AUTH_LOGOUT, properties) } diff --git a/packages/backend-core/src/events/publishers/automation.ts b/packages/backend-core/src/events/publishers/automation.ts index 6eb36ab067..419d4136bd 100644 --- a/packages/backend-core/src/events/publishers/automation.ts +++ b/packages/backend-core/src/events/publishers/automation.ts @@ -18,6 +18,9 @@ async function created(automation: Automation, timestamp?: string | number) { automationId: automation._id as string, triggerId: automation.definition?.trigger?.id, triggerType: automation.definition?.trigger?.stepId, + audited: { + name: automation.name, + }, } await publishEvent(Event.AUTOMATION_CREATED, properties, timestamp) } @@ -38,6 +41,9 @@ async function deleted(automation: Automation) { automationId: automation._id as string, triggerId: automation.definition?.trigger?.id, triggerType: automation.definition?.trigger?.stepId, + audited: { + name: automation.name, + }, } await publishEvent(Event.AUTOMATION_DELETED, properties) } @@ -71,6 +77,9 @@ async function stepCreated( triggerType: automation.definition?.trigger?.stepId, stepId: step.id!, stepType: step.stepId, + audited: { + name: automation.name, + }, } await publishEvent(Event.AUTOMATION_STEP_CREATED, properties, timestamp) } @@ -83,6 +92,9 @@ async function stepDeleted(automation: Automation, step: AutomationStep) { triggerType: automation.definition?.trigger?.stepId, stepId: step.id!, stepType: step.stepId, + audited: { + name: automation.name, + }, } await publishEvent(Event.AUTOMATION_STEP_DELETED, properties) } diff --git a/packages/backend-core/src/events/publishers/backup.ts b/packages/backend-core/src/events/publishers/backup.ts index 12263fe1ff..d7d87f09f1 100644 --- a/packages/backend-core/src/events/publishers/backup.ts +++ b/packages/backend-core/src/events/publishers/backup.ts @@ -13,6 +13,7 @@ async function appBackupRestored(backup: AppBackup) { appId: backup.appId, restoreId: backup._id!, backupCreatedAt: backup.timestamp, + name: backup.name as string, } await publishEvent(Event.APP_BACKUP_RESTORED, properties) @@ -22,13 +23,15 @@ async function appBackupTriggered( appId: string, backupId: string, type: AppBackupType, - trigger: AppBackupTrigger + trigger: AppBackupTrigger, + name: string ) { const properties: AppBackupTriggeredEvent = { appId: appId, backupId, type, trigger, + name, } await publishEvent(Event.APP_BACKUP_TRIGGERED, properties) } diff --git a/packages/backend-core/src/events/publishers/group.ts b/packages/backend-core/src/events/publishers/group.ts index d79920562b..a000b880a2 100644 --- a/packages/backend-core/src/events/publishers/group.ts +++ b/packages/backend-core/src/events/publishers/group.ts @@ -8,12 +8,16 @@ import { GroupUsersAddedEvent, GroupUsersDeletedEvent, GroupAddedOnboardingEvent, + GroupPermissionsEditedEvent, UserGroupRoles, } from "@budibase/types" async function created(group: UserGroup, timestamp?: number) { const properties: GroupCreatedEvent = { groupId: group._id as string, + audited: { + name: group.name, + }, } await publishEvent(Event.USER_GROUP_CREATED, properties, timestamp) } @@ -21,6 +25,9 @@ async function created(group: UserGroup, timestamp?: number) { async function updated(group: UserGroup) { const properties: GroupUpdatedEvent = { groupId: group._id as string, + audited: { + name: group.name, + }, } await publishEvent(Event.USER_GROUP_UPDATED, properties) } @@ -28,6 +35,9 @@ async function updated(group: UserGroup) { async function deleted(group: UserGroup) { const properties: GroupDeletedEvent = { groupId: group._id as string, + audited: { + name: group.name, + }, } await publishEvent(Event.USER_GROUP_DELETED, properties) } @@ -36,6 +46,9 @@ async function usersAdded(count: number, group: UserGroup) { const properties: GroupUsersAddedEvent = { count, groupId: group._id as string, + audited: { + name: group.name, + }, } await publishEvent(Event.USER_GROUP_USERS_ADDED, properties) } @@ -44,6 +57,9 @@ async function usersDeleted(count: number, group: UserGroup) { const properties: GroupUsersDeletedEvent = { count, groupId: group._id as string, + audited: { + name: group.name, + }, } await publishEvent(Event.USER_GROUP_USERS_REMOVED, properties) } @@ -56,9 +72,13 @@ async function createdOnboarding(groupId: string) { await publishEvent(Event.USER_GROUP_ONBOARDING, properties) } -async function permissionsEdited(roles: UserGroupRoles) { - const properties: UserGroupRoles = { - ...roles, +async function permissionsEdited(group: UserGroup) { + const properties: GroupPermissionsEditedEvent = { + permissions: group.roles!, + groupId: group._id as string, + audited: { + name: group.name, + }, } await publishEvent(Event.USER_GROUP_PERMISSIONS_EDITED, properties) } diff --git a/packages/backend-core/src/events/publishers/index.ts b/packages/backend-core/src/events/publishers/index.ts index 34e47b2990..87a34bf3f1 100644 --- a/packages/backend-core/src/events/publishers/index.ts +++ b/packages/backend-core/src/events/publishers/index.ts @@ -21,3 +21,4 @@ export { default as group } from "./group" export { default as plugin } from "./plugin" export { default as backup } from "./backup" export { default as environmentVariable } from "./environmentVariable" +export { default as auditLog } from "./auditLog" diff --git a/packages/backend-core/src/events/publishers/screen.ts b/packages/backend-core/src/events/publishers/screen.ts index 27264b5847..df486029e8 100644 --- a/packages/backend-core/src/events/publishers/screen.ts +++ b/packages/backend-core/src/events/publishers/screen.ts @@ -11,6 +11,9 @@ async function created(screen: Screen, timestamp?: string | number) { layoutId: screen.layoutId, screenId: screen._id as string, roleId: screen.routing.roleId, + audited: { + name: screen.routing?.route, + }, } await publishEvent(Event.SCREEN_CREATED, properties, timestamp) } @@ -20,6 +23,9 @@ async function deleted(screen: Screen) { layoutId: screen.layoutId, screenId: screen._id as string, roleId: screen.routing.roleId, + audited: { + name: screen.routing?.route, + }, } await publishEvent(Event.SCREEN_DELETED, properties) } diff --git a/packages/backend-core/src/events/publishers/table.ts b/packages/backend-core/src/events/publishers/table.ts index d50f4df0e1..dc3200291a 100644 --- a/packages/backend-core/src/events/publishers/table.ts +++ b/packages/backend-core/src/events/publishers/table.ts @@ -13,6 +13,9 @@ import { async function created(table: Table, timestamp?: string | number) { const properties: TableCreatedEvent = { tableId: table._id as string, + audited: { + name: table.name, + }, } await publishEvent(Event.TABLE_CREATED, properties, timestamp) } @@ -20,6 +23,9 @@ async function created(table: Table, timestamp?: string | number) { async function updated(table: Table) { const properties: TableUpdatedEvent = { tableId: table._id as string, + audited: { + name: table.name, + }, } await publishEvent(Event.TABLE_UPDATED, properties) } @@ -27,6 +33,9 @@ async function updated(table: Table) { async function deleted(table: Table) { const properties: TableDeletedEvent = { tableId: table._id as string, + audited: { + name: table.name, + }, } await publishEvent(Event.TABLE_DELETED, properties) } @@ -35,6 +44,9 @@ async function exported(table: Table, format: TableExportFormat) { const properties: TableExportedEvent = { tableId: table._id as string, format, + audited: { + name: table.name, + }, } await publishEvent(Event.TABLE_EXPORTED, properties) } @@ -42,6 +54,9 @@ async function exported(table: Table, format: TableExportFormat) { async function imported(table: Table) { const properties: TableImportedEvent = { tableId: table._id as string, + audited: { + name: table.name, + }, } await publishEvent(Event.TABLE_IMPORTED, properties) } diff --git a/packages/backend-core/src/events/publishers/user.ts b/packages/backend-core/src/events/publishers/user.ts index 1fe50149b5..8dbc494d1e 100644 --- a/packages/backend-core/src/events/publishers/user.ts +++ b/packages/backend-core/src/events/publishers/user.ts @@ -19,6 +19,9 @@ import { async function created(user: User, timestamp?: number) { const properties: UserCreatedEvent = { userId: user._id as string, + audited: { + email: user.email, + }, } await publishEvent(Event.USER_CREATED, properties, timestamp) } @@ -26,6 +29,9 @@ async function created(user: User, timestamp?: number) { async function updated(user: User) { const properties: UserUpdatedEvent = { userId: user._id as string, + audited: { + email: user.email, + }, } await publishEvent(Event.USER_UPDATED, properties) } @@ -33,6 +39,9 @@ async function updated(user: User) { async function deleted(user: User) { const properties: UserDeletedEvent = { userId: user._id as string, + audited: { + email: user.email, + }, } await publishEvent(Event.USER_DELETED, properties) } @@ -40,6 +49,9 @@ async function deleted(user: User) { export async function onboardingComplete(user: User) { const properties: UserOnboardingEvent = { userId: user._id as string, + audited: { + email: user.email, + }, } await publishEvent(Event.USER_ONBOARDING_COMPLETE, properties) } @@ -49,6 +61,9 @@ export async function onboardingComplete(user: User) { async function permissionAdminAssigned(user: User, timestamp?: number) { const properties: UserPermissionAssignedEvent = { userId: user._id as string, + audited: { + email: user.email, + }, } await publishEvent( Event.USER_PERMISSION_ADMIN_ASSIGNED, @@ -60,6 +75,9 @@ async function permissionAdminAssigned(user: User, timestamp?: number) { async function permissionAdminRemoved(user: User) { const properties: UserPermissionRemovedEvent = { userId: user._id as string, + audited: { + email: user.email, + }, } await publishEvent(Event.USER_PERMISSION_ADMIN_REMOVED, properties) } @@ -67,6 +85,9 @@ async function permissionAdminRemoved(user: User) { async function permissionBuilderAssigned(user: User, timestamp?: number) { const properties: UserPermissionAssignedEvent = { userId: user._id as string, + audited: { + email: user.email, + }, } await publishEvent( Event.USER_PERMISSION_BUILDER_ASSIGNED, @@ -78,20 +99,30 @@ async function permissionBuilderAssigned(user: User, timestamp?: number) { async function permissionBuilderRemoved(user: User) { const properties: UserPermissionRemovedEvent = { userId: user._id as string, + audited: { + email: user.email, + }, } await publishEvent(Event.USER_PERMISSION_BUILDER_REMOVED, properties) } // INVITE -async function invited() { - const properties: UserInvitedEvent = {} +async function invited(email: string) { + const properties: UserInvitedEvent = { + audited: { + email, + }, + } await publishEvent(Event.USER_INVITED, properties) } async function inviteAccepted(user: User) { const properties: UserInviteAcceptedEvent = { userId: user._id as string, + audited: { + email: user.email, + }, } await publishEvent(Event.USER_INVITED_ACCEPTED, properties) } @@ -101,6 +132,9 @@ async function inviteAccepted(user: User) { async function passwordForceReset(user: User) { const properties: UserPasswordForceResetEvent = { userId: user._id as string, + audited: { + email: user.email, + }, } await publishEvent(Event.USER_PASSWORD_FORCE_RESET, properties) } @@ -108,6 +142,9 @@ async function passwordForceReset(user: User) { async function passwordUpdated(user: User) { const properties: UserPasswordUpdatedEvent = { userId: user._id as string, + audited: { + email: user.email, + }, } await publishEvent(Event.USER_PASSWORD_UPDATED, properties) } @@ -115,6 +152,9 @@ async function passwordUpdated(user: User) { async function passwordResetRequested(user: User) { const properties: UserPasswordResetRequestedEvent = { userId: user._id as string, + audited: { + email: user.email, + }, } await publishEvent(Event.USER_PASSWORD_RESET_REQUESTED, properties) } @@ -122,6 +162,9 @@ async function passwordResetRequested(user: User) { async function passwordReset(user: User) { const properties: UserPasswordResetEvent = { userId: user._id as string, + audited: { + email: user.email, + }, } await publishEvent(Event.USER_PASSWORD_RESET, properties) } diff --git a/packages/backend-core/src/index.ts b/packages/backend-core/src/index.ts index 601b14a661..48569548e3 100644 --- a/packages/backend-core/src/index.ts +++ b/packages/backend-core/src/index.ts @@ -21,11 +21,11 @@ export * as context from "./context" export * as cache from "./cache" export * as objectStore from "./objectStore" export * as redis from "./redis" -export * as locks from "./redis/redlock" +export * as locks from "./redis/redlockImpl" export * as utils from "./utils" export * as errors from "./errors" export { default as env } from "./environment" - +export { SearchParams } from "./db" // Add context to tenancy for backwards compatibility // only do this for external usages to prevent internal // circular dependencies diff --git a/packages/backend-core/src/middleware/authenticated.ts b/packages/backend-core/src/middleware/authenticated.ts index 23fb87df8b..d7e6346b3f 100644 --- a/packages/backend-core/src/middleware/authenticated.ts +++ b/packages/backend-core/src/middleware/authenticated.ts @@ -8,7 +8,7 @@ import { getGlobalDB, doInTenant } from "../context" import { decrypt } from "../security/encryption" import * as identity from "../context/identity" import env from "../environment" -import { BBContext, EndpointMatcher } from "@budibase/types" +import { Ctx, EndpointMatcher } from "@budibase/types" const ONE_MINUTE = env.SESSION_UPDATE_PERIOD ? parseInt(env.SESSION_UPDATE_PERIOD) @@ -73,7 +73,7 @@ export default function ( } ) { const noAuthOptions = noAuthPatterns ? buildMatcherRegex(noAuthPatterns) : [] - return async (ctx: BBContext | any, next: any) => { + return async (ctx: Ctx | any, next: any) => { let publicEndpoint = false const version = ctx.request.headers[Header.API_VER] // the path is not authenticated @@ -149,7 +149,7 @@ export default function ( finalise(ctx, { authenticated, user, internal, version, publicEndpoint }) if (user && user.email) { - return identity.doInUserContext(user, next) + return identity.doInUserContext(user, ctx, next) } else { return next() } diff --git a/packages/backend-core/src/middleware/index.ts b/packages/backend-core/src/middleware/index.ts index de609f9a3e..addeac6a1a 100644 --- a/packages/backend-core/src/middleware/index.ts +++ b/packages/backend-core/src/middleware/index.ts @@ -17,4 +17,5 @@ export { default as builderOrAdmin } from "./builderOrAdmin" export { default as builderOnly } from "./builderOnly" export { default as logging } from "./logging" export { default as errorHandling } from "./errorHandling" +export { default as querystringToBody } from "./querystringToBody" export * as joiValidator from "./joi-validator" diff --git a/packages/backend-core/src/middleware/querystringToBody.ts b/packages/backend-core/src/middleware/querystringToBody.ts new file mode 100644 index 0000000000..b6f109231a --- /dev/null +++ b/packages/backend-core/src/middleware/querystringToBody.ts @@ -0,0 +1,28 @@ +import { Ctx } from "@budibase/types" + +/** + * Expects a standard "query" query string property which is the JSON body + * of the request, which has to be sent via query string due to the requirement + * of making an endpoint a GET request e.g. downloading a file stream. + */ +export default function (ctx: Ctx, next: any) { + const queryString = ctx.request.query?.query as string | undefined + if (ctx.request.method.toLowerCase() !== "get") { + ctx.throw( + 500, + "Query to download middleware can only be used for get requests." + ) + } + if (!queryString) { + return next() + } + const decoded = decodeURIComponent(queryString) + let json + try { + json = JSON.parse(decoded) + } catch (err) { + return next() + } + ctx.request.body = json + return next() +} diff --git a/packages/backend-core/src/platform/tenants.ts b/packages/backend-core/src/platform/tenants.ts index b9f946a735..b6bc3410d8 100644 --- a/packages/backend-core/src/platform/tenants.ts +++ b/packages/backend-core/src/platform/tenants.ts @@ -1,7 +1,7 @@ import { StaticDatabases } from "../constants" import { getPlatformDB } from "./platformDb" import { LockName, LockOptions, LockType, Tenants } from "@budibase/types" -import * as locks from "../redis/redlock" +import * as locks from "../redis/redlockImpl" const TENANT_DOC = StaticDatabases.PLATFORM_INFO.docs.tenants diff --git a/packages/backend-core/src/queue/constants.ts b/packages/backend-core/src/queue/constants.ts index e8323dacb8..9261ed1176 100644 --- a/packages/backend-core/src/queue/constants.ts +++ b/packages/backend-core/src/queue/constants.ts @@ -1,4 +1,5 @@ export enum JobQueue { AUTOMATION = "automationQueue", APP_BACKUP = "appBackupQueue", + AUDIT_LOG = "auditLogQueue", } diff --git a/packages/backend-core/src/queue/queue.ts b/packages/backend-core/src/queue/queue.ts index 8e1fc1fbf3..c57ebafb1f 100644 --- a/packages/backend-core/src/queue/queue.ts +++ b/packages/backend-core/src/queue/queue.ts @@ -40,8 +40,10 @@ export function createQueue( } export async function shutdown() { - if (QUEUES.length) { + if (cleanupInterval) { clearInterval(cleanupInterval) + } + if (QUEUES.length) { for (let queue of QUEUES) { await queue.close() } diff --git a/packages/backend-core/src/redis/index.ts b/packages/backend-core/src/redis/index.ts index 5bf2c65c39..6585d6e4fa 100644 --- a/packages/backend-core/src/redis/index.ts +++ b/packages/backend-core/src/redis/index.ts @@ -3,4 +3,4 @@ export { default as Client } from "./redis" export * as utils from "./utils" export * as clients from "./init" -export * as locks from "./redlock" +export * as locks from "./redlockImpl" diff --git a/packages/backend-core/src/redis/redlock.ts b/packages/backend-core/src/redis/redlockImpl.ts similarity index 100% rename from packages/backend-core/src/redis/redlock.ts rename to packages/backend-core/src/redis/redlockImpl.ts diff --git a/packages/backend-core/src/users.ts b/packages/backend-core/src/users.ts index ef76af390d..9ef2c5c31f 100644 --- a/packages/backend-core/src/users.ts +++ b/packages/backend-core/src/users.ts @@ -10,14 +10,38 @@ import { BulkDocsResponse, User } from "@budibase/types" import { getGlobalDB } from "./context" import * as context from "./context" -export const bulkGetGlobalUsersById = async (userIds: string[]) => { +type GetOpts = { cleanup?: boolean } + +function removeUserPassword(users: User | User[]) { + if (Array.isArray(users)) { + return users.map(user => { + if (user) { + delete user.password + return user + } + }) + } else if (users) { + delete users.password + return users + } + return users +} + +export const bulkGetGlobalUsersById = async ( + userIds: string[], + opts?: GetOpts +) => { const db = getGlobalDB() - return ( + let users = ( await db.allDocs({ keys: userIds, include_docs: true, }) ).rows.map(row => row.doc) as User[] + if (opts?.cleanup) { + users = removeUserPassword(users) as User[] + } + return users } export const bulkUpdateGlobalUsers = async (users: User[]) => { @@ -25,18 +49,22 @@ export const bulkUpdateGlobalUsers = async (users: User[]) => { return (await db.bulkDocs(users)) as BulkDocsResponse } -export async function getById(id: string): Promise { +export async function getById(id: string, opts?: GetOpts): Promise { const db = context.getGlobalDB() - return db.get(id) + let user = await db.get(id) + if (opts?.cleanup) { + user = removeUserPassword(user) + } + return user } /** * Given an email address this will use a view to search through * all the users to find one with this email address. - * @param {string} email the email to lookup the user by. */ export const getGlobalUserByEmail = async ( - email: String + email: String, + opts?: GetOpts ): Promise => { if (email == null) { throw "Must supply an email address to view" @@ -52,10 +80,19 @@ export const getGlobalUserByEmail = async ( throw new Error(`Multiple users found with email address: ${email}`) } - return response + let user = response as User + if (opts?.cleanup) { + user = removeUserPassword(user) as User + } + + return user } -export const searchGlobalUsersByApp = async (appId: any, opts: any) => { +export const searchGlobalUsersByApp = async ( + appId: any, + opts: any, + getOpts?: GetOpts +) => { if (typeof appId !== "string") { throw new Error("Must provide a string based app ID") } @@ -67,7 +104,11 @@ export const searchGlobalUsersByApp = async (appId: any, opts: any) => { if (!response) { response = [] } - return Array.isArray(response) ? response : [response] + let users: User[] = Array.isArray(response) ? response : [response] + if (getOpts?.cleanup) { + users = removeUserPassword(users) as User[] + } + return users } export const getGlobalUserByAppPage = (appId: string, user: User) => { @@ -80,7 +121,11 @@ export const getGlobalUserByAppPage = (appId: string, user: User) => { /** * Performs a starts with search on the global email view. */ -export const searchGlobalUsersByEmail = async (email: string, opts: any) => { +export const searchGlobalUsersByEmail = async ( + email: string, + opts: any, + getOpts?: GetOpts +) => { if (typeof email !== "string") { throw new Error("Must provide a string to search by") } @@ -95,5 +140,9 @@ export const searchGlobalUsersByEmail = async (email: string, opts: any) => { if (!response) { response = [] } - return Array.isArray(response) ? response : [response] + let users: User[] = Array.isArray(response) ? response : [response] + if (getOpts?.cleanup) { + users = removeUserPassword(users) as User[] + } + return users } diff --git a/packages/backend-core/src/utils/utils.ts b/packages/backend-core/src/utils/utils.ts index 3731e134ad..3efd40ca80 100644 --- a/packages/backend-core/src/utils/utils.ts +++ b/packages/backend-core/src/utils/utils.ts @@ -10,7 +10,13 @@ import { import env from "../environment" import * as tenancy from "../tenancy" import * as context from "../context" -import { App, Ctx, TenantResolutionStrategy } from "@budibase/types" +import { + App, + AuditedEventFriendlyName, + Ctx, + Event, + TenantResolutionStrategy, +} from "@budibase/types" import { SetOption } from "cookies" const jwt = require("jsonwebtoken") @@ -217,3 +223,7 @@ export async function getBuildersCount() { export function timeout(timeMs: number) { return new Promise(resolve => setTimeout(resolve, timeMs)) } + +export function isAudited(event: Event) { + return !!AuditedEventFriendlyName[event] +} diff --git a/packages/backend-core/tests/utilities/mocks/licenses.ts b/packages/backend-core/tests/utilities/mocks/licenses.ts index 109fcdf9c6..2ca41616e4 100644 --- a/packages/backend-core/tests/utilities/mocks/licenses.ts +++ b/packages/backend-core/tests/utilities/mocks/licenses.ts @@ -82,6 +82,10 @@ export const useEnvironmentVariables = () => { return useFeature(Feature.ENVIRONMENT_VARIABLES) } +export const useAuditLogs = () => { + return useFeature(Feature.AUDIT_LOGS) +} + // QUOTAS export const setAutomationLogsQuota = (value: number) => { diff --git a/packages/backend-core/tests/utilities/structures/generator.ts b/packages/backend-core/tests/utilities/structures/generator.ts new file mode 100644 index 0000000000..51567b152e --- /dev/null +++ b/packages/backend-core/tests/utilities/structures/generator.ts @@ -0,0 +1,2 @@ +import Chance from "chance" +export const generator = new Chance() diff --git a/packages/backend-core/tests/utilities/structures/index.ts b/packages/backend-core/tests/utilities/structures/index.ts index d0073ba851..ca77f476d0 100644 --- a/packages/backend-core/tests/utilities/structures/index.ts +++ b/packages/backend-core/tests/utilities/structures/index.ts @@ -1,8 +1,4 @@ export * from "./common" - -import Chance from "chance" -export const generator = new Chance() - export * as accounts from "./accounts" export * as apps from "./apps" export * as db from "./db" @@ -12,3 +8,4 @@ export * as plugins from "./plugins" export * as sso from "./sso" export * as tenant from "./tenants" export * as users from "./users" +export { generator } from "./generator" diff --git a/packages/backend-core/tests/utilities/structures/shared.ts b/packages/backend-core/tests/utilities/structures/shared.ts new file mode 100644 index 0000000000..de0e19486c --- /dev/null +++ b/packages/backend-core/tests/utilities/structures/shared.ts @@ -0,0 +1,19 @@ +import { User } from "@budibase/types" +import { generator } from "./generator" +import { uuid } from "./common" + +export const newEmail = () => { + return `${uuid()}@test.com` +} + +export const user = (userProps?: any): User => { + return { + email: newEmail(), + password: "test", + roles: { app_test: "admin" }, + firstName: generator.first(), + lastName: generator.last(), + pictureUrl: "http://test.com", + ...userProps, + } +} diff --git a/packages/backend-core/tests/utilities/structures/sso.ts b/packages/backend-core/tests/utilities/structures/sso.ts index a63d26cbb3..a5957c9233 100644 --- a/packages/backend-core/tests/utilities/structures/sso.ts +++ b/packages/backend-core/tests/utilities/structures/sso.ts @@ -8,8 +8,36 @@ import { SSOProviderType, User, } from "@budibase/types" -import { uuid, generator, users, email } from "./index" +import { generator } from "./generator" +import { uuid, email } from "./common" +import * as shared from "./shared" import _ from "lodash" +import { user } from "./shared" + +export function authDetails(userDoc?: User): SSOAuthDetails { + if (!userDoc) { + userDoc = user() + } + + const userId = userDoc._id || uuid() + const provider = generator.string() + + const profile = ssoProfile(userDoc) + profile.provider = provider + profile.id = userId + + return { + email: userDoc.email, + oauth2: { + refreshToken: generator.string(), + accessToken: generator.string(), + }, + profile, + provider, + providerType: providerType(), + userId, + } +} export function providerType(): SSOProviderType { return _.sample(Object.values(SSOProviderType)) as SSOProviderType @@ -17,7 +45,7 @@ export function providerType(): SSOProviderType { export function ssoProfile(user?: User): SSOProfile { if (!user) { - user = users.user() + user = shared.user() } return { id: user._id!, @@ -33,31 +61,6 @@ export function ssoProfile(user?: User): SSOProfile { } } -export function authDetails(user?: User): SSOAuthDetails { - if (!user) { - user = users.user() - } - - const userId = user._id || uuid() - const provider = generator.string() - - const profile = ssoProfile(user) - profile.provider = provider - profile.id = userId - - return { - email: user.email, - oauth2: { - refreshToken: generator.string(), - accessToken: generator.string(), - }, - profile, - provider, - providerType: providerType(), - userId, - } -} - // OIDC export function oidcConfig(): OIDCInnerConfig { diff --git a/packages/backend-core/tests/utilities/structures/users.ts b/packages/backend-core/tests/utilities/structures/users.ts index 332c27ca12..7a6b4f0d80 100644 --- a/packages/backend-core/tests/utilities/structures/users.ts +++ b/packages/backend-core/tests/utilities/structures/users.ts @@ -1,29 +1,13 @@ -import { generator } from "../" import { AdminUser, BuilderUser, SSOAuthDetails, SSOUser, - User, } from "@budibase/types" -import { v4 as uuid } from "uuid" -import * as sso from "./sso" +import { user } from "./shared" +import { authDetails } from "./sso" -export const newEmail = () => { - return `${uuid()}@test.com` -} - -export const user = (userProps?: any): User => { - return { - email: newEmail(), - password: "test", - roles: { app_test: "admin" }, - firstName: generator.first(), - lastName: generator.last(), - pictureUrl: "http://test.com", - ...userProps, - } -} +export { user, newEmail } from "./shared" export const adminUser = (userProps?: any): AdminUser => { return { @@ -53,7 +37,7 @@ export function ssoUser( delete base.password if (!opts.details) { - opts.details = sso.authDetails(base) + opts.details = authDetails(base) } return { diff --git a/packages/backend-core/tests/utilities/testContainerUtils.ts b/packages/backend-core/tests/utilities/testContainerUtils.ts index 11c5fca806..f6c702f7ef 100644 --- a/packages/backend-core/tests/utilities/testContainerUtils.ts +++ b/packages/backend-core/tests/utilities/testContainerUtils.ts @@ -1,3 +1,31 @@ +import { execSync } from "child_process" + +let dockerPsResult: string | undefined + +function formatDockerPsResult(serverName: string, port: number) { + const lines = dockerPsResult?.split("\n") + let first = true + if (!lines) { + return null + } + for (let line of lines) { + if (first) { + first = false + continue + } + let toLookFor = serverName.split("-service")[0] + if (!line.includes(toLookFor)) { + continue + } + const regex = new RegExp(`0.0.0.0:([0-9]*)->${port}`, "g") + const found = line.match(regex) + if (found) { + return found[0].split(":")[1].split("->")[0] + } + } + return null +} + function getTestContainerSettings( serverName: string, key: string @@ -14,10 +42,22 @@ function getTestContainerSettings( } function getContainerInfo(containerName: string, port: number) { - const assignedPort = getTestContainerSettings( + let assignedPort = getTestContainerSettings( containerName.toUpperCase(), `PORT_${port}` ) + if (!dockerPsResult) { + try { + const outputBuffer = execSync("docker ps") + dockerPsResult = outputBuffer.toString("utf8") + } catch (err) { + //no-op + } + } + const possiblePort = formatDockerPsResult(containerName, port) + if (possiblePort) { + assignedPort = possiblePort + } const host = getTestContainerSettings(containerName.toUpperCase(), "IP") return { port: assignedPort, @@ -39,12 +79,15 @@ function getRedisConfig() { } export function setupEnv(...envs: any[]) { + const couch = getCouchConfig(), + minio = getCouchConfig(), + redis = getRedisConfig() const configs = [ - { key: "COUCH_DB_PORT", value: getCouchConfig().port }, - { key: "COUCH_DB_URL", value: getCouchConfig().url }, - { key: "MINIO_PORT", value: getMinioConfig().port }, - { key: "MINIO_URL", value: getMinioConfig().url }, - { key: "REDIS_URL", value: getRedisConfig().url }, + { key: "COUCH_DB_PORT", value: couch.port }, + { key: "COUCH_DB_URL", value: couch.url }, + { key: "MINIO_PORT", value: minio.port }, + { key: "MINIO_URL", value: minio.url }, + { key: "REDIS_URL", value: redis.url }, ] for (const config of configs.filter(x => !!x.value)) { diff --git a/packages/bbui/src/Form/Core/Multiselect.svelte b/packages/bbui/src/Form/Core/Multiselect.svelte index d6c4dc23ac..8860fcbf0b 100644 --- a/packages/bbui/src/Form/Core/Multiselect.svelte +++ b/packages/bbui/src/Form/Core/Multiselect.svelte @@ -14,6 +14,8 @@ export let autocomplete = false export let sort = false export let autoWidth = false + export let fetchTerm = null + export let customPopoverHeight const dispatch = createEventDispatcher() @@ -83,10 +85,12 @@ {options} isPlaceholder={!value?.length} {autocomplete} + bind:fetchTerm {isOptionSelected} {getOptionLabel} {getOptionValue} onSelectOption={toggleOption} {sort} {autoWidth} + {customPopoverHeight} /> diff --git a/packages/bbui/src/Form/Core/Picker.svelte b/packages/bbui/src/Form/Core/Picker.svelte index 32cfcf3310..5cef0f9213 100644 --- a/packages/bbui/src/Form/Core/Picker.svelte +++ b/packages/bbui/src/Form/Core/Picker.svelte @@ -31,7 +31,8 @@ export let autoWidth = false export let autocomplete = false export let sort = false - + export let fetchTerm = null + export let customPopoverHeight const dispatch = createEventDispatcher() let searchTerm = null @@ -71,7 +72,7 @@ } const getFilteredOptions = (options, term, getLabel) => { - if (autocomplete && term) { + if (autocomplete && term && !fetchTerm) { const lowerCaseTerm = term.toLowerCase() return options.filter(option => { return `${getLabel(option)}`.toLowerCase().includes(lowerCaseTerm) @@ -136,6 +137,7 @@ on:close={() => (open = false)} useAnchorWidth={!autoWidth} maxWidth={autoWidth ? 400 : null} + customHeight={customPopoverHeight} >
{#if autocomplete} (searchTerm = event.detail)} + value={fetchTerm ? fetchTerm : searchTerm} + on:change={event => + fetchTerm ? (fetchTerm = event.detail) : (searchTerm = event.detail)} {disabled} placeholder="Search" /> @@ -247,7 +250,7 @@ } .popover-content.auto-width .spectrum-Menu-itemLabel { white-space: nowrap; - overflow: hidden; + overflow: none; text-overflow: ellipsis; } .popover-content:not(.auto-width) .spectrum-Menu-itemLabel { diff --git a/packages/bbui/src/Form/Multiselect.svelte b/packages/bbui/src/Form/Multiselect.svelte index 7bcf22aa06..2237cd1dce 100644 --- a/packages/bbui/src/Form/Multiselect.svelte +++ b/packages/bbui/src/Form/Multiselect.svelte @@ -15,6 +15,10 @@ export let getOptionValue = option => option export let sort = false export let autoWidth = false + export let autocomplete = false + export let fetchTerm = null + export let customPopoverHeight + const dispatch = createEventDispatcher() const onChange = e => { value = e.detail @@ -34,6 +38,9 @@ {getOptionLabel} {getOptionValue} {autoWidth} + {autocomplete} + {customPopoverHeight} + bind:fetchTerm on:change={onChange} on:click /> diff --git a/packages/bbui/src/Form/Select.svelte b/packages/bbui/src/Form/Select.svelte index 69126e648d..76fe613c92 100644 --- a/packages/bbui/src/Form/Select.svelte +++ b/packages/bbui/src/Form/Select.svelte @@ -20,6 +20,8 @@ export let autoWidth = false export let sort = false export let tooltip = "" + export let autocomplete = false + export let customPopoverHeight const dispatch = createEventDispatcher() const onChange = e => { @@ -51,6 +53,8 @@ {getOptionIcon} {getOptionColour} {isOptionEnabled} + {autocomplete} + {customPopoverHeight} on:change={onChange} on:click /> diff --git a/packages/bbui/src/Popover/Popover.svelte b/packages/bbui/src/Popover/Popover.svelte index 8f6ef06591..081e3a34df 100644 --- a/packages/bbui/src/Popover/Popover.svelte +++ b/packages/bbui/src/Popover/Popover.svelte @@ -18,6 +18,7 @@ export let useAnchorWidth = false export let dismissible = true export let offset = 5 + export let customHeight $: target = portalTarget || getContext(Context.PopoverRoot) || ".spectrum" @@ -74,6 +75,7 @@ on:keydown={handleEscape} class="spectrum-Popover is-open" role="presentation" + style="height: {customHeight}" transition:fly|local={{ y: -20, duration: 200 }} > diff --git a/packages/builder/src/pages/builder/portal/_components/LockedFeature.svelte b/packages/builder/src/pages/builder/portal/_components/LockedFeature.svelte new file mode 100644 index 0000000000..e6f4075e2e --- /dev/null +++ b/packages/builder/src/pages/builder/portal/_components/LockedFeature.svelte @@ -0,0 +1,76 @@ + + + + +
+ {title} + {#if !enabled} + + {planType} + + {/if} +
+ {description} +
+ + + {#if enabled} + + {:else} +
+ + + +
+ {/if} +
+ + diff --git a/packages/builder/src/pages/builder/portal/account/_layout.svelte b/packages/builder/src/pages/builder/portal/account/_layout.svelte index 3083b574a8..892e853aad 100644 --- a/packages/builder/src/pages/builder/portal/account/_layout.svelte +++ b/packages/builder/src/pages/builder/portal/account/_layout.svelte @@ -4,12 +4,13 @@ import { Content, SideNav, SideNavItem } from "components/portal/page" import { menu } from "stores/portal" + $: wide = $isActive("./auditLogs") $: pages = $menu.find(x => x.title === "Account")?.subPages || [] $: !pages.length && $goto("../") - +
{#each pages as { title, href }} diff --git a/packages/builder/src/pages/builder/portal/account/auditLogs/_components/AppColumnRenderer.svelte b/packages/builder/src/pages/builder/portal/account/auditLogs/_components/AppColumnRenderer.svelte new file mode 100644 index 0000000000..08260ecc5c --- /dev/null +++ b/packages/builder/src/pages/builder/portal/account/auditLogs/_components/AppColumnRenderer.svelte @@ -0,0 +1,5 @@ + + +
{value?.name || ""}
diff --git a/packages/builder/src/pages/builder/portal/account/auditLogs/_components/TimeRenderer.svelte b/packages/builder/src/pages/builder/portal/account/auditLogs/_components/TimeRenderer.svelte new file mode 100644 index 0000000000..b6c0262b47 --- /dev/null +++ b/packages/builder/src/pages/builder/portal/account/auditLogs/_components/TimeRenderer.svelte @@ -0,0 +1,11 @@ + + +
+ {dayjs(row.timestamp).fromNow()} +
diff --git a/packages/builder/src/pages/builder/portal/account/auditLogs/_components/UserRenderer.svelte b/packages/builder/src/pages/builder/portal/account/auditLogs/_components/UserRenderer.svelte new file mode 100644 index 0000000000..16e2a7be04 --- /dev/null +++ b/packages/builder/src/pages/builder/portal/account/auditLogs/_components/UserRenderer.svelte @@ -0,0 +1,45 @@ + + +
(showTooltip = true)} + on:focus={() => (showTooltip = true)} + on:mouseleave={() => (showTooltip = false)} +> + +
+{#if showTooltip} +
+ +
+{/if} + + diff --git a/packages/builder/src/pages/builder/portal/account/auditLogs/_components/ViewDetailsRenderer.svelte b/packages/builder/src/pages/builder/portal/account/auditLogs/_components/ViewDetailsRenderer.svelte new file mode 100644 index 0000000000..b62ee48cb8 --- /dev/null +++ b/packages/builder/src/pages/builder/portal/account/auditLogs/_components/ViewDetailsRenderer.svelte @@ -0,0 +1,13 @@ + + +Details diff --git a/packages/builder/src/pages/builder/portal/account/auditLogs/index.svelte b/packages/builder/src/pages/builder/portal/account/auditLogs/index.svelte new file mode 100644 index 0000000000..739877db26 --- /dev/null +++ b/packages/builder/src/pages/builder/portal/account/auditLogs/index.svelte @@ -0,0 +1,501 @@ + + + + { + $licensing.goToUpgradePage() + }} +> +
+
+ user._id} + getOptionLabel={user => user.email} + options={sortedUsers} + /> +
+
+ app.instance._id} + getOptionLabel={app => app.name} + options={sortedApps} + bind:value={selectedApps} + /> +
+
+ event.id} + getOptionLabel={event => event.label} + options={sortedEvents} + placeholder="All events" + label="Events" + bind:value={selectedEvents} + /> +
+ +
+ { + if (e.detail[0]?.length === 1) { + startDate = e.detail[0][0].toISOString() + endDate = "" + } else if (e.detail[0]?.length > 1) { + startDate = e.detail[0][0].toISOString() + endDate = e.detail[0][1].toISOString() + } else { + startDate = "" + endDate = "" + } + }} + /> +
+
+ debounce(e.detail)} /> +
+ +
+ downloadLogs()} /> +
+
+ + viewDetails(detail)} + {customRenderers} + data={$auditLogs.logs.data} + allowEditColumns={false} + allowEditRows={false} + allowSelectRows={false} + {schema} + /> + + + + +{#if selectedLog} +
{ + sidePanelVisible = false + }} + > +
+ Audit Log +
+ { + wideSidePanel = !wideSidePanel + }} + /> + { + sidePanelVisible = false + }} + /> +
+
+ + +
+
copyToClipboard(JSON.stringify(selectedLog.metadata))} + class="copy-icon" + > + +
+ +
+
+{/if} + + diff --git a/packages/builder/src/pages/builder/portal/overview/[appId]/backups/index.svelte b/packages/builder/src/pages/builder/portal/overview/[appId]/backups/index.svelte index f2592ec805..4ff9ea386a 100644 --- a/packages/builder/src/pages/builder/portal/overview/[appId]/backups/index.svelte +++ b/packages/builder/src/pages/builder/portal/overview/[appId]/backups/index.svelte @@ -185,7 +185,7 @@ {#if !$licensing.backupsEnabled} - {#if !$auth.accountPortalAccess && !$licensing.groupsEnabled && $admin.cloud} + {#if !$auth.accountPortalAccess && $admin.cloud} Contact your account holder to upgrade your plan. {/if}
diff --git a/packages/builder/src/pages/builder/portal/settings/environment/index.svelte b/packages/builder/src/pages/builder/portal/settings/environment/index.svelte index 3c170235e9..cff578febd 100644 --- a/packages/builder/src/pages/builder/portal/settings/environment/index.svelte +++ b/packages/builder/src/pages/builder/portal/settings/environment/index.svelte @@ -1,21 +1,17 @@ - - -
- Environment Variables - {#if !$licensing.environmentVariablesEnabled} - - Business plan - - {/if} -
- Add and manage environment variables for development and production -
- - - {#if $licensing.environmentVariablesEnabled} - {#if noEncryptionKey} - - {/if} -
- -
- - -
- - {:else} -
- - - -
+ { + await environment.upgradePanelOpened() + $licensing.goToUpgradePage() + }} +> + {#if noEncryptionKey} + {/if} - +
+ +
+ +
+ + diff --git a/packages/builder/src/stores/portal/auditLogs.js b/packages/builder/src/stores/portal/auditLogs.js new file mode 100644 index 0000000000..9abf8ec11b --- /dev/null +++ b/packages/builder/src/stores/portal/auditLogs.js @@ -0,0 +1,43 @@ +import { writable, get } from "svelte/store" +import { API } from "api" +import { licensing } from "stores/portal" + +export function createAuditLogsStore() { + const { subscribe, update } = writable({ + events: {}, + logs: {}, + }) + + async function search(opts = {}) { + if (get(licensing).auditLogsEnabled) { + const paged = await API.searchAuditLogs(opts) + + update(state => { + return { ...state, logs: { ...paged, opts } } + }) + + return paged + } + } + + async function getEventDefinitions() { + const events = await API.getEventDefinitions() + + update(state => { + return { ...state, ...events } + }) + } + + function getDownloadUrl(opts = {}) { + return API.getDownloadUrl(opts) + } + + return { + subscribe, + search, + getEventDefinitions, + getDownloadUrl, + } +} + +export const auditLogs = createAuditLogsStore() diff --git a/packages/builder/src/stores/portal/index.js b/packages/builder/src/stores/portal/index.js index 8d704a5e15..e8f7853165 100644 --- a/packages/builder/src/stores/portal/index.js +++ b/packages/builder/src/stores/portal/index.js @@ -13,3 +13,4 @@ export { backups } from "./backups" export { overview } from "./overview" export { environment } from "./environment" export { menu } from "./menu" +export { auditLogs } from "./auditLogs" diff --git a/packages/builder/src/stores/portal/licensing.js b/packages/builder/src/stores/portal/licensing.js index 42c150c893..df63e9455b 100644 --- a/packages/builder/src/stores/portal/licensing.js +++ b/packages/builder/src/stores/portal/licensing.js @@ -67,6 +67,9 @@ export const createLicensingStore = () => { Constants.Features.ENFORCEABLE_SSO ) + const auditLogsEnabled = license.features.includes( + Constants.Features.AUDIT_LOGS + ) store.update(state => { return { ...state, @@ -75,6 +78,7 @@ export const createLicensingStore = () => { groupsEnabled, backupsEnabled, environmentVariablesEnabled, + auditLogsEnabled, enforceableSSO, } }) diff --git a/packages/builder/src/stores/portal/menu.js b/packages/builder/src/stores/portal/menu.js index 8eea36c08c..56956fc330 100644 --- a/packages/builder/src/stores/portal/menu.js +++ b/packages/builder/src/stores/portal/menu.js @@ -75,6 +75,10 @@ export const menu = derived([admin, auth], ([$admin, $auth]) => { title: "Usage", href: "/builder/portal/account/usage", }, + { + title: "Audit Logs", + href: "/builder/portal/account/auditLogs", + }, ] if ($admin.cloud && $auth?.user?.accountPortalAccess) { accountSubPages.push({ @@ -87,6 +91,7 @@ export const menu = derived([admin, auth], ([$admin, $auth]) => { href: "/builder/portal/account/upgrade", }) } + // add license check here if ( $auth?.user?.accountPortalAccess && $auth.user.account.stripeCustomerId diff --git a/packages/frontend-core/src/api/auditLogs.js b/packages/frontend-core/src/api/auditLogs.js new file mode 100644 index 0000000000..c4230df6d9 --- /dev/null +++ b/packages/frontend-core/src/api/auditLogs.js @@ -0,0 +1,63 @@ +const buildOpts = ({ + bookmark, + userIds, + appIds, + startDate, + endDate, + fullSearch, + events, +}) => { + const opts = {} + + if (bookmark) { + opts.bookmark = bookmark + } + + if (startDate && endDate) { + opts.startDate = startDate + opts.endDate = endDate + } else if (startDate && !endDate) { + opts.startDate = startDate + } + + if (fullSearch) { + opts.fullSearch = fullSearch + } + + if (events.length) { + opts.events = events + } + + if (userIds.length) { + opts.userIds = userIds + } + + if (appIds.length) { + opts.appIds = appIds + } + + return opts +} + +export const buildAuditLogsEndpoints = API => ({ + /** + * Gets a list of users in the current tenant. + */ + searchAuditLogs: async opts => { + return await API.post({ + url: `/api/global/auditlogs/search`, + body: buildOpts(opts), + }) + }, + + getEventDefinitions: async () => { + return await API.get({ + url: `/api/global/auditlogs/definitions`, + }) + }, + + getDownloadUrl: opts => { + const query = encodeURIComponent(JSON.stringify(opts)) + return `/api/global/auditlogs/download?query=${query}` + }, +}) diff --git a/packages/frontend-core/src/api/index.js b/packages/frontend-core/src/api/index.js index e2935b416b..f8eee45cb8 100644 --- a/packages/frontend-core/src/api/index.js +++ b/packages/frontend-core/src/api/index.js @@ -28,6 +28,8 @@ import { buildPluginEndpoints } from "./plugins" import { buildBackupsEndpoints } from "./backups" import { buildEnvironmentVariableEndpoints } from "./environmentVariables" import { buildEventEndpoints } from "./events" +import { buildAuditLogsEndpoints } from "./auditLogs" + const defaultAPIClientConfig = { /** * Certain definitions can't change at runtime for client apps, such as the @@ -250,5 +252,6 @@ export const createAPIClient = config => { ...buildBackupsEndpoints(API), ...buildEnvironmentVariableEndpoints(API), ...buildEventEndpoints(API), + ...buildAuditLogsEndpoints(API), } } diff --git a/packages/frontend-core/src/constants.js b/packages/frontend-core/src/constants.js index ec0b447159..09b7a00aec 100644 --- a/packages/frontend-core/src/constants.js +++ b/packages/frontend-core/src/constants.js @@ -115,6 +115,7 @@ export const Features = { USER_GROUPS: "userGroups", BACKUPS: "appBackups", ENVIRONMENT_VARIABLES: "environmentVariables", + AUDIT_LOGS: "auditLogs", ENFORCEABLE_SSO: "enforceableSSO", } diff --git a/packages/server/jest.config.ts b/packages/server/jest.config.ts index 331912aa19..f247e90bd1 100644 --- a/packages/server/jest.config.ts +++ b/packages/server/jest.config.ts @@ -39,6 +39,7 @@ const config: Config.InitialOptions = { ], collectCoverageFrom: [ "src/**/*.{js,ts}", + "../backend-core/src/**/*.{js,ts}", // The use of coverage with couchdb view functions breaks tests "!src/db/views/staticViews.*", ], diff --git a/packages/server/package.json b/packages/server/package.json index 5c970de929..84337f0113 100644 --- a/packages/server/package.json +++ b/packages/server/package.json @@ -87,6 +87,7 @@ "koa-send": "5.0.0", "koa-session": "5.12.0", "koa-static": "5.0.0", + "koa-useragent": "^4.1.0", "koa2-ratelimit": "1.1.1", "lodash": "4.17.21", "memorystream": "0.3.1", diff --git a/packages/server/src/api/controllers/row/ExternalRequest.ts b/packages/server/src/api/controllers/row/ExternalRequest.ts index c3b7636636..bdf8d485f2 100644 --- a/packages/server/src/api/controllers/row/ExternalRequest.ts +++ b/packages/server/src/api/controllers/row/ExternalRequest.ts @@ -24,8 +24,7 @@ import { breakExternalTableId, isSQL } from "../../../integrations/utils" import { processObjectSync } from "@budibase/string-templates" import { cloneDeep } from "lodash/fp" import { processFormulas, processDates } from "../../../utilities/rowProcessor" -import { context } from "@budibase/backend-core" -import { removeKeyNumbering } from "./utils" +import { db as dbCore } from "@budibase/backend-core" import sdk from "../../../sdk" export interface ManyRelationship { @@ -61,7 +60,7 @@ function buildFilters( let prefix = 1 for (let operator of Object.values(filters)) { for (let field of Object.keys(operator || {})) { - if (removeKeyNumbering(field) === "_id") { + if (dbCore.removeKeyNumbering(field) === "_id") { if (primary) { const parts = breakRowIdField(operator[field]) for (let field of primary) { diff --git a/packages/server/src/api/controllers/row/internalSearch.ts b/packages/server/src/api/controllers/row/internalSearch.ts index 7068aabc5a..9dc12342d6 100644 --- a/packages/server/src/api/controllers/row/internalSearch.ts +++ b/packages/server/src/api/controllers/row/internalSearch.ts @@ -1,531 +1,18 @@ -import { SearchIndexes } from "../../../db/utils" -import { removeKeyNumbering } from "./utils" -import fetch from "node-fetch" -import { db as dbCore, context } from "@budibase/backend-core" -import { SearchFilters, Row } from "@budibase/types" +import { db as dbCore, context, SearchParams } from "@budibase/backend-core" +import { SearchFilters, Row, SearchIndex } from "@budibase/types" -type SearchParams = { - tableId: string - sort?: string - sortOrder?: string - sortType?: string - limit?: number - bookmark?: string - version?: string - rows?: Row[] -} - -/** - * Class to build lucene query URLs. - * Optionally takes a base lucene query object. - */ -export class QueryBuilder { - query: SearchFilters - limit: number - sort?: string - bookmark?: string - sortOrder: string - sortType: string - includeDocs: boolean - version?: string - - constructor(base?: SearchFilters) { - this.query = { - allOr: false, - string: {}, - fuzzy: {}, - range: {}, - equal: {}, - notEqual: {}, - empty: {}, - notEmpty: {}, - oneOf: {}, - contains: {}, - notContains: {}, - containsAny: {}, - ...base, - } - this.limit = 50 - this.sortOrder = "ascending" - this.sortType = "string" - this.includeDocs = true - } - - setVersion(version?: string) { - if (version != null) { - this.version = version - } - return this - } - - setTable(tableId: string) { - this.query.equal!.tableId = tableId - return this - } - - setLimit(limit?: number) { - if (limit != null) { - this.limit = limit - } - return this - } - - setSort(sort?: string) { - if (sort != null) { - this.sort = sort - } - return this - } - - setSortOrder(sortOrder?: string) { - if (sortOrder != null) { - this.sortOrder = sortOrder - } - return this - } - - setSortType(sortType?: string) { - if (sortType != null) { - this.sortType = sortType - } - return this - } - - setBookmark(bookmark?: string) { - if (bookmark != null) { - this.bookmark = bookmark - } - return this - } - - excludeDocs() { - this.includeDocs = false - return this - } - - addString(key: string, partial: string) { - this.query.string![key] = partial - return this - } - - addFuzzy(key: string, fuzzy: string) { - this.query.fuzzy![key] = fuzzy - return this - } - - addRange(key: string, low: string | number, high: string | number) { - this.query.range![key] = { - low, - high, - } - return this - } - - addEqual(key: string, value: any) { - this.query.equal![key] = value - return this - } - - addNotEqual(key: string, value: any) { - this.query.notEqual![key] = value - return this - } - - addEmpty(key: string, value: any) { - this.query.empty![key] = value - return this - } - - addNotEmpty(key: string, value: any) { - this.query.notEmpty![key] = value - return this - } - - addOneOf(key: string, value: any) { - this.query.oneOf![key] = value - return this - } - - addContains(key: string, value: any) { - this.query.contains![key] = value - return this - } - - addNotContains(key: string, value: any) { - this.query.notContains![key] = value - return this - } - - addContainsAny(key: string, value: any) { - this.query.containsAny![key] = value - return this - } - - /** - * Preprocesses a value before going into a lucene search. - * Transforms strings to lowercase and wraps strings and bools in quotes. - * @param value The value to process - * @param options The preprocess options - * @returns {string|*} - */ - preprocess(value: any, { escape, lowercase, wrap, type }: any = {}) { - const hasVersion = !!this.version - // Determine if type needs wrapped - const originalType = typeof value - // Convert to lowercase - if (value && lowercase) { - value = value.toLowerCase ? value.toLowerCase() : value - } - // Escape characters - if (escape && originalType === "string") { - value = `${value}`.replace(/[ #+\-&|!(){}\]^"~*?:\\]/g, "\\$&") - } - - // Wrap in quotes - if (originalType === "string" && !isNaN(value) && !type) { - value = `"${value}"` - } else if (hasVersion && wrap) { - value = originalType === "number" ? value : `"${value}"` - } - return value - } - - buildSearchQuery() { - const builder = this - let allOr = this.query && this.query.allOr - let query = allOr ? "" : "*:*" - const allPreProcessingOpts = { escape: true, lowercase: true, wrap: true } - let tableId - if (this.query.equal!.tableId) { - tableId = this.query.equal!.tableId - delete this.query.equal!.tableId - } - - const equal = (key: string, value: any) => { - // 0 evaluates to false, which means we would return all rows if we don't check it - if (!value && value !== 0) { - return null - } - return `${key}:${builder.preprocess(value, allPreProcessingOpts)}` - } - - const contains = (key: string, value: any, mode = "AND") => { - if (Array.isArray(value) && value.length === 0) { - return null - } - if (!Array.isArray(value)) { - return `${key}:${value}` - } - let statement = `${builder.preprocess(value[0], { escape: true })}` - for (let i = 1; i < value.length; i++) { - statement += ` ${mode} ${builder.preprocess(value[i], { - escape: true, - })}` - } - return `${key}:(${statement})` - } - - const notContains = (key: string, value: any) => { - // @ts-ignore - const allPrefix = allOr === "" ? "*:* AND" : "" - return allPrefix + "NOT " + contains(key, value) - } - - const containsAny = (key: string, value: any) => { - return contains(key, value, "OR") - } - - const oneOf = (key: string, value: any) => { - if (!Array.isArray(value)) { - if (typeof value === "string") { - value = value.split(",") - } else { - return "" - } - } - let orStatement = `${builder.preprocess(value[0], allPreProcessingOpts)}` - for (let i = 1; i < value.length; i++) { - orStatement += ` OR ${builder.preprocess( - value[i], - allPreProcessingOpts - )}` - } - return `${key}:(${orStatement})` - } - - function build(structure: any, queryFn: any) { - for (let [key, value] of Object.entries(structure)) { - // check for new format - remove numbering if needed - key = removeKeyNumbering(key) - key = builder.preprocess(key.replace(/ /g, "_"), { - escape: true, - }) - const expression = queryFn(key, value) - if (expression == null) { - continue - } - if (query.length > 0) { - query += ` ${allOr ? "OR" : "AND"} ` - } - query += expression - } - } - - // Construct the actual lucene search query string from JSON structure - if (this.query.string) { - build(this.query.string, (key: string, value: any) => { - if (!value) { - return null - } - value = builder.preprocess(value, { - escape: true, - lowercase: true, - type: "string", - }) - return `${key}:${value}*` - }) - } - if (this.query.range) { - build(this.query.range, (key: string, value: any) => { - if (!value) { - return null - } - if (value.low == null || value.low === "") { - return null - } - if (value.high == null || value.high === "") { - return null - } - const low = builder.preprocess(value.low, allPreProcessingOpts) - const high = builder.preprocess(value.high, allPreProcessingOpts) - return `${key}:[${low} TO ${high}]` - }) - } - if (this.query.fuzzy) { - build(this.query.fuzzy, (key: string, value: any) => { - if (!value) { - return null - } - value = builder.preprocess(value, { - escape: true, - lowercase: true, - type: "fuzzy", - }) - return `${key}:${value}~` - }) - } - if (this.query.equal) { - build(this.query.equal, equal) - } - if (this.query.notEqual) { - build(this.query.notEqual, (key: string, value: any) => { - if (!value) { - return null - } - return `!${key}:${builder.preprocess(value, allPreProcessingOpts)}` - }) - } - if (this.query.empty) { - build(this.query.empty, (key: string) => `!${key}:["" TO *]`) - } - if (this.query.notEmpty) { - build(this.query.notEmpty, (key: string) => `${key}:["" TO *]`) - } - if (this.query.oneOf) { - build(this.query.oneOf, oneOf) - } - if (this.query.contains) { - build(this.query.contains, contains) - } - if (this.query.notContains) { - build(this.query.notContains, notContains) - } - if (this.query.containsAny) { - build(this.query.containsAny, containsAny) - } - // make sure table ID is always added as an AND - if (tableId) { - query = `(${query})` - allOr = false - build({ tableId }, equal) - } - return query - } - - buildSearchBody() { - let body: any = { - q: this.buildSearchQuery(), - limit: Math.min(this.limit, 200), - include_docs: this.includeDocs, - } - if (this.bookmark) { - body.bookmark = this.bookmark - } - if (this.sort) { - const order = this.sortOrder === "descending" ? "-" : "" - const type = `<${this.sortType}>` - body.sort = `${order}${this.sort.replace(/ /g, "_")}${type}` - } - return body - } - - async run() { - const appId = context.getAppId() - const { url, cookie } = dbCore.getCouchInfo() - const fullPath = `${url}/${appId}/_design/database/_search/${SearchIndexes.ROWS}` - const body = this.buildSearchBody() - return await runQuery(fullPath, body, cookie) - } -} - -/** - * Executes a lucene search query. - * @param url The query URL - * @param body The request body defining search criteria - * @param cookie The auth cookie for CouchDB - * @returns {Promise<{rows: []}>} - */ -const runQuery = async (url: string, body: any, cookie: string) => { - const response = await fetch(url, { - body: JSON.stringify(body), - method: "POST", - headers: { - Authorization: cookie, - }, - }) - const json = await response.json() - - let output: any = { - rows: [], - } - if (json.rows != null && json.rows.length > 0) { - output.rows = json.rows.map((row: any) => row.doc) - } - if (json.bookmark) { - output.bookmark = json.bookmark - } - return output -} - -/** - * Gets round the fixed limit of 200 results from a query by fetching as many - * pages as required and concatenating the results. This recursively operates - * until enough results have been found. - * @param query {object} The JSON query structure - * @param params {object} The search params including: - * tableId {string} The table ID to search - * sort {string} The sort column - * sortOrder {string} The sort order ("ascending" or "descending") - * sortType {string} Whether to treat sortable values as strings or - * numbers. ("string" or "number") - * limit {number} The number of results to fetch - * bookmark {string|null} Current bookmark in the recursive search - * rows {array|null} Current results in the recursive search - * @returns {Promise<*[]|*>} - */ -async function recursiveSearch(query: any, params: any): Promise { - const bookmark = params.bookmark - const rows = params.rows || [] - if (rows.length >= params.limit) { - return rows - } - let pageSize = 200 - if (rows.length > params.limit - 200) { - pageSize = params.limit - rows.length - } - const page = await new QueryBuilder(query) - .setVersion(params.version) - .setTable(params.tableId) - .setBookmark(bookmark) - .setLimit(pageSize) - .setSort(params.sort) - .setSortOrder(params.sortOrder) - .setSortType(params.sortType) - .run() - if (!page.rows.length) { - return rows - } - if (page.rows.length < 200) { - return [...rows, ...page.rows] - } - const newParams = { - ...params, - bookmark: page.bookmark, - rows: [...rows, ...page.rows], - } - return await recursiveSearch(query, newParams) -} - -/** - * Performs a paginated search. A bookmark will be returned to allow the next - * page to be fetched. There is a max limit off 200 results per page in a - * paginated search. - * @param query {object} The JSON query structure - * @param params {object} The search params including: - * tableId {string} The table ID to search - * sort {string} The sort column - * sortOrder {string} The sort order ("ascending" or "descending") - * sortType {string} Whether to treat sortable values as strings or - * numbers. ("string" or "number") - * limit {number} The desired page size - * bookmark {string} The bookmark to resume from - * @returns {Promise<{hasNextPage: boolean, rows: *[]}>} - */ export async function paginatedSearch( query: SearchFilters, - params: SearchParams + params: SearchParams ) { - let limit = params.limit - if (limit == null || isNaN(limit) || limit < 0) { - limit = 50 - } - limit = Math.min(limit, 200) - const search = new QueryBuilder(query) - .setVersion(params.version) - .setTable(params.tableId) - .setSort(params.sort) - .setSortOrder(params.sortOrder) - .setSortType(params.sortType) - const searchResults = await search - .setBookmark(params.bookmark) - .setLimit(limit) - .run() - - // Try fetching 1 row in the next page to see if another page of results - // exists or not - const nextResults = await search - .setTable(params.tableId) - .setBookmark(searchResults.bookmark) - .setLimit(1) - .run() - - return { - ...searchResults, - hasNextPage: nextResults.rows && nextResults.rows.length > 0, - } + const appId = context.getAppId() + return dbCore.paginatedSearch(appId!, SearchIndex.ROWS, query, params) } -/** - * Performs a full search, fetching multiple pages if required to return the - * desired amount of results. There is a limit of 1000 results to avoid - * heavy performance hits, and to avoid client components breaking from - * handling too much data. - * @param query {object} The JSON query structure - * @param params {object} The search params including: - * tableId {string} The table ID to search - * sort {string} The sort column - * sortOrder {string} The sort order ("ascending" or "descending") - * sortType {string} Whether to treat sortable values as strings or - * numbers. ("string" or "number") - * limit {number} The desired number of results - * @returns {Promise<{rows: *}>} - */ -export async function fullSearch(query: SearchFilters, params: SearchParams) { - let limit = params.limit - if (limit == null || isNaN(limit) || limit < 0) { - limit = 1000 - } - params.limit = Math.min(limit, 1000) - const rows = await recursiveSearch(query, params) - return { rows } +export async function fullSearch( + query: SearchFilters, + params: SearchParams +) { + const appId = context.getAppId() + return dbCore.fullSearch(appId!, SearchIndex.ROWS, query, params) } diff --git a/packages/server/src/api/controllers/row/utils.ts b/packages/server/src/api/controllers/row/utils.ts index f0f1075205..82232b7f98 100644 --- a/packages/server/src/api/controllers/row/utils.ts +++ b/packages/server/src/api/controllers/row/utils.ts @@ -3,8 +3,7 @@ import * as userController from "../user" import { FieldTypes } from "../../../constants" import { context } from "@budibase/backend-core" import { makeExternalQuery } from "../../../integrations/base/query" -import { BBContext, Row, Table } from "@budibase/types" -export { removeKeyNumbering } from "../../../integrations/base/utils" +import { Row, Table } from "@budibase/types" const validateJs = require("validate.js") const { cloneDeep } = require("lodash/fp") import { Format } from "../view/exporters" diff --git a/packages/server/src/app.ts b/packages/server/src/app.ts index 03dce4f875..00f2aca7fc 100644 --- a/packages/server/src/app.ts +++ b/packages/server/src/app.ts @@ -32,6 +32,7 @@ import { initialise as initialiseWebsockets } from "./websocket" import { startup } from "./startup" const Sentry = require("@sentry/node") const destroyable = require("server-destroy") +const { userAgent } = require("koa-useragent") const app = new Koa() @@ -53,6 +54,7 @@ app.use( ) app.use(middleware.logging) +app.use(userAgent) if (env.isProd()) { env._set("NODE_ENV", "production") diff --git a/packages/server/src/db/index.ts b/packages/server/src/db/index.ts index fa8027dcc1..157c2f4fb3 100644 --- a/packages/server/src/db/index.ts +++ b/packages/server/src/db/index.ts @@ -1,4 +1,4 @@ -import { init as coreInit } from "@budibase/backend-core" +import * as core from "@budibase/backend-core" import env from "../environment" export function init() { @@ -12,5 +12,5 @@ export function init() { dbConfig.allDbs = true } - coreInit({ db: dbConfig }) + core.init({ db: dbConfig }) } diff --git a/packages/server/src/db/utils.ts b/packages/server/src/db/utils.ts index ac5a6162b9..50341e4abc 100644 --- a/packages/server/src/db/utils.ts +++ b/packages/server/src/db/utils.ts @@ -9,10 +9,6 @@ export const AppStatus = { DEPLOYED: "published", } -export const SearchIndexes = { - ROWS: "rows", -} - export const BudibaseInternalDB = { _id: "bb_internal", type: dbCore.BUDIBASE_DATASOURCE_TYPE, diff --git a/packages/server/src/db/views/staticViews.ts b/packages/server/src/db/views/staticViews.ts index 4bccfebeee..730e5b1e66 100644 --- a/packages/server/src/db/views/staticViews.ts +++ b/packages/server/src/db/views/staticViews.ts @@ -1,6 +1,6 @@ import { context } from "@budibase/backend-core" -import { DocumentType, SEPARATOR, ViewName, SearchIndexes } from "../utils" -import { LinkDocument, Row } from "@budibase/types" +import { DocumentType, SEPARATOR, ViewName } from "../utils" +import { LinkDocument, Row, SearchIndex } from "@budibase/types" const SCREEN_PREFIX = DocumentType.SCREEN + SEPARATOR /************************************************** @@ -91,7 +91,7 @@ async function searchIndex(indexName: string, fnString: string) { export async function createAllSearchIndex() { await searchIndex( - SearchIndexes.ROWS, + SearchIndex.ROWS, function (doc: Row) { function idx(input: Row, prev?: string) { for (let key of Object.keys(input)) { diff --git a/packages/server/src/integrations/base/sql.ts b/packages/server/src/integrations/base/sql.ts index 6871d7b0ba..00e62d3ce4 100644 --- a/packages/server/src/integrations/base/sql.ts +++ b/packages/server/src/integrations/base/sql.ts @@ -6,11 +6,11 @@ import { SearchFilters, SortDirection, } from "@budibase/types" +import { db as dbCore } from "@budibase/backend-core" import { QueryOptions } from "../../definitions/datasource" import { isIsoDateString, SqlClient } from "../utils" import SqlTableQueryBuilder from "./sqlTable" import environment from "../../environment" -import { removeKeyNumbering } from "./utils" const envLimit = environment.SQL_MAX_ROWS ? parseInt(environment.SQL_MAX_ROWS) @@ -136,7 +136,7 @@ class InternalBuilder { fn: (key: string, value: any) => void ) { for (let [key, value] of Object.entries(structure)) { - const updatedKey = removeKeyNumbering(key) + const updatedKey = dbCore.removeKeyNumbering(key) const isRelationshipField = updatedKey.includes(".") if (!opts.relationship && !isRelationshipField) { fn(`${opts.tableName}.${updatedKey}`, value) diff --git a/packages/server/src/integrations/base/utils.ts b/packages/server/src/integrations/base/utils.ts deleted file mode 100644 index 54efdb91a0..0000000000 --- a/packages/server/src/integrations/base/utils.ts +++ /dev/null @@ -1,12 +0,0 @@ -const QUERY_START_REGEX = /\d[0-9]*:/g - -export function removeKeyNumbering(key: any): string { - if (typeof key === "string" && key.match(QUERY_START_REGEX) != null) { - const parts = key.split(":") - // remove the number - parts.shift() - return parts.join(":") - } else { - return key - } -} diff --git a/packages/server/src/startup.ts b/packages/server/src/startup.ts index f70694226d..6cdbf87c2c 100644 --- a/packages/server/src/startup.ts +++ b/packages/server/src/startup.ts @@ -5,7 +5,7 @@ import { generateApiKey, getChecklist, } from "./utilities/workerRequests" -import { installation, tenancy, logging } from "@budibase/backend-core" +import { installation, tenancy, logging, events } from "@budibase/backend-core" import fs from "fs" import { watch } from "./watch" import * as automations from "./automations" @@ -124,6 +124,9 @@ export async function startup(app?: any, server?: any) { // get the references to the queue promises, don't await as // they will never end, unless the processing stops let queuePromises = [] + // configure events to use the pro audit log write + // can't integrate directly into backend-core due to cyclic issues + queuePromises.push(events.processors.init(pro.sdk.auditLogs.write)) queuePromises.push(automations.init()) queuePromises.push(initPro()) if (app) { diff --git a/packages/server/yarn.lock b/packages/server/yarn.lock index e4ad525977..f39c5387d7 100644 --- a/packages/server/yarn.lock +++ b/packages/server/yarn.lock @@ -6122,20 +6122,13 @@ dir-glob@^3.0.1: dependencies: path-type "^4.0.0" -docker-compose@0.23.17: +docker-compose@0.23.17, docker-compose@^0.23.5: version "0.23.17" resolved "https://registry.yarnpkg.com/docker-compose/-/docker-compose-0.23.17.tgz#8816bef82562d9417dc8c790aa4871350f93a2ba" integrity sha512-YJV18YoYIcxOdJKeFcCFihE6F4M2NExWM/d4S1ITcS9samHKnNUihz9kjggr0dNtsrbpFNc7/Yzd19DWs+m1xg== dependencies: yaml "^1.10.2" -docker-compose@^0.23.5: - version "0.23.19" - resolved "https://registry.yarnpkg.com/docker-compose/-/docker-compose-0.23.19.tgz#9947726e2fe67bdfa9e8efe1ff15aa0de2e10eb8" - integrity sha512-v5vNLIdUqwj4my80wxFDkNH+4S85zsRuH29SO7dCWVWPCMt/ohZBsGN6g6KXWifT0pzQ7uOxqEKCYCDPJ8Vz4g== - dependencies: - yaml "^1.10.2" - docker-modem@^3.0.0: version "3.0.6" resolved "https://registry.yarnpkg.com/docker-modem/-/docker-modem-3.0.6.tgz#8c76338641679e28ec2323abb65b3276fb1ce597" @@ -6983,6 +6976,11 @@ expose-loader@^3.1.0: resolved "https://registry.yarnpkg.com/expose-loader/-/expose-loader-3.1.0.tgz#7a0bdecb345b921ca238a8c4715a4ea7e227213f" integrity sha512-2RExSo0yJiqP+xiUue13jQa2IHE8kLDzTI7b6kn+vUlBVvlzNSiLDzo4e5Pp5J039usvTUnxZ8sUOhv0Kg15NA== +express-useragent@^1.0.15: + version "1.0.15" + resolved "https://registry.yarnpkg.com/express-useragent/-/express-useragent-1.0.15.tgz#cefda5fa4904345d51d3368b117a8dd4124985d9" + integrity sha512-eq5xMiYCYwFPoekffMjvEIk+NWdlQY9Y38OsTyl13IvA728vKT+q/CSERYWzcw93HGBJcIqMIsZC5CZGARPVdg== + ext-list@^2.0.0: version "2.2.2" resolved "https://registry.yarnpkg.com/ext-list/-/ext-list-2.2.2.tgz#0b98e64ed82f5acf0f2931babf69212ef52ddd37" @@ -10243,6 +10241,13 @@ koa-static@5.0.0, koa-static@^5.0.0: debug "^3.1.0" koa-send "^5.0.0" +koa-useragent@^4.1.0: + version "4.1.0" + resolved "https://registry.yarnpkg.com/koa-useragent/-/koa-useragent-4.1.0.tgz#d3f128b552c6da3e5e9e9e9c887b2922b16e4468" + integrity sha512-x/HUDZ1zAmNNh5hA9hHbPm9p3UVg2prlpHzxCXQCzbibrNS0kmj7MkCResCbAbG7ZT6FVxNSMjR94ZGamdMwxA== + dependencies: + express-useragent "^1.0.15" + koa-views@^7.0.1: version "7.0.2" resolved "https://registry.yarnpkg.com/koa-views/-/koa-views-7.0.2.tgz#c96fd9e2143ef00c29dc5160c5ed639891aa723d" @@ -10706,7 +10711,12 @@ lru-cache@^6.0.0: dependencies: yallist "^4.0.0" -lru-cache@^7.14.0, lru-cache@^7.14.1: +lru-cache@^7.14.0: + version "7.16.1" + resolved "https://registry.yarnpkg.com/lru-cache/-/lru-cache-7.16.1.tgz#7acea16fecd9ed11430e78443c2bb81a06d3dea9" + integrity sha512-9kkuMZHnLH/8qXARvYSjNvq8S1GYFFzynQTAfKeaJ0sIrR3PUPuu37Z+EiIANiZBvpfTf2B5y8ecDLSMWlLv+w== + +lru-cache@^7.14.1: version "7.14.1" resolved "https://registry.yarnpkg.com/lru-cache/-/lru-cache-7.14.1.tgz#8da8d2f5f59827edb388e63e459ac23d6d408fea" integrity sha512-ysxwsnTKdAx96aTRdhDOCQfDgbHnt8SK0KY8SEjO0wHinhWOFTESbjVCMPbU1uGXg/ch4lifqx0wfjOawU2+WA== diff --git a/packages/types/src/api/web/global/auditLogs.ts b/packages/types/src/api/web/global/auditLogs.ts new file mode 100644 index 0000000000..8ce0e742d2 --- /dev/null +++ b/packages/types/src/api/web/global/auditLogs.ts @@ -0,0 +1,44 @@ +import { Event, AuditedEventFriendlyName } from "../../../sdk" +import { + PaginationResponse, + PaginationRequest, + BasicPaginationRequest, +} from "../" +import { User, App } from "../../../" + +export interface AuditLogSearchParams { + userIds?: string[] + appIds?: string[] + events?: Event[] + startDate?: string + endDate?: string + fullSearch?: string + bookmark?: string +} + +export interface DownloadAuditLogsRequest extends AuditLogSearchParams {} + +export interface SearchAuditLogsRequest + extends BasicPaginationRequest, + AuditLogSearchParams {} + +export enum AuditLogResourceStatus { + DELETED = "deleted", +} + +export interface AuditLogEnriched { + app?: App | { _id: string; status: AuditLogResourceStatus } + user: User | { _id: string; status: AuditLogResourceStatus } + event: Event + timestamp: string + name: string + metadata: any +} + +export interface SearchAuditLogsResponse extends PaginationResponse { + data: AuditLogEnriched[] +} + +export interface DefinitionsAuditLogsResponse { + events: Record +} diff --git a/packages/types/src/api/web/global/index.ts b/packages/types/src/api/web/global/index.ts index d6e87d3a0d..21a5de3727 100644 --- a/packages/types/src/api/web/global/index.ts +++ b/packages/types/src/api/web/global/index.ts @@ -1,3 +1,4 @@ export * from "./environmentVariables" +export * from "./auditLogs" export * from "./events" export * from "./configs" diff --git a/packages/types/src/api/web/index.ts b/packages/types/src/api/web/index.ts index 8ed5b0aad4..141119c837 100644 --- a/packages/types/src/api/web/index.ts +++ b/packages/types/src/api/web/index.ts @@ -5,3 +5,4 @@ export * from "./errors" export * from "./schedule" export * from "./app" export * from "./global" +export * from "./pagination" diff --git a/packages/types/src/api/web/pagination.ts b/packages/types/src/api/web/pagination.ts new file mode 100644 index 0000000000..ae4c56971a --- /dev/null +++ b/packages/types/src/api/web/pagination.ts @@ -0,0 +1,27 @@ +export enum SortOrder { + ASCENDING = "ascending", + DESCENDING = "descending", +} + +export enum SortType { + STRING = "string", + number = "number", +} + +export interface BasicPaginationRequest { + bookmark?: string +} + +export interface PaginationRequest extends BasicPaginationRequest { + limit?: number + sort?: { + order: SortOrder + column: string + type: SortType + } +} + +export interface PaginationResponse { + bookmark: string + hasNextPage: boolean +} diff --git a/packages/types/src/documents/global/auditLogs.ts b/packages/types/src/documents/global/auditLogs.ts new file mode 100644 index 0000000000..7902997b39 --- /dev/null +++ b/packages/types/src/documents/global/auditLogs.ts @@ -0,0 +1,13 @@ +import { Document } from "../document" +import { Event } from "../../sdk" + +export const AuditLogSystemUser = "SYSTEM" + +export interface AuditLogDoc extends Document { + appId?: string + event: Event + userId: string + timestamp: string + metadata: any + name: string +} diff --git a/packages/types/src/documents/global/index.ts b/packages/types/src/documents/global/index.ts index 11ce7513f2..b728439dd6 100644 --- a/packages/types/src/documents/global/index.ts +++ b/packages/types/src/documents/global/index.ts @@ -6,3 +6,4 @@ export * from "./quotas" export * from "./schedule" export * from "./templates" export * from "./environmentVariables" +export * from "./auditLogs" diff --git a/packages/types/src/sdk/auditLogs.ts b/packages/types/src/sdk/auditLogs.ts new file mode 100644 index 0000000000..0322d2e862 --- /dev/null +++ b/packages/types/src/sdk/auditLogs.ts @@ -0,0 +1,21 @@ +import { Event, HostInfo } from "./events" +import { AuditLogDoc } from "../documents" + +export type AuditWriteOpts = { + appId?: string + timestamp?: string | number + userId?: string + hostInfo?: HostInfo +} + +export type AuditLogFn = ( + event: Event, + metadata: any, + opts: AuditWriteOpts +) => Promise + +export type AuditLogQueueEvent = { + event: Event + properties: any + opts: AuditWriteOpts +} diff --git a/packages/types/src/sdk/context.ts b/packages/types/src/sdk/context.ts index b3403df8af..c8345de196 100644 --- a/packages/types/src/sdk/context.ts +++ b/packages/types/src/sdk/context.ts @@ -1,5 +1,5 @@ import { User, Account } from "../documents" -import { IdentityType } from "./events" +import { IdentityType, HostInfo } from "./events" export interface BaseContext { _id: string @@ -16,6 +16,7 @@ export interface UserContext extends BaseContext, User { _id: string tenantId: string account?: Account + hostInfo: HostInfo } export type IdentityContext = BaseContext | AccountUserContext | UserContext diff --git a/packages/types/src/sdk/db.ts b/packages/types/src/sdk/db.ts index 35d198ccb2..6e213b5831 100644 --- a/packages/types/src/sdk/db.ts +++ b/packages/types/src/sdk/db.ts @@ -1,5 +1,11 @@ import Nano from "@budibase/nano" import { AllDocsResponse, AnyDocument, Document } from "../" +import { Writable } from "stream" + +export enum SearchIndex { + ROWS = "rows", + AUDIT = "audit", +} export type PouchOptions = { inMemory?: boolean @@ -63,6 +69,18 @@ export const isDocument = (doc: any): doc is Document => { return typeof doc === "object" && doc._id && doc._rev } +export interface DatabaseDumpOpts { + filter?: (doc: AnyDocument) => boolean + batch_size?: number + batch_limit?: number + style?: "main_only" | "all_docs" + timeout?: number + doc_ids?: string[] + query_params?: any + view?: string + selector?: any +} + export interface Database { name: string @@ -87,7 +105,7 @@ export interface Database { compact(): Promise // these are all PouchDB related functions that are rarely used - in future // should be replaced by better typed/non-pouch implemented methods - dump(...args: any[]): Promise + dump(stream: Writable, opts?: DatabaseDumpOpts): Promise load(...args: any[]): Promise createIndex(...args: any[]): Promise deleteIndex(...args: any[]): Promise diff --git a/packages/types/src/sdk/events/app.ts b/packages/types/src/sdk/events/app.ts index 73d491070f..602dd8b6a5 100644 --- a/packages/types/src/sdk/events/app.ts +++ b/packages/types/src/sdk/events/app.ts @@ -3,50 +3,83 @@ import { BaseEvent } from "./event" export interface AppCreatedEvent extends BaseEvent { appId: string version: string + audited: { + name: string + } } export interface AppUpdatedEvent extends BaseEvent { appId: string version: string + audited: { + name: string + } } export interface AppDeletedEvent extends BaseEvent { appId: string + audited: { + name: string + } } export interface AppPublishedEvent extends BaseEvent { appId: string + audited: { + name: string + } } export interface AppUnpublishedEvent extends BaseEvent { appId: string + audited: { + name: string + } } export interface AppFileImportedEvent extends BaseEvent { appId: string + audited: { + name: string + } } export interface AppTemplateImportedEvent extends BaseEvent { appId: string templateKey: string + audited: { + name: string + } } export interface AppVersionUpdatedEvent extends BaseEvent { appId: string currentVersion: string updatedToVersion: string + audited: { + name: string + } } export interface AppVersionRevertedEvent extends BaseEvent { appId: string currentVersion: string revertedToVersion: string + audited: { + name: string + } } export interface AppRevertedEvent extends BaseEvent { appId: string + audited: { + name: string + } } export interface AppExportedEvent extends BaseEvent { appId: string + audited: { + name: string + } } diff --git a/packages/types/src/sdk/events/auditLog.ts b/packages/types/src/sdk/events/auditLog.ts new file mode 100644 index 0000000000..5f3edfb826 --- /dev/null +++ b/packages/types/src/sdk/events/auditLog.ts @@ -0,0 +1,10 @@ +import { BaseEvent } from "./event" +import { AuditLogSearchParams } from "../../api" + +export interface AuditLogFilteredEvent extends BaseEvent { + filters: AuditLogSearchParams +} + +export interface AuditLogDownloadedEvent extends BaseEvent { + filters: AuditLogSearchParams +} diff --git a/packages/types/src/sdk/events/auth.ts b/packages/types/src/sdk/events/auth.ts index eb9f3148a3..c7171d923d 100644 --- a/packages/types/src/sdk/events/auth.ts +++ b/packages/types/src/sdk/events/auth.ts @@ -7,10 +7,16 @@ export type SSOType = ConfigType.OIDC | ConfigType.GOOGLE export interface LoginEvent extends BaseEvent { userId: string source: LoginSource + audited: { + email: string + } } export interface LogoutEvent extends BaseEvent { userId: string + audited: { + email?: string + } } export interface SSOCreatedEvent extends BaseEvent { diff --git a/packages/types/src/sdk/events/automation.ts b/packages/types/src/sdk/events/automation.ts index beabdbb9eb..a3f4d15186 100644 --- a/packages/types/src/sdk/events/automation.ts +++ b/packages/types/src/sdk/events/automation.ts @@ -5,6 +5,9 @@ export interface AutomationCreatedEvent extends BaseEvent { automationId: string triggerId: string triggerType: string + audited: { + name: string + } } export interface AutomationTriggerUpdatedEvent extends BaseEvent { @@ -19,6 +22,9 @@ export interface AutomationDeletedEvent extends BaseEvent { automationId: string triggerId: string triggerType: string + audited: { + name: string + } } export interface AutomationTestedEvent extends BaseEvent { @@ -35,6 +41,9 @@ export interface AutomationStepCreatedEvent extends BaseEvent { triggerType: string stepId: string stepType: string + audited: { + name: string + } } export interface AutomationStepDeletedEvent extends BaseEvent { @@ -44,6 +53,9 @@ export interface AutomationStepDeletedEvent extends BaseEvent { triggerType: string stepId: string stepType: string + audited: { + name: string + } } export interface AutomationsRunEvent extends BaseEvent { diff --git a/packages/types/src/sdk/events/backup.ts b/packages/types/src/sdk/events/backup.ts index 1dddc109cc..23863cf8ac 100644 --- a/packages/types/src/sdk/events/backup.ts +++ b/packages/types/src/sdk/events/backup.ts @@ -5,6 +5,7 @@ export interface AppBackupRestoreEvent extends BaseEvent { appId: string restoreId: string backupCreatedAt: string + name: string } export interface AppBackupTriggeredEvent extends BaseEvent { @@ -12,4 +13,5 @@ export interface AppBackupTriggeredEvent extends BaseEvent { appId: string trigger: AppBackupTrigger type: AppBackupType + name: string } diff --git a/packages/types/src/sdk/events/event.ts b/packages/types/src/sdk/events/event.ts index f509682add..3d0d3122b5 100644 --- a/packages/types/src/sdk/events/event.ts +++ b/packages/types/src/sdk/events/event.ts @@ -180,6 +180,190 @@ export enum Event { ENVIRONMENT_VARIABLE_CREATED = "environment_variable:created", ENVIRONMENT_VARIABLE_DELETED = "environment_variable:deleted", ENVIRONMENT_VARIABLE_UPGRADE_PANEL_OPENED = "environment_variable:upgrade_panel_opened", + + // AUDIT LOG + AUDIT_LOGS_FILTERED = "audit_log:filtered", + AUDIT_LOGS_DOWNLOADED = "audit_log:downloaded", +} + +// all events that are not audited have been added to this record as undefined, this means +// that Typescript can protect us against new events being added and auditing of those +// events not being considered. This might be a little ugly, but provides a level of +// Typescript build protection for the audit log feature, any new event also needs to be +// added to this map, during which the developer will need to consider if it should be +// a user facing event or not. +export const AuditedEventFriendlyName: Record = { + // USER + [Event.USER_CREATED]: `User "{{ email }}" created`, + [Event.USER_UPDATED]: `User "{{ email }}" updated`, + [Event.USER_DELETED]: `User "{{ email }}" deleted`, + [Event.USER_PERMISSION_ADMIN_ASSIGNED]: `User "{{ email }}" admin role assigned`, + [Event.USER_PERMISSION_ADMIN_REMOVED]: `User "{{ email }}" admin role removed`, + [Event.USER_PERMISSION_BUILDER_ASSIGNED]: `User "{{ email }}" builder role assigned`, + [Event.USER_PERMISSION_BUILDER_REMOVED]: `User "{{ email }}" builder role removed`, + [Event.USER_INVITED]: `User "{{ email }}" invited`, + [Event.USER_INVITED_ACCEPTED]: `User "{{ email }}" accepted invite`, + [Event.USER_PASSWORD_UPDATED]: `User "{{ email }}" password updated`, + [Event.USER_PASSWORD_RESET_REQUESTED]: `User "{{ email }}" password reset requested`, + [Event.USER_PASSWORD_RESET]: `User "{{ email }}" password reset`, + [Event.USER_GROUP_CREATED]: `User group "{{ name }}" created`, + [Event.USER_GROUP_UPDATED]: `User group "{{ name }}" updated`, + [Event.USER_GROUP_DELETED]: `User group "{{ name }}" deleted`, + [Event.USER_GROUP_USERS_ADDED]: `User group "{{ name }}" {{ count }} users added`, + [Event.USER_GROUP_USERS_REMOVED]: `User group "{{ name }}" {{ count }} users removed`, + [Event.USER_GROUP_PERMISSIONS_EDITED]: `User group "{{ name }}" permissions edited`, + [Event.USER_PASSWORD_FORCE_RESET]: undefined, + [Event.USER_GROUP_ONBOARDING]: undefined, + [Event.USER_ONBOARDING_COMPLETE]: undefined, + + // EMAIL + [Event.EMAIL_SMTP_CREATED]: `Email configuration created`, + [Event.EMAIL_SMTP_UPDATED]: `Email configuration updated`, + + // AUTH + [Event.AUTH_SSO_CREATED]: `SSO configuration created`, + [Event.AUTH_SSO_UPDATED]: `SSO configuration updated`, + [Event.AUTH_SSO_ACTIVATED]: `SSO configuration activated`, + [Event.AUTH_SSO_DEACTIVATED]: `SSO configuration deactivated`, + [Event.AUTH_LOGIN]: `User "{{ email }}" logged in`, + [Event.AUTH_LOGOUT]: `User "{{ email }}" logged out`, + + // ORG + [Event.ORG_NAME_UPDATED]: `Organisation name updated`, + [Event.ORG_LOGO_UPDATED]: `Organisation logo updated`, + [Event.ORG_PLATFORM_URL_UPDATED]: `Organisation platform URL updated`, + + // APP + [Event.APP_CREATED]: `App "{{ name }}" created`, + [Event.APP_UPDATED]: `App "{{ name }}" updated`, + [Event.APP_DELETED]: `App "{{ name }}" deleted`, + [Event.APP_PUBLISHED]: `App "{{ name }}" published`, + [Event.APP_UNPUBLISHED]: `App "{{ name }}" unpublished`, + [Event.APP_TEMPLATE_IMPORTED]: `App "{{ name }}" template imported`, + [Event.APP_FILE_IMPORTED]: `App "{{ name }}" file imported`, + [Event.APP_VERSION_UPDATED]: `App "{{ name }}" version updated`, + [Event.APP_VERSION_REVERTED]: `App "{{ name }}" version reverted`, + [Event.APP_REVERTED]: `App "{{ name }}" reverted`, + [Event.APP_EXPORTED]: `App "{{ name }}" exported`, + [Event.APP_BACKUP_RESTORED]: `App backup "{{ name }}" restored`, + [Event.APP_BACKUP_TRIGGERED]: `App backup "{{ name }}" triggered`, + + // DATASOURCE + [Event.DATASOURCE_CREATED]: `Datasource created`, + [Event.DATASOURCE_UPDATED]: `Datasource updated`, + [Event.DATASOURCE_DELETED]: `Datasource deleted`, + + // QUERY + [Event.QUERY_CREATED]: `Query created`, + [Event.QUERY_UPDATED]: `Query updated`, + [Event.QUERY_DELETED]: `Query deleted`, + [Event.QUERY_IMPORT]: `Query import`, + [Event.QUERIES_RUN]: undefined, + [Event.QUERY_PREVIEWED]: undefined, + + // TABLE + [Event.TABLE_CREATED]: `Table "{{ name }}" created`, + [Event.TABLE_UPDATED]: `Table "{{ name }}" updated`, + [Event.TABLE_DELETED]: `Table "{{ name }}" deleted`, + [Event.TABLE_EXPORTED]: `Table "{{ name }}" exported`, + [Event.TABLE_IMPORTED]: `Table "{{ name }}" imported`, + [Event.TABLE_DATA_IMPORTED]: `Data imported to table`, + + // ROWS + [Event.ROWS_CREATED]: `Rows created`, + [Event.ROWS_IMPORTED]: `Rows imported`, + + // AUTOMATION + [Event.AUTOMATION_CREATED]: `Automation "{{ name }}" created`, + [Event.AUTOMATION_DELETED]: `Automation "{{ name }}" deleted`, + [Event.AUTOMATION_STEP_CREATED]: `Automation "{{ name }}" step added`, + [Event.AUTOMATION_STEP_DELETED]: `Automation "{{ name }}" step removed`, + [Event.AUTOMATION_TESTED]: undefined, + [Event.AUTOMATIONS_RUN]: undefined, + [Event.AUTOMATION_TRIGGER_UPDATED]: undefined, + + // SCREEN + [Event.SCREEN_CREATED]: `Screen "{{ name }}" created`, + [Event.SCREEN_DELETED]: `Screen "{{ name }}" deleted`, + + // COMPONENT + [Event.COMPONENT_CREATED]: `Component created`, + [Event.COMPONENT_DELETED]: `Component deleted`, + + // ENVIRONMENT VARIABLE + [Event.ENVIRONMENT_VARIABLE_CREATED]: `Environment variable created`, + [Event.ENVIRONMENT_VARIABLE_DELETED]: `Environment variable deleted`, + [Event.ENVIRONMENT_VARIABLE_UPGRADE_PANEL_OPENED]: undefined, + + // PLUGIN + [Event.PLUGIN_IMPORTED]: `Plugin imported`, + [Event.PLUGIN_DELETED]: `Plugin deleted`, + [Event.PLUGIN_INIT]: undefined, + + // ROLE - NOT AUDITED + [Event.ROLE_CREATED]: undefined, + [Event.ROLE_UPDATED]: undefined, + [Event.ROLE_DELETED]: undefined, + [Event.ROLE_ASSIGNED]: undefined, + [Event.ROLE_UNASSIGNED]: undefined, + + // LICENSE - NOT AUDITED + [Event.LICENSE_PLAN_CHANGED]: undefined, + [Event.LICENSE_TIER_CHANGED]: undefined, + [Event.LICENSE_ACTIVATED]: undefined, + [Event.LICENSE_PAYMENT_FAILED]: undefined, + [Event.LICENSE_PAYMENT_RECOVERED]: undefined, + [Event.LICENSE_CHECKOUT_OPENED]: undefined, + [Event.LICENSE_CHECKOUT_SUCCESS]: undefined, + [Event.LICENSE_PORTAL_OPENED]: undefined, + + // ACCOUNT - NOT AUDITED + [Event.ACCOUNT_CREATED]: undefined, + [Event.ACCOUNT_DELETED]: undefined, + [Event.ACCOUNT_VERIFIED]: undefined, + + // BACKFILL - NOT AUDITED + [Event.APP_BACKFILL_SUCCEEDED]: undefined, + [Event.APP_BACKFILL_FAILED]: undefined, + [Event.TENANT_BACKFILL_SUCCEEDED]: undefined, + [Event.TENANT_BACKFILL_FAILED]: undefined, + [Event.INSTALLATION_BACKFILL_SUCCEEDED]: undefined, + [Event.INSTALLATION_BACKFILL_FAILED]: undefined, + + // LAYOUT - NOT AUDITED + [Event.LAYOUT_CREATED]: undefined, + [Event.LAYOUT_DELETED]: undefined, + + // VIEW - NOT AUDITED + [Event.VIEW_CREATED]: undefined, + [Event.VIEW_UPDATED]: undefined, + [Event.VIEW_DELETED]: undefined, + [Event.VIEW_EXPORTED]: undefined, + [Event.VIEW_FILTER_CREATED]: undefined, + [Event.VIEW_FILTER_UPDATED]: undefined, + [Event.VIEW_FILTER_DELETED]: undefined, + [Event.VIEW_CALCULATION_CREATED]: undefined, + [Event.VIEW_CALCULATION_UPDATED]: undefined, + [Event.VIEW_CALCULATION_DELETED]: undefined, + + // SERVED - NOT AUDITED + [Event.SERVED_BUILDER]: undefined, + [Event.SERVED_APP]: undefined, + [Event.SERVED_APP_PREVIEW]: undefined, + + // ANALYTICS - NOT AUDITED + [Event.ANALYTICS_OPT_OUT]: undefined, + [Event.ANALYTICS_OPT_IN]: undefined, + + // INSTALLATION - NOT AUDITED + [Event.INSTALLATION_VERSION_CHECKED]: undefined, + [Event.INSTALLATION_VERSION_UPGRADED]: undefined, + [Event.INSTALLATION_VERSION_DOWNGRADED]: undefined, + [Event.INSTALLATION_FIRST_STARTUP]: undefined, + + // AUDIT LOG - NOT AUDITED + [Event.AUDIT_LOGS_FILTERED]: undefined, + [Event.AUDIT_LOGS_DOWNLOADED]: undefined, } // properties added at the final stage of the event pipeline @@ -191,6 +375,11 @@ export interface BaseEvent { installationId?: string tenantId?: string hosting?: Hosting + // any props in the audited section will be removed before passing events + // up out of system (purely for use with auditing) + audited?: { + [key: string]: any + } } export type TableExportFormat = "json" | "csv" diff --git a/packages/types/src/sdk/events/identification.ts b/packages/types/src/sdk/events/identification.ts index 3f4e7ec9d4..627254882e 100644 --- a/packages/types/src/sdk/events/identification.ts +++ b/packages/types/src/sdk/events/identification.ts @@ -34,6 +34,11 @@ export enum IdentityType { INSTALLATION = "installation", } +export interface HostInfo { + ipAddress?: string + userAgent?: string +} + export interface Identity { id: string type: IdentityType @@ -41,6 +46,7 @@ export interface Identity { environment: string installationId?: string tenantId?: string + hostInfo?: HostInfo } export interface UserIdentity extends Identity { diff --git a/packages/types/src/sdk/events/index.ts b/packages/types/src/sdk/events/index.ts index 009d9beac4..745f84d2a3 100644 --- a/packages/types/src/sdk/events/index.ts +++ b/packages/types/src/sdk/events/index.ts @@ -22,3 +22,4 @@ export * from "./userGroup" export * from "./plugin" export * from "./backup" export * from "./environmentVariable" +export * from "./auditLog" diff --git a/packages/types/src/sdk/events/screen.ts b/packages/types/src/sdk/events/screen.ts index 23c468b7ad..e6b1d4d360 100644 --- a/packages/types/src/sdk/events/screen.ts +++ b/packages/types/src/sdk/events/screen.ts @@ -4,10 +4,16 @@ export interface ScreenCreatedEvent extends BaseEvent { screenId: string layoutId?: string roleId: string + audited: { + name: string + } } export interface ScreenDeletedEvent extends BaseEvent { screenId: string layoutId?: string roleId: string + audited: { + name: string + } } diff --git a/packages/types/src/sdk/events/table.ts b/packages/types/src/sdk/events/table.ts index da3c7edf47..8df2a95796 100644 --- a/packages/types/src/sdk/events/table.ts +++ b/packages/types/src/sdk/events/table.ts @@ -2,21 +2,36 @@ import { BaseEvent, TableExportFormat } from "./event" export interface TableCreatedEvent extends BaseEvent { tableId: string + audited: { + name: string + } } export interface TableUpdatedEvent extends BaseEvent { tableId: string + audited: { + name: string + } } export interface TableDeletedEvent extends BaseEvent { tableId: string + audited: { + name: string + } } export interface TableExportedEvent extends BaseEvent { tableId: string format: TableExportFormat + audited: { + name: string + } } export interface TableImportedEvent extends BaseEvent { tableId: string + audited: { + name: string + } } diff --git a/packages/types/src/sdk/events/user.ts b/packages/types/src/sdk/events/user.ts index 3f8f72801c..ab4b4d9724 100644 --- a/packages/types/src/sdk/events/user.ts +++ b/packages/types/src/sdk/events/user.ts @@ -2,47 +2,84 @@ import { BaseEvent } from "./event" export interface UserCreatedEvent extends BaseEvent { userId: string + audited: { + email: string + } } export interface UserUpdatedEvent extends BaseEvent { userId: string + audited: { + email: string + } } export interface UserDeletedEvent extends BaseEvent { userId: string + audited: { + email: string + } } export interface UserOnboardingEvent extends BaseEvent { userId: string step?: string + audited: { + email: string + } } export interface UserPermissionAssignedEvent extends BaseEvent { userId: string + audited: { + email: string + } } export interface UserPermissionRemovedEvent extends BaseEvent { userId: string + audited: { + email: string + } } -export interface UserInvitedEvent extends BaseEvent {} +export interface UserInvitedEvent extends BaseEvent { + audited: { + email: string + } +} export interface UserInviteAcceptedEvent extends BaseEvent { userId: string + audited: { + email: string + } } export interface UserPasswordForceResetEvent extends BaseEvent { userId: string + audited: { + email: string + } } export interface UserPasswordUpdatedEvent extends BaseEvent { userId: string + audited: { + email: string + } } export interface UserPasswordResetRequestedEvent extends BaseEvent { userId: string + audited: { + email: string + } } export interface UserPasswordResetEvent extends BaseEvent { userId: string + audited: { + email: string + } } diff --git a/packages/types/src/sdk/events/userGroup.ts b/packages/types/src/sdk/events/userGroup.ts index 2ce642e274..d82ab70b4c 100644 --- a/packages/types/src/sdk/events/userGroup.ts +++ b/packages/types/src/sdk/events/userGroup.ts @@ -2,27 +2,50 @@ import { BaseEvent } from "./event" export interface GroupCreatedEvent extends BaseEvent { groupId: string + audited: { + name: string + } } export interface GroupUpdatedEvent extends BaseEvent { groupId: string + audited: { + name: string + } } export interface GroupDeletedEvent extends BaseEvent { groupId: string + audited: { + name: string + } } export interface GroupUsersAddedEvent extends BaseEvent { count: number groupId: string + audited: { + name: string + } } export interface GroupUsersDeletedEvent extends BaseEvent { count: number groupId: string + audited: { + name: string + } } export interface GroupAddedOnboardingEvent extends BaseEvent { groupId: string onboarding: boolean } + +export interface GroupPermissionsEditedEvent extends BaseEvent { + permissions: Record + groupId: string + audited: { + name: string + } +} diff --git a/packages/types/src/sdk/index.ts b/packages/types/src/sdk/index.ts index be12d45527..ccd59bf720 100644 --- a/packages/types/src/sdk/index.ts +++ b/packages/types/src/sdk/index.ts @@ -12,5 +12,6 @@ export * from "./db" export * from "./middleware" export * from "./featureFlag" export * from "./environmentVariables" +export * from "./auditLogs" export * from "./sso" export * from "./user" diff --git a/packages/types/src/sdk/licensing/feature.ts b/packages/types/src/sdk/licensing/feature.ts index f28c3c0709..9f6090623b 100644 --- a/packages/types/src/sdk/licensing/feature.ts +++ b/packages/types/src/sdk/licensing/feature.ts @@ -2,5 +2,6 @@ export enum Feature { USER_GROUPS = "userGroups", APP_BACKUPS = "appBackups", ENVIRONMENT_VARIABLES = "environmentVariables", + AUDIT_LOGS = "auditLogs", ENFORCEABLE_SSO = "enforceableSSO", } diff --git a/packages/worker/jest.config.ts b/packages/worker/jest.config.ts index cdacfa411a..3655479d82 100644 --- a/packages/worker/jest.config.ts +++ b/packages/worker/jest.config.ts @@ -7,7 +7,7 @@ const config: Config.InitialOptions = { preset: "@trendyol/jest-testcontainers", setupFiles: ["./src/tests/jestEnv.ts"], setupFilesAfterEnv: ["./src/tests/jestSetup.ts"], - collectCoverageFrom: ["src/**/*.{js,ts}"], + collectCoverageFrom: ["src/**/*.{js,ts}", "../backend-core/src/**/*.{js,ts}"], coverageReporters: ["lcov", "json", "clover"], transform: { "^.+\\.ts?$": "@swc/jest", diff --git a/packages/worker/package.json b/packages/worker/package.json index 1f5d4fb36a..84b5d69ee0 100644 --- a/packages/worker/package.json +++ b/packages/worker/package.json @@ -60,6 +60,7 @@ "koa-send": "5.0.1", "koa-session": "5.13.1", "koa-static": "5.0.0", + "koa-useragent": "^4.1.0", "node-fetch": "2.6.7", "nodemailer": "6.7.2", "passport-google-oauth": "2.0.0", diff --git a/packages/worker/src/api/controllers/global/auth.ts b/packages/worker/src/api/controllers/global/auth.ts index 9336ba0d5b..92cf014a48 100644 --- a/packages/worker/src/api/controllers/global/auth.ts +++ b/packages/worker/src/api/controllers/global/auth.ts @@ -69,8 +69,8 @@ export const login = async (ctx: Ctx, next: any) => { "local", async (err: any, user: User, info: any) => { await passportCallback(ctx, user, err, info) - await context.identity.doInUserContext(user, async () => { - await events.auth.login("local") + await context.identity.doInUserContext(user, ctx, async () => { + await events.auth.login("local", user.email) }) ctx.status = 200 } @@ -207,8 +207,8 @@ export const googleCallback = async (ctx: any, next: any) => { { successRedirect: "/", failureRedirect: "/error" }, async (err: any, user: SSOUser, info: any) => { await passportCallback(ctx, user, err, info) - await context.identity.doInUserContext(user, async () => { - await events.auth.login("google-internal") + await context.identity.doInUserContext(user, ctx, async () => { + await events.auth.login("google-internal", user.email) }) ctx.redirect("/") } @@ -272,8 +272,8 @@ export const oidcCallback = async (ctx: any, next: any) => { { successRedirect: "/", failureRedirect: "/error" }, async (err: any, user: SSOUser, info: any) => { await passportCallback(ctx, user, err, info) - await context.identity.doInUserContext(user, async () => { - await events.auth.login("oidc") + await context.identity.doInUserContext(user, ctx, async () => { + await events.auth.login("oidc", user.email) }) ctx.redirect("/") } diff --git a/packages/worker/src/api/routes/global/tests/auditLogs.spec.ts b/packages/worker/src/api/routes/global/tests/auditLogs.spec.ts new file mode 100644 index 0000000000..19e3cd64b4 --- /dev/null +++ b/packages/worker/src/api/routes/global/tests/auditLogs.spec.ts @@ -0,0 +1,111 @@ +import { mocks, structures } from "@budibase/backend-core/tests" +import { context, events } from "@budibase/backend-core" +import { Event, IdentityType } from "@budibase/types" +import { TestConfiguration } from "../../../../tests" + +mocks.licenses.useAuditLogs() + +const BASE_IDENTITY = { + account: undefined, + type: IdentityType.USER, +} +const USER_AUDIT_LOG_COUNT = 3 +const APP_ID = "app_1" + +describe("/api/global/auditlogs", () => { + const config = new TestConfiguration() + + beforeAll(async () => { + await config.beforeAll() + }) + + afterAll(async () => { + await config.afterAll() + }) + + describe("POST /api/global/auditlogs/search", () => { + it("should be able to fire some events (create audit logs)", async () => { + await context.doInTenant(config.tenantId, async () => { + const userId = config.user!._id! + const identity = { + ...BASE_IDENTITY, + _id: userId, + tenantId: config.tenantId, + } + await context.doInIdentityContext(identity, async () => { + for (let i = 0; i < USER_AUDIT_LOG_COUNT; i++) { + await events.user.created(structures.users.user()) + } + await context.doInAppContext(APP_ID, async () => { + await events.app.created(structures.apps.app(APP_ID)) + }) + // fetch the user created events + const response = await config.api.auditLogs.search({ + events: [Event.USER_CREATED], + }) + expect(response.data).toBeDefined() + // there will be an initial event which comes from the default user creation + expect(response.data.length).toBe(USER_AUDIT_LOG_COUNT + 1) + }) + }) + }) + + it("should be able to search by event", async () => { + const response = await config.api.auditLogs.search({ + events: [Event.USER_CREATED], + }) + expect(response.data.length).toBeGreaterThan(0) + for (let log of response.data) { + expect(log.event).toBe(Event.USER_CREATED) + } + }) + + it("should be able to search by time range (frozen)", async () => { + // this is frozen, only need to add 1 and minus 1 + const now = new Date() + const start = new Date() + start.setSeconds(now.getSeconds() - 1) + const end = new Date() + end.setSeconds(now.getSeconds() + 1) + const response = await config.api.auditLogs.search({ + startDate: start.toISOString(), + endDate: end.toISOString(), + }) + expect(response.data.length).toBeGreaterThan(0) + for (let log of response.data) { + expect(log.timestamp).toBe(now.toISOString()) + } + }) + + it("should be able to search by user ID", async () => { + const userId = config.user!._id! + const response = await config.api.auditLogs.search({ + userIds: [userId], + }) + expect(response.data.length).toBeGreaterThan(0) + for (let log of response.data) { + expect(log.user._id).toBe(userId) + } + }) + + it("should be able to search by app ID", async () => { + const response = await config.api.auditLogs.search({ + appIds: [APP_ID], + }) + expect(response.data.length).toBeGreaterThan(0) + for (let log of response.data) { + expect(log.app?._id).toBe(APP_ID) + } + }) + + it("should be able to search by full string", async () => { + const response = await config.api.auditLogs.search({ + fullSearch: "User", + }) + expect(response.data.length).toBeGreaterThan(0) + for (let log of response.data) { + expect(log.name.includes("User")).toBe(true) + } + }) + }) +}) diff --git a/packages/worker/src/api/routes/global/tests/auth.spec.ts b/packages/worker/src/api/routes/global/tests/auth.spec.ts index b6be1fd7af..9b5392fc73 100644 --- a/packages/worker/src/api/routes/global/tests/auth.spec.ts +++ b/packages/worker/src/api/routes/global/tests/auth.spec.ts @@ -367,7 +367,7 @@ describe("/api/global/auth", () => { const res = await config.api.configs.OIDCCallback(configId, preAuthRes) - expect(events.auth.login).toBeCalledWith("oidc") + expect(events.auth.login).toBeCalledWith("oidc", "oauth@example.com") expect(events.auth.login).toBeCalledTimes(1) expect(res.status).toBe(302) const location: string = res.get("location") diff --git a/packages/worker/src/api/routes/index.ts b/packages/worker/src/api/routes/index.ts index 3aa9422238..c64ad44423 100644 --- a/packages/worker/src/api/routes/index.ts +++ b/packages/worker/src/api/routes/index.ts @@ -18,6 +18,8 @@ import accountRoutes from "./system/accounts" import restoreRoutes from "./system/restore" let userGroupRoutes = api.groups +let auditLogRoutes = api.auditLogs + export const routes: Router[] = [ configRoutes, userRoutes, @@ -32,6 +34,7 @@ export const routes: Router[] = [ selfRoutes, licenseRoutes, userGroupRoutes, + auditLogRoutes, migrationRoutes, accountRoutes, restoreRoutes, diff --git a/packages/worker/src/db/index.ts b/packages/worker/src/db/index.ts index d74d00d910..157c2f4fb3 100644 --- a/packages/worker/src/db/index.ts +++ b/packages/worker/src/db/index.ts @@ -1,10 +1,16 @@ import * as core from "@budibase/backend-core" import env from "../environment" -export const init = () => { - const dbConfig: any = {} +export function init() { + const dbConfig: any = { + replication: true, + find: true, + } + if (env.isTest() && !env.COUCH_DB_URL) { dbConfig.inMemory = true + dbConfig.allDbs = true } + core.init({ db: dbConfig }) } diff --git a/packages/worker/src/index.ts b/packages/worker/src/index.ts index 6e3a88bf20..04413e8429 100644 --- a/packages/worker/src/index.ts +++ b/packages/worker/src/index.ts @@ -13,11 +13,13 @@ import { Event } from "@sentry/types/dist/event" import Application from "koa" import { bootstrap } from "global-agent" import * as db from "./db" +import { sdk as proSdk } from "@budibase/pro" import { auth, logging, events, middleware, + queue, env as coreEnv, } from "@budibase/backend-core" db.init() @@ -29,8 +31,14 @@ import * as redis from "./utilities/redis" const Sentry = require("@sentry/node") const koaSession = require("koa-session") const logger = require("koa-pino-logger") +const { userAgent } = require("koa-useragent") + import destroyable from "server-destroy" +// configure events to use the pro audit log write +// can't integrate directly into backend-core due to cyclic issues +events.processors.init(proSdk.auditLogs.write) + if (coreEnv.ENABLE_SSO_MAINTENANCE_MODE) { console.warn( "Warning: ENABLE_SSO_MAINTENANCE_MODE is set. It is recommended this flag is disabled if maintenance is not in progress" @@ -49,6 +57,7 @@ app.use(koaBody({ multipart: true })) app.use(koaSession(app)) app.use(middleware.logging) app.use(logger(logging.pinoSettings())) +app.use(userAgent) // authentication app.use(auth.passport.initialize()) @@ -84,6 +93,7 @@ server.on("close", async () => { console.log("Server Closed") await redis.shutdown() await events.shutdown() + await queue.shutdown() if (!env.isTest()) { process.exit(errCode) } diff --git a/packages/worker/src/sdk/users/users.ts b/packages/worker/src/sdk/users/users.ts index b0ca22e44a..18d5a04cda 100644 --- a/packages/worker/src/sdk/users/users.ts +++ b/packages/worker/src/sdk/users/users.ts @@ -637,7 +637,7 @@ export const invite = async ( } await sendEmail(user.email, EmailTemplatePurpose.INVITATION, opts) response.successful.push({ email: user.email }) - await events.user.invited() + await events.user.invited(user.email) } catch (e) { console.error(`Failed to send email invitation email=${user.email}`, e) response.unsuccessful.push({ diff --git a/packages/worker/src/tests/api/auditLogs.ts b/packages/worker/src/tests/api/auditLogs.ts new file mode 100644 index 0000000000..d7bc4d99fb --- /dev/null +++ b/packages/worker/src/tests/api/auditLogs.ts @@ -0,0 +1,26 @@ +import { AuditLogSearchParams, SearchAuditLogsResponse } from "@budibase/types" +import TestConfiguration from "../TestConfiguration" +import { TestAPI } from "./base" + +export class AuditLogAPI extends TestAPI { + constructor(config: TestConfiguration) { + super(config) + } + + search = async (search: AuditLogSearchParams) => { + const res = await this.request + .post("/api/global/auditlogs/search") + .send(search) + .set(this.config.defaultHeaders()) + .expect("Content-Type", /json/) + .expect(200) + return res.body as SearchAuditLogsResponse + } + + download = (search: AuditLogSearchParams) => { + const query = encodeURIComponent(JSON.stringify(search)) + return this.request + .get(`/api/global/auditlogs/download?query=${query}`) + .set(this.config.defaultHeaders()) + } +} diff --git a/packages/worker/src/tests/api/index.ts b/packages/worker/src/tests/api/index.ts index 0bd0308e2f..166996e792 100644 --- a/packages/worker/src/tests/api/index.ts +++ b/packages/worker/src/tests/api/index.ts @@ -14,6 +14,7 @@ import { GroupsAPI } from "./groups" import { RolesAPI } from "./roles" import { TemplatesAPI } from "./templates" import { LicenseAPI } from "./license" +import { AuditLogAPI } from "./auditLogs" export default class API { accounts: AccountAPI auth: AuthAPI @@ -30,6 +31,7 @@ export default class API { roles: RolesAPI templates: TemplatesAPI license: LicenseAPI + auditLogs: AuditLogAPI constructor(config: TestConfiguration) { this.accounts = new AccountAPI(config) @@ -47,5 +49,6 @@ export default class API { this.roles = new RolesAPI(config) this.templates = new TemplatesAPI(config) this.license = new LicenseAPI(config) + this.auditLogs = new AuditLogAPI(config) } } diff --git a/packages/worker/yarn.lock b/packages/worker/yarn.lock index 4abc720cf1..dc2b635259 100644 --- a/packages/worker/yarn.lock +++ b/packages/worker/yarn.lock @@ -3653,6 +3653,11 @@ expect@^28.1.3: jest-message-util "^28.1.3" jest-util "^28.1.3" +express-useragent@^1.0.15: + version "1.0.15" + resolved "https://registry.yarnpkg.com/express-useragent/-/express-useragent-1.0.15.tgz#cefda5fa4904345d51d3368b117a8dd4124985d9" + integrity sha512-eq5xMiYCYwFPoekffMjvEIk+NWdlQY9Y38OsTyl13IvA728vKT+q/CSERYWzcw93HGBJcIqMIsZC5CZGARPVdg== + extend@~3.0.2: version "3.0.2" resolved "https://registry.yarnpkg.com/extend/-/extend-3.0.2.tgz#f8b1136b4071fbd8eb140aff858b1019ec2915fa" @@ -5467,6 +5472,13 @@ koa-static@5.0.0: debug "^3.1.0" koa-send "^5.0.0" +koa-useragent@^4.1.0: + version "4.1.0" + resolved "https://registry.yarnpkg.com/koa-useragent/-/koa-useragent-4.1.0.tgz#d3f128b552c6da3e5e9e9e9c887b2922b16e4468" + integrity sha512-x/HUDZ1zAmNNh5hA9hHbPm9p3UVg2prlpHzxCXQCzbibrNS0kmj7MkCResCbAbG7ZT6FVxNSMjR94ZGamdMwxA== + dependencies: + express-useragent "^1.0.15" + koa@2.13.4, koa@^2.13.4: version "2.13.4" resolved "https://registry.yarnpkg.com/koa/-/koa-2.13.4.tgz#ee5b0cb39e0b8069c38d115139c774833d32462e" diff --git a/scripts/link-dependencies.sh b/scripts/link-dependencies.sh index d2a501162b..9926f3dd2b 100755 --- a/scripts/link-dependencies.sh +++ b/scripts/link-dependencies.sh @@ -44,6 +44,9 @@ if [ -d "../budibase-pro" ]; then echo "Linking types to pro" yarn link '@budibase/types' + echo "Linking string-templates to pro" + yarn link '@budibase/string-templates' + cd ../../../budibase echo "Linking pro to worker" diff --git a/yarn.lock b/yarn.lock index f80cde1b6f..bcd33ad2b6 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1064,6 +1064,11 @@ resolved "https://registry.yarnpkg.com/@typescript-eslint/types/-/types-5.45.0.tgz#794760b9037ee4154c09549ef5a96599621109c5" integrity sha512-QQij+u/vgskA66azc9dCmx+rev79PzX8uDHpsqSjEFtfF2gBUTRCpvYMh2gw2ghkJabNkPlSUCimsyBEQZd1DA== +"@typescript-eslint/types@5.53.0": + version "5.53.0" + resolved "https://registry.yarnpkg.com/@typescript-eslint/types/-/types-5.53.0.tgz#f79eca62b97e518ee124086a21a24f3be267026f" + integrity sha512-5kcDL9ZUIP756K6+QOAfPkigJmCPHcLN7Zjdz76lQWWDdzfOhZDTj1irs6gPBKiXx5/6O3L0+AvupAut3z7D2A== + "@typescript-eslint/typescript-estree@5.45.0": version "5.45.0" resolved "https://registry.yarnpkg.com/@typescript-eslint/typescript-estree/-/typescript-estree-5.45.0.tgz#f70a0d646d7f38c0dfd6936a5e171a77f1e5291d" @@ -1090,6 +1095,19 @@ semver "^7.3.5" tsutils "^3.21.0" +"@typescript-eslint/typescript-estree@^5.13.0": + version "5.53.0" + resolved "https://registry.yarnpkg.com/@typescript-eslint/typescript-estree/-/typescript-estree-5.53.0.tgz#bc651dc28cf18ab248ecd18a4c886c744aebd690" + integrity sha512-eKmipH7QyScpHSkhbptBBYh9v8FxtngLquq292YTEQ1pxVs39yFBlLC1xeIZcPPz1RWGqb7YgERJRGkjw8ZV7w== + dependencies: + "@typescript-eslint/types" "5.53.0" + "@typescript-eslint/visitor-keys" "5.53.0" + debug "^4.3.4" + globby "^11.1.0" + is-glob "^4.0.3" + semver "^7.3.7" + tsutils "^3.21.0" + "@typescript-eslint/visitor-keys@4.33.0": version "4.33.0" resolved "https://registry.yarnpkg.com/@typescript-eslint/visitor-keys/-/visitor-keys-4.33.0.tgz#2a22f77a41604289b7a186586e9ec48ca92ef1dd" @@ -1106,6 +1124,14 @@ "@typescript-eslint/types" "5.45.0" eslint-visitor-keys "^3.3.0" +"@typescript-eslint/visitor-keys@5.53.0": + version "5.53.0" + resolved "https://registry.yarnpkg.com/@typescript-eslint/visitor-keys/-/visitor-keys-5.53.0.tgz#8a5126623937cdd909c30d8fa72f79fa56cc1a9f" + integrity sha512-JqNLnX3leaHFZEN0gCh81sIvgrp/2GOACZNgO4+Tkf64u51kTpAyWFOY8XHx8XuXr3N2C9zgPPHtcpMg6z1g0w== + dependencies: + "@typescript-eslint/types" "5.53.0" + eslint-visitor-keys "^3.3.0" + JSONStream@^1.0.4, JSONStream@^1.3.4: version "1.3.5" resolved "https://registry.yarnpkg.com/JSONStream/-/JSONStream-1.3.5.tgz#3208c1f08d3a4d99261ab64f92302bc15e111ca0" @@ -1214,6 +1240,11 @@ ansi-styles@^4.0.0, ansi-styles@^4.1.0: dependencies: color-convert "^2.0.1" +any-promise@^1.1.0: + version "1.3.0" + resolved "https://registry.yarnpkg.com/any-promise/-/any-promise-1.3.0.tgz#abc6afeedcea52e809cdc0376aed3ce39635d17f" + integrity sha512-7UvmKalWRt1wgjL1RrGxoSJW/0QZFIegpeGvZG9kjp8vrRu55XTHbwnqq2GpXm9uLbcuhxm3IqX9OB4MZR1b2A== + app-module-path@^2.2.0: version "2.2.0" resolved "https://registry.yarnpkg.com/app-module-path/-/app-module-path-2.2.0.tgz#641aa55dfb7d6a6f0a8141c4b9c0aa50b6c24dd5" @@ -1793,6 +1824,11 @@ commander@^7.2.0: resolved "https://registry.yarnpkg.com/commander/-/commander-7.2.0.tgz#a36cb57d0b501ce108e4d20559a150a391d97ab7" integrity sha512-QrWXB+ZQSVPmIWIhtEO9H+gwHaMGYiF5ChvoJ+K9ZGHG/sVsa6yiesAD1GC/x46sET00Xlwo1u49RVVVzvcSkw== +commander@^9.1.0: + version "9.5.0" + resolved "https://registry.yarnpkg.com/commander/-/commander-9.5.0.tgz#bc08d1eb5cedf7ccb797a96199d41c7bc3e60d30" + integrity sha512-KRs7WVDKg86PWiuAqhDrAQnTXZKraVcCc6vFdL14qrZ/DcWwuRo7VoiYXalXO7S5GKpqYiVEwCbgFDfxNHKJBQ== + commondir@^1.0.1: version "1.0.1" resolved "https://registry.yarnpkg.com/commondir/-/commondir-1.0.1.tgz#ddd800da0c66127393cca5950ea968a3aaf1253b" @@ -2139,16 +2175,16 @@ delegates@^1.0.0: resolved "https://registry.yarnpkg.com/delegates/-/delegates-1.0.0.tgz#84c6e159b81904fdca59a0ef44cd870d31250f9a" integrity sha512-bd2L678uiWATM6m5Z1VzNCErI3jiGzt6HGY8OVICs40JQq/HALfbyNJmp0UDakEY4pMMaN0Ly5om/B1VI/+xfQ== -dependency-tree@^8.1.1: - version "8.1.2" - resolved "https://registry.yarnpkg.com/dependency-tree/-/dependency-tree-8.1.2.tgz#c9e652984f53bd0239bc8a3e50cbd52f05b2e770" - integrity sha512-c4CL1IKxkKng0oT5xrg4uNiiMVFqTGOXqHSFx7XEFdgSsp6nw3AGGruICppzJUrfad/r7GLqt26rmWU4h4j39A== +dependency-tree@^9.0.0: + version "9.0.0" + resolved "https://registry.yarnpkg.com/dependency-tree/-/dependency-tree-9.0.0.tgz#9288dd6daf35f6510c1ea30d9894b75369aa50a2" + integrity sha512-osYHZJ1fBSon3lNLw70amAXsQ+RGzXsPvk9HbBgTLbp/bQBmpH5mOmsUvqXU+YEWVU0ZLewsmzOET/8jWswjDQ== dependencies: commander "^2.20.3" debug "^4.3.1" filing-cabinet "^3.0.1" - precinct "^8.0.0" - typescript "^3.9.7" + precinct "^9.0.0" + typescript "^4.0.0" deprecation@^2.0.0, deprecation@^2.3.1: version "2.3.1" @@ -2170,6 +2206,16 @@ detective-amd@^3.1.0: get-amd-module-type "^3.0.0" node-source-walk "^4.2.0" +detective-amd@^4.0.1: + version "4.0.1" + resolved "https://registry.yarnpkg.com/detective-amd/-/detective-amd-4.0.1.tgz#1a827d9e4fa2f832506bd87aa392f124155bca3a" + integrity sha512-bDo22IYbJ8yzALB0Ow5CQLtyhU1BpDksLB9dsWHI9Eh0N3OQR6aQqhjPsNDd69ncYwRfL1sTo7OA9T3VRVSe2Q== + dependencies: + ast-module-types "^3.0.0" + escodegen "^2.0.0" + get-amd-module-type "^4.0.0" + node-source-walk "^5.0.0" + detective-cjs@^3.1.1: version "3.1.3" resolved "https://registry.yarnpkg.com/detective-cjs/-/detective-cjs-3.1.3.tgz#50e107d67b37f459b0ec02966ceb7e20a73f268b" @@ -2178,13 +2224,28 @@ detective-cjs@^3.1.1: ast-module-types "^3.0.0" node-source-walk "^4.0.0" -detective-es6@^2.2.0, detective-es6@^2.2.1: +detective-cjs@^4.0.0: + version "4.0.0" + resolved "https://registry.yarnpkg.com/detective-cjs/-/detective-cjs-4.0.0.tgz#316e2c7ae14276a5dcfe0c43dc05d8cf9a0e5cf9" + integrity sha512-VsD6Yo1+1xgxJWoeDRyut7eqZ8EWaJI70C5eanSAPcBHzenHZx0uhjxaaEfIm0cHII7dBiwU98Orh44bwXN2jg== + dependencies: + ast-module-types "^3.0.0" + node-source-walk "^5.0.0" + +detective-es6@^2.2.1: version "2.2.2" resolved "https://registry.yarnpkg.com/detective-es6/-/detective-es6-2.2.2.tgz#ee5f880981d9fecae9a694007029a2f6f26d8d28" integrity sha512-eZUKCUsbHm8xoeoCM0z6JFwvDfJ5Ww5HANo+jPR7AzkFpW9Mun3t/TqIF2jjeWa2TFbAiGaWESykf2OQp3oeMw== dependencies: node-source-walk "^4.0.0" +detective-es6@^3.0.0: + version "3.0.0" + resolved "https://registry.yarnpkg.com/detective-es6/-/detective-es6-3.0.0.tgz#78c9ef5492d5b59748b411aecaaf52b5ca0f0bc2" + integrity sha512-Uv2b5Uih7vorYlqGzCX+nTPUb4CMzUAn3VPHTV5p5lBkAN4cAApLGgUz4mZE2sXlBfv4/LMmeP7qzxHV/ZcfWA== + dependencies: + node-source-walk "^5.0.0" + detective-less@^1.0.2: version "1.0.2" resolved "https://registry.yarnpkg.com/detective-less/-/detective-less-1.0.2.tgz#a68af9ca5f69d74b7d0aa190218b211d83b4f7e3" @@ -2204,14 +2265,14 @@ detective-postcss@^4.0.0: postcss "^8.1.7" postcss-values-parser "^2.0.1" -detective-postcss@^5.0.0: - version "5.1.1" - resolved "https://registry.yarnpkg.com/detective-postcss/-/detective-postcss-5.1.1.tgz#ec23ac3818f8be95ac3a38a8b9f3b6d43103ef87" - integrity sha512-YJMsvA0Y6/ST9abMNcQytl9iFQ2bfu4I7B74IUiAvyThfaI9Y666yipL+SrqfReoIekeIEwmGH72oeqX63mwUw== +detective-postcss@^6.0.1, detective-postcss@^6.1.0: + version "6.1.0" + resolved "https://registry.yarnpkg.com/detective-postcss/-/detective-postcss-6.1.0.tgz#01ca6f83dbc3158540fb0e6a6c631f3998946e54" + integrity sha512-ZFZnEmUrL2XHAC0j/4D1fdwZbo/anAcK84soJh7qc7xfx2Kc8gFO5Bk5I9jU7NLC/OAF1Yho1GLxEDnmQnRH2A== dependencies: is-url "^1.2.4" - postcss "^8.4.6" - postcss-values-parser "^5.0.0" + postcss "^8.4.12" + postcss-values-parser "^6.0.2" detective-sass@^3.0.1: version "3.0.2" @@ -2221,6 +2282,14 @@ detective-sass@^3.0.1: gonzales-pe "^4.3.0" node-source-walk "^4.0.0" +detective-sass@^4.0.1: + version "4.0.1" + resolved "https://registry.yarnpkg.com/detective-sass/-/detective-sass-4.0.1.tgz#2a9e303a0bd472d00aaa79512334845e3acbaffc" + integrity sha512-80zfpxux1krOrkxCHbtwvIs2gNHUBScnSqlGl0FvUuHVz8HD6vD2ov66OroMctyvzhM67fxhuEeVjIk18s6yTQ== + dependencies: + gonzales-pe "^4.3.0" + node-source-walk "^5.0.0" + detective-scss@^2.0.1: version "2.0.2" resolved "https://registry.yarnpkg.com/detective-scss/-/detective-scss-2.0.2.tgz#7d2a642616d44bf677963484fa8754d9558b8235" @@ -2229,11 +2298,24 @@ detective-scss@^2.0.1: gonzales-pe "^4.3.0" node-source-walk "^4.0.0" +detective-scss@^3.0.0: + version "3.0.0" + resolved "https://registry.yarnpkg.com/detective-scss/-/detective-scss-3.0.0.tgz#c3c7bc4799f51515a4f0ed1e8ca491151364230f" + integrity sha512-37MB/mhJyS45ngqfzd6eTbuLMoDgdZnH03ZOMW2m9WqJ/Rlbuc8kZAr0Ypovaf1DJiTRzy5mmxzOTja85jbzlA== + dependencies: + gonzales-pe "^4.3.0" + node-source-walk "^5.0.0" + detective-stylus@^1.0.0: version "1.0.3" resolved "https://registry.yarnpkg.com/detective-stylus/-/detective-stylus-1.0.3.tgz#20a702936c9fd7d4203fd7a903314b5dd43ac713" integrity sha512-4/bfIU5kqjwugymoxLXXLltzQNeQfxGoLm2eIaqtnkWxqbhap9puDVpJPVDx96hnptdERzS5Cy6p9N8/08A69Q== +detective-stylus@^2.0.0, detective-stylus@^2.0.1: + version "2.0.1" + resolved "https://registry.yarnpkg.com/detective-stylus/-/detective-stylus-2.0.1.tgz#d528dfa7ef3c4eb2fbc9a7249d54906ec4e05d09" + integrity sha512-/Tvs1pWLg8eYwwV6kZQY5IslGaYqc/GACxjcaGudiNtN5nKCH6o2WnJK3j0gA3huCnoQcbv8X7oz/c1lnvE3zQ== + detective-typescript@^7.0.0: version "7.0.2" resolved "https://registry.yarnpkg.com/detective-typescript/-/detective-typescript-7.0.2.tgz#c6e00b4c28764741ef719662250e6b014a5f3c8e" @@ -2244,6 +2326,16 @@ detective-typescript@^7.0.0: node-source-walk "^4.2.0" typescript "^3.9.10" +detective-typescript@^9.0.0: + version "9.0.0" + resolved "https://registry.yarnpkg.com/detective-typescript/-/detective-typescript-9.0.0.tgz#57d674cec49ec775460ab975b5bcbb5c2d32ff8e" + integrity sha512-lR78AugfUSBojwlSRZBeEqQ1l8LI7rbxOl1qTUnGLcjZQDjZmrZCb7R46rK8U8B5WzFvJrxa7fEBA8FoD/n5fA== + dependencies: + "@typescript-eslint/typescript-estree" "^5.13.0" + ast-module-types "^3.0.0" + node-source-walk "^5.0.0" + typescript "^4.5.5" + dezalgo@^1.0.0: version "1.0.4" resolved "https://registry.yarnpkg.com/dezalgo/-/dezalgo-1.0.4.tgz#751235260469084c132157dfa857f386d4c33d81" @@ -2937,6 +3029,14 @@ get-amd-module-type@^3.0.0: ast-module-types "^3.0.0" node-source-walk "^4.2.2" +get-amd-module-type@^4.0.0: + version "4.0.0" + resolved "https://registry.yarnpkg.com/get-amd-module-type/-/get-amd-module-type-4.0.0.tgz#3d4e5b44eec81f8337157d7c52c4fa9389aff78b" + integrity sha512-GbBawUCuA2tY8ztiMiVo3e3P95gc2TVrfYFfpUHdHQA8WyxMCckK29bQsVKhYX8SUf+w6JLhL2LG8tSC0ANt9Q== + dependencies: + ast-module-types "^3.0.0" + node-source-walk "^5.0.0" + get-caller-file@^1.0.1: version "1.0.3" resolved "https://registry.yarnpkg.com/get-caller-file/-/get-caller-file-1.0.3.tgz#f978fa4c90d1dfe7ff2d6beda2a515e713bdcf4a" @@ -3148,13 +3248,6 @@ graceful-fs@^4.1.11, graceful-fs@^4.1.15, graceful-fs@^4.1.2, graceful-fs@^4.1.6 resolved "https://registry.yarnpkg.com/graceful-fs/-/graceful-fs-4.2.10.tgz#147d3a006da4ca3ce14728c7aefc287c367d7a6c" integrity sha512-9ByhssR2fPVsNZj478qUUbKfmL0+t5BDVyjShtyZZLiK7ZDAArFFfopyOTj0M05wE2tJPisA4iTnnXl2YoPvOA== -graphviz@0.0.9: - version "0.0.9" - resolved "https://registry.yarnpkg.com/graphviz/-/graphviz-0.0.9.tgz#0bbf1df588c6a92259282da35323622528c4bbc4" - integrity sha512-SmoY2pOtcikmMCqCSy2NO1YsRfu9OO0wpTlOYW++giGjfX1a6gax/m1Fo8IdUd0/3H15cTOfR1SMKwohj4LKsg== - dependencies: - temp "~0.4.0" - handlebars@^4.7.6: version "4.7.7" resolved "https://registry.yarnpkg.com/handlebars/-/handlebars-4.7.7.tgz#9ce33416aad02dbd6c8fafa8240d5d98004945a1" @@ -4167,31 +4260,32 @@ macos-release@^2.2.0: resolved "https://registry.yarnpkg.com/macos-release/-/macos-release-2.5.0.tgz#067c2c88b5f3fb3c56a375b2ec93826220fa1ff2" integrity sha512-EIgv+QZ9r+814gjJj0Bt5vSLJLzswGmSUbUpbi9AIr/fsN2IWFBl2NucV9PAiek+U1STK468tEkxmVYUtuAN3g== -madge@^5.0.1: - version "5.0.1" - resolved "https://registry.yarnpkg.com/madge/-/madge-5.0.1.tgz#2096d9006558ea0669b3ade89c2cda708a24e22b" - integrity sha512-krmSWL9Hkgub74bOjnjWRoFPAJvPwSG6Dbta06qhWOq6X/n/FPzO3ESZvbFYVIvG2g4UHXvCJN1b+RZLaSs9nA== +madge@^6.0.0: + version "6.0.0" + resolved "https://registry.yarnpkg.com/madge/-/madge-6.0.0.tgz#d17d68d1023376318cae89abd16629d329f4ed0a" + integrity sha512-dddxP62sj5pL+l9MJnq9C34VFqmRj+2+uSOdn/7lOTSliLRH0WyQ8uCEF3VxkPRNOBvMKK2xumnIE15HRSAL9A== dependencies: chalk "^4.1.1" commander "^7.2.0" commondir "^1.0.1" debug "^4.3.1" - dependency-tree "^8.1.1" - detective-amd "^3.1.0" - detective-cjs "^3.1.1" - detective-es6 "^2.2.0" + dependency-tree "^9.0.0" + detective-amd "^4.0.1" + detective-cjs "^4.0.0" + detective-es6 "^3.0.0" detective-less "^1.0.2" - detective-postcss "^5.0.0" - detective-sass "^3.0.1" - detective-scss "^2.0.1" - detective-stylus "^1.0.0" - detective-typescript "^7.0.0" - graphviz "0.0.9" + detective-postcss "^6.1.0" + detective-sass "^4.0.1" + detective-scss "^3.0.0" + detective-stylus "^2.0.1" + detective-typescript "^9.0.0" ora "^5.4.1" pluralize "^8.0.0" precinct "^8.1.0" pretty-ms "^7.0.1" rc "^1.2.7" + stream-to-array "^2.3.0" + ts-graphviz "^1.5.0" typescript "^3.9.5" walkdir "^0.4.1" @@ -4481,6 +4575,14 @@ module-definition@^3.3.1: ast-module-types "^3.0.0" node-source-walk "^4.0.0" +module-definition@^4.0.0: + version "4.0.0" + resolved "https://registry.yarnpkg.com/module-definition/-/module-definition-4.0.0.tgz#5b39cca9be28c5b0fec768eb2d9fd8de08a2550b" + integrity sha512-wntiAHV4lDn24BQn2kX6LKq0y85phHLHiv3aOPDF+lIs06kVjEMTe/ZTdrbVLnQV5FQsjik21taknvMhKY1Cug== + dependencies: + ast-module-types "^3.0.0" + node-source-walk "^5.0.0" + module-lookup-amd@^7.0.1: version "7.0.1" resolved "https://registry.yarnpkg.com/module-lookup-amd/-/module-lookup-amd-7.0.1.tgz#d67c1a93f2ff8e38b8774b99a638e9a4395774b2" @@ -4616,6 +4718,13 @@ node-source-walk@^4.0.0, node-source-walk@^4.2.0, node-source-walk@^4.2.2: dependencies: "@babel/parser" "^7.0.0" +node-source-walk@^5.0.0: + version "5.0.0" + resolved "https://registry.yarnpkg.com/node-source-walk/-/node-source-walk-5.0.0.tgz#7cf93a0d12408081531fc440a00d7019eb3d5665" + integrity sha512-58APXoMXpmmU+oVBJFajhTCoD8d/OGtngnVAWzIo2A8yn0IXwBzvIVIsTzoie/SrA37u+1hnpNz2HMWx/VIqlw== + dependencies: + "@babel/parser" "^7.0.0" + "nopt@2 || 3": version "3.0.6" resolved "https://registry.yarnpkg.com/nopt/-/nopt-3.0.6.tgz#c6465dbf08abcd4db359317f79ac68a646b28ff9" @@ -5246,16 +5355,16 @@ postcss-values-parser@^2.0.1: indexes-of "^1.0.1" uniq "^1.0.1" -postcss-values-parser@^5.0.0: - version "5.0.0" - resolved "https://registry.yarnpkg.com/postcss-values-parser/-/postcss-values-parser-5.0.0.tgz#10c61ac3f488e4de25746b829ea8d8894e9ac3d2" - integrity sha512-2viDDjMMrt21W2izbeiJxl3kFuD/+asgB0CBwPEgSyhCmBnDIa/y+pLaoyX+q3I3DHH0oPPL3cgjVTQvlS1Maw== +postcss-values-parser@^6.0.2: + version "6.0.2" + resolved "https://registry.yarnpkg.com/postcss-values-parser/-/postcss-values-parser-6.0.2.tgz#636edc5b86c953896f1bb0d7a7a6615df00fb76f" + integrity sha512-YLJpK0N1brcNJrs9WatuJFtHaV9q5aAOj+S4DI5S7jgHlRfm0PIbDCAFRYMQD5SHq7Fy6xsDhyutgS0QOAs0qw== dependencies: color-name "^1.1.4" is-url-superb "^4.0.0" quote-unquote "^1.0.0" -postcss@^8.1.7, postcss@^8.4.6: +postcss@^8.1.7: version "8.4.16" resolved "https://registry.yarnpkg.com/postcss/-/postcss-8.4.16.tgz#33a1d675fac39941f5f445db0de4db2b6e01d43c" integrity sha512-ipHE1XBvKzm5xI7hiHCZJCSugxvsdq2mPnsq5+UF+VHCjiBvtDrlxJfMBToWaP9D5XlgNmcFGqoHmUn0EYEaRQ== @@ -5264,7 +5373,16 @@ postcss@^8.1.7, postcss@^8.4.6: picocolors "^1.0.0" source-map-js "^1.0.2" -precinct@^8.0.0, precinct@^8.1.0: +postcss@^8.4.12: + version "8.4.21" + resolved "https://registry.yarnpkg.com/postcss/-/postcss-8.4.21.tgz#c639b719a57efc3187b13a1d765675485f4134f4" + integrity sha512-tP7u/Sn/dVxK2NnruI4H9BG+x+Wxz6oeZ1cJ8P6G/PZY0IKk4k/63TDsQf2kQq3+qoJeLm2kIBUNlZe3zgb4Zg== + dependencies: + nanoid "^3.3.4" + picocolors "^1.0.0" + source-map-js "^1.0.2" + +precinct@^8.1.0: version "8.3.1" resolved "https://registry.yarnpkg.com/precinct/-/precinct-8.3.1.tgz#94b99b623df144eed1ce40e0801c86078466f0dc" integrity sha512-pVppfMWLp2wF68rwHqBIpPBYY8Kd12lDhk8LVQzOwqllifVR15qNFyod43YLyFpurKRZQKnE7E4pofAagDOm2Q== @@ -5283,6 +5401,24 @@ precinct@^8.0.0, precinct@^8.1.0: module-definition "^3.3.1" node-source-walk "^4.2.0" +precinct@^9.0.0: + version "9.0.1" + resolved "https://registry.yarnpkg.com/precinct/-/precinct-9.0.1.tgz#64e7ea0de4bea1b73572e50531dfe297c7b8a7c7" + integrity sha512-hVNS6JvfvlZ64B3ezKeGAcVhIuOvuAiSVzagHX/+KjVPkYWoCNkfyMgCl1bjDtAFQSlzi95NcS9ykUWrl1L1vA== + dependencies: + commander "^9.1.0" + detective-amd "^4.0.1" + detective-cjs "^4.0.0" + detective-es6 "^3.0.0" + detective-less "^1.0.2" + detective-postcss "^6.0.1" + detective-sass "^4.0.1" + detective-scss "^3.0.0" + detective-stylus "^2.0.0" + detective-typescript "^9.0.0" + module-definition "^4.0.0" + node-source-walk "^5.0.0" + prelude-ls@^1.2.1: version "1.2.1" resolved "https://registry.yarnpkg.com/prelude-ls/-/prelude-ls-1.2.1.tgz#debc6489d7a6e6b0e7611888cec880337d316396" @@ -6168,6 +6304,13 @@ stream-shift@^1.0.0: resolved "https://registry.yarnpkg.com/stream-shift/-/stream-shift-1.0.1.tgz#d7088281559ab2778424279b0877da3c392d5a3d" integrity sha512-AiisoFqQ0vbGcZgQPY1cdP2I76glaVA/RauYR4G4thNFgkTqr90yXTo4LYX60Jl+sIlPNHHdGSwo01AvbKUSVQ== +stream-to-array@^2.3.0: + version "2.3.0" + resolved "https://registry.yarnpkg.com/stream-to-array/-/stream-to-array-2.3.0.tgz#bbf6b39f5f43ec30bc71babcb37557acecf34353" + integrity sha512-UsZtOYEn4tWU2RGLOXr/o/xjRBftZRlG3dEWoaHr8j4GuypJ3isitGbVyjQKAuMu+xbiop8q224TjiZWc4XTZA== + dependencies: + any-promise "^1.1.0" + strict-uri-encode@^2.0.0: version "2.0.0" resolved "https://registry.yarnpkg.com/strict-uri-encode/-/strict-uri-encode-2.0.0.tgz#b9c7330c7042862f6b142dc274bbcc5866ce3546" @@ -6401,11 +6544,6 @@ temp-write@^3.4.0: temp-dir "^1.0.0" uuid "^3.0.1" -temp@~0.4.0: - version "0.4.0" - resolved "https://registry.yarnpkg.com/temp/-/temp-0.4.0.tgz#671ad63d57be0fe9d7294664b3fc400636678a60" - integrity sha512-IsFisGgDKk7qzK9erMIkQe/XwiSUdac7z3wYOsjcLkhPBy3k1SlvLoIh2dAHIlEpgA971CgguMrx9z8fFg7tSA== - text-extensions@^1.0.0: version "1.9.0" resolved "https://registry.yarnpkg.com/text-extensions/-/text-extensions-1.9.0.tgz#1853e45fee39c945ce6f6c36b2d659b5aabc2a26" @@ -6523,6 +6661,11 @@ trim-newlines@^3.0.0: resolved "https://registry.yarnpkg.com/trim-newlines/-/trim-newlines-3.0.1.tgz#260a5d962d8b752425b32f3a7db0dcacd176c144" integrity sha512-c1PTsA3tYrIsLGkJkzHF+w9F2EyxfXGo4UyJc4pFL++FMjnq0HJS69T3M7d//gKrFKwy429bouPescbjecU+Zw== +ts-graphviz@^1.5.0: + version "1.5.4" + resolved "https://registry.yarnpkg.com/ts-graphviz/-/ts-graphviz-1.5.4.tgz#61a3059afeac4f6d4be3c6729a4d88546ca9e095" + integrity sha512-oxhI6wfPQBJC8WSiP0AjDI8z+9hcc9yl1etaynQJmQ2Yivn0KlT0oImLzJct7Es0TR/6xphzvJBAhGzgOjTGmQ== + tsconfig-paths@^3.10.1: version "3.14.1" resolved "https://registry.yarnpkg.com/tsconfig-paths/-/tsconfig-paths-3.14.1.tgz#ba0734599e8ea36c862798e920bcf163277b137a" @@ -6606,6 +6749,11 @@ typescript@^3.9.10, typescript@^3.9.5, typescript@^3.9.7: resolved "https://registry.yarnpkg.com/typescript/-/typescript-3.9.10.tgz#70f3910ac7a51ed6bef79da7800690b19bf778b8" integrity sha512-w6fIxVE/H1PkLKcCPsFqKE7Kv7QUwhU8qQY2MueZXWx5cPZdwFupLgKK3vntcK98BtNHZtAF4LA/yl2a7k8R6Q== +typescript@^4.0.0, typescript@^4.5.5: + version "4.9.5" + resolved "https://registry.yarnpkg.com/typescript/-/typescript-4.9.5.tgz#095979f9bcc0d09da324d58d03ce8f8374cbe65a" + integrity sha512-1FXk9E2Hm+QzZQ7z+McJiHL4NW1F2EzMu9Nq9i3zAaGqibafqYwCVU6WyWAuyQRRzOlxou8xZSyXLEN8oKj24g== + uglify-js@^3.1.4: version "3.16.1" resolved "https://registry.yarnpkg.com/uglify-js/-/uglify-js-3.16.1.tgz#0e7ec928b3d0b1e1d952bce634c384fd56377317"