MirrorSave
/
budibase
mirror of https://github.com/Budibase/budibase.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Required work to support the new lucene audit logs search indexing.

This commit is contained in:
mike12345567 2023-02-17 16:26:55 +00:00
parent 2afd3e1580
commit 6ec5e97ce9
9 changed files with 211 additions and 91 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
packages/backend-core/src/constants/db.ts
View File

@ -24,6 +24,11 @@ export enum ViewName {
APP_BACKUP_BY_TRIGGER = "by_trigger",
}
export const SearchIndexes = {
ROWS: "rows",
AUDIT: "audit",
}
export const DeprecatedViews = {
[ViewName.USER_BY_EMAIL]: [
// removed due to inaccuracy in view doc filter logic

1
packages/backend-core/src/constants/misc.ts
View File

@ -41,5 +41,6 @@ export enum Config {
OIDC_LOGOS = "logos_oidc",
}
export const MIN_VALID_DATE = new Date(-2147483647000)
export const MAX_VALID_DATE = new Date(2147483647000)
export const DEFAULT_TENANT_ID = "default"

12
packages/backend-core/src/context/mainContext.ts
View File

@ -34,17 +34,19 @@ export function getAuditLogDBName(tenantId?: string) {
if (!tenantId) {
tenantId = getTenantId()
}
return `${tenantId}${SEPARATOR}${StaticDatabases.AUDIT_LOGS.name}`
if (tenantId === DEFAULT_TENANT_ID) {
return StaticDatabases.AUDIT_LOGS.name
} else {
return `${tenantId}${SEPARATOR}${StaticDatabases.AUDIT_LOGS.name}`
}
}
export function baseGlobalDBName(tenantId: string | undefined | null) {
let dbName
if (!tenantId || tenantId === DEFAULT_TENANT_ID) {
dbName = StaticDatabases.GLOBAL.name
return StaticDatabases.GLOBAL.name
} else {
dbName = `${tenantId}${SEPARATOR}${StaticDatabases.GLOBAL.name}`
return `${tenantId}${SEPARATOR}${StaticDatabases.GLOBAL.name}`
}
return dbName
}
export function isMultiTenant() {

39
packages/backend-core/src/db/lucene.ts
View File

@ -41,6 +41,8 @@ export class QueryBuilder {
sortType: string
includeDocs: boolean
version?: string
indexBuilder?: () => Promise<any>
noEscaping = false
constructor(dbName: string, index: string, base?: SearchFilters) {
this.dbName = dbName
@ -66,6 +68,14 @@ export class QueryBuilder {
this.includeDocs = true
}
disableEscaping() {
this.noEscaping = true
}
setIndexBuilder(builderFn: () => Promise<any>) {
this.indexBuilder = builderFn
}
setVersion(version?: string) {
if (version != null) {
this.version = version
@ -176,6 +186,14 @@ export class QueryBuilder {
return this
}
handleSpaces(input: string) {
if (this.noEscaping) {
return input
} else {
return input.replace(/ /g, "_")
}
}
/**
* Preprocesses a value before going into a lucene search.
* Transforms strings to lowercase and wraps strings and bools in quotes.
@ -192,7 +210,7 @@ export class QueryBuilder {
value = value.toLowerCase ? value.toLowerCase() : value
}
// Escape characters
if (escape && originalType === "string") {
if (!this.noEscaping && escape && originalType === "string") {
value = `${value}`.replace(/[ #+\-&|!(){}\]^"~*?:\\]/g, "\\$&")
}
@ -272,7 +290,7 @@ export class QueryBuilder {
for (let [key, value] of Object.entries(structure)) {
// check for new format - remove numbering if needed
key = removeKeyNumbering(key)
key = builder.preprocess(key.replace(/ /g, "_"), {
key = builder.preprocess(builder.handleSpaces(key), {
escape: true,
})
const expression = queryFn(key, value)
@ -379,7 +397,7 @@ export class QueryBuilder {
if (this.sort) {
const order = this.sortOrder === "descending" ? "-" : ""
const type = `<${this.sortType}>`
body.sort = `${order}${this.sort.replace(/ /g, "_")}${type}`
body.sort = `${order}${this.handleSpaces(this.sort)}${type}`
}
return body
}
@ -388,7 +406,16 @@ export class QueryBuilder {
const { url, cookie } = dbCore.getCouchInfo()
const fullPath = `${url}/${this.dbName}/_design/database/_search/${this.index}`
const body = this.buildSearchBody()
return await runQuery(fullPath, body, cookie)
try {
return await runQuery(fullPath, body, cookie)
} catch (err: any) {
if (err.status === 404 && this.indexBuilder) {
await this.indexBuilder()
return await runQuery(fullPath, body, cookie)
} else {
throw err
}
}
}
}
@ -407,6 +434,10 @@ const runQuery = async (url: string, body: any, cookie: string) => {
Authorization: cookie,
},
})
if (response.status === 404) {
throw response
}
const json = await response.json()
let output: any = {

10
packages/server/src/api/controllers/row/internalSearch.ts
View File

@ -1,4 +1,3 @@
import { SearchIndexes } from "../../../db/utils"
import { db as dbCore, context, SearchParams } from "@budibase/backend-core"
import { SearchFilters } from "@budibase/types"
@ -7,10 +6,15 @@ export async function paginatedSearch(
params: SearchParams
) {
const appId = context.getAppId()
return dbCore.paginatedSearch(appId!, SearchIndexes.ROWS, query, params)
return dbCore.paginatedSearch(
appId!,
dbCore.SearchIndexes.ROWS,
query,
params
)
}
export async function fullSearch(query: SearchFilters, params: SearchParams) {
const appId = context.getAppId()
return dbCore.fullSearch(appId!, SearchIndexes.ROWS, query, params)
return dbCore.fullSearch(appId!, dbCore.SearchIndexes.ROWS, query, params)
}

4
packages/server/src/db/utils.ts
View File

@ -9,10 +9,6 @@ export const AppStatus = {
DEPLOYED: "published",
}
export const SearchIndexes = {
ROWS: "rows",
}
export const BudibaseInternalDB = {
_id: "bb_internal",
type: dbCore.BUDIBASE_DATASOURCE_TYPE,

6
packages/server/src/db/views/staticViews.ts
View File

@ -1,5 +1,5 @@
import { context } from "@budibase/backend-core"
import { DocumentType, SEPARATOR, ViewName, SearchIndexes } from "../utils"
import { context, db as dbCore } from "@budibase/backend-core"
import { DocumentType, SEPARATOR, ViewName } from "../utils"
import { LinkDocument, Row } from "@budibase/types"
const SCREEN_PREFIX = DocumentType.SCREEN + SEPARATOR
@ -91,7 +91,7 @@ async function searchIndex(indexName: string, fnString: string) {
export async function createAllSearchIndex() {
await searchIndex(
SearchIndexes.ROWS,
dbCore.SearchIndexes.ROWS,
function (doc: Row) {
function idx(input: Row, prev?: string) {
for (let key of Object.keys(input)) {

222
packages/types/src/sdk/events/event.ts
View File

@ -182,102 +182,180 @@ export enum Event {
ENVIRONMENT_VARIABLE_UPGRADE_PANEL_OPENED = "environment_variable:upgrade_panel_opened",
}
export class AuditedEventFriendlyName {
// all events that are not audited have been added to this record as undefined, this means
// that Typescript can protect us against new events being added and auditing of those
// events not being considered. This might be a little ugly, but provides a level of
// Typescript build protection for the audit log feature, any new event also needs to be
// added to this map, during which the developer will need to consider if it should be
// a user facing event or not.
export const AuditedEventFriendlyName: Record<Event, string | undefined> = {
// USER
static USER_CREATED = "User {{ email }} created"
static USER_UPDATED = "User {{ email }} updated"
static USER_DELETED = "User {{ email }} deleted"
static USER_PERMISSION_ADMIN_ASSIGNED = "User {{ email }} admin role assigned"
static USER_PERMISSION_ADMIN_REMOVED = "User {{ email }} admin role removed"
static USER_PERMISSION_BUILDER_ASSIGNED =
"User {{ email }} builder role assigned"
static USER_PERMISSION_BUILDER_REMOVED =
"User {{ email }} builder role removed"
static USER_INVITED = "User {{ email }} invited"
static USER_INVITED_ACCEPTED = "User {{ email }} accepted invite"
static USER_PASSWORD_UPDATED = "User {{ email }} password updated"
static USER_PASSWORD_RESET_REQUESTED =
"User {{ email }} password reset requested"
static USER_PASSWORD_RESET = "User {{ email }} password reset"
static USER_GROUP_CREATED = "User group {{ name }} created"
static USER_GROUP_UPDATED = "User group {{ name }} updated"
static USER_GROUP_DELETED = "User group {{ name }} deleted"
static USER_GROUP_USERS_ADDED =
"User group {{ name }} {{ count }} users added"
static USER_GROUP_USERS_REMOVED =
"User group {{ name }} {{ count }} users removed"
static USER_GROUP_PERMISSIONS_EDITED =
"User group {{ name }} permissions edited"
[Event.USER_CREATED]: `User "{{ email }}" created`,
[Event.USER_UPDATED]: `User "{{ email }}" updated`,
[Event.USER_DELETED]: `User "{{ email }}" deleted`,
[Event.USER_PERMISSION_ADMIN_ASSIGNED]: `User "{{ email }}" admin role assigned`,
[Event.USER_PERMISSION_ADMIN_REMOVED]: `User "{{ email }}" admin role removed`,
[Event.USER_PERMISSION_BUILDER_ASSIGNED]: `User "{{ email }}" builder role assigned`,
[Event.USER_PERMISSION_BUILDER_REMOVED]: `User "{{ email }}" builder role removed`,
[Event.USER_INVITED]: `User "{{ email }} invited`,
[Event.USER_INVITED_ACCEPTED]: `User "{{ email }}" accepted invite`,
[Event.USER_PASSWORD_UPDATED]: `User "{{ email }}" password updated`,
[Event.USER_PASSWORD_RESET_REQUESTED]: `User "{{ email }}" password reset requested`,
[Event.USER_PASSWORD_RESET]: `User "{{ email }}" password reset`,
[Event.USER_GROUP_CREATED]: `User group "{{ name }}" created`,
[Event.USER_GROUP_UPDATED]: `User group "{{ name }}" updated`,
[Event.USER_GROUP_DELETED]: `User group "{{ name }}" deleted`,
[Event.USER_GROUP_USERS_ADDED]: `User group "{{ name }}" {{ count }} users added`,
[Event.USER_GROUP_USERS_REMOVED]: `User group "{{ name }}" {{ count }} users removed`,
[Event.USER_GROUP_PERMISSIONS_EDITED]: `User group "{{ name }}" permissions edited`,
[Event.USER_PASSWORD_FORCE_RESET]: undefined,
[Event.USER_GROUP_ONBOARDING]: undefined,
[Event.USER_ONBOARDING_COMPLETE]: undefined,
// EMAIL
static EMAIL_SMTP_CREATED = "Email configuration created"
static EMAIL_SMTP_UPDATED = "Email configuration updated"
[Event.EMAIL_SMTP_CREATED]: `Email configuration created`,
[Event.EMAIL_SMTP_UPDATED]: `Email configuration updated`,
// AUTH
static AUTH_SSO_CREATED = "SSO configuration created"
static AUTH_SSO_UPDATED = "SSO configuration updated"
static AUTH_SSO_ACTIVATED = "SSO configuration activated"
static AUTH_SSO_DEACTIVATED = "SSO configuration deactivated"
static AUTH_LOGIN = "User {{ email }} logged in"
static AUTH_LOGOUT = "User {{ email }} logged out"
[Event.AUTH_SSO_CREATED]: `SSO configuration created`,
[Event.AUTH_SSO_UPDATED]: `SSO configuration updated`,
[Event.AUTH_SSO_ACTIVATED]: `SSO configuration activated`,
[Event.AUTH_SSO_DEACTIVATED]: `SSO configuration deactivated`,
[Event.AUTH_LOGIN]: `User "{{ email }}" logged in`,
[Event.AUTH_LOGOUT]: `User "{{ email }}" logged out`,
// ORG
static ORG_NAME_UPDATED = "Organisation name updated"
static ORG_LOGO_UPDATED = "Organisation logo updated"
static ORG_PLATFORM_URL_UPDATED = "Organisation platform URL updated"
[Event.ORG_NAME_UPDATED]: `Organisation name updated`,
[Event.ORG_LOGO_UPDATED]: `Organisation logo updated`,
[Event.ORG_PLATFORM_URL_UPDATED]: `Organisation platform URL updated`,
// APP
static APP_CREATED = "App {{ name }} created"
static APP_UPDATED = "App {{ name }} updated"
static APP_DELETED = "App {{ name }} deleted"
static APP_PUBLISHED = "App {{ name }} published"
static APP_UNPUBLISHED = "App {{ name }} unpublished"
static APP_TEMPLATE_IMPORTED = "App {{ name }} template imported"
static APP_FILE_IMPORTED = "App {{ name }} file imported"
static APP_VERSION_UPDATED = "App {{ name }} version updated"
static APP_VERSION_REVERTED = "App {{ name }} version reverted"
static APP_REVERTED = "App {{ name }} reverted"
static APP_EXPORTED = "App {{ name }} exported"
static APP_BACKUP_RESTORED = "App backup {{ name }} restored"
static APP_BACKUP_TRIGGERED = "App backup {{ name }} triggered"
[Event.APP_CREATED]: `App "{{ name }}" created`,
[Event.APP_UPDATED]: `App "{{ name }}" updated`,
[Event.APP_DELETED]: `App "{{ name }}" deleted`,
[Event.APP_PUBLISHED]: `App "{{ name }}" published`,
[Event.APP_UNPUBLISHED]: `App "{{ name }}" unpublished`,
[Event.APP_TEMPLATE_IMPORTED]: `App "{{ name }}" template imported`,
[Event.APP_FILE_IMPORTED]: `App "{{ name }}" file imported`,
[Event.APP_VERSION_UPDATED]: `App "{{ name }}" version updated`,
[Event.APP_VERSION_REVERTED]: `App "{{ name }}" version reverted`,
[Event.APP_REVERTED]: `App "{{ name }}" reverted`,
[Event.APP_EXPORTED]: `App "{{ name }}" exported`,
[Event.APP_BACKUP_RESTORED]: `App backup "{{ name }}" restored`,
[Event.APP_BACKUP_TRIGGERED]: `App backup "{{ name }}" triggered`,
// DATASOURCE
static DATASOURCE_CREATED = "Datasource created"
static DATASOURCE_UPDATED = "Datasource updated"
static DATASOURCE_DELETED = "Datasource deleted"
[Event.DATASOURCE_CREATED]: `Datasource created`,
[Event.DATASOURCE_UPDATED]: `Datasource updated`,
[Event.DATASOURCE_DELETED]: `Datasource deleted`,
// QUERY
static QUERY_CREATED = "Query created"
static QUERY_UPDATED = "Query updated"
static QUERY_DELETED = "Query deleted"
static QUERY_IMPORT = "Query import"
[Event.QUERY_CREATED]: `Query created`,
[Event.QUERY_UPDATED]: `Query updated`,
[Event.QUERY_DELETED]: `Query deleted`,
[Event.QUERY_IMPORT]: `Query import`,
[Event.QUERIES_RUN]: undefined,
[Event.QUERY_PREVIEWED]: undefined,
// TABLE
static TABLE_CREATED = "Table created"
static TABLE_UPDATED = "Table updated"
static TABLE_DELETED = "Table deleted"
static TABLE_EXPORTED = "Table exported"
static TABLE_IMPORTED = "Table imported"
static TABLE_DATA_IMPORTED = "Data imported to table"
[Event.TABLE_CREATED]: `Table created`,
[Event.TABLE_UPDATED]: `Table updated`,
[Event.TABLE_DELETED]: `Table deleted`,
[Event.TABLE_EXPORTED]: `Table exported`,
[Event.TABLE_IMPORTED]: `Table imported`,
[Event.TABLE_DATA_IMPORTED]: `Data imported to table`,
// ROWS
static ROWS_CREATED = "Rows created"
static ROWS_IMPORTED = "Rows imported"
[Event.ROWS_CREATED]: `Rows created`,
[Event.ROWS_IMPORTED]: `Rows imported`,
// AUTOMATION
static AUTOMATION_CREATED = "Automation created"
static AUTOMATION_DELETED = "Automation deleted"
[Event.AUTOMATION_CREATED]: `Automation created`,
[Event.AUTOMATION_DELETED]: `Automation deleted`,
[Event.AUTOMATION_TESTED]: undefined,
[Event.AUTOMATIONS_RUN]: undefined,
[Event.AUTOMATION_STEP_CREATED]: undefined,
[Event.AUTOMATION_STEP_DELETED]: undefined,
[Event.AUTOMATION_TRIGGER_UPDATED]: undefined,
// SCREEN
static SCREEN_CREATED = "Screen created"
static SCREEN_DELETED = "Screen deleted"
[Event.SCREEN_CREATED]: `Screen created`,
[Event.SCREEN_DELETED]: `Screen deleted`,
// COMPONENT
static COMPONENT_CREATED = "Component created"
static COMPONENT_DELETED = "Component deleted"
[Event.COMPONENT_CREATED]: `Component created`,
[Event.COMPONENT_DELETED]: `Component deleted`,
static ENVIRONMENT_VARIABLE_CREATED = "Environment variable created"
static ENVIRONMENT_VARIABLE_DELETED = "Environment variable deleted"
// ENVIRONMENT VARIABLE
[Event.ENVIRONMENT_VARIABLE_CREATED]: `Environment variable created`,
[Event.ENVIRONMENT_VARIABLE_DELETED]: `Environment variable deleted`,
[Event.ENVIRONMENT_VARIABLE_UPGRADE_PANEL_OPENED]: undefined,
// PLUGIN
[Event.PLUGIN_IMPORTED]: `Plugin imported`,
[Event.PLUGIN_DELETED]: `Plugin deleted`,
[Event.PLUGIN_INIT]: undefined,
// ROLE - NOT AUDITED
[Event.ROLE_CREATED]: undefined,
[Event.ROLE_UPDATED]: undefined,
[Event.ROLE_DELETED]: undefined,
[Event.ROLE_ASSIGNED]: undefined,
[Event.ROLE_UNASSIGNED]: undefined,
// LICENSE - NOT AUDITED
[Event.LICENSE_PLAN_CHANGED]: undefined,
[Event.LICENSE_TIER_CHANGED]: undefined,
[Event.LICENSE_ACTIVATED]: undefined,
[Event.LICENSE_PAYMENT_FAILED]: undefined,
[Event.LICENSE_PAYMENT_RECOVERED]: undefined,
[Event.LICENSE_CHECKOUT_OPENED]: undefined,
[Event.LICENSE_CHECKOUT_SUCCESS]: undefined,
[Event.LICENSE_PORTAL_OPENED]: undefined,
// ACCOUNT - NOT AUDITED
[Event.ACCOUNT_CREATED]: undefined,
[Event.ACCOUNT_DELETED]: undefined,
[Event.ACCOUNT_VERIFIED]: undefined,
// BACKFILL - NOT AUDITED
[Event.APP_BACKFILL_SUCCEEDED]: undefined,
[Event.APP_BACKFILL_FAILED]: undefined,
[Event.TENANT_BACKFILL_SUCCEEDED]: undefined,
[Event.TENANT_BACKFILL_FAILED]: undefined,
[Event.INSTALLATION_BACKFILL_SUCCEEDED]: undefined,
[Event.INSTALLATION_BACKFILL_FAILED]: undefined,
// LAYOUT - NOT AUDITED
[Event.LAYOUT_CREATED]: undefined,
[Event.LAYOUT_DELETED]: undefined,
// VIEW - NOT AUDITED
[Event.VIEW_CREATED]: undefined,
[Event.VIEW_UPDATED]: undefined,
[Event.VIEW_DELETED]: undefined,
[Event.VIEW_EXPORTED]: undefined,
[Event.VIEW_FILTER_CREATED]: undefined,
[Event.VIEW_FILTER_UPDATED]: undefined,
[Event.VIEW_FILTER_DELETED]: undefined,
[Event.VIEW_CALCULATION_CREATED]: undefined,
[Event.VIEW_CALCULATION_UPDATED]: undefined,
[Event.VIEW_CALCULATION_DELETED]: undefined,
// SERVED - NOT AUDITED
[Event.SERVED_BUILDER]: undefined,
[Event.SERVED_APP]: undefined,
[Event.SERVED_APP_PREVIEW]: undefined,
// ANALYTICS - NOT AUDITED
[Event.ANALYTICS_OPT_OUT]: undefined,
[Event.ANALYTICS_OPT_IN]: undefined,
// INSTALLATION - NOT AUDITED
[Event.INSTALLATION_VERSION_CHECKED]: undefined,
[Event.INSTALLATION_VERSION_UPGRADED]: undefined,
[Event.INSTALLATION_VERSION_DOWNGRADED]: undefined,
[Event.INSTALLATION_FIRST_STARTUP]: undefined,
}
// properties added at the final stage of the event pipeline

3
scripts/link-dependencies.sh
View File

@ -44,6 +44,9 @@ if [ -d "../budibase-pro" ]; then
echo "Linking types to pro"
yarn link '@budibase/types'
echo "Linking string-templates to pro"
yarn link '@budibase/string-templates'
cd ../../../budibase
echo "Linking pro to worker"