From 6a4c601ae8685978006cf64cbf1b914b7b790a56 Mon Sep 17 00:00:00 2001 From: Rory Powell Date: Wed, 25 May 2022 00:15:52 +0100 Subject: [PATCH] timestamp support, logging updates, migration progress indicators, identification updates --- packages/backend-core/context.js | 2 ++ packages/backend-core/src/events/events.ts | 8 +++-- .../backend-core/src/events/identification.ts | 7 ++-- .../events/processors/AnalyticsProcessor.ts | 5 +-- .../src/events/processors/LoggingProcessor.ts | 6 +++- .../src/events/processors/PosthogProcessor.ts | 9 +++-- .../src/events/processors/Processors.ts | 5 +-- .../src/events/processors/types.ts | 7 +++- packages/backend-core/src/migrations/index.js | 33 ++++++++++++++++--- .../src/migrations/functions/backfill/app.ts | 8 +++++ .../migrations/functions/backfill/global.ts | 10 ++++++ .../functions/backfill/global/users.ts | 1 + packages/server/src/migrations/index.ts | 14 +++++--- packages/types/src/events/identification.ts | 3 +- 14 files changed, 94 insertions(+), 24 deletions(-) diff --git a/packages/backend-core/context.js b/packages/backend-core/context.js index 4ba3e6bffe..0d86c530a7 100644 --- a/packages/backend-core/context.js +++ b/packages/backend-core/context.js @@ -6,6 +6,7 @@ const { updateAppId, doInAppContext, doInUserContext, + doInTenant, } = require("./src/context") module.exports = { @@ -16,4 +17,5 @@ module.exports = { updateAppId, doInAppContext, doInUserContext, + doInTenant, } diff --git a/packages/backend-core/src/events/events.ts b/packages/backend-core/src/events/events.ts index 4ae90be745..8d81608365 100644 --- a/packages/backend-core/src/events/events.ts +++ b/packages/backend-core/src/events/events.ts @@ -2,8 +2,12 @@ import { Event } from "@budibase/types" import { processors } from "./processors" import * as identification from "./identification" -export const publishEvent = async (event: Event, properties: any) => { +export const publishEvent = async ( + event: Event, + properties: any, + timestamp?: string | number +) => { // in future this should use async events via a distributed queue. const identity = await identification.getCurrentIdentity() - await processors.processEvent(event, identity, properties) + await processors.processEvent(event, identity, properties, timestamp) } diff --git a/packages/backend-core/src/events/identification.ts b/packages/backend-core/src/events/identification.ts index d22cfe7720..d6aca63e68 100644 --- a/packages/backend-core/src/events/identification.ts +++ b/packages/backend-core/src/events/identification.ts @@ -23,18 +23,22 @@ export const getCurrentIdentity = async (): Promise => { const user: SessionUser | undefined = context.getUser() let tenantId = context.getTenantId() let id: string + let type: IdentityType if (user) { id = user._id + type = IdentityType.USER } else { const global = await getGlobalIdentifiers(tenantId) id = global.id tenantId = global.tenantId + type = IdentityType.TENANT } return { id, tenantId, + type, } } @@ -113,14 +117,13 @@ export const identifyAccount = async (account: Account) => { let id = account.accountId const tenantId = account.tenantId const hosting = account.hosting - let type = IdentityType.ACCOUNT + let type = IdentityType.USER let providerType = isSSOAccount(account) ? account.providerType : undefined if (isCloudAccount(account)) { if (account.budibaseUserId) { // use the budibase user as the id if set id = account.budibaseUserId - type = IdentityType.USER } } diff --git a/packages/backend-core/src/events/processors/AnalyticsProcessor.ts b/packages/backend-core/src/events/processors/AnalyticsProcessor.ts index d2f82d59fb..63b7d3ed97 100644 --- a/packages/backend-core/src/events/processors/AnalyticsProcessor.ts +++ b/packages/backend-core/src/events/processors/AnalyticsProcessor.ts @@ -16,13 +16,14 @@ export default class AnalyticsProcessor implements EventProcessor { async processEvent( event: Event, identity: Identity, - properties: any + properties: any, + timestamp?: string ): Promise { if (!(await analytics.enabled())) { return } if (this.posthog) { - this.posthog.processEvent(event, identity, properties) + this.posthog.processEvent(event, identity, properties, timestamp) } } diff --git a/packages/backend-core/src/events/processors/LoggingProcessor.ts b/packages/backend-core/src/events/processors/LoggingProcessor.ts index 1094411298..102a50f8d9 100644 --- a/packages/backend-core/src/events/processors/LoggingProcessor.ts +++ b/packages/backend-core/src/events/processors/LoggingProcessor.ts @@ -1,5 +1,6 @@ import { Event, Identity } from "@budibase/types" import { EventProcessor } from "./types" +import env from "../../environment" export default class LoggingProcessor implements EventProcessor { async processEvent( @@ -7,8 +8,11 @@ export default class LoggingProcessor implements EventProcessor { identity: Identity, properties: any ): Promise { + if (env.SELF_HOSTED && !env.isDev()) { + return + } console.log( - `[audit] [tenant=${identity.tenantId}] [identity=${identity.id}] ${event}` + `[audit] [tenant=${identity.tenantId}] [identityType=${identity.type}] [identity=${identity.id}] ${event}` ) } diff --git a/packages/backend-core/src/events/processors/PosthogProcessor.ts b/packages/backend-core/src/events/processors/PosthogProcessor.ts index 499f69f581..dbbb034428 100644 --- a/packages/backend-core/src/events/processors/PosthogProcessor.ts +++ b/packages/backend-core/src/events/processors/PosthogProcessor.ts @@ -15,9 +15,14 @@ export default class PosthogProcessor implements EventProcessor { async processEvent( event: Event, identity: Identity, - properties: any + properties: any, + timestamp?: string | number ): Promise { - this.posthog.capture({ distinctId: identity.id, event, properties }) + const payload: any = { distinctId: identity.id, event, properties } + if (timestamp) { + payload.timestamp = new Date(timestamp) + } + this.posthog.capture(payload) } async identify(identity: Identity) { diff --git a/packages/backend-core/src/events/processors/Processors.ts b/packages/backend-core/src/events/processors/Processors.ts index ed23af4780..d2ee44a4a6 100644 --- a/packages/backend-core/src/events/processors/Processors.ts +++ b/packages/backend-core/src/events/processors/Processors.ts @@ -12,10 +12,11 @@ export default class Processor implements EventProcessor { async processEvent( event: Event, identity: Identity, - properties: any + properties: any, + timestamp?: string | number ): Promise { for (const eventProcessor of this.processors) { - await eventProcessor.processEvent(event, identity, properties) + await eventProcessor.processEvent(event, identity, properties, timestamp) } } diff --git a/packages/backend-core/src/events/processors/types.ts b/packages/backend-core/src/events/processors/types.ts index 0be1606351..9b48b2beee 100644 --- a/packages/backend-core/src/events/processors/types.ts +++ b/packages/backend-core/src/events/processors/types.ts @@ -6,6 +6,11 @@ export enum EventProcessorType { } export interface EventProcessor { - processEvent(event: Event, identity: Identity, properties: any): Promise + processEvent( + event: Event, + identity: Identity, + properties: any, + timestamp?: string | number + ): Promise shutdown(): void } diff --git a/packages/backend-core/src/migrations/index.js b/packages/backend-core/src/migrations/index.js index eb51613e60..26be4c90bc 100644 --- a/packages/backend-core/src/migrations/index.js +++ b/packages/backend-core/src/migrations/index.js @@ -31,6 +31,13 @@ exports.runMigration = async (migration, options = {}) => { const tenantId = getTenantId() const migrationType = migration.type const migrationName = migration.name + const silent = migration.silent + + const log = message => { + if (!silent) { + console.log(message) + } + } // get the db to store the migration in let dbNames @@ -45,8 +52,14 @@ exports.runMigration = async (migration, options = {}) => { ) } + const length = dbNames.length + let count = 0 + // run the migration against each db for (const dbName of dbNames) { + count++ + const lengthStatement = length > 1 ? `[${count}/${length}]` : "" + await doWithDB(dbName, async db => { try { const doc = await exports.getMigrationsDoc(db) @@ -58,7 +71,7 @@ exports.runMigration = async (migration, options = {}) => { options.force[migrationType] && options.force[migrationType].includes(migrationName) ) { - console.log( + log( `[Tenant: ${tenantId}] [Migration: ${migrationName}] [DB: ${dbName}] Forcing` ) } else { @@ -67,12 +80,12 @@ exports.runMigration = async (migration, options = {}) => { } } - console.log( - `[Tenant: ${tenantId}] [Migration: ${migrationName}] [DB: ${dbName}] Running` + log( + `[Tenant: ${tenantId}] [Migration: ${migrationName}] [DB: ${dbName}] Running ${lengthStatement}` ) // run the migration with tenant context await migration.fn(db) - console.log( + log( `[Tenant: ${tenantId}] [Migration: ${migrationName}] [DB: ${dbName}] Complete` ) @@ -91,7 +104,6 @@ exports.runMigration = async (migration, options = {}) => { } exports.runMigrations = async (migrations, options = {}) => { - console.log("Running migrations") let tenantIds if (environment.MULTI_TENANCY) { if (!options.tenantIds || !options.tenantIds.length) { @@ -105,8 +117,19 @@ exports.runMigrations = async (migrations, options = {}) => { tenantIds = [DEFAULT_TENANT_ID] } + if (tenantIds.length > 1) { + console.log(`Checking migrations for ${tenantIds.length} tenants`) + } else { + console.log("Checking migrations") + } + + let count = 0 // for all tenants for (const tenantId of tenantIds) { + count++ + if (tenantIds.length > 1) { + console.log(`Progress [${count}/${tenantIds.length}]`) + } // for all migrations for (const migration of migrations) { // run the migration diff --git a/packages/server/src/migrations/functions/backfill/app.ts b/packages/server/src/migrations/functions/backfill/app.ts index ba8d889343..ec905d6efb 100644 --- a/packages/server/src/migrations/functions/backfill/app.ts +++ b/packages/server/src/migrations/functions/backfill/app.ts @@ -6,6 +6,7 @@ import * as queries from "./app/queries" import * as roles from "./app/roles" import * as tables from "./app/tables" import * as screens from "./app/screens" +import * as global from "./global" /** * Date: @@ -16,6 +17,13 @@ import * as screens from "./app/screens" */ export const run = async (appDb: any) => { + if (await global.isComplete()) { + // make sure new apps aren't backfilled + // return if the global migration for this tenant is complete + // which runs after the app migrations + return + } + await apps.backfill(appDb) await automations.backfill(appDb) await datasources.backfill(appDb) diff --git a/packages/server/src/migrations/functions/backfill/global.ts b/packages/server/src/migrations/functions/backfill/global.ts index a723e03a75..44aa29a561 100644 --- a/packages/server/src/migrations/functions/backfill/global.ts +++ b/packages/server/src/migrations/functions/backfill/global.ts @@ -1,6 +1,7 @@ import * as users from "./global/users" import * as rows from "./global/rows" import * as configs from "./global/configs" +import { tenancy, events, migrations } from "@budibase/backend-core" /** * Date: @@ -11,7 +12,16 @@ import * as configs from "./global/configs" */ export const run = async (db: any) => { + const tenantId = tenancy.getTenantId() + await events.identification.identifyTenant(tenantId) + await users.backfill(db) await rows.backfill() await configs.backfill(db) } + +export const isComplete = async (): Promise => { + const globalDb = tenancy.getGlobalDB() + const migrationsDoc = await migrations.getMigrationsDoc(globalDb) + return !!migrationsDoc.event_global_backfill +} diff --git a/packages/server/src/migrations/functions/backfill/global/users.ts b/packages/server/src/migrations/functions/backfill/global/users.ts index 60ae9c699c..1816c970db 100644 --- a/packages/server/src/migrations/functions/backfill/global/users.ts +++ b/packages/server/src/migrations/functions/backfill/global/users.ts @@ -19,6 +19,7 @@ export const backfill = async (globalDb: any) => { const users = await getUsers(globalDb) for (const user of users) { + await events.identification.identifyUser(user) await events.user.created(user) if (user.admin?.global) { diff --git a/packages/server/src/migrations/index.ts b/packages/server/src/migrations/index.ts index 991cde650d..92e17ed24e 100644 --- a/packages/server/src/migrations/index.ts +++ b/packages/server/src/migrations/index.ts @@ -7,12 +7,14 @@ import * as appUrls from "./functions/appUrls" import * as developerQuota from "./functions/developerQuota" import * as publishedAppsQuota from "./functions/publishedAppsQuota" import * as backfill from "./functions/backfill" +import env from "../environment" export interface Migration { type: string name: string opts?: object fn: Function + silent?: boolean } /** @@ -58,16 +60,18 @@ export const MIGRATIONS: Migration[] = [ name: "published_apps_quota", fn: publishedAppsQuota.run, }, - { - type: migrations.MIGRATION_TYPES.GLOBAL, - name: "event_global_backfill", - fn: backfill.global.run, - }, { type: migrations.MIGRATION_TYPES.APP, name: "event_app_backfill", opts: { all: true }, fn: backfill.app.run, + silent: !!env.SELF_HOSTED, // reduce noisy logging + }, + { + type: migrations.MIGRATION_TYPES.GLOBAL, + name: "event_global_backfill", + fn: backfill.global.run, + silent: !!env.SELF_HOSTED, // reduce noisy logging }, ] diff --git a/packages/types/src/events/identification.ts b/packages/types/src/events/identification.ts index e35167d00f..1c20fd2ed6 100644 --- a/packages/types/src/events/identification.ts +++ b/packages/types/src/events/identification.ts @@ -2,18 +2,17 @@ import { Hosting } from "../core" export enum IdentityType { USER = "user", // cloud and self hosted users - ACCOUNT = "account", // self hosted accounts TENANT = "tenant", // cloud and self hosted tenants } export interface Identity { id: string tenantId: string + type: IdentityType } export interface TenantIdentity extends Identity { hosting: Hosting - type: IdentityType } export interface UserIdentity extends TenantIdentity {