From b986b689d6eddb1240a7a99d1cf5bbdf8593c47f Mon Sep 17 00:00:00 2001 From: Rory Powell Date: Wed, 25 May 2022 21:32:08 +0100 Subject: [PATCH] Historical timestamps --- .../backend-core/src/events/identification.ts | 15 ++++++---- .../events/processors/AnalyticsProcessor.ts | 6 ++-- .../src/events/processors/PosthogProcessor.ts | 8 ++++-- .../backend-core/src/events/publishers/app.ts | 8 +++--- .../src/events/publishers/auth.ts | 8 +++--- .../src/events/publishers/automation.ts | 9 +++--- .../src/events/publishers/datasource.ts | 4 +-- .../src/events/publishers/email.ts | 7 +++-- .../src/events/publishers/layout.ts | 4 +-- .../backend-core/src/events/publishers/org.ts | 12 ++++---- .../src/events/publishers/query.ts | 8 ++++-- .../src/events/publishers/role.ts | 8 +++--- .../src/events/publishers/rows.ts | 4 +-- .../src/events/publishers/screen.ts | 4 +-- .../src/events/publishers/table.ts | 4 +-- .../src/events/publishers/user.ts | 23 +++++++++++---- .../src/events/publishers/view.ts | 15 ++++++---- .../src/migrations/functions/backfill/app.ts | 28 +++++++++++++------ .../migrations/functions/backfill/app/apps.ts | 14 ---------- .../functions/backfill/app/automations.ts | 16 +++++------ .../functions/backfill/app/datasources.ts | 12 ++++---- .../functions/backfill/app/layouts.ts | 12 ++++---- .../functions/backfill/app/queries.ts | 20 ++++++------- .../functions/backfill/app/roles.ts | 12 ++++---- .../functions/backfill/app/screens.ts | 12 ++++---- .../functions/backfill/app/tables.ts | 28 +++++++++---------- .../migrations/functions/backfill/global.ts | 24 ++++++++++++++-- .../functions/backfill/global/configs.ts | 25 +++++++++-------- .../functions/backfill/global/rows.ts | 20 ++++++++++--- .../functions/backfill/global/users.ts | 17 +++++------ packages/server/tsconfig.build.json | 3 +- packages/server/tsconfig.json | 3 -- packages/types/src/documents/document.ts | 2 +- packages/types/src/documents/global/user.ts | 5 ++-- packages/types/tsconfig.json | 1 + packages/worker/tsconfig.json | 3 +- 36 files changed, 226 insertions(+), 178 deletions(-) delete mode 100644 packages/server/src/migrations/functions/backfill/app/apps.ts diff --git a/packages/backend-core/src/events/identification.ts b/packages/backend-core/src/events/identification.ts index d6aca63e68..edf1b658bc 100644 --- a/packages/backend-core/src/events/identification.ts +++ b/packages/backend-core/src/events/identification.ts @@ -79,7 +79,10 @@ const getHostingFromEnv = () => { return env.SELF_HOSTED ? Hosting.SELF : Hosting.CLOUD } -export const identifyTenant = async (tenantId: string) => { +export const identifyTenant = async ( + tenantId: string, + timestamp?: string | number +) => { const global = await getGlobalIdentifiers(tenantId) const identity: TenantIdentity = { @@ -88,10 +91,10 @@ export const identifyTenant = async (tenantId: string) => { hosting: getHostingFromEnv(), type: IdentityType.TENANT, } - await identify(identity) + await identify(identity, timestamp) } -export const identifyUser = async (user: User) => { +export const identifyUser = async (user: User, timestamp?: string | number) => { const id = user._id as string const tenantId = user.tenantId const hosting = env.SELF_HOSTED ? Hosting.SELF : Hosting.CLOUD @@ -110,7 +113,7 @@ export const identifyUser = async (user: User) => { providerType, } - await identify(identity) + await identify(identity, timestamp) } export const identifyAccount = async (account: Account) => { @@ -141,6 +144,6 @@ export const identifyAccount = async (account: Account) => { await identify(identity) } -const identify = async (identity: Identity) => { - await analyticsProcessor.identify(identity) +const identify = async (identity: Identity, timestamp?: string | number) => { + await analyticsProcessor.identify(identity, timestamp) } diff --git a/packages/backend-core/src/events/processors/AnalyticsProcessor.ts b/packages/backend-core/src/events/processors/AnalyticsProcessor.ts index 63b7d3ed97..eaa5ae1135 100644 --- a/packages/backend-core/src/events/processors/AnalyticsProcessor.ts +++ b/packages/backend-core/src/events/processors/AnalyticsProcessor.ts @@ -17,7 +17,7 @@ export default class AnalyticsProcessor implements EventProcessor { event: Event, identity: Identity, properties: any, - timestamp?: string + timestamp?: string | number ): Promise { if (!(await analytics.enabled())) { return @@ -27,12 +27,12 @@ export default class AnalyticsProcessor implements EventProcessor { } } - async identify(identity: Identity) { + async identify(identity: Identity, timestamp?: string | number) { if (!(await analytics.enabled())) { return } if (this.posthog) { - this.posthog.identify(identity) + this.posthog.identify(identity, timestamp) } } diff --git a/packages/backend-core/src/events/processors/PosthogProcessor.ts b/packages/backend-core/src/events/processors/PosthogProcessor.ts index dbbb034428..54b5133dfb 100644 --- a/packages/backend-core/src/events/processors/PosthogProcessor.ts +++ b/packages/backend-core/src/events/processors/PosthogProcessor.ts @@ -25,8 +25,12 @@ export default class PosthogProcessor implements EventProcessor { this.posthog.capture(payload) } - async identify(identity: Identity) { - this.posthog.identify({ distinctId: identity.id, properties: identity }) + async identify(identity: Identity, timestamp?: string | number) { + const payload: any = { distinctId: identity.id, properties: identity } + if (timestamp) { + payload.timestamp = new Date(timestamp) + } + this.posthog.identify(payload) } shutdown() { diff --git a/packages/backend-core/src/events/publishers/app.ts b/packages/backend-core/src/events/publishers/app.ts index 00df1f18bc..b297b2a33e 100644 --- a/packages/backend-core/src/events/publishers/app.ts +++ b/packages/backend-core/src/events/publishers/app.ts @@ -15,9 +15,9 @@ import { AppExportedEvent, } from "@budibase/types" -export const created = async (app: App) => { +export const created = async (app: App, timestamp?: string | number) => { const properties: AppCreatedEvent = {} - await publishEvent(Event.APP_CREATED, properties) + await publishEvent(Event.APP_CREATED, properties, timestamp) } export async function updated(app: App) { @@ -30,9 +30,9 @@ export async function deleted(app: App) { await publishEvent(Event.APP_DELETED, properties) } -export async function published(app: App) { +export async function published(app: App, timestamp?: string | number) { const properties: AppPublishedEvent = {} - await publishEvent(Event.APP_PUBLISHED, properties) + await publishEvent(Event.APP_PUBLISHED, properties, timestamp) } export async function unpublished(app: App) { diff --git a/packages/backend-core/src/events/publishers/auth.ts b/packages/backend-core/src/events/publishers/auth.ts index 32af58258d..a1f15470f9 100644 --- a/packages/backend-core/src/events/publishers/auth.ts +++ b/packages/backend-core/src/events/publishers/auth.ts @@ -23,11 +23,11 @@ export async function logout() { await publishEvent(Event.AUTH_LOGOUT, properties) } -export async function SSOCreated(type: SSOType) { +export async function SSOCreated(type: SSOType, timestamp?: string | number) { const properties: SSOCreatedEvent = { type, } - await publishEvent(Event.AUTH_SSO_CREATED, properties) + await publishEvent(Event.AUTH_SSO_CREATED, properties, timestamp) } export async function SSOUpdated(type: SSOType) { @@ -37,11 +37,11 @@ export async function SSOUpdated(type: SSOType) { await publishEvent(Event.AUTH_SSO_UPDATED, properties) } -export async function SSOActivated(type: SSOType) { +export async function SSOActivated(type: SSOType, timestamp?: string | number) { const properties: SSOActivatedEvent = { type, } - await publishEvent(Event.AUTH_SSO_ACTIVATED, properties) + await publishEvent(Event.AUTH_SSO_ACTIVATED, properties, timestamp) } export async function SSODeactivated(type: SSOType) { diff --git a/packages/backend-core/src/events/publishers/automation.ts b/packages/backend-core/src/events/publishers/automation.ts index 2a20a90929..ba4477618b 100644 --- a/packages/backend-core/src/events/publishers/automation.ts +++ b/packages/backend-core/src/events/publishers/automation.ts @@ -12,9 +12,9 @@ import { AutomationTriggerUpdatedEvent, } from "@budibase/types" -export async function created(automation: Automation) { +export async function created(automation: Automation, timestamp?: string) { const properties: AutomationCreatedEvent = {} - await publishEvent(Event.AUTOMATION_CREATED, properties) + await publishEvent(Event.AUTOMATION_CREATED, properties, timestamp) } export async function deleted(automation: Automation) { @@ -29,10 +29,11 @@ export async function tested(automation: Automation) { export async function stepCreated( automation: Automation, - step: AutomationStep + step: AutomationStep, + timestamp?: string ) { const properties: AutomationStepCreatedEvent = {} - await publishEvent(Event.AUTOMATION_STEP_CREATED, properties) + await publishEvent(Event.AUTOMATION_STEP_CREATED, properties, timestamp) } export async function stepDeleted( diff --git a/packages/backend-core/src/events/publishers/datasource.ts b/packages/backend-core/src/events/publishers/datasource.ts index e5392d3bb8..95f47f7947 100644 --- a/packages/backend-core/src/events/publishers/datasource.ts +++ b/packages/backend-core/src/events/publishers/datasource.ts @@ -7,9 +7,9 @@ import { DatasourceDeletedEvent, } from "@budibase/types" -export async function created(datasource: Datasource) { +export async function created(datasource: Datasource, timestamp?: string) { const properties: DatasourceCreatedEvent = {} - await publishEvent(Event.DATASOURCE_CREATED, properties) + await publishEvent(Event.DATASOURCE_CREATED, properties, timestamp) } export async function updated(datasource: Datasource) { diff --git a/packages/backend-core/src/events/publishers/email.ts b/packages/backend-core/src/events/publishers/email.ts index 76745453f9..102cd2dedb 100644 --- a/packages/backend-core/src/events/publishers/email.ts +++ b/packages/backend-core/src/events/publishers/email.ts @@ -6,9 +6,12 @@ import { SMTPUpdatedEvent, } from "@budibase/types" -export async function SMTPCreated(config: SMTPConfig) { +export async function SMTPCreated( + config: SMTPConfig, + timestamp?: string | number +) { const properties: SMTPCreatedEvent = {} - await publishEvent(Event.EMAIL_SMTP_CREATED, properties) + await publishEvent(Event.EMAIL_SMTP_CREATED, properties, timestamp) } export async function SMTPUpdated(config: SMTPConfig) { diff --git a/packages/backend-core/src/events/publishers/layout.ts b/packages/backend-core/src/events/publishers/layout.ts index 109f23571a..c7d4f20225 100644 --- a/packages/backend-core/src/events/publishers/layout.ts +++ b/packages/backend-core/src/events/publishers/layout.ts @@ -6,9 +6,9 @@ import { LayoutDeletedEvent, } from "@budibase/types" -export async function created(layout: Layout) { +export async function created(layout: Layout, timestamp?: string) { const properties: LayoutCreatedEvent = {} - await publishEvent(Event.LAYOUT_CREATED, properties) + await publishEvent(Event.LAYOUT_CREATED, properties, timestamp) } export async function deleted(layout: Layout) { diff --git a/packages/backend-core/src/events/publishers/org.ts b/packages/backend-core/src/events/publishers/org.ts index ae2670d4da..075ab1592b 100644 --- a/packages/backend-core/src/events/publishers/org.ts +++ b/packages/backend-core/src/events/publishers/org.ts @@ -1,19 +1,19 @@ import { publishEvent } from "../events" import { Event, VersionCheckedEvent } from "@budibase/types" -export async function nameUpdated() { +export async function nameUpdated(timestamp?: string | number) { const properties = {} - await publishEvent(Event.ORG_NAME_UPDATED, properties) + await publishEvent(Event.ORG_NAME_UPDATED, properties, timestamp) } -export async function logoUpdated() { +export async function logoUpdated(timestamp?: string | number) { const properties = {} - await publishEvent(Event.ORG_LOGO_UPDATED, properties) + await publishEvent(Event.ORG_LOGO_UPDATED, properties, timestamp) } -export async function platformURLUpdated() { +export async function platformURLUpdated(timestamp?: string | number) { const properties = {} - await publishEvent(Event.ORG_PLATFORM_URL_UPDATED, properties) + await publishEvent(Event.ORG_PLATFORM_URL_UPDATED, properties, timestamp) } export async function versionChecked(version: number) { diff --git a/packages/backend-core/src/events/publishers/query.ts b/packages/backend-core/src/events/publishers/query.ts index e012d5c830..e659f5c2be 100644 --- a/packages/backend-core/src/events/publishers/query.ts +++ b/packages/backend-core/src/events/publishers/query.ts @@ -12,9 +12,13 @@ import { /* eslint-disable */ -export const created = async (datasource: Datasource, query: Query) => { +export const created = async ( + datasource: Datasource, + query: Query, + timestamp?: string +) => { const properties: QueryCreatedEvent = {} - await publishEvent(Event.QUERY_CREATED, properties) + await publishEvent(Event.QUERY_CREATED, properties, timestamp) } export const updated = async (datasource: Datasource, query: Query) => { diff --git a/packages/backend-core/src/events/publishers/role.ts b/packages/backend-core/src/events/publishers/role.ts index d257eadf2b..0002499814 100644 --- a/packages/backend-core/src/events/publishers/role.ts +++ b/packages/backend-core/src/events/publishers/role.ts @@ -12,9 +12,9 @@ import { /* eslint-disable */ -export async function created(role: Role) { +export async function created(role: Role, timestamp?: string) { const properties: RoleCreatedEvent = {} - await publishEvent(Event.ROLE_CREATED, properties) + await publishEvent(Event.ROLE_CREATED, properties, timestamp) } export async function updated(role: Role) { @@ -27,9 +27,9 @@ export async function deleted(role: Role) { await publishEvent(Event.ROLE_DELETED, properties) } -export async function assigned(user: User, role: string) { +export async function assigned(user: User, role: string, timestamp?: number) { const properties: RoleAssignedEvent = {} - await publishEvent(Event.ROLE_ASSIGNED, properties) + await publishEvent(Event.ROLE_ASSIGNED, properties, timestamp) } export async function unassigned(user: User, role: string) { diff --git a/packages/backend-core/src/events/publishers/rows.ts b/packages/backend-core/src/events/publishers/rows.ts index 0877b77b7a..4affd5d989 100644 --- a/packages/backend-core/src/events/publishers/rows.ts +++ b/packages/backend-core/src/events/publishers/rows.ts @@ -9,11 +9,11 @@ import { /* eslint-disable */ -export const created = async (count: number) => { +export const created = async (count: number, timestamp?: string) => { const properties: RowsCreatedEvent = { count, } - await publishEvent(Event.ROWS_CREATED, properties) + await publishEvent(Event.ROWS_CREATED, properties, timestamp) } export const imported = async ( diff --git a/packages/backend-core/src/events/publishers/screen.ts b/packages/backend-core/src/events/publishers/screen.ts index efb7d9538f..038f7bec3a 100644 --- a/packages/backend-core/src/events/publishers/screen.ts +++ b/packages/backend-core/src/events/publishers/screen.ts @@ -6,9 +6,9 @@ import { ScreenDeletedEvent, } from "@budibase/types" -export async function created(screen: Screen) { +export async function created(screen: Screen, timestamp?: string) { const properties: ScreenCreatedEvent = {} - await publishEvent(Event.SCREEN_CREATED, properties) + await publishEvent(Event.SCREEN_CREATED, properties, timestamp) } export async function deleted(screen: Screen) { diff --git a/packages/backend-core/src/events/publishers/table.ts b/packages/backend-core/src/events/publishers/table.ts index cfa4b28835..c1f050b25e 100644 --- a/packages/backend-core/src/events/publishers/table.ts +++ b/packages/backend-core/src/events/publishers/table.ts @@ -13,9 +13,9 @@ import { /* eslint-disable */ -export async function created(table: Table) { +export async function created(table: Table, timestamp?: string) { const properties: TableCreatedEvent = {} - await publishEvent(Event.TABLE_CREATED, properties) + await publishEvent(Event.TABLE_CREATED, properties, timestamp) } export async function updated(table: Table) { diff --git a/packages/backend-core/src/events/publishers/user.ts b/packages/backend-core/src/events/publishers/user.ts index eeea5234cb..c5c90fb4e1 100644 --- a/packages/backend-core/src/events/publishers/user.ts +++ b/packages/backend-core/src/events/publishers/user.ts @@ -17,9 +17,9 @@ import { /* eslint-disable */ -export async function created(user: User) { +export async function created(user: User, timestamp?: number) { const properties: UserCreatedEvent = {} - await publishEvent(Event.USER_CREATED, properties) + await publishEvent(Event.USER_CREATED, properties, timestamp) } export async function updated(user: User) { @@ -34,9 +34,13 @@ export async function deleted(user: User) { // PERMISSIONS -export async function permissionAdminAssigned(user: User) { +export async function permissionAdminAssigned(user: User, timestamp?: number) { const properties: UserPermissionAssignedEvent = {} - await publishEvent(Event.USER_PERMISSION_ADMIN_ASSIGNED, properties) + await publishEvent( + Event.USER_PERMISSION_ADMIN_ASSIGNED, + properties, + timestamp + ) } export async function permissionAdminRemoved(user: User) { @@ -44,9 +48,16 @@ export async function permissionAdminRemoved(user: User) { await publishEvent(Event.USER_PERMISSION_ADMIN_REMOVED, properties) } -export async function permissionBuilderAssigned(user: User) { +export async function permissionBuilderAssigned( + user: User, + timestamp?: number +) { const properties: UserPermissionAssignedEvent = {} - await publishEvent(Event.USER_PERMISSION_BUILDER_ASSIGNED, properties) + await publishEvent( + Event.USER_PERMISSION_BUILDER_ASSIGNED, + properties, + timestamp + ) } export async function permissionBuilderRemoved(user: User) { diff --git a/packages/backend-core/src/events/publishers/view.ts b/packages/backend-core/src/events/publishers/view.ts index ff0c8a7f0d..e6f82579cd 100644 --- a/packages/backend-core/src/events/publishers/view.ts +++ b/packages/backend-core/src/events/publishers/view.ts @@ -19,9 +19,9 @@ import { /* eslint-disable */ -export async function created(view: View) { +export async function created(view: View, timestamp?: string) { const properties: ViewCreatedEvent = {} - await publishEvent(Event.VIEW_CREATED, properties) + await publishEvent(Event.VIEW_CREATED, properties, timestamp) } export async function updated(view: View) { @@ -39,9 +39,9 @@ export async function exported(table: Table, format: TableExportFormat) { await publishEvent(Event.VIEW_EXPORTED, properties) } -export async function filterCreated() { +export async function filterCreated(timestamp?: string) { const properties: ViewFilterCreatedEvent = {} - await publishEvent(Event.VIEW_FILTER_CREATED, properties) + await publishEvent(Event.VIEW_FILTER_CREATED, properties, timestamp) } export async function filterUpdated() { @@ -54,9 +54,12 @@ export async function filterDeleted() { await publishEvent(Event.VIEW_FILTER_DELETED, properties) } -export async function calculationCreated(calculation: ViewCalculation) { +export async function calculationCreated( + calculation: ViewCalculation, + timestamp?: string +) { const properties: ViewCalculationCreatedEvent = {} - await publishEvent(Event.VIEW_CALCULATION_CREATED, properties) + await publishEvent(Event.VIEW_CALCULATION_CREATED, properties, timestamp) } export async function calculationUpdated() { diff --git a/packages/server/src/migrations/functions/backfill/app.ts b/packages/server/src/migrations/functions/backfill/app.ts index ec905d6efb..60ff953aa0 100644 --- a/packages/server/src/migrations/functions/backfill/app.ts +++ b/packages/server/src/migrations/functions/backfill/app.ts @@ -1,4 +1,3 @@ -import * as apps from "./app/apps" import * as automations from "./app/automations" import * as datasources from "./app/datasources" import * as layouts from "./app/layouts" @@ -7,6 +6,8 @@ import * as roles from "./app/roles" import * as tables from "./app/tables" import * as screens from "./app/screens" import * as global from "./global" +import { App } from "@budibase/types" +import { db as dbUtils, events } from "@budibase/backend-core" /** * Date: @@ -24,12 +25,21 @@ export const run = async (appDb: any) => { return } - await apps.backfill(appDb) - await automations.backfill(appDb) - await datasources.backfill(appDb) - await layouts.backfill(appDb) - await queries.backfill(appDb) - await roles.backfill(appDb) - await tables.backfill(appDb) - await screens.backfill(appDb) + const app: App = await appDb.get(dbUtils.DocumentTypes.APP_METADATA) + const timestamp = app.createdAt as string + + if (dbUtils.isDevAppID(app.appId)) { + await events.app.created(app, timestamp) + await automations.backfill(appDb, timestamp) + await datasources.backfill(appDb, timestamp) + await layouts.backfill(appDb, timestamp) + await queries.backfill(appDb, timestamp) + await roles.backfill(appDb, timestamp) + await tables.backfill(appDb, timestamp) + await screens.backfill(appDb, timestamp) + } + + if (dbUtils.isProdAppID(app.appId)) { + await events.app.published(app, timestamp) + } } diff --git a/packages/server/src/migrations/functions/backfill/app/apps.ts b/packages/server/src/migrations/functions/backfill/app/apps.ts deleted file mode 100644 index 35536198aa..0000000000 --- a/packages/server/src/migrations/functions/backfill/app/apps.ts +++ /dev/null @@ -1,14 +0,0 @@ -import { events, db } from "@budibase/backend-core" -import { App } from "@budibase/types" - -export const backfill = async (appDb: any) => { - const app: App = await appDb.get(db.DocumentTypes.APP_METADATA) - - if (db.isDevAppID(app.appId)) { - await events.app.created(app) - } - - if (db.isProdAppID(app.appId)) { - await events.app.published(app) - } -} diff --git a/packages/server/src/migrations/functions/backfill/app/automations.ts b/packages/server/src/migrations/functions/backfill/app/automations.ts index ccdc1cbdb0..f455e10fc8 100644 --- a/packages/server/src/migrations/functions/backfill/app/automations.ts +++ b/packages/server/src/migrations/functions/backfill/app/automations.ts @@ -1,4 +1,4 @@ -import { events, db } from "@budibase/backend-core" +import { events } from "@budibase/backend-core" import { getAutomationParams } from "../../../../db/utils" import { Automation } from "@budibase/types" @@ -11,16 +11,14 @@ const getAutomations = async (appDb: any): Promise => { return response.rows.map((row: any) => row.doc) } -export const backfill = async (appDb: any) => { - if (db.isDevAppID(appDb.name)) { - const automations = await getAutomations(appDb) +export const backfill = async (appDb: any, timestamp: string) => { + const automations = await getAutomations(appDb) - for (const automation of automations) { - await events.automation.created(automation) + for (const automation of automations) { + await events.automation.created(automation, timestamp) - for (const step of automation.definition.steps) { - await events.automation.stepCreated(automation, step) - } + for (const step of automation.definition.steps) { + await events.automation.stepCreated(automation, step, timestamp) } } } diff --git a/packages/server/src/migrations/functions/backfill/app/datasources.ts b/packages/server/src/migrations/functions/backfill/app/datasources.ts index c1ad9d02e3..1919231a88 100644 --- a/packages/server/src/migrations/functions/backfill/app/datasources.ts +++ b/packages/server/src/migrations/functions/backfill/app/datasources.ts @@ -1,4 +1,4 @@ -import { events, db } from "@budibase/backend-core" +import { events } from "@budibase/backend-core" import { getDatasourceParams } from "../../../../db/utils" import { Datasource } from "@budibase/types" @@ -11,12 +11,10 @@ const getDatasources = async (appDb: any): Promise => { return response.rows.map((row: any) => row.doc) } -export const backfill = async (appDb: any) => { - if (db.isDevAppID(appDb.name)) { - const datasources: Datasource[] = await getDatasources(appDb) +export const backfill = async (appDb: any, timestamp: string) => { + const datasources: Datasource[] = await getDatasources(appDb) - for (const datasource of datasources) { - await events.datasource.created(datasource) - } + for (const datasource of datasources) { + await events.datasource.created(datasource, timestamp) } } diff --git a/packages/server/src/migrations/functions/backfill/app/layouts.ts b/packages/server/src/migrations/functions/backfill/app/layouts.ts index fb0db1b5a6..a02eeb7123 100644 --- a/packages/server/src/migrations/functions/backfill/app/layouts.ts +++ b/packages/server/src/migrations/functions/backfill/app/layouts.ts @@ -1,4 +1,4 @@ -import { events, db } from "@budibase/backend-core" +import { events } from "@budibase/backend-core" import { getLayoutParams } from "../../../../db/utils" import { Layout } from "@budibase/types" @@ -11,12 +11,10 @@ const getLayouts = async (appDb: any): Promise => { return response.rows.map((row: any) => row.doc) } -export const backfill = async (appDb: any) => { - if (db.isDevAppID(appDb.name)) { - const layouts: Layout[] = await getLayouts(appDb) +export const backfill = async (appDb: any, timestamp: string) => { + const layouts: Layout[] = await getLayouts(appDb) - for (const layout of layouts) { - await events.layout.created(layout) - } + for (const layout of layouts) { + await events.layout.created(layout, timestamp) } } diff --git a/packages/server/src/migrations/functions/backfill/app/queries.ts b/packages/server/src/migrations/functions/backfill/app/queries.ts index dfc236ec8b..9e83e6a371 100644 --- a/packages/server/src/migrations/functions/backfill/app/queries.ts +++ b/packages/server/src/migrations/functions/backfill/app/queries.ts @@ -1,4 +1,4 @@ -import { events, db } from "@budibase/backend-core" +import { events } from "@budibase/backend-core" import { getQueryParams } from "../../../../db/utils" import { Query, Datasource } from "@budibase/types" @@ -18,16 +18,14 @@ const getDatasource = async ( return appDb.get(datasourceId) } -export const backfill = async (appDb: any) => { - if (db.isDevAppID(appDb.name)) { - const queries: Query[] = await getQueries(appDb) +export const backfill = async (appDb: any, timestamp: string) => { + const queries: Query[] = await getQueries(appDb) - for (const query of queries) { - const datasource: Datasource = await getDatasource( - appDb, - query.datasourceId - ) - await events.query.created(datasource, query) - } + for (const query of queries) { + const datasource: Datasource = await getDatasource( + appDb, + query.datasourceId + ) + await events.query.created(datasource, query, timestamp) } } diff --git a/packages/server/src/migrations/functions/backfill/app/roles.ts b/packages/server/src/migrations/functions/backfill/app/roles.ts index 4672030b55..4fa57b4165 100644 --- a/packages/server/src/migrations/functions/backfill/app/roles.ts +++ b/packages/server/src/migrations/functions/backfill/app/roles.ts @@ -1,4 +1,4 @@ -import { events, db } from "@budibase/backend-core" +import { events } from "@budibase/backend-core" import { getRoleParams } from "../../../../db/utils" import { Role } from "@budibase/types" @@ -11,12 +11,10 @@ const getRoles = async (appDb: any): Promise => { return response.rows.map((row: any) => row.doc) } -export const backfill = async (appDb: any) => { - if (db.isDevAppID(appDb.name)) { - const roles = await getRoles(appDb) +export const backfill = async (appDb: any, timestamp: string) => { + const roles = await getRoles(appDb) - for (const role of roles) { - await events.role.created(role) - } + for (const role of roles) { + await events.role.created(role, timestamp) } } diff --git a/packages/server/src/migrations/functions/backfill/app/screens.ts b/packages/server/src/migrations/functions/backfill/app/screens.ts index 340187ae40..b1d9791c2d 100644 --- a/packages/server/src/migrations/functions/backfill/app/screens.ts +++ b/packages/server/src/migrations/functions/backfill/app/screens.ts @@ -1,4 +1,4 @@ -import { events, db } from "@budibase/backend-core" +import { events } from "@budibase/backend-core" import { getScreenParams } from "../../../../db/utils" import { Screen } from "@budibase/types" @@ -11,12 +11,10 @@ const getScreens = async (appDb: any): Promise => { return response.rows.map((row: any) => row.doc) } -export const backfill = async (appDb: any) => { - if (db.isDevAppID(appDb.name)) { - const screens = await getScreens(appDb) +export const backfill = async (appDb: any, timestamp: string) => { + const screens = await getScreens(appDb) - for (const screen of screens) { - await events.screen.created(screen) - } + for (const screen of screens) { + await events.screen.created(screen, timestamp) } } diff --git a/packages/server/src/migrations/functions/backfill/app/tables.ts b/packages/server/src/migrations/functions/backfill/app/tables.ts index c7401e921d..23d0dff702 100644 --- a/packages/server/src/migrations/functions/backfill/app/tables.ts +++ b/packages/server/src/migrations/functions/backfill/app/tables.ts @@ -1,4 +1,4 @@ -import { events, db } from "@budibase/backend-core" +import { events } from "@budibase/backend-core" import { getTableParams } from "../../../../db/utils" import { Table } from "@budibase/types" @@ -11,24 +11,22 @@ const getTables = async (appDb: any): Promise => { return response.rows.map((row: any) => row.doc) } -export const backfill = async (appDb: any) => { - if (db.isDevAppID(appDb.name)) { - const tables = await getTables(appDb) +export const backfill = async (appDb: any, timestamp: string) => { + const tables = await getTables(appDb) - for (const table of tables) { - await events.table.created(table) + for (const table of tables) { + await events.table.created(table, timestamp) - if (table.views) { - for (const view of Object.values(table.views)) { - await events.view.created(view) + if (table.views) { + for (const view of Object.values(table.views)) { + await events.view.created(view, timestamp) - if (view.calculation) { - await events.view.calculationCreated(view.calculation) - } + if (view.calculation) { + await events.view.calculationCreated(view.calculation, timestamp) + } - if (view.filters?.length) { - await events.view.filterCreated() - } + if (view.filters?.length) { + await events.view.filterCreated(timestamp) } } } diff --git a/packages/server/src/migrations/functions/backfill/global.ts b/packages/server/src/migrations/functions/backfill/global.ts index 44aa29a561..d723844d5a 100644 --- a/packages/server/src/migrations/functions/backfill/global.ts +++ b/packages/server/src/migrations/functions/backfill/global.ts @@ -13,11 +13,14 @@ import { tenancy, events, migrations } from "@budibase/backend-core" export const run = async (db: any) => { const tenantId = tenancy.getTenantId() - await events.identification.identifyTenant(tenantId) + const installTimestamp = (await getInstallTimestamp(db)) as number + await events.identification.identifyTenant(tenantId, installTimestamp) + await configs.backfill(db, installTimestamp) + + // users and rows provide their own timestamp await users.backfill(db) await rows.backfill() - await configs.backfill(db) } export const isComplete = async (): Promise => { @@ -25,3 +28,20 @@ export const isComplete = async (): Promise => { const migrationsDoc = await migrations.getMigrationsDoc(globalDb) return !!migrationsDoc.event_global_backfill } + +const getInstallTimestamp = async (db: any): Promise => { + const allUsers = await users.getUsers(db) + + // get the olders user timestamp + const timestamps = allUsers + .map(user => user.createdAt) + .filter(timestamp => !!timestamp) + .sort( + (a, b) => + new Date(a as number).getTime() - new Date(b as number).getTime() + ) + + if (timestamps.length) { + return timestamps[0] + } +} diff --git a/packages/server/src/migrations/functions/backfill/global/configs.ts b/packages/server/src/migrations/functions/backfill/global/configs.ts index 4705e49a38..94ac838e20 100644 --- a/packages/server/src/migrations/functions/backfill/global/configs.ts +++ b/packages/server/src/migrations/functions/backfill/global/configs.ts @@ -1,4 +1,4 @@ -import { events, db } from "@budibase/backend-core" +import { events, db as dbUtils } from "@budibase/backend-core" import { Config, isSMTPConfig, @@ -10,7 +10,7 @@ import env from "./../../../../environment" const getConfigs = async (globalDb: any): Promise => { const response = await globalDb.allDocs( - db.getConfigParams( + dbUtils.getConfigParams( {}, { include_docs: true, @@ -20,34 +20,37 @@ const getConfigs = async (globalDb: any): Promise => { return response.rows.map((row: any) => row.doc) } -export const backfill = async (globalDb: any) => { +export const backfill = async ( + globalDb: any, + timestamp: string | number | undefined +) => { const configs = await getConfigs(globalDb) for (const config of configs) { if (isSMTPConfig(config)) { - await events.email.SMTPCreated(config) + await events.email.SMTPCreated(config, timestamp) } if (isGoogleConfig(config)) { - await events.auth.SSOCreated("google") + await events.auth.SSOCreated("google", timestamp) if (config.config.activated) { - await events.auth.SSOActivated("google") + await events.auth.SSOActivated("google", timestamp) } } if (isOIDCConfig(config)) { - await events.auth.SSOCreated("oidc") + await events.auth.SSOCreated("oidc", timestamp) if (config.config.configs[0].activated) { - await events.auth.SSOActivated("oidc") + await events.auth.SSOActivated("oidc", timestamp) } } if (isSettingsConfig(config)) { const company = config.config.company if (company && company !== "Budibase") { - await events.org.nameUpdated() + await events.org.nameUpdated(timestamp) } const logoUrl = config.config.logoUrl if (logoUrl) { - await events.org.logoUpdated() + await events.org.logoUpdated(timestamp) } const platformUrl = config.config.platformUrl @@ -56,7 +59,7 @@ export const backfill = async (globalDb: any) => { platformUrl !== "http://localhost:10000" && env.SELF_HOSTED ) { - await events.org.platformURLUpdated() + await events.org.platformURLUpdated(timestamp) } } } diff --git a/packages/server/src/migrations/functions/backfill/global/rows.ts b/packages/server/src/migrations/functions/backfill/global/rows.ts index 4aaa47c1ee..b336668d64 100644 --- a/packages/server/src/migrations/functions/backfill/global/rows.ts +++ b/packages/server/src/migrations/functions/backfill/global/rows.ts @@ -1,16 +1,28 @@ -import { events, db } from "@budibase/backend-core" -import { Row } from "@budibase/types" +import { events, db as dbUtils } from "@budibase/backend-core" +import { Row, App } from "@budibase/types" import { getUniqueRows } from "../../../../utilities/usageQuota/rows" // Rows is a special circumstance where we get rows across all apps // therefore migration is performed at the global level +const getOldestCreatedAt = (allApps: App[]): string | undefined => { + const timestamps = allApps + .filter(app => !!app.createdAt) + .map(app => app.createdAt as string) + .sort((a, b) => new Date(a).getTime() - new Date(b).getTime()) + + if (timestamps.length) { + return timestamps[0] + } +} + export const backfill = async () => { - const allApps = await db.getAllApps({ all: true }) + const allApps: App[] = await dbUtils.getAllApps({ dev: true }) + const timestamp = getOldestCreatedAt(allApps) const appIds = allApps ? allApps.map((app: { appId: any }) => app.appId) : [] const rows: Row[] = await getUniqueRows(appIds) const rowCount = rows ? rows.length : 0 if (rowCount) { - await events.rows.created(rowCount) + await events.rows.created(rowCount, timestamp) } } diff --git a/packages/server/src/migrations/functions/backfill/global/users.ts b/packages/server/src/migrations/functions/backfill/global/users.ts index 1816c970db..c7c1b40df2 100644 --- a/packages/server/src/migrations/functions/backfill/global/users.ts +++ b/packages/server/src/migrations/functions/backfill/global/users.ts @@ -1,12 +1,12 @@ -import { events, db } from "@budibase/backend-core" +import { events, db as dbUtils } from "@budibase/backend-core" import { User } from "@budibase/types" // manually define user doc params - normally server doesn't read users from the db const getUserParams = (props: any) => { - return db.getDocParams(db.DocumentTypes.USER, null, props) + return dbUtils.getDocParams(dbUtils.DocumentTypes.USER, null, props) } -const getUsers = async (globalDb: any): Promise => { +export const getUsers = async (globalDb: any): Promise => { const response = await globalDb.allDocs( getUserParams({ include_docs: true, @@ -19,20 +19,21 @@ export const backfill = async (globalDb: any) => { const users = await getUsers(globalDb) for (const user of users) { - await events.identification.identifyUser(user) - await events.user.created(user) + const timestamp = user.createdAt as number + await events.identification.identifyUser(user, timestamp) + await events.user.created(user, timestamp) if (user.admin?.global) { - await events.user.permissionAdminAssigned(user) + await events.user.permissionAdminAssigned(user, timestamp) } if (user.builder?.global) { - await events.user.permissionBuilderAssigned(user) + await events.user.permissionBuilderAssigned(user, timestamp) } if (user.roles) { for (const [appId, role] of Object.entries(user.roles)) { - await events.role.assigned(user, role) + await events.role.assigned(user, role, timestamp) } } } diff --git a/packages/server/tsconfig.build.json b/packages/server/tsconfig.build.json index 836a726805..0f606f3a1d 100644 --- a/packages/server/tsconfig.build.json +++ b/packages/server/tsconfig.build.json @@ -2,7 +2,8 @@ // Used for building with tsc "extends": "./tsconfig.json", "references": [ - { "path": "../backend-core/tsconfig.build.json" } + { "path": "../backend-core/tsconfig.build.json" }, + { "path": "../types" } ], "exclude": [ "node_modules", diff --git a/packages/server/tsconfig.json b/packages/server/tsconfig.json index 4412d07b1d..d0fed6e824 100644 --- a/packages/server/tsconfig.json +++ b/packages/server/tsconfig.json @@ -16,9 +16,6 @@ "./src/**/*", "./src/module.d.ts" ], - "references": [ - { "path": "../backend-core/tsconfig.build.json" } - ], "exclude": [ "node_modules", "**/*.json", diff --git a/packages/types/src/documents/document.ts b/packages/types/src/documents/document.ts index b1b41394f4..fea05918d5 100644 --- a/packages/types/src/documents/document.ts +++ b/packages/types/src/documents/document.ts @@ -1,6 +1,6 @@ export interface Document { _id?: string _rev?: string - createdAt?: string + createdAt?: string | number updatedAt?: string } diff --git a/packages/types/src/documents/global/user.ts b/packages/types/src/documents/global/user.ts index 61a942fdb8..2de9ff7719 100644 --- a/packages/types/src/documents/global/user.ts +++ b/packages/types/src/documents/global/user.ts @@ -1,6 +1,8 @@ import { Document } from "../document" export interface User extends Document { + tenantId: string + email: string roles: UserRoles builder?: { global: boolean @@ -9,10 +11,9 @@ export interface User extends Document { global: boolean } providerType?: string - tenantId: string - email: string password?: string status?: string + createdAt?: number // override the default createdAt behaviour - users sdk historically set this to Date.now() } export interface UserRoles { diff --git a/packages/types/tsconfig.json b/packages/types/tsconfig.json index 86a066fec8..1571b1ea3c 100644 --- a/packages/types/tsconfig.json +++ b/packages/types/tsconfig.json @@ -10,6 +10,7 @@ "resolveJsonModule": true, "incremental": true, "types": [ "node"], + "composite": true }, "include": [ "**/*.ts", diff --git a/packages/worker/tsconfig.json b/packages/worker/tsconfig.json index 080766172a..bb5c2ca8f8 100644 --- a/packages/worker/tsconfig.json +++ b/packages/worker/tsconfig.json @@ -13,7 +13,8 @@ "types": [ "node", "jest"], }, "references": [ - { "path": "../backend-core/tsconfig.build.json" } + { "path": "../backend-core/tsconfig.build.json" }, + { "path": "../types" } ], "include": [ "./src/**/*"