From 7f26a60cc97b6d61eb3133003d1f481723185312 Mon Sep 17 00:00:00 2001 From: Rory Powell Date: Wed, 1 Jun 2022 14:44:12 +0100 Subject: [PATCH] Default to historical timestamp --- .../backend-core/src/events/publishers/automation.ts | 7 +++++-- .../backend-core/src/events/publishers/datasource.ts | 5 ++++- .../backend-core/src/events/publishers/layout.ts | 2 +- packages/backend-core/src/events/publishers/query.ts | 2 +- packages/backend-core/src/events/publishers/role.ts | 2 +- packages/backend-core/src/events/publishers/rows.ts | 2 +- .../backend-core/src/events/publishers/screen.ts | 2 +- packages/backend-core/src/events/publishers/table.ts | 2 +- packages/backend-core/src/events/publishers/view.ts | 9 ++++++--- .../server/src/migrations/functions/backfill/app.ts | 6 +++++- .../migrations/functions/backfill/app/automations.ts | 2 +- .../migrations/functions/backfill/app/datasources.ts | 2 +- .../src/migrations/functions/backfill/app/layouts.ts | 2 +- .../src/migrations/functions/backfill/app/queries.ts | 2 +- .../src/migrations/functions/backfill/app/roles.ts | 2 +- .../src/migrations/functions/backfill/app/screens.ts | 2 +- .../src/migrations/functions/backfill/app/tables.ts | 2 +- .../src/migrations/functions/backfill/global.ts | 12 ++++++++---- .../migrations/functions/backfill/global/quotas.ts | 11 +++++++++-- .../migrations/functions/backfill/global/users.ts | 6 +++++- .../src/migrations/functions/backfill/index.ts | 4 ++++ .../migrations/functions/backfill/installation.ts | 11 +++++++++-- 22 files changed, 68 insertions(+), 29 deletions(-) diff --git a/packages/backend-core/src/events/publishers/automation.ts b/packages/backend-core/src/events/publishers/automation.ts index e1df28ead8..95f9cb8db6 100644 --- a/packages/backend-core/src/events/publishers/automation.ts +++ b/packages/backend-core/src/events/publishers/automation.ts @@ -12,7 +12,10 @@ import { AutomationsRunEvent, } from "@budibase/types" -export async function created(automation: Automation, timestamp?: string) { +export async function created( + automation: Automation, + timestamp?: string | number +) { const properties: AutomationCreatedEvent = { appId: automation.appId, automationId: automation._id as string, @@ -62,7 +65,7 @@ export const run = async (count: number, timestamp?: string | number) => { export async function stepCreated( automation: Automation, step: AutomationStep, - timestamp?: string + timestamp?: string | number ) { const properties: AutomationStepCreatedEvent = { appId: automation.appId, diff --git a/packages/backend-core/src/events/publishers/datasource.ts b/packages/backend-core/src/events/publishers/datasource.ts index 4cff0c9a18..3cd68033fc 100644 --- a/packages/backend-core/src/events/publishers/datasource.ts +++ b/packages/backend-core/src/events/publishers/datasource.ts @@ -7,7 +7,10 @@ import { DatasourceDeletedEvent, } from "@budibase/types" -export async function created(datasource: Datasource, timestamp?: string) { +export async function created( + datasource: Datasource, + timestamp?: string | number +) { const properties: DatasourceCreatedEvent = { datasourceId: datasource._id as string, source: datasource.source, diff --git a/packages/backend-core/src/events/publishers/layout.ts b/packages/backend-core/src/events/publishers/layout.ts index 82e9f613ca..1eede40143 100644 --- a/packages/backend-core/src/events/publishers/layout.ts +++ b/packages/backend-core/src/events/publishers/layout.ts @@ -6,7 +6,7 @@ import { LayoutDeletedEvent, } from "@budibase/types" -export async function created(layout: Layout, timestamp?: string) { +export async function created(layout: Layout, timestamp?: string | number) { const properties: LayoutCreatedEvent = { layoutId: layout._id as string, } diff --git a/packages/backend-core/src/events/publishers/query.ts b/packages/backend-core/src/events/publishers/query.ts index c031885ac3..1bcf561fd0 100644 --- a/packages/backend-core/src/events/publishers/query.ts +++ b/packages/backend-core/src/events/publishers/query.ts @@ -16,7 +16,7 @@ import { export const created = async ( datasource: Datasource, query: Query, - timestamp?: string + timestamp?: string | number ) => { const properties: QueryCreatedEvent = { queryId: query._id as string, diff --git a/packages/backend-core/src/events/publishers/role.ts b/packages/backend-core/src/events/publishers/role.ts index 4cd5f91703..99074d76a5 100644 --- a/packages/backend-core/src/events/publishers/role.ts +++ b/packages/backend-core/src/events/publishers/role.ts @@ -10,7 +10,7 @@ import { User, } from "@budibase/types" -export async function created(role: Role, timestamp?: string) { +export async function created(role: Role, timestamp?: string | number) { const properties: RoleCreatedEvent = { roleId: role._id as string, permissionId: role.permissionId, diff --git a/packages/backend-core/src/events/publishers/rows.ts b/packages/backend-core/src/events/publishers/rows.ts index 9ff5b4d3d7..487e7ea074 100644 --- a/packages/backend-core/src/events/publishers/rows.ts +++ b/packages/backend-core/src/events/publishers/rows.ts @@ -9,7 +9,7 @@ import { /* eslint-disable */ -export const created = async (count: number, timestamp?: string) => { +export const created = async (count: number, timestamp?: string | number) => { const properties: RowsCreatedEvent = { count, } diff --git a/packages/backend-core/src/events/publishers/screen.ts b/packages/backend-core/src/events/publishers/screen.ts index cbd19327e6..966bf72e52 100644 --- a/packages/backend-core/src/events/publishers/screen.ts +++ b/packages/backend-core/src/events/publishers/screen.ts @@ -6,7 +6,7 @@ import { ScreenDeletedEvent, } from "@budibase/types" -export async function created(screen: Screen, timestamp?: string) { +export async function created(screen: Screen, timestamp?: string | number) { const properties: ScreenCreatedEvent = { layoutId: screen.layoutId, screenId: screen._id as string, diff --git a/packages/backend-core/src/events/publishers/table.ts b/packages/backend-core/src/events/publishers/table.ts index 9853be7eab..ba53163ca3 100644 --- a/packages/backend-core/src/events/publishers/table.ts +++ b/packages/backend-core/src/events/publishers/table.ts @@ -11,7 +11,7 @@ import { TableImportedEvent, } from "@budibase/types" -export async function created(table: Table, timestamp?: string) { +export async function created(table: Table, timestamp?: string | number) { const properties: TableCreatedEvent = { tableId: table._id as string, } diff --git a/packages/backend-core/src/events/publishers/view.ts b/packages/backend-core/src/events/publishers/view.ts index 1a382ec00e..17947758d5 100644 --- a/packages/backend-core/src/events/publishers/view.ts +++ b/packages/backend-core/src/events/publishers/view.ts @@ -19,7 +19,7 @@ import { /* eslint-disable */ -export async function created(view: View, timestamp?: string) { +export async function created(view: View, timestamp?: string | number) { const properties: ViewCreatedEvent = { tableId: view.tableId, } @@ -48,7 +48,7 @@ export async function exported(table: Table, format: TableExportFormat) { await publishEvent(Event.VIEW_EXPORTED, properties) } -export async function filterCreated(view: View, timestamp?: string) { +export async function filterCreated(view: View, timestamp?: string | number) { const properties: ViewFilterCreatedEvent = { tableId: view.tableId, } @@ -69,7 +69,10 @@ export async function filterDeleted(view: View) { await publishEvent(Event.VIEW_FILTER_DELETED, properties) } -export async function calculationCreated(view: View, timestamp?: string) { +export async function calculationCreated( + view: View, + timestamp?: string | number +) { const properties: ViewCalculationCreatedEvent = { tableId: view.tableId, calculation: view.calculation as ViewCalculation, diff --git a/packages/server/src/migrations/functions/backfill/app.ts b/packages/server/src/migrations/functions/backfill/app.ts index 3c562ecc1d..0f53e4d0ad 100644 --- a/packages/server/src/migrations/functions/backfill/app.ts +++ b/packages/server/src/migrations/functions/backfill/app.ts @@ -9,6 +9,7 @@ import * as global from "./global" import { App, AppBackfillSucceededEvent, Event } from "@budibase/types" import { db as dbUtils, events } from "@budibase/backend-core" import env from "../../../environment" +import { DEFAULT_TIMESTAMP } from "." const failGraceful = env.SELF_HOSTED && !env.isDev() @@ -59,8 +60,11 @@ export const run = async (appDb: any) => { // events for this tenant await events.backfillCache.start(EVENTS) + let timestamp: string | number = DEFAULT_TIMESTAMP const app: App = await appDb.get(dbUtils.DocumentTypes.APP_METADATA) - const timestamp = app.createdAt as string + if (app.createdAt) { + timestamp = app.createdAt as string + } if (dbUtils.isProdAppID(app.appId)) { await events.app.published(app, timestamp) diff --git a/packages/server/src/migrations/functions/backfill/app/automations.ts b/packages/server/src/migrations/functions/backfill/app/automations.ts index e4c3ad19f3..20da8fd3c0 100644 --- a/packages/server/src/migrations/functions/backfill/app/automations.ts +++ b/packages/server/src/migrations/functions/backfill/app/automations.ts @@ -11,7 +11,7 @@ const getAutomations = async (appDb: any): Promise => { return response.rows.map((row: any) => row.doc) } -export const backfill = async (appDb: any, timestamp: string) => { +export const backfill = async (appDb: any, timestamp: string | number) => { const automations = await getAutomations(appDb) for (const automation of automations) { diff --git a/packages/server/src/migrations/functions/backfill/app/datasources.ts b/packages/server/src/migrations/functions/backfill/app/datasources.ts index 1245e5d745..5d7e1ad866 100644 --- a/packages/server/src/migrations/functions/backfill/app/datasources.ts +++ b/packages/server/src/migrations/functions/backfill/app/datasources.ts @@ -11,7 +11,7 @@ const getDatasources = async (appDb: any): Promise => { return response.rows.map((row: any) => row.doc) } -export const backfill = async (appDb: any, timestamp: string) => { +export const backfill = async (appDb: any, timestamp: string | number) => { const datasources: Datasource[] = await getDatasources(appDb) for (const datasource of datasources) { diff --git a/packages/server/src/migrations/functions/backfill/app/layouts.ts b/packages/server/src/migrations/functions/backfill/app/layouts.ts index d2a551096d..ee5806459b 100644 --- a/packages/server/src/migrations/functions/backfill/app/layouts.ts +++ b/packages/server/src/migrations/functions/backfill/app/layouts.ts @@ -11,7 +11,7 @@ const getLayouts = async (appDb: any): Promise => { return response.rows.map((row: any) => row.doc) } -export const backfill = async (appDb: any, timestamp: string) => { +export const backfill = async (appDb: any, timestamp: string | number) => { const layouts: Layout[] = await getLayouts(appDb) for (const layout of layouts) { diff --git a/packages/server/src/migrations/functions/backfill/app/queries.ts b/packages/server/src/migrations/functions/backfill/app/queries.ts index fb0e561363..e07dec8c06 100644 --- a/packages/server/src/migrations/functions/backfill/app/queries.ts +++ b/packages/server/src/migrations/functions/backfill/app/queries.ts @@ -18,7 +18,7 @@ const getDatasource = async ( return appDb.get(datasourceId) } -export const backfill = async (appDb: any, timestamp: string) => { +export const backfill = async (appDb: any, timestamp: string | number) => { const queries: Query[] = await getQueries(appDb) for (const query of queries) { diff --git a/packages/server/src/migrations/functions/backfill/app/roles.ts b/packages/server/src/migrations/functions/backfill/app/roles.ts index 1c90b0f658..494b6f6923 100644 --- a/packages/server/src/migrations/functions/backfill/app/roles.ts +++ b/packages/server/src/migrations/functions/backfill/app/roles.ts @@ -11,7 +11,7 @@ const getRoles = async (appDb: any): Promise => { return response.rows.map((row: any) => row.doc) } -export const backfill = async (appDb: any, timestamp: string) => { +export const backfill = async (appDb: any, timestamp: string | number) => { const roles = await getRoles(appDb) for (const role of roles) { diff --git a/packages/server/src/migrations/functions/backfill/app/screens.ts b/packages/server/src/migrations/functions/backfill/app/screens.ts index a17be76ea3..ab3b4b9d3c 100644 --- a/packages/server/src/migrations/functions/backfill/app/screens.ts +++ b/packages/server/src/migrations/functions/backfill/app/screens.ts @@ -11,7 +11,7 @@ const getScreens = async (appDb: any): Promise => { return response.rows.map((row: any) => row.doc) } -export const backfill = async (appDb: any, timestamp: string) => { +export const backfill = async (appDb: any, timestamp: string | number) => { const screens = await getScreens(appDb) for (const screen of screens) { diff --git a/packages/server/src/migrations/functions/backfill/app/tables.ts b/packages/server/src/migrations/functions/backfill/app/tables.ts index 484a179bbe..150a3b4d4a 100644 --- a/packages/server/src/migrations/functions/backfill/app/tables.ts +++ b/packages/server/src/migrations/functions/backfill/app/tables.ts @@ -11,7 +11,7 @@ const getTables = async (appDb: any): Promise => { return response.rows.map((row: any) => row.doc) } -export const backfill = async (appDb: any, timestamp: string) => { +export const backfill = async (appDb: any, timestamp: string | number) => { const tables = await getTables(appDb) for (const table of tables) { diff --git a/packages/server/src/migrations/functions/backfill/global.ts b/packages/server/src/migrations/functions/backfill/global.ts index 8b58b15c7e..cac38b8066 100644 --- a/packages/server/src/migrations/functions/backfill/global.ts +++ b/packages/server/src/migrations/functions/backfill/global.ts @@ -16,6 +16,7 @@ import { Event, } from "@budibase/types" import env from "../../../environment" +import { DEFAULT_TIMESTAMP } from "." const failGraceful = env.SELF_HOSTED && !env.isDev() @@ -88,13 +89,16 @@ const EVENTS = [ export const run = async (db: any) => { try { const tenantId = tenancy.getTenantId() - let installTimestamp + let timestamp: string | number = DEFAULT_TIMESTAMP const totals: any = {} const errors: any = [] try { - installTimestamp = await getInstallTimestamp(db) + const installTimestamp = await getInstallTimestamp(db) + if (installTimestamp) { + timestamp = installTimestamp + } } catch (e) { handleError(e, errors) } @@ -108,7 +112,7 @@ export const run = async (db: any) => { await events.identification.identifyTenantGroup( tenantId, account, - installTimestamp + timestamp ) } catch (e) { handleError(e, errors) @@ -119,7 +123,7 @@ export const run = async (db: any) => { await events.backfillCache.start(EVENTS) try { - await configs.backfill(db, installTimestamp) + await configs.backfill(db, timestamp) } catch (e) { handleError(e, errors) } diff --git a/packages/server/src/migrations/functions/backfill/global/quotas.ts b/packages/server/src/migrations/functions/backfill/global/quotas.ts index 274804adad..505274a318 100644 --- a/packages/server/src/migrations/functions/backfill/global/quotas.ts +++ b/packages/server/src/migrations/functions/backfill/global/quotas.ts @@ -1,3 +1,4 @@ +import { DEFAULT_TIMESTAMP } from "./../index" import { events } from "@budibase/backend-core" import { quotas } from "@budibase/pro" import { App } from "@budibase/types" @@ -33,8 +34,14 @@ export const backfill = async (allApps: App[]) => { const usage = await quotas.getQuotaUsage() const rows = usage.usageQuota.rows - const rowsTimestamp = getOldestCreatedAt(allApps) - await events.rows.created(rows, rowsTimestamp) + let timestamp: string | number = DEFAULT_TIMESTAMP + + const oldestAppTimestamp = getOldestCreatedAt(allApps) + if (oldestAppTimestamp) { + timestamp = oldestAppTimestamp + } + + await events.rows.created(rows, timestamp) for (const [monthString, quotas] of Object.entries(usage.monthly)) { if (monthString === "current") { diff --git a/packages/server/src/migrations/functions/backfill/global/users.ts b/packages/server/src/migrations/functions/backfill/global/users.ts index 0a59bec630..5f811614b4 100644 --- a/packages/server/src/migrations/functions/backfill/global/users.ts +++ b/packages/server/src/migrations/functions/backfill/global/users.ts @@ -1,5 +1,6 @@ import { events, db as dbUtils } from "@budibase/backend-core" import { User, CloudAccount } from "@budibase/types" +import { DEFAULT_TIMESTAMP } from ".." // manually define user doc params - normally server doesn't read users from the db const getUserParams = (props: any) => { @@ -22,7 +23,10 @@ export const backfill = async ( const users = await getUsers(globalDb) for (const user of users) { - const timestamp = user.createdAt as number + let timestamp: string | number = DEFAULT_TIMESTAMP + if (user.createdAt) { + timestamp = user.createdAt + } await events.identification.identifyUser(user, account, timestamp) await events.user.created(user, timestamp) diff --git a/packages/server/src/migrations/functions/backfill/index.ts b/packages/server/src/migrations/functions/backfill/index.ts index dee9b82e1a..00c04722b4 100644 --- a/packages/server/src/migrations/functions/backfill/index.ts +++ b/packages/server/src/migrations/functions/backfill/index.ts @@ -1,3 +1,7 @@ export * as app from "./app" export * as global from "./global" export * as installation from "./installation" + +// historical events are free in posthog - make sure we default to a +// historical time if no other can be found +export const DEFAULT_TIMESTAMP = new Date(2022, 0, 1).getTime() diff --git a/packages/server/src/migrations/functions/backfill/installation.ts b/packages/server/src/migrations/functions/backfill/installation.ts index 425ec80f0d..430fe2edd1 100644 --- a/packages/server/src/migrations/functions/backfill/installation.ts +++ b/packages/server/src/migrations/functions/backfill/installation.ts @@ -1,3 +1,4 @@ +import { DEFAULT_TIMESTAMP } from "./index" import { events, tenancy, installation } from "@budibase/backend-core" import { Installation } from "@budibase/types" import * as global from "./global" @@ -28,11 +29,17 @@ export const run = async () => { // need to use the default tenant to try to get the installation time await tenancy.doInTenant(tenancy.DEFAULT_TENANT_ID, async () => { const db = tenancy.getGlobalDB() - const installTimestamp = (await global.getInstallTimestamp(db)) as number + let timestamp: string | number = DEFAULT_TIMESTAMP + + const installTimestamp = await global.getInstallTimestamp(db) + if (installTimestamp) { + timestamp = installTimestamp + } + const install: Installation = await installation.getInstall() await events.identification.identifyInstallationGroup( install.installId, - installTimestamp + timestamp ) }) await events.backfill.installationSucceeded()