From 3332f2fa228e958f2a2541ed7c9715a3d2544867 Mon Sep 17 00:00:00 2001 From: Martin McKeaveney Date: Thu, 5 Dec 2024 17:32:07 +0000 Subject: [PATCH] metrics for view joins and grouped filters --- .../processors/posthog/PosthogProcessor.ts | 4 +- .../src/events/publishers/table.ts | 33 +++++++++--- .../src/events/publishers/view.ts | 17 ++++-- .../src/api/controllers/table/external.ts | 4 +- .../server/src/api/controllers/table/index.ts | 11 +++- .../src/api/controllers/table/internal.ts | 4 +- .../server/src/api/controllers/view/views.ts | 54 +++++++------------ .../src/api/controllers/view/viewsV2.ts | 11 ++-- .../functions/backfill/app/tables.ts | 2 +- .../src/sdk/app/tables/external/index.ts | 2 +- .../src/sdk/app/tables/internal/index.ts | 2 +- packages/server/src/sdk/app/views/external.ts | 4 +- packages/server/src/sdk/app/views/index.ts | 5 +- packages/server/src/sdk/app/views/internal.ts | 4 +- packages/types/src/sdk/events/table.ts | 3 ++ packages/types/src/sdk/events/view.ts | 7 +-- 16 files changed, 100 insertions(+), 67 deletions(-) diff --git a/packages/backend-core/src/events/processors/posthog/PosthogProcessor.ts b/packages/backend-core/src/events/processors/posthog/PosthogProcessor.ts index 12d2bb7e2c..687fe54256 100644 --- a/packages/backend-core/src/events/processors/posthog/PosthogProcessor.ts +++ b/packages/backend-core/src/events/processors/posthog/PosthogProcessor.ts @@ -13,8 +13,8 @@ const EXCLUDED_EVENTS: Event[] = [ Event.ROLE_UPDATED, Event.DATASOURCE_UPDATED, Event.QUERY_UPDATED, - Event.TABLE_UPDATED, - Event.VIEW_UPDATED, + // Event.TABLE_UPDATED, + // Event.VIEW_UPDATED, Event.VIEW_FILTER_UPDATED, Event.VIEW_CALCULATION_UPDATED, Event.AUTOMATION_TRIGGER_UPDATED, diff --git a/packages/backend-core/src/events/publishers/table.ts b/packages/backend-core/src/events/publishers/table.ts index dc3200291a..de765e2cdd 100644 --- a/packages/backend-core/src/events/publishers/table.ts +++ b/packages/backend-core/src/events/publishers/table.ts @@ -1,13 +1,14 @@ import { publishEvent } from "../events" import { Event, - TableExportFormat, + FieldType, Table, TableCreatedEvent, - TableUpdatedEvent, TableDeletedEvent, TableExportedEvent, + TableExportFormat, TableImportedEvent, + TableUpdatedEvent, } from "@budibase/types" async function created(table: Table, timestamp?: string | number) { @@ -20,14 +21,34 @@ async function created(table: Table, timestamp?: string | number) { await publishEvent(Event.TABLE_CREATED, properties, timestamp) } -async function updated(table: Table) { +async function updated(oldTable: Table, newTable: Table) { + // only publish the event if it has fields we are interested in + let defaultValues, aiColumn + + // check that new fields have been added + for (const key in newTable.schema) { + if (!oldTable.schema[key]) { + const newColumn = newTable.schema[key] + if ("default" in newColumn) { + defaultValues = true + } + if (newColumn.type === FieldType.AI) { + aiColumn = newColumn.operation + } + } + } + const properties: TableUpdatedEvent = { - tableId: table._id as string, + tableId: newTable._id as string, + defaultValues, + aiColumn, audited: { - name: table.name, + name: newTable.name, }, } - await publishEvent(Event.TABLE_UPDATED, properties) + if (defaultValues || aiColumn) { + await publishEvent(Event.TABLE_UPDATED, properties) + } } async function deleted(table: Table) { diff --git a/packages/backend-core/src/events/publishers/view.ts b/packages/backend-core/src/events/publishers/view.ts index 0ec8f67485..6d107d58f3 100644 --- a/packages/backend-core/src/events/publishers/view.ts +++ b/packages/backend-core/src/events/publishers/view.ts @@ -11,6 +11,7 @@ import { ViewFilterDeletedEvent, ViewFilterUpdatedEvent, ViewUpdatedEvent, + View, ViewV2, ViewCalculation, Table, @@ -19,17 +20,27 @@ import { /* eslint-disable */ -async function created(view: Partial, timestamp?: string | number) { +async function created(view: ViewV2, timestamp?: string | number) { const properties: ViewCreatedEvent = { name: view.name, type: view.type, + tableId: view.tableId, } await publishEvent(Event.VIEW_CREATED, properties, timestamp) } -async function updated(view: View) { +async function updated(newView: ViewV2) { + // // check whether any of the fields are different + let viewJoins = 0 + for (const key in newView.schema) { + if (newView.schema[key]?.columns) { + viewJoins += Object.keys(newView.schema[key]?.columns).length + } + } const properties: ViewUpdatedEvent = { - tableId: view.tableId, + tableId: newView.tableId, + groupedFilters: newView.queryUI?.groups?.length || 0, + viewJoins, } await publishEvent(Event.VIEW_UPDATED, properties) } diff --git a/packages/server/src/api/controllers/table/external.ts b/packages/server/src/api/controllers/table/external.ts index 6f09bf4a61..d3f5ef99f6 100644 --- a/packages/server/src/api/controllers/table/external.ts +++ b/packages/server/src/api/controllers/table/external.ts @@ -45,13 +45,13 @@ export async function updateTable( inputs.created = true } try { - const { datasource, table } = await sdk.tables.external.save( + const { datasource, oldTable, table } = await sdk.tables.external.save( datasourceId!, inputs, { tableId, renaming } ) builderSocket?.emitDatasourceUpdate(ctx, datasource) - return table + return { table, oldTable } } catch (err: any) { if (err instanceof Error) { ctx.throw(400, err.message) diff --git a/packages/server/src/api/controllers/table/index.ts b/packages/server/src/api/controllers/table/index.ts index 77c1f3923a..0b0c9531e9 100644 --- a/packages/server/src/api/controllers/table/index.ts +++ b/packages/server/src/api/controllers/table/index.ts @@ -119,8 +119,15 @@ export async function save(ctx: UserCtx) { await events.table.created(savedTable) } else { const api = pickApi({ table }) - savedTable = await api.updateTable(ctx, renaming) - await events.table.updated(savedTable) + const { table: updatedTable, oldTable } = await api.updateTable( + ctx, + renaming + ) + savedTable = updatedTable + + if (oldTable) { + await events.table.updated(oldTable, savedTable) + } } if (renaming) { await sdk.views.renameLinkedViews(savedTable, renaming) diff --git a/packages/server/src/api/controllers/table/internal.ts b/packages/server/src/api/controllers/table/internal.ts index 40ce5e279d..67c4ec100c 100644 --- a/packages/server/src/api/controllers/table/internal.ts +++ b/packages/server/src/api/controllers/table/internal.ts @@ -30,14 +30,14 @@ export async function updateTable( } try { - const { table } = await sdk.tables.internal.save(tableToSave, { + const { table, oldTable } = await sdk.tables.internal.save(tableToSave, { userId: ctx.user._id, rowsToImport: rows, tableId: ctx.request.body._id, renaming, }) - return table + return { table, oldTable } } catch (err: any) { if (err instanceof Error) { ctx.throw(400, err.message) diff --git a/packages/server/src/api/controllers/view/views.ts b/packages/server/src/api/controllers/view/views.ts index b1f1f6c154..f1aa219aca 100644 --- a/packages/server/src/api/controllers/view/views.ts +++ b/packages/server/src/api/controllers/view/views.ts @@ -60,35 +60,31 @@ export async function save(ctx: Ctx) { existingTable.views[viewName] = existingTable.views[originalName] } await db.put(table) - await handleViewEvents( - existingTable.views[viewName] as View, - table.views[viewName] - ) ctx.body = table.views[viewName] builderSocket?.emitTableUpdate(ctx, table) } -export async function calculationEvents(existingView: View, newView: View) { - const existingCalculation = existingView && existingView.calculation - const newCalculation = newView && newView.calculation - - if (existingCalculation && !newCalculation) { - await events.view.calculationDeleted(existingView) - } - - if (!existingCalculation && newCalculation) { - await events.view.calculationCreated(newView) - } - - if ( - existingCalculation && - newCalculation && - existingCalculation !== newCalculation - ) { - await events.view.calculationUpdated(newView) - } -} +// export async function calculationEvents(existingView: View, newView: View) { +// const existingCalculation = existingView && existingView.calculation +// const newCalculation = newView && newView.calculation +// +// if (existingCalculation && !newCalculation) { +// await events.view.calculationDeleted(existingView) +// } +// +// if (!existingCalculation && newCalculation) { +// await events.view.calculationCreated(newView) +// } +// +// if ( +// existingCalculation && +// newCalculation && +// existingCalculation !== newCalculation +// ) { +// await events.view.calculationUpdated(newView) +// } +// } export async function filterEvents(existingView: View, newView: View) { const hasExistingFilters = !!( @@ -115,16 +111,6 @@ export async function filterEvents(existingView: View, newView: View) { } } -async function handleViewEvents(existingView: View, newView: View) { - if (!existingView) { - await events.view.created(newView) - } else { - await events.view.updated(newView) - } - await calculationEvents(existingView, newView) - await filterEvents(existingView, newView) -} - export async function destroy(ctx: Ctx) { const db = context.getAppDB() const viewName = decodeURIComponent(ctx.params.viewName) diff --git a/packages/server/src/api/controllers/view/viewsV2.ts b/packages/server/src/api/controllers/view/viewsV2.ts index 46c44ed577..38983a978e 100644 --- a/packages/server/src/api/controllers/view/viewsV2.ts +++ b/packages/server/src/api/controllers/view/viewsV2.ts @@ -151,7 +151,7 @@ export async function create(ctx: Ctx) { } const result = await sdk.views.create(tableId, parsedView) - await events.view.created(view) + await events.view.created(result) ctx.status = 201 ctx.body = { @@ -190,10 +190,11 @@ export async function update(ctx: Ctx) { primaryDisplay: view.primaryDisplay, } - const result = await sdk.views.update(tableId, parsedView) - ctx.body = { - data: result, - } + const { view: result } = await sdk.views.update(tableId, parsedView) + + await events.view.updated(result) + + ctx.body = { data: result } const table = await sdk.tables.getTable(tableId) builderSocket?.emitTableUpdate(ctx, table) diff --git a/packages/server/src/migrations/functions/backfill/app/tables.ts b/packages/server/src/migrations/functions/backfill/app/tables.ts index 081b81ede5..c6e46173dc 100644 --- a/packages/server/src/migrations/functions/backfill/app/tables.ts +++ b/packages/server/src/migrations/functions/backfill/app/tables.ts @@ -14,7 +14,7 @@ export const backfill = async (appDb: Database, timestamp: string | number) => { continue } - await events.view.created(view, timestamp) + // await events.view.created(view, timestamp) if (view.calculation) { await events.view.calculationCreated(view, timestamp) diff --git a/packages/server/src/sdk/app/tables/external/index.ts b/packages/server/src/sdk/app/tables/external/index.ts index 941d193b94..a7a15cac22 100644 --- a/packages/server/src/sdk/app/tables/external/index.ts +++ b/packages/server/src/sdk/app/tables/external/index.ts @@ -282,7 +282,7 @@ export async function save( tableToSave.sql = true } - return { datasource: updatedDatasource, table: tableToSave } + return { datasource: updatedDatasource, table: tableToSave, oldTable } } export async function destroy(datasourceId: string, table: Table) { diff --git a/packages/server/src/sdk/app/tables/internal/index.ts b/packages/server/src/sdk/app/tables/internal/index.ts index fbcbed03dc..5b9f346e93 100644 --- a/packages/server/src/sdk/app/tables/internal/index.ts +++ b/packages/server/src/sdk/app/tables/internal/index.ts @@ -171,7 +171,7 @@ export async function save( } // has to run after, make sure it has _id await runStaticFormulaChecks(table, { oldTable, deletion: false }) - return { table } + return { table, oldTable } } export async function destroy(table: Table) { diff --git a/packages/server/src/sdk/app/views/external.ts b/packages/server/src/sdk/app/views/external.ts index bee153a910..9016a3bd61 100644 --- a/packages/server/src/sdk/app/views/external.ts +++ b/packages/server/src/sdk/app/views/external.ts @@ -63,7 +63,7 @@ export async function create( export async function update( tableId: string, view: Readonly -): Promise { +): Promise<{ view: ViewV2; existingView: ViewV2 }> { const db = context.getAppDB() const { datasourceId, tableName } = breakExternalTableId(tableId) @@ -87,7 +87,7 @@ export async function update( delete views[existingView.name] views[view.name] = view await db.put(ds) - return view + return { view, existingView } } export async function remove(viewId: string): Promise { diff --git a/packages/server/src/sdk/app/views/index.ts b/packages/server/src/sdk/app/views/index.ts index 58537c96ad..f483ebc0bc 100644 --- a/packages/server/src/sdk/app/views/index.ts +++ b/packages/server/src/sdk/app/views/index.ts @@ -315,7 +315,10 @@ export async function create( return view } -export async function update(tableId: string, view: ViewV2): Promise { +export async function update( + tableId: string, + view: ViewV2 +): Promise<{ view: ViewV2; existingView: ViewV2 }> { await guardViewSchema(tableId, view) return pickApi(tableId).update(tableId, view) diff --git a/packages/server/src/sdk/app/views/internal.ts b/packages/server/src/sdk/app/views/internal.ts index 63807bcfd4..ec152abe54 100644 --- a/packages/server/src/sdk/app/views/internal.ts +++ b/packages/server/src/sdk/app/views/internal.ts @@ -54,7 +54,7 @@ export async function create( export async function update( tableId: string, view: Readonly -): Promise { +): Promise<{ view: ViewV2; existingView: ViewV2 }> { const db = context.getAppDB() const table = await sdk.tables.getTable(tableId) table.views ??= {} @@ -76,7 +76,7 @@ export async function update( delete table.views[existingView.name] table.views[view.name] = view await db.put(table) - return view + return { view, existingView } } export async function remove(viewId: string): Promise { diff --git a/packages/types/src/sdk/events/table.ts b/packages/types/src/sdk/events/table.ts index 8df2a95796..4a5880b1db 100644 --- a/packages/types/src/sdk/events/table.ts +++ b/packages/types/src/sdk/events/table.ts @@ -1,4 +1,5 @@ import { BaseEvent, TableExportFormat } from "./event" +import { AIOperationEnum } from "../ai" export interface TableCreatedEvent extends BaseEvent { tableId: string @@ -9,6 +10,8 @@ export interface TableCreatedEvent extends BaseEvent { export interface TableUpdatedEvent extends BaseEvent { tableId: string + defaultValues: boolean | undefined + aiColumn: AIOperationEnum | undefined audited: { name: string } diff --git a/packages/types/src/sdk/events/view.ts b/packages/types/src/sdk/events/view.ts index 84fcb0facc..c73a591a34 100644 --- a/packages/types/src/sdk/events/view.ts +++ b/packages/types/src/sdk/events/view.ts @@ -1,15 +1,16 @@ -import { ViewCalculation, ViewV2Schema, ViewV2Type } from "../../documents" +import { ViewCalculation, ViewV2Type } from "../../documents" import { BaseEvent, TableExportFormat } from "./event" -import { LegacyFilter, SortOrder, SortType, UISearchFilter } from "../../api" -import { SearchFilters } from "../search" export interface ViewCreatedEvent extends BaseEvent { name: string type?: ViewV2Type + tableId: string } export interface ViewUpdatedEvent extends BaseEvent { tableId: string + groupedFilters: number + viewJoins: number } export interface ViewDeletedEvent extends BaseEvent {