From cf37563c8935ed8665a546f065bd8be85178ea0f Mon Sep 17 00:00:00 2001 From: Rory Powell Date: Tue, 9 Aug 2022 13:31:12 +0100 Subject: [PATCH 1/2] Fix served events --- .../src/events/async/BBEventEmitter.ts | 27 +++++++++++++ .../backend-core/src/events/async/index.ts | 1 + packages/backend-core/src/events/events.ts | 40 +++++++++++++++++++ .../events/processors/AnalyticsProcessor.ts | 6 +-- .../events/processors/posthog/rateLimiting.ts | 11 +++-- .../controllers/static/{index.js => index.ts} | 20 +++++----- 6 files changed, 88 insertions(+), 17 deletions(-) create mode 100644 packages/backend-core/src/events/async/BBEventEmitter.ts create mode 100644 packages/backend-core/src/events/async/index.ts rename packages/server/src/api/controllers/static/{index.js => index.ts} (89%) diff --git a/packages/backend-core/src/events/async/BBEventEmitter.ts b/packages/backend-core/src/events/async/BBEventEmitter.ts new file mode 100644 index 0000000000..41668858e1 --- /dev/null +++ b/packages/backend-core/src/events/async/BBEventEmitter.ts @@ -0,0 +1,27 @@ +import { EventEmitter } from "events" +import * as context from "../../context" +import { Identity, Event } from "@budibase/types" + +export interface EmittedEvent { + tenantId: string + identity: Identity + appId: string | undefined + properties: any +} + +class BBEventEmitter extends EventEmitter { + emitEvent(event: Event, properties: any, identity: Identity) { + const tenantId = context.getTenantId() + const appId = context.getAppId() + + const emittedEvent: EmittedEvent = { + tenantId, + identity, + appId, + properties, + } + this.emit(event, emittedEvent) + } +} + +export const emitter = new BBEventEmitter() diff --git a/packages/backend-core/src/events/async/index.ts b/packages/backend-core/src/events/async/index.ts new file mode 100644 index 0000000000..60e47f7a13 --- /dev/null +++ b/packages/backend-core/src/events/async/index.ts @@ -0,0 +1 @@ +export * from "./BBEventEmitter" diff --git a/packages/backend-core/src/events/events.ts b/packages/backend-core/src/events/events.ts index cda90d12c9..1d532c092b 100644 --- a/packages/backend-core/src/events/events.ts +++ b/packages/backend-core/src/events/events.ts @@ -2,6 +2,41 @@ import { Event } from "@budibase/types" import { processors } from "./processors" import * as identification from "./identification" import * as backfill from "./backfill" +import { emitter, EmittedEvent } from "./async" +import * as context from "../context" +import * as logging from "../logging" + +const USE_ASYNC: any[] = [ + Event.SERVED_BUILDER, + Event.SERVED_APP, + Event.SERVED_APP_PREVIEW, +] + +for (let event of USE_ASYNC) { + emitter.on(event, async (props: EmittedEvent) => { + try { + await context.doInTenant(props.tenantId, async () => { + if (props.appId) { + await context.doInAppContext(props.appId, async () => { + await processors.processEvent( + event as Event, + props.identity, + props.properties + ) + }) + } else { + await processors.processEvent( + event as Event, + props.identity, + props.properties + ) + } + }) + } catch (e) { + logging.logAlert(`Unable to process async event ${event}`, e) + } + }) +} export const publishEvent = async ( event: Event, @@ -11,6 +46,11 @@ export const publishEvent = async ( // in future this should use async events via a distributed queue. const identity = await identification.getCurrentIdentity() + if (USE_ASYNC.includes(event)) { + emitter.emitEvent(event, properties, identity) + return + } + const backfilling = await backfill.isBackfillingEvent(event) // no backfill - send the event and exit if (!backfilling) { diff --git a/packages/backend-core/src/events/processors/AnalyticsProcessor.ts b/packages/backend-core/src/events/processors/AnalyticsProcessor.ts index 602407400e..f9d7547120 100644 --- a/packages/backend-core/src/events/processors/AnalyticsProcessor.ts +++ b/packages/backend-core/src/events/processors/AnalyticsProcessor.ts @@ -32,7 +32,7 @@ export default class AnalyticsProcessor implements EventProcessor { return } if (this.posthog) { - this.posthog.processEvent(event, identity, properties, timestamp) + await this.posthog.processEvent(event, identity, properties, timestamp) } } @@ -45,14 +45,14 @@ export default class AnalyticsProcessor implements EventProcessor { return } if (this.posthog) { - this.posthog.identify(identity, timestamp) + await this.posthog.identify(identity, timestamp) } } async identifyGroup(group: Group, timestamp?: string | number) { // Group indentifications (tenant and installation) always on if (this.posthog) { - this.posthog.identifyGroup(group, timestamp) + await this.posthog.identifyGroup(group, timestamp) } } diff --git a/packages/backend-core/src/events/processors/posthog/rateLimiting.ts b/packages/backend-core/src/events/processors/posthog/rateLimiting.ts index e2a02eb28e..9c7b7876d6 100644 --- a/packages/backend-core/src/events/processors/posthog/rateLimiting.ts +++ b/packages/backend-core/src/events/processors/posthog/rateLimiting.ts @@ -45,7 +45,7 @@ export const limited = async (event: Event): Promise => { return false } - const cachedEvent = (await readEvent(event)) as EventProperties + const cachedEvent = await readEvent(event) if (cachedEvent) { const timestamp = new Date(cachedEvent.timestamp) const limit = RATE_LIMITS[event] @@ -76,14 +76,17 @@ export const limited = async (event: Event): Promise => { const eventKey = (event: RateLimitedEvent) => { let key = `${CacheKeys.EVENTS_RATE_LIMIT}:${event}` if (isPerApp(event)) { - key = key + context.getAppId() + key = key + ":" + context.getAppId() } return key } -const readEvent = async (event: RateLimitedEvent) => { +const readEvent = async ( + event: RateLimitedEvent +): Promise => { const key = eventKey(event) - return cache.get(key) + const result = await cache.get(key) + return result as EventProperties } const recordEvent = async ( diff --git a/packages/server/src/api/controllers/static/index.js b/packages/server/src/api/controllers/static/index.ts similarity index 89% rename from packages/server/src/api/controllers/static/index.js rename to packages/server/src/api/controllers/static/index.ts index 86bce89b4a..b0e36b8943 100644 --- a/packages/server/src/api/controllers/static/index.js +++ b/packages/server/src/api/controllers/static/index.ts @@ -18,14 +18,14 @@ const { DocumentTypes, isDevAppID } = require("../../../db/utils") const { getAppDB, getAppId } = require("@budibase/backend-core/context") const { setCookie, clearCookie } = require("@budibase/backend-core/utils") const AWS = require("aws-sdk") -const { events } = require("@budibase/backend-core") +import { events } from "@budibase/backend-core" const fs = require("fs") const { downloadTarballDirect, } = require("../../../utilities/fileSystem/utilities") -async function prepareUpload({ s3Key, bucket, metadata, file }) { +async function prepareUpload({ s3Key, bucket, metadata, file }: any) { const response = await upload({ bucket, metadata, @@ -44,7 +44,7 @@ async function prepareUpload({ s3Key, bucket, metadata, file }) { } } -exports.toggleBetaUiFeature = async function (ctx) { +export const toggleBetaUiFeature = async function (ctx: any) { const cookieName = `beta:${ctx.params.feature}` if (ctx.cookies.get(cookieName)) { @@ -72,21 +72,21 @@ exports.toggleBetaUiFeature = async function (ctx) { } } -exports.serveBuilder = async function (ctx) { +export const serveBuilder = async function (ctx: any) { const builderPath = resolve(TOP_LEVEL_PATH, "builder") await send(ctx, ctx.file, { root: builderPath }) - if (!ctx.file.includes("assets/")) { + if (ctx.file === "index.html") { await events.serve.servedBuilder() } } -exports.uploadFile = async function (ctx) { +export const uploadFile = async function (ctx: any) { let files = ctx.request.files.file.length > 1 ? Array.from(ctx.request.files.file) : [ctx.request.files.file] - const uploads = files.map(async file => { + const uploads = files.map(async (file: any) => { const fileExtension = [...file.name.split(".")].pop() // filenames converted to UUIDs so they are unique const processedFileName = `${uuid.v4()}.${fileExtension}` @@ -101,7 +101,7 @@ exports.uploadFile = async function (ctx) { ctx.body = await Promise.all(uploads) } -exports.serveApp = async function (ctx) { +export const serveApp = async function (ctx: any) { const db = getAppDB({ skip_setup: true }) const appInfo = await db.get(DocumentTypes.APP_METADATA) let appId = getAppId() @@ -134,13 +134,13 @@ exports.serveApp = async function (ctx) { } } -exports.serveClientLibrary = async function (ctx) { +export const serveClientLibrary = async function (ctx: any) { return send(ctx, "budibase-client.js", { root: join(NODE_MODULES_PATH, "@budibase", "client", "dist"), }) } -exports.getSignedUploadURL = async function (ctx) { +export const getSignedUploadURL = async function (ctx: any) { const database = getAppDB() // Ensure datasource is valid From 1a31dff984bfa747abaca90b0599452a31d28df2 Mon Sep 17 00:00:00 2001 From: Rory Powell Date: Tue, 9 Aug 2022 14:01:55 +0100 Subject: [PATCH 2/2] Rename async to emit --- .../src/events/{async => emit}/BBEventEmitter.ts | 0 packages/backend-core/src/events/{async => emit}/index.ts | 0 packages/backend-core/src/events/events.ts | 8 ++++---- 3 files changed, 4 insertions(+), 4 deletions(-) rename packages/backend-core/src/events/{async => emit}/BBEventEmitter.ts (100%) rename packages/backend-core/src/events/{async => emit}/index.ts (100%) diff --git a/packages/backend-core/src/events/async/BBEventEmitter.ts b/packages/backend-core/src/events/emit/BBEventEmitter.ts similarity index 100% rename from packages/backend-core/src/events/async/BBEventEmitter.ts rename to packages/backend-core/src/events/emit/BBEventEmitter.ts diff --git a/packages/backend-core/src/events/async/index.ts b/packages/backend-core/src/events/emit/index.ts similarity index 100% rename from packages/backend-core/src/events/async/index.ts rename to packages/backend-core/src/events/emit/index.ts diff --git a/packages/backend-core/src/events/events.ts b/packages/backend-core/src/events/events.ts index 1d532c092b..63fedf4754 100644 --- a/packages/backend-core/src/events/events.ts +++ b/packages/backend-core/src/events/events.ts @@ -2,17 +2,17 @@ import { Event } from "@budibase/types" import { processors } from "./processors" import * as identification from "./identification" import * as backfill from "./backfill" -import { emitter, EmittedEvent } from "./async" +import { emitter, EmittedEvent } from "./emit" import * as context from "../context" import * as logging from "../logging" -const USE_ASYNC: any[] = [ +const USE_EMITTER: any[] = [ Event.SERVED_BUILDER, Event.SERVED_APP, Event.SERVED_APP_PREVIEW, ] -for (let event of USE_ASYNC) { +for (let event of USE_EMITTER) { emitter.on(event, async (props: EmittedEvent) => { try { await context.doInTenant(props.tenantId, async () => { @@ -46,7 +46,7 @@ export const publishEvent = async ( // in future this should use async events via a distributed queue. const identity = await identification.getCurrentIdentity() - if (USE_ASYNC.includes(event)) { + if (USE_EMITTER.includes(event)) { emitter.emitEvent(event, properties, identity) return }