From ae342910c20ff522bc488e4eae1950299692fa32 Mon Sep 17 00:00:00 2001 From: Rory Powell Date: Mon, 8 Aug 2022 15:44:29 +0100 Subject: [PATCH] Rate limit served:X events --- packages/backend-core/src/cache/generic.js | 1 + .../events/processors/AnalyticsProcessor.ts | 2 +- .../{ => posthog}/PosthogProcessor.ts | 13 ++- .../src/events/processors/posthog/index.ts | 2 + .../events/processors/posthog/rateLimiting.ts | 95 ++++++++++++++++++ .../posthog/tests/PosthogProcessor.spec.ts | 96 +++++++++++++++++++ .../processors/tests/PosthogProcessor.spec.ts | 40 -------- 7 files changed, 204 insertions(+), 45 deletions(-) rename packages/backend-core/src/events/processors/{ => posthog}/PosthogProcessor.ts (89%) create mode 100644 packages/backend-core/src/events/processors/posthog/index.ts create mode 100644 packages/backend-core/src/events/processors/posthog/rateLimiting.ts create mode 100644 packages/backend-core/src/events/processors/posthog/tests/PosthogProcessor.spec.ts delete mode 100644 packages/backend-core/src/events/processors/tests/PosthogProcessor.spec.ts diff --git a/packages/backend-core/src/cache/generic.js b/packages/backend-core/src/cache/generic.js index e2f3915339..26ef0c6bb0 100644 --- a/packages/backend-core/src/cache/generic.js +++ b/packages/backend-core/src/cache/generic.js @@ -9,6 +9,7 @@ exports.CacheKeys = { UNIQUE_TENANT_ID: "uniqueTenantId", EVENTS: "events", BACKFILL_METADATA: "backfillMetadata", + EVENTS_RATE_LIMIT: "eventsRateLimit", } exports.TTL = { diff --git a/packages/backend-core/src/events/processors/AnalyticsProcessor.ts b/packages/backend-core/src/events/processors/AnalyticsProcessor.ts index 2ee7a02afa..602407400e 100644 --- a/packages/backend-core/src/events/processors/AnalyticsProcessor.ts +++ b/packages/backend-core/src/events/processors/AnalyticsProcessor.ts @@ -2,7 +2,7 @@ import { Event, Identity, Group, IdentityType } from "@budibase/types" import { EventProcessor } from "./types" import env from "../../environment" import * as analytics from "../analytics" -import PosthogProcessor from "./PosthogProcessor" +import PosthogProcessor from "./posthog" /** * Events that are always captured. diff --git a/packages/backend-core/src/events/processors/PosthogProcessor.ts b/packages/backend-core/src/events/processors/posthog/PosthogProcessor.ts similarity index 89% rename from packages/backend-core/src/events/processors/PosthogProcessor.ts rename to packages/backend-core/src/events/processors/posthog/PosthogProcessor.ts index 9d68d3919a..593e5ff082 100644 --- a/packages/backend-core/src/events/processors/PosthogProcessor.ts +++ b/packages/backend-core/src/events/processors/posthog/PosthogProcessor.ts @@ -1,9 +1,10 @@ import PostHog from "posthog-node" import { Event, Identity, Group, BaseEvent } from "@budibase/types" -import { EventProcessor } from "./types" -import env from "../../environment" -import * as context from "../../context" -const pkg = require("../../../package.json") +import { EventProcessor } from "../types" +import env from "../../../environment" +import * as context from "../../../context" +import * as rateLimiting from "./rateLimiting" +const pkg = require("../../../../package.json") const EXCLUDED_EVENTS: Event[] = [ Event.USER_UPDATED, @@ -42,6 +43,10 @@ export default class PosthogProcessor implements EventProcessor { return } + if (await rateLimiting.limited(event)) { + return + } + properties.version = pkg.version properties.service = env.SERVICE properties.environment = identity.environment diff --git a/packages/backend-core/src/events/processors/posthog/index.ts b/packages/backend-core/src/events/processors/posthog/index.ts new file mode 100644 index 0000000000..dceb10d2cd --- /dev/null +++ b/packages/backend-core/src/events/processors/posthog/index.ts @@ -0,0 +1,2 @@ +import PosthogProcessor from "./PosthogProcessor" +export default PosthogProcessor diff --git a/packages/backend-core/src/events/processors/posthog/rateLimiting.ts b/packages/backend-core/src/events/processors/posthog/rateLimiting.ts new file mode 100644 index 0000000000..f61d34d0a4 --- /dev/null +++ b/packages/backend-core/src/events/processors/posthog/rateLimiting.ts @@ -0,0 +1,95 @@ +import { Event } from "@budibase/types" +import { CacheKeys, TTL } from "../../../cache/generic" +import * as cache from "../../../cache/generic" + +type RateLimitedEvent = + | Event.SERVED_BUILDER + | Event.SERVED_APP_PREVIEW + | Event.SERVED_APP + +const isRateLimited = (event: Event): event is RateLimitedEvent => { + return ( + event === Event.SERVED_BUILDER || + event === Event.SERVED_APP_PREVIEW || + event === Event.SERVED_APP + ) +} + +interface EventProperties { + timestamp: number +} + +enum RateLimit { + CALENDAR_DAY = "calendarDay", +} + +const RATE_LIMITS = { + [Event.SERVED_APP]: RateLimit.CALENDAR_DAY, + [Event.SERVED_APP_PREVIEW]: RateLimit.CALENDAR_DAY, + [Event.SERVED_BUILDER]: RateLimit.CALENDAR_DAY, +} + +/** + * Check if this event should be sent right now + * Return false to signal the event SHOULD be sent + * Return true to signal the event should NOT be sent + */ +export const limited = async (event: Event): Promise => { + // not a rate limited event -- send + if (!isRateLimited(event)) { + return false + } + + const cachedEvent = (await readEvent(event)) as EventProperties + if (cachedEvent) { + const timestamp = new Date(cachedEvent.timestamp) + const limit = RATE_LIMITS[event] + switch (limit) { + case RateLimit.CALENDAR_DAY: { + // get midnight at the start of the next day for the timestamp + timestamp.setDate(timestamp.getDate() + 1) + timestamp.setHours(0, 0, 0, 0) + + // if we have passed the threshold into the next day + if (Date.now() > timestamp.getTime()) { + // update the timestamp in the event -- send + await recordEvent(event, { timestamp: Date.now() }) + return false + } else { + // still within the limited period -- don't send + return true + } + } + } + } else { + // no event present i.e. expired -- send + await recordEvent(event, { timestamp: Date.now() }) + return false + } +} + +const eventKey = (event: RateLimitedEvent) => { + return `${CacheKeys.EVENTS_RATE_LIMIT}:${event}` +} + +const readEvent = async (event: RateLimitedEvent) => { + const key = eventKey(event) + return cache.get(key) +} + +const recordEvent = async ( + event: RateLimitedEvent, + properties: EventProperties +) => { + const key = `${CacheKeys.EVENTS_RATE_LIMIT}:${event}` + + const limit = RATE_LIMITS[event] + let ttl + switch (limit) { + case RateLimit.CALENDAR_DAY: { + ttl = TTL.ONE_DAY + } + } + + await cache.store(key, properties, ttl) +} diff --git a/packages/backend-core/src/events/processors/posthog/tests/PosthogProcessor.spec.ts b/packages/backend-core/src/events/processors/posthog/tests/PosthogProcessor.spec.ts new file mode 100644 index 0000000000..7ddee7279b --- /dev/null +++ b/packages/backend-core/src/events/processors/posthog/tests/PosthogProcessor.spec.ts @@ -0,0 +1,96 @@ +import PosthogProcessor from "../PosthogProcessor" +import { Event, IdentityType, Hosting } from "@budibase/types" +const tk = require("timekeeper") +import * as Cache from "../../../../cache/generic" + +const newIdentity = () => { + return { + id: "test", + type: IdentityType.USER, + hosting: Hosting.SELF, + environment: "test", + } +} + +describe("PosthogProcessor", () => { + beforeEach(() => { + jest.clearAllMocks() + }) + + describe("processEvent", () => { + it("processes event", async () => { + const processor = new PosthogProcessor("test") + + const identity = newIdentity() + const properties = {} + + await processor.processEvent(Event.APP_CREATED, identity, properties) + + expect(processor.posthog.capture).toHaveBeenCalledTimes(1) + }) + + it("honours exclusions", async () => { + const processor = new PosthogProcessor("test") + + const identity = newIdentity() + const properties = {} + + await processor.processEvent(Event.AUTH_SSO_UPDATED, identity, properties) + expect(processor.posthog.capture).toHaveBeenCalledTimes(0) + }) + + describe("rate limiting", () => { + it("sends daily event once in same day", async () => { + const processor = new PosthogProcessor("test") + const identity = newIdentity() + const properties = {} + + tk.freeze(new Date(2022, 0, 1, 14, 0)) + await processor.processEvent(Event.SERVED_BUILDER, identity, properties) + // go forward one hour + tk.freeze(new Date(2022, 0, 1, 15, 0)) + await processor.processEvent(Event.SERVED_BUILDER, identity, properties) + + expect(processor.posthog.capture).toHaveBeenCalledTimes(1) + }) + + it("sends daily event once per unique day", async () => { + const processor = new PosthogProcessor("test") + const identity = newIdentity() + const properties = {} + + tk.freeze(new Date(2022, 0, 1, 14, 0)) + await processor.processEvent(Event.SERVED_BUILDER, identity, properties) + // go forward into next day + tk.freeze(new Date(2022, 0, 2, 9, 0)) + await processor.processEvent(Event.SERVED_BUILDER, identity, properties) + // go forward into next day + tk.freeze(new Date(2022, 0, 3, 5, 0)) + await processor.processEvent(Event.SERVED_BUILDER, identity, properties) + // go forward one hour + tk.freeze(new Date(2022, 0, 3, 6, 0)) + await processor.processEvent(Event.SERVED_BUILDER, identity, properties) + + expect(processor.posthog.capture).toHaveBeenCalledTimes(4) + }) + + it("sends event again after cache expires", async () => { + const processor = new PosthogProcessor("test") + const identity = newIdentity() + const properties = {} + + tk.freeze(new Date(2022, 0, 1, 14, 0)) + await processor.processEvent(Event.SERVED_BUILDER, identity, properties) + + await Cache.bustCache( + `${Cache.CacheKeys.EVENTS_RATE_LIMIT}:${Event.SERVED_BUILDER}` + ) + + tk.freeze(new Date(2022, 0, 1, 14, 0)) + await processor.processEvent(Event.SERVED_BUILDER, identity, properties) + + expect(processor.posthog.capture).toHaveBeenCalledTimes(2) + }) + }) + }) +}) diff --git a/packages/backend-core/src/events/processors/tests/PosthogProcessor.spec.ts b/packages/backend-core/src/events/processors/tests/PosthogProcessor.spec.ts deleted file mode 100644 index 4a6d55b272..0000000000 --- a/packages/backend-core/src/events/processors/tests/PosthogProcessor.spec.ts +++ /dev/null @@ -1,40 +0,0 @@ -import PosthogProcessor from "../PosthogProcessor" -import { Event, IdentityType, Hosting } from "@budibase/types" - -const newIdentity = () => { - return { - id: "test", - type: IdentityType.USER, - hosting: Hosting.SELF, - environment: "test", - } -} - -describe("PosthogProcessor", () => { - beforeEach(() => { - jest.clearAllMocks() - }) - - describe("processEvent", () => { - it("processes event", () => { - const processor = new PosthogProcessor("test") - - const identity = newIdentity() - const properties = {} - - processor.processEvent(Event.APP_CREATED, identity, properties) - - expect(processor.posthog.capture).toHaveBeenCalledTimes(1) - }) - - it("honours exclusions", () => { - const processor = new PosthogProcessor("test") - - const identity = newIdentity() - const properties = {} - - processor.processEvent(Event.AUTH_SSO_UPDATED, identity, properties) - expect(processor.posthog.capture).toHaveBeenCalledTimes(0) - }) - }) -})