diff --git a/hosting/docker-compose.yaml b/hosting/docker-compose.yaml index f669f9261d..5281155545 100644 --- a/hosting/docker-compose.yaml +++ b/hosting/docker-compose.yaml @@ -75,7 +75,9 @@ services: ports: - "${MAIN_PORT}:10000" container_name: bbproxy - image: budibase/proxy + image: proxy-service + environment: + - PROXY_RATE_LIMIT_WEBHOOKS_PER_SECOND=10 depends_on: - minio-service - worker-service diff --git a/hosting/nginx.prod.conf.hbs b/hosting/nginx.prod.conf.hbs index ac35a2020d..4213626309 100644 --- a/hosting/nginx.prod.conf.hbs +++ b/hosting/nginx.prod.conf.hbs @@ -9,7 +9,11 @@ events { } http { + # rate limiting + limit_req_status 429; limit_req_zone $binary_remote_addr zone=ratelimit:10m rate=20r/s; + limit_req_zone $binary_remote_addr zone=webhooks:10m rate=${PROXY_RATE_LIMIT_WEBHOOKS_PER_SECOND}r/s; + include /etc/nginx/mime.types; default_type application/octet-stream; proxy_set_header Host $host; @@ -126,6 +130,25 @@ http { proxy_pass http://$apps:4002; } + location /api/webhooks/ { + # calls to webhooks are rate limited + limit_req zone=webhooks nodelay; + + # Rest of configuration copied from /api/ location above + # 120s timeout on API requests + proxy_read_timeout 120s; + proxy_connect_timeout 120s; + proxy_send_timeout 120s; + + proxy_http_version 1.1; + proxy_set_header Connection $connection_upgrade; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + + proxy_pass http://$apps:4002; + } + location /db/ { proxy_pass http://$couchdb:5984; rewrite ^/db/(.*)$ /$1 break; diff --git a/hosting/proxy/Dockerfile b/hosting/proxy/Dockerfile index a2b17d3333..d9b33e3e9a 100644 --- a/hosting/proxy/Dockerfile +++ b/hosting/proxy/Dockerfile @@ -1,3 +1,13 @@ FROM nginx:latest -COPY .generated-nginx.prod.conf /etc/nginx/nginx.conf -COPY error.html /usr/share/nginx/html/error.html \ No newline at end of file + +# nginx.conf +# use the default nginx behaviour for *.template files which are processed with envsubst +# override the output dir to output directly to /etc/nginx instead of /etc/nginx/conf.d +ENV NGINX_ENVSUBST_OUTPUT_DIR=/etc/nginx +COPY .generated-nginx.prod.conf /etc/nginx/templates/nginx.conf.template + +# Error handling +COPY error.html /usr/share/nginx/html/error.html + +# Default environment +ENV PROXY_RATE_LIMIT_WEBHOOKS_PER_SECOND=10 \ No newline at end of file diff --git a/lerna.json b/lerna.json index 26cb79285e..15a771a46f 100644 --- a/lerna.json +++ b/lerna.json @@ -1,5 +1,5 @@ { - "version": "1.2.26-alpha.0", + "version": "1.2.27", "npmClient": "yarn", "packages": [ "packages/*" diff --git a/packages/backend-core/package.json b/packages/backend-core/package.json index 9fe0dcc943..3e2386df2c 100644 --- a/packages/backend-core/package.json +++ b/packages/backend-core/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/backend-core", - "version": "1.2.26-alpha.0", + "version": "1.2.27", "description": "Budibase backend core libraries used in server and worker", "main": "dist/src/index.js", "types": "dist/src/index.d.ts", @@ -20,7 +20,7 @@ "test:watch": "jest --watchAll" }, "dependencies": { - "@budibase/types": "1.2.26-alpha.0", + "@budibase/types": "^1.2.27", "@techpass/passport-openidconnect": "0.3.2", "aws-sdk": "2.1030.0", "bcrypt": "5.0.1", diff --git a/packages/backend-core/src/events/emit/BBEventEmitter.ts b/packages/backend-core/src/events/emit/BBEventEmitter.ts new file mode 100644 index 0000000000..41668858e1 --- /dev/null +++ b/packages/backend-core/src/events/emit/BBEventEmitter.ts @@ -0,0 +1,27 @@ +import { EventEmitter } from "events" +import * as context from "../../context" +import { Identity, Event } from "@budibase/types" + +export interface EmittedEvent { + tenantId: string + identity: Identity + appId: string | undefined + properties: any +} + +class BBEventEmitter extends EventEmitter { + emitEvent(event: Event, properties: any, identity: Identity) { + const tenantId = context.getTenantId() + const appId = context.getAppId() + + const emittedEvent: EmittedEvent = { + tenantId, + identity, + appId, + properties, + } + this.emit(event, emittedEvent) + } +} + +export const emitter = new BBEventEmitter() diff --git a/packages/backend-core/src/events/emit/index.ts b/packages/backend-core/src/events/emit/index.ts new file mode 100644 index 0000000000..60e47f7a13 --- /dev/null +++ b/packages/backend-core/src/events/emit/index.ts @@ -0,0 +1 @@ +export * from "./BBEventEmitter" diff --git a/packages/backend-core/src/events/events.ts b/packages/backend-core/src/events/events.ts index cda90d12c9..63fedf4754 100644 --- a/packages/backend-core/src/events/events.ts +++ b/packages/backend-core/src/events/events.ts @@ -2,6 +2,41 @@ import { Event } from "@budibase/types" import { processors } from "./processors" import * as identification from "./identification" import * as backfill from "./backfill" +import { emitter, EmittedEvent } from "./emit" +import * as context from "../context" +import * as logging from "../logging" + +const USE_EMITTER: any[] = [ + Event.SERVED_BUILDER, + Event.SERVED_APP, + Event.SERVED_APP_PREVIEW, +] + +for (let event of USE_EMITTER) { + emitter.on(event, async (props: EmittedEvent) => { + try { + await context.doInTenant(props.tenantId, async () => { + if (props.appId) { + await context.doInAppContext(props.appId, async () => { + await processors.processEvent( + event as Event, + props.identity, + props.properties + ) + }) + } else { + await processors.processEvent( + event as Event, + props.identity, + props.properties + ) + } + }) + } catch (e) { + logging.logAlert(`Unable to process async event ${event}`, e) + } + }) +} export const publishEvent = async ( event: Event, @@ -11,6 +46,11 @@ export const publishEvent = async ( // in future this should use async events via a distributed queue. const identity = await identification.getCurrentIdentity() + if (USE_EMITTER.includes(event)) { + emitter.emitEvent(event, properties, identity) + return + } + const backfilling = await backfill.isBackfillingEvent(event) // no backfill - send the event and exit if (!backfilling) { diff --git a/packages/backend-core/src/events/processors/AnalyticsProcessor.ts b/packages/backend-core/src/events/processors/AnalyticsProcessor.ts index 602407400e..f9d7547120 100644 --- a/packages/backend-core/src/events/processors/AnalyticsProcessor.ts +++ b/packages/backend-core/src/events/processors/AnalyticsProcessor.ts @@ -32,7 +32,7 @@ export default class AnalyticsProcessor implements EventProcessor { return } if (this.posthog) { - this.posthog.processEvent(event, identity, properties, timestamp) + await this.posthog.processEvent(event, identity, properties, timestamp) } } @@ -45,14 +45,14 @@ export default class AnalyticsProcessor implements EventProcessor { return } if (this.posthog) { - this.posthog.identify(identity, timestamp) + await this.posthog.identify(identity, timestamp) } } async identifyGroup(group: Group, timestamp?: string | number) { // Group indentifications (tenant and installation) always on if (this.posthog) { - this.posthog.identifyGroup(group, timestamp) + await this.posthog.identifyGroup(group, timestamp) } } diff --git a/packages/backend-core/src/events/processors/posthog/rateLimiting.ts b/packages/backend-core/src/events/processors/posthog/rateLimiting.ts index e2a02eb28e..9c7b7876d6 100644 --- a/packages/backend-core/src/events/processors/posthog/rateLimiting.ts +++ b/packages/backend-core/src/events/processors/posthog/rateLimiting.ts @@ -45,7 +45,7 @@ export const limited = async (event: Event): Promise => { return false } - const cachedEvent = (await readEvent(event)) as EventProperties + const cachedEvent = await readEvent(event) if (cachedEvent) { const timestamp = new Date(cachedEvent.timestamp) const limit = RATE_LIMITS[event] @@ -76,14 +76,17 @@ export const limited = async (event: Event): Promise => { const eventKey = (event: RateLimitedEvent) => { let key = `${CacheKeys.EVENTS_RATE_LIMIT}:${event}` if (isPerApp(event)) { - key = key + context.getAppId() + key = key + ":" + context.getAppId() } return key } -const readEvent = async (event: RateLimitedEvent) => { +const readEvent = async ( + event: RateLimitedEvent +): Promise => { const key = eventKey(event) - return cache.get(key) + const result = await cache.get(key) + return result as EventProperties } const recordEvent = async ( diff --git a/packages/bbui/package.json b/packages/bbui/package.json index eb96cdef3e..c80fa643e6 100644 --- a/packages/bbui/package.json +++ b/packages/bbui/package.json @@ -1,7 +1,7 @@ { "name": "@budibase/bbui", "description": "A UI solution used in the different Budibase projects.", - "version": "1.2.26-alpha.0", + "version": "1.2.27", "license": "MPL-2.0", "svelte": "src/index.js", "module": "dist/bbui.es.js", @@ -38,7 +38,7 @@ ], "dependencies": { "@adobe/spectrum-css-workflow-icons": "^1.2.1", - "@budibase/string-templates": "1.2.26-alpha.0", + "@budibase/string-templates": "^1.2.27", "@spectrum-css/actionbutton": "^1.0.1", "@spectrum-css/actiongroup": "^1.0.1", "@spectrum-css/avatar": "^3.0.2", diff --git a/packages/builder/package.json b/packages/builder/package.json index 2a9c94c357..d2e0bc0fe9 100644 --- a/packages/builder/package.json +++ b/packages/builder/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/builder", - "version": "1.2.26-alpha.0", + "version": "1.2.27", "license": "GPL-3.0", "private": true, "scripts": { @@ -69,10 +69,10 @@ } }, "dependencies": { - "@budibase/bbui": "1.2.26-alpha.0", - "@budibase/client": "1.2.26-alpha.0", - "@budibase/frontend-core": "1.2.26-alpha.0", - "@budibase/string-templates": "1.2.26-alpha.0", + "@budibase/bbui": "^1.2.27", + "@budibase/client": "^1.2.27", + "@budibase/frontend-core": "^1.2.27", + "@budibase/string-templates": "^1.2.27", "@sentry/browser": "5.19.1", "@spectrum-css/page": "^3.0.1", "@spectrum-css/vars": "^3.0.1", diff --git a/packages/cli/package.json b/packages/cli/package.json index 709683af4c..e92ea4a86d 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/cli", - "version": "1.2.26-alpha.0", + "version": "1.2.27", "description": "Budibase CLI, for developers, self hosting and migrations.", "main": "src/index.js", "bin": { diff --git a/packages/client/package.json b/packages/client/package.json index b11d1cfb48..9a83975e60 100644 --- a/packages/client/package.json +++ b/packages/client/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/client", - "version": "1.2.26-alpha.0", + "version": "1.2.27", "license": "MPL-2.0", "module": "dist/budibase-client.js", "main": "dist/budibase-client.js", @@ -19,9 +19,9 @@ "dev:builder": "rollup -cw" }, "dependencies": { - "@budibase/bbui": "1.2.26-alpha.0", - "@budibase/frontend-core": "1.2.26-alpha.0", - "@budibase/string-templates": "1.2.26-alpha.0", + "@budibase/bbui": "^1.2.27", + "@budibase/frontend-core": "^1.2.27", + "@budibase/string-templates": "^1.2.27", "@spectrum-css/button": "^3.0.3", "@spectrum-css/card": "^3.0.3", "@spectrum-css/divider": "^1.0.3", diff --git a/packages/frontend-core/package.json b/packages/frontend-core/package.json index 68b436bc48..7d0ee0c42f 100644 --- a/packages/frontend-core/package.json +++ b/packages/frontend-core/package.json @@ -1,12 +1,12 @@ { "name": "@budibase/frontend-core", - "version": "1.2.26-alpha.0", + "version": "1.2.27", "description": "Budibase frontend core libraries used in builder and client", "author": "Budibase", "license": "MPL-2.0", "svelte": "src/index.js", "dependencies": { - "@budibase/bbui": "1.2.26-alpha.0", + "@budibase/bbui": "^1.2.27", "lodash": "^4.17.21", "svelte": "^3.46.2" } diff --git a/packages/server/package.json b/packages/server/package.json index fd70df44ba..b76fa66dbb 100644 --- a/packages/server/package.json +++ b/packages/server/package.json @@ -1,7 +1,7 @@ { "name": "@budibase/server", "email": "hi@budibase.com", - "version": "1.2.26-alpha.0", + "version": "1.2.27", "description": "Budibase Web Server", "main": "src/index.ts", "repository": { @@ -77,11 +77,11 @@ "license": "GPL-3.0", "dependencies": { "@apidevtools/swagger-parser": "10.0.3", - "@budibase/backend-core": "1.2.26-alpha.0", - "@budibase/client": "1.2.26-alpha.0", - "@budibase/pro": "1.2.26-alpha.0", - "@budibase/string-templates": "1.2.26-alpha.0", - "@budibase/types": "1.2.26-alpha.0", + "@budibase/backend-core": "^1.2.27", + "@budibase/client": "^1.2.27", + "@budibase/pro": "1.2.27", + "@budibase/string-templates": "^1.2.27", + "@budibase/types": "^1.2.27", "@bull-board/api": "3.7.0", "@bull-board/koa": "3.9.4", "@elastic/elasticsearch": "7.10.0", diff --git a/packages/server/src/api/controllers/static/index.js b/packages/server/src/api/controllers/static/index.ts similarity index 89% rename from packages/server/src/api/controllers/static/index.js rename to packages/server/src/api/controllers/static/index.ts index 86bce89b4a..b0e36b8943 100644 --- a/packages/server/src/api/controllers/static/index.js +++ b/packages/server/src/api/controllers/static/index.ts @@ -18,14 +18,14 @@ const { DocumentTypes, isDevAppID } = require("../../../db/utils") const { getAppDB, getAppId } = require("@budibase/backend-core/context") const { setCookie, clearCookie } = require("@budibase/backend-core/utils") const AWS = require("aws-sdk") -const { events } = require("@budibase/backend-core") +import { events } from "@budibase/backend-core" const fs = require("fs") const { downloadTarballDirect, } = require("../../../utilities/fileSystem/utilities") -async function prepareUpload({ s3Key, bucket, metadata, file }) { +async function prepareUpload({ s3Key, bucket, metadata, file }: any) { const response = await upload({ bucket, metadata, @@ -44,7 +44,7 @@ async function prepareUpload({ s3Key, bucket, metadata, file }) { } } -exports.toggleBetaUiFeature = async function (ctx) { +export const toggleBetaUiFeature = async function (ctx: any) { const cookieName = `beta:${ctx.params.feature}` if (ctx.cookies.get(cookieName)) { @@ -72,21 +72,21 @@ exports.toggleBetaUiFeature = async function (ctx) { } } -exports.serveBuilder = async function (ctx) { +export const serveBuilder = async function (ctx: any) { const builderPath = resolve(TOP_LEVEL_PATH, "builder") await send(ctx, ctx.file, { root: builderPath }) - if (!ctx.file.includes("assets/")) { + if (ctx.file === "index.html") { await events.serve.servedBuilder() } } -exports.uploadFile = async function (ctx) { +export const uploadFile = async function (ctx: any) { let files = ctx.request.files.file.length > 1 ? Array.from(ctx.request.files.file) : [ctx.request.files.file] - const uploads = files.map(async file => { + const uploads = files.map(async (file: any) => { const fileExtension = [...file.name.split(".")].pop() // filenames converted to UUIDs so they are unique const processedFileName = `${uuid.v4()}.${fileExtension}` @@ -101,7 +101,7 @@ exports.uploadFile = async function (ctx) { ctx.body = await Promise.all(uploads) } -exports.serveApp = async function (ctx) { +export const serveApp = async function (ctx: any) { const db = getAppDB({ skip_setup: true }) const appInfo = await db.get(DocumentTypes.APP_METADATA) let appId = getAppId() @@ -134,13 +134,13 @@ exports.serveApp = async function (ctx) { } } -exports.serveClientLibrary = async function (ctx) { +export const serveClientLibrary = async function (ctx: any) { return send(ctx, "budibase-client.js", { root: join(NODE_MODULES_PATH, "@budibase", "client", "dist"), }) } -exports.getSignedUploadURL = async function (ctx) { +export const getSignedUploadURL = async function (ctx: any) { const database = getAppDB() // Ensure datasource is valid diff --git a/packages/string-templates/package.json b/packages/string-templates/package.json index bfafc094a2..726b5207b7 100644 --- a/packages/string-templates/package.json +++ b/packages/string-templates/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/string-templates", - "version": "1.2.26-alpha.0", + "version": "1.2.27", "description": "Handlebars wrapper for Budibase templating.", "main": "src/index.cjs", "module": "dist/bundle.mjs", diff --git a/packages/types/package.json b/packages/types/package.json index fadfc35297..8e15aae144 100644 --- a/packages/types/package.json +++ b/packages/types/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/types", - "version": "1.2.26-alpha.0", + "version": "1.2.27", "description": "Budibase types", "main": "dist/index.js", "types": "dist/index.d.ts", diff --git a/packages/worker/package.json b/packages/worker/package.json index 95697d7d26..a1d20c090f 100644 --- a/packages/worker/package.json +++ b/packages/worker/package.json @@ -1,7 +1,7 @@ { "name": "@budibase/worker", "email": "hi@budibase.com", - "version": "1.2.26-alpha.0", + "version": "1.2.27", "description": "Budibase background service", "main": "src/index.ts", "repository": { @@ -35,10 +35,10 @@ "author": "Budibase", "license": "GPL-3.0", "dependencies": { - "@budibase/backend-core": "1.2.26-alpha.0", - "@budibase/pro": "1.2.26-alpha.0", - "@budibase/string-templates": "1.2.26-alpha.0", - "@budibase/types": "1.2.26-alpha.0", + "@budibase/backend-core": "^1.2.27", + "@budibase/pro": "1.2.27", + "@budibase/string-templates": "^1.2.27", + "@budibase/types": "^1.2.27", "@koa/router": "8.0.8", "@sentry/node": "6.17.7", "@techpass/passport-openidconnect": "0.3.2",