Merge branch 'master' of github.com:Budibase/budibase into develop

hosting/docker-compose.yaml

@@ -75,7 +75,9 @@ services:
     ports:
       - "${MAIN_PORT}:10000"
     container_name: bbproxy
-    image: budibase/proxy
+    image: proxy-service
+    environment:
+      - PROXY_RATE_LIMIT_WEBHOOKS_PER_SECOND=10
     depends_on:
       - minio-service
       - worker-service

hosting/nginx.prod.conf.hbs

@@ -9,7 +9,11 @@ events {
 }
 
 http {
+  # rate limiting
+  limit_req_status 429;
   limit_req_zone $binary_remote_addr zone=ratelimit:10m rate=20r/s;
+  limit_req_zone $binary_remote_addr zone=webhooks:10m rate=${PROXY_RATE_LIMIT_WEBHOOKS_PER_SECOND}r/s;
+
   include       /etc/nginx/mime.types;
   default_type  application/octet-stream;
   proxy_set_header Host $host;
@@ -126,6 +130,25 @@ http {
       proxy_pass      http://$apps:4002;
     }
 
+    location /api/webhooks/ {
+      # calls to webhooks are rate limited
+      limit_req zone=webhooks nodelay;
+
+      # Rest of configuration copied from /api/ location above
+      # 120s timeout on API requests
+      proxy_read_timeout 120s;
+      proxy_connect_timeout 120s;
+      proxy_send_timeout 120s;
+
+      proxy_http_version  1.1;
+      proxy_set_header    Connection          $connection_upgrade;
+      proxy_set_header    Upgrade             $http_upgrade;
+      proxy_set_header    X-Real-IP           $remote_addr;
+      proxy_set_header    X-Forwarded-For     $proxy_add_x_forwarded_for;
+
+      proxy_pass      http://$apps:4002;
+    }
+
     location /db/ {
       proxy_pass      http://$couchdb:5984;
       rewrite ^/db/(.*)$ /$1 break;

hosting/proxy/Dockerfile

@@ -1,3 +1,13 @@
 FROM nginx:latest
-COPY .generated-nginx.prod.conf /etc/nginx/nginx.conf
+
+# nginx.conf
+# use the default nginx behaviour for *.template files which are processed with envsubst
+# override the output dir to output directly to /etc/nginx instead of /etc/nginx/conf.d
+ENV NGINX_ENVSUBST_OUTPUT_DIR=/etc/nginx
+COPY .generated-nginx.prod.conf /etc/nginx/templates/nginx.conf.template
+
+# Error handling
 COPY error.html /usr/share/nginx/html/error.html
+
+# Default environment
+ENV PROXY_RATE_LIMIT_WEBHOOKS_PER_SECOND=10

lerna.json

@@ -1,5 +1,5 @@
 {
-  "version": "1.2.26-alpha.0",
+  "version": "1.2.27",
   "npmClient": "yarn",
   "packages": [
     "packages/*"

packages/backend-core/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@budibase/backend-core",
-  "version": "1.2.26-alpha.0",
+  "version": "1.2.27",
   "description": "Budibase backend core libraries used in server and worker",
   "main": "dist/src/index.js",
   "types": "dist/src/index.d.ts",
@@ -20,7 +20,7 @@
     "test:watch": "jest --watchAll"
   },
   "dependencies": {
-    "@budibase/types": "1.2.26-alpha.0",
+    "@budibase/types": "^1.2.27",
     "@techpass/passport-openidconnect": "0.3.2",
     "aws-sdk": "2.1030.0",
     "bcrypt": "5.0.1",

packages/backend-core/src/events/emit/BBEventEmitter.ts

@@ -0,0 +1,27 @@
+import { EventEmitter } from "events"
+import * as context from "../../context"
+import { Identity, Event } from "@budibase/types"
+
+export interface EmittedEvent {
+  tenantId: string
+  identity: Identity
+  appId: string | undefined
+  properties: any
+}
+
+class BBEventEmitter extends EventEmitter {
+  emitEvent(event: Event, properties: any, identity: Identity) {
+    const tenantId = context.getTenantId()
+    const appId = context.getAppId()
+
+    const emittedEvent: EmittedEvent = {
+      tenantId,
+      identity,
+      appId,
+      properties,
+    }
+    this.emit(event, emittedEvent)
+  }
+}
+
+export const emitter = new BBEventEmitter()

packages/backend-core/src/events/emit/index.ts

@@ -0,0 +1 @@
+export * from "./BBEventEmitter"

packages/backend-core/src/events/events.ts

@@ -2,6 +2,41 @@ import { Event } from "@budibase/types"
 import { processors } from "./processors"
 import * as identification from "./identification"
 import * as backfill from "./backfill"
+import { emitter, EmittedEvent } from "./emit"
+import * as context from "../context"
+import * as logging from "../logging"
+
+const USE_EMITTER: any[] = [
+  Event.SERVED_BUILDER,
+  Event.SERVED_APP,
+  Event.SERVED_APP_PREVIEW,
+]
+
+for (let event of USE_EMITTER) {
+  emitter.on(event, async (props: EmittedEvent) => {
+    try {
+      await context.doInTenant(props.tenantId, async () => {
+        if (props.appId) {
+          await context.doInAppContext(props.appId, async () => {
+            await processors.processEvent(
+              event as Event,
+              props.identity,
+              props.properties
+            )
+          })
+        } else {
+          await processors.processEvent(
+            event as Event,
+            props.identity,
+            props.properties
+          )
+        }
+      })
+    } catch (e) {
+      logging.logAlert(`Unable to process async event ${event}`, e)
+    }
+  })
+}
 
 export const publishEvent = async (
   event: Event,
@@ -11,6 +46,11 @@ export const publishEvent = async (
   // in future this should use async events via a distributed queue.
   const identity = await identification.getCurrentIdentity()
 
+  if (USE_EMITTER.includes(event)) {
+    emitter.emitEvent(event, properties, identity)
+    return
+  }
+
   const backfilling = await backfill.isBackfillingEvent(event)
   // no backfill - send the event and exit
   if (!backfilling) {

packages/backend-core/src/events/processors/AnalyticsProcessor.ts

@@ -32,7 +32,7 @@ export default class AnalyticsProcessor implements EventProcessor {
       return
     }
     if (this.posthog) {
-      this.posthog.processEvent(event, identity, properties, timestamp)
+      await this.posthog.processEvent(event, identity, properties, timestamp)
     }
   }
 
@@ -45,14 +45,14 @@ export default class AnalyticsProcessor implements EventProcessor {
       return
     }
     if (this.posthog) {
-      this.posthog.identify(identity, timestamp)
+      await this.posthog.identify(identity, timestamp)
     }
   }
 
   async identifyGroup(group: Group, timestamp?: string | number) {
     // Group indentifications (tenant and installation) always on
     if (this.posthog) {
-      this.posthog.identifyGroup(group, timestamp)
+      await this.posthog.identifyGroup(group, timestamp)
     }
   }
 

packages/backend-core/src/events/processors/posthog/rateLimiting.ts

@@ -45,7 +45,7 @@ export const limited = async (event: Event): Promise<boolean> => {
     return false
   }
 
-  const cachedEvent = (await readEvent(event)) as EventProperties
+  const cachedEvent = await readEvent(event)
   if (cachedEvent) {
     const timestamp = new Date(cachedEvent.timestamp)
     const limit = RATE_LIMITS[event]
@@ -76,14 +76,17 @@ export const limited = async (event: Event): Promise<boolean> => {
 const eventKey = (event: RateLimitedEvent) => {
   let key = `${CacheKeys.EVENTS_RATE_LIMIT}:${event}`
   if (isPerApp(event)) {
-    key = key + context.getAppId()
+    key = key + ":" + context.getAppId()
   }
   return key
 }
 
-const readEvent = async (event: RateLimitedEvent) => {
+const readEvent = async (
+  event: RateLimitedEvent
+): Promise<EventProperties | undefined> => {
   const key = eventKey(event)
-  return cache.get(key)
+  const result = await cache.get(key)
+  return result as EventProperties
 }
 
 const recordEvent = async (

packages/bbui/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@budibase/bbui",
   "description": "A UI solution used in the different Budibase projects.",
-  "version": "1.2.26-alpha.0",
+  "version": "1.2.27",
   "license": "MPL-2.0",
   "svelte": "src/index.js",
   "module": "dist/bbui.es.js",
@@ -38,7 +38,7 @@
   ],
   "dependencies": {
     "@adobe/spectrum-css-workflow-icons": "^1.2.1",
-    "@budibase/string-templates": "1.2.26-alpha.0",
+    "@budibase/string-templates": "^1.2.27",
     "@spectrum-css/actionbutton": "^1.0.1",
     "@spectrum-css/actiongroup": "^1.0.1",
     "@spectrum-css/avatar": "^3.0.2",

packages/builder/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@budibase/builder",
-  "version": "1.2.26-alpha.0",
+  "version": "1.2.27",
   "license": "GPL-3.0",
   "private": true,
   "scripts": {
@@ -69,10 +69,10 @@
     }
   },
   "dependencies": {
-    "@budibase/bbui": "1.2.26-alpha.0",
+    "@budibase/bbui": "^1.2.27",
-    "@budibase/client": "1.2.26-alpha.0",
+    "@budibase/client": "^1.2.27",
-    "@budibase/frontend-core": "1.2.26-alpha.0",
+    "@budibase/frontend-core": "^1.2.27",
-    "@budibase/string-templates": "1.2.26-alpha.0",
+    "@budibase/string-templates": "^1.2.27",
     "@sentry/browser": "5.19.1",
     "@spectrum-css/page": "^3.0.1",
     "@spectrum-css/vars": "^3.0.1",

packages/cli/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@budibase/cli",
-  "version": "1.2.26-alpha.0",
+  "version": "1.2.27",
   "description": "Budibase CLI, for developers, self hosting and migrations.",
   "main": "src/index.js",
   "bin": {

packages/client/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@budibase/client",
-  "version": "1.2.26-alpha.0",
+  "version": "1.2.27",
   "license": "MPL-2.0",
   "module": "dist/budibase-client.js",
   "main": "dist/budibase-client.js",
@@ -19,9 +19,9 @@
     "dev:builder": "rollup -cw"
   },
   "dependencies": {
-    "@budibase/bbui": "1.2.26-alpha.0",
+    "@budibase/bbui": "^1.2.27",
-    "@budibase/frontend-core": "1.2.26-alpha.0",
+    "@budibase/frontend-core": "^1.2.27",
-    "@budibase/string-templates": "1.2.26-alpha.0",
+    "@budibase/string-templates": "^1.2.27",
     "@spectrum-css/button": "^3.0.3",
     "@spectrum-css/card": "^3.0.3",
     "@spectrum-css/divider": "^1.0.3",

packages/frontend-core/package.json

@@ -1,12 +1,12 @@
 {
   "name": "@budibase/frontend-core",
-  "version": "1.2.26-alpha.0",
+  "version": "1.2.27",
   "description": "Budibase frontend core libraries used in builder and client",
   "author": "Budibase",
   "license": "MPL-2.0",
   "svelte": "src/index.js",
   "dependencies": {
-    "@budibase/bbui": "1.2.26-alpha.0",
+    "@budibase/bbui": "^1.2.27",
     "lodash": "^4.17.21",
     "svelte": "^3.46.2"
   }

packages/server/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@budibase/server",
   "email": "hi@budibase.com",
-  "version": "1.2.26-alpha.0",
+  "version": "1.2.27",
   "description": "Budibase Web Server",
   "main": "src/index.ts",
   "repository": {
@@ -77,11 +77,11 @@
   "license": "GPL-3.0",
   "dependencies": {
     "@apidevtools/swagger-parser": "10.0.3",
-    "@budibase/backend-core": "1.2.26-alpha.0",
+    "@budibase/backend-core": "^1.2.27",
-    "@budibase/client": "1.2.26-alpha.0",
+    "@budibase/client": "^1.2.27",
-    "@budibase/pro": "1.2.26-alpha.0",
+    "@budibase/pro": "1.2.27",
-    "@budibase/string-templates": "1.2.26-alpha.0",
+    "@budibase/string-templates": "^1.2.27",
-    "@budibase/types": "1.2.26-alpha.0",
+    "@budibase/types": "^1.2.27",
     "@bull-board/api": "3.7.0",
     "@bull-board/koa": "3.9.4",
     "@elastic/elasticsearch": "7.10.0",

packages/server/src/api/controllers/static/index.ts

@@ -18,14 +18,14 @@ const { DocumentTypes, isDevAppID } = require("../../../db/utils")
 const { getAppDB, getAppId } = require("@budibase/backend-core/context")
 const { setCookie, clearCookie } = require("@budibase/backend-core/utils")
 const AWS = require("aws-sdk")
-const { events } = require("@budibase/backend-core")
+import { events } from "@budibase/backend-core"
 
 const fs = require("fs")
 const {
   downloadTarballDirect,
 } = require("../../../utilities/fileSystem/utilities")
 
-async function prepareUpload({ s3Key, bucket, metadata, file }) {
+async function prepareUpload({ s3Key, bucket, metadata, file }: any) {
   const response = await upload({
     bucket,
     metadata,
@@ -44,7 +44,7 @@ async function prepareUpload({ s3Key, bucket, metadata, file }) {
   }
 }
 
-exports.toggleBetaUiFeature = async function (ctx) {
+export const toggleBetaUiFeature = async function (ctx: any) {
   const cookieName = `beta:${ctx.params.feature}`
 
   if (ctx.cookies.get(cookieName)) {
@@ -72,21 +72,21 @@ exports.toggleBetaUiFeature = async function (ctx) {
   }
 }
 
-exports.serveBuilder = async function (ctx) {
+export const serveBuilder = async function (ctx: any) {
   const builderPath = resolve(TOP_LEVEL_PATH, "builder")
   await send(ctx, ctx.file, { root: builderPath })
-  if (!ctx.file.includes("assets/")) {
+  if (ctx.file === "index.html") {
     await events.serve.servedBuilder()
   }
 }
 
-exports.uploadFile = async function (ctx) {
+export const uploadFile = async function (ctx: any) {
   let files =
     ctx.request.files.file.length > 1
       ? Array.from(ctx.request.files.file)
       : [ctx.request.files.file]
 
-  const uploads = files.map(async file => {
+  const uploads = files.map(async (file: any) => {
     const fileExtension = [...file.name.split(".")].pop()
     // filenames converted to UUIDs so they are unique
     const processedFileName = `${uuid.v4()}.${fileExtension}`
@@ -101,7 +101,7 @@ exports.uploadFile = async function (ctx) {
   ctx.body = await Promise.all(uploads)
 }
 
-exports.serveApp = async function (ctx) {
+export const serveApp = async function (ctx: any) {
   const db = getAppDB({ skip_setup: true })
   const appInfo = await db.get(DocumentTypes.APP_METADATA)
   let appId = getAppId()
@@ -134,13 +134,13 @@ exports.serveApp = async function (ctx) {
   }
 }
 
-exports.serveClientLibrary = async function (ctx) {
+export const serveClientLibrary = async function (ctx: any) {
   return send(ctx, "budibase-client.js", {
     root: join(NODE_MODULES_PATH, "@budibase", "client", "dist"),
   })
 }
 
-exports.getSignedUploadURL = async function (ctx) {
+export const getSignedUploadURL = async function (ctx: any) {
   const database = getAppDB()
 
   // Ensure datasource is valid

packages/string-templates/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@budibase/string-templates",
-  "version": "1.2.26-alpha.0",
+  "version": "1.2.27",
   "description": "Handlebars wrapper for Budibase templating.",
   "main": "src/index.cjs",
   "module": "dist/bundle.mjs",

packages/types/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@budibase/types",
-  "version": "1.2.26-alpha.0",
+  "version": "1.2.27",
   "description": "Budibase types",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",

packages/worker/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@budibase/worker",
   "email": "hi@budibase.com",
-  "version": "1.2.26-alpha.0",
+  "version": "1.2.27",
   "description": "Budibase background service",
   "main": "src/index.ts",
   "repository": {
@@ -35,10 +35,10 @@
   "author": "Budibase",
   "license": "GPL-3.0",
   "dependencies": {
-    "@budibase/backend-core": "1.2.26-alpha.0",
+    "@budibase/backend-core": "^1.2.27",
-    "@budibase/pro": "1.2.26-alpha.0",
+    "@budibase/pro": "1.2.27",
-    "@budibase/string-templates": "1.2.26-alpha.0",
+    "@budibase/string-templates": "^1.2.27",
-    "@budibase/types": "1.2.26-alpha.0",
+    "@budibase/types": "^1.2.27",
     "@koa/router": "8.0.8",
     "@sentry/node": "6.17.7",
     "@techpass/passport-openidconnect": "0.3.2",