MirrorSave
/
budibase
mirror of https://github.com/Budibase/budibase.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' of github.com:Budibase/budibase into develop

This commit is contained in:
mike12345567 2022-08-10 11:55:53 +01:00
parent 527bf06f48 a4a2229957
commit 3a340379e9
20 changed files with 156 additions and 50 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
hosting/docker-compose.yaml
View File

@ -75,7 +75,9 @@ services:
ports:
- "${MAIN_PORT}:10000"
container_name: bbproxy
image: budibase/proxy
image: proxy-service
environment:
- PROXY_RATE_LIMIT_WEBHOOKS_PER_SECOND=10
depends_on:
- minio-service
- worker-service

23
hosting/nginx.prod.conf.hbs
View File

@ -9,7 +9,11 @@ events {
}
http {
# rate limiting
limit_req_status 429;
limit_req_zone $binary_remote_addr zone=ratelimit:10m rate=20r/s;
limit_req_zone $binary_remote_addr zone=webhooks:10m rate=${PROXY_RATE_LIMIT_WEBHOOKS_PER_SECOND}r/s;
include /etc/nginx/mime.types;
default_type application/octet-stream;
proxy_set_header Host $host;
@ -126,6 +130,25 @@ http {
proxy_pass http://$apps:4002;
}
location /api/webhooks/ {
# calls to webhooks are rate limited
limit_req zone=webhooks nodelay;
# Rest of configuration copied from /api/ location above
# 120s timeout on API requests
proxy_read_timeout 120s;
proxy_connect_timeout 120s;
proxy_send_timeout 120s;
proxy_http_version 1.1;
proxy_set_header Connection $connection_upgrade;
proxy_set_header Upgrade $http_upgrade;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_pass http://$apps:4002;
}
location /db/ {
proxy_pass http://$couchdb:5984;
rewrite ^/db/(.*)$ /$1 break;

12
hosting/proxy/Dockerfile
View File

@ -1,3 +1,13 @@
FROM nginx:latest
COPY .generated-nginx.prod.conf /etc/nginx/nginx.conf
# nginx.conf
# use the default nginx behaviour for *.template files which are processed with envsubst
# override the output dir to output directly to /etc/nginx instead of /etc/nginx/conf.d
ENV NGINX_ENVSUBST_OUTPUT_DIR=/etc/nginx
COPY .generated-nginx.prod.conf /etc/nginx/templates/nginx.conf.template
# Error handling
COPY error.html /usr/share/nginx/html/error.html
# Default environment
ENV PROXY_RATE_LIMIT_WEBHOOKS_PER_SECOND=10

2
lerna.json
View File

@ -1,5 +1,5 @@
{
"version": "1.2.26-alpha.0",
"version": "1.2.27",
"npmClient": "yarn",
"packages": [
"packages/*"

4
packages/backend-core/package.json
View File

@ -1,6 +1,6 @@
{
"name": "@budibase/backend-core",
"version": "1.2.26-alpha.0",
"version": "1.2.27",
"description": "Budibase backend core libraries used in server and worker",
"main": "dist/src/index.js",
"types": "dist/src/index.d.ts",
@ -20,7 +20,7 @@
"test:watch": "jest --watchAll"
},
"dependencies": {
"@budibase/types": "1.2.26-alpha.0",
"@budibase/types": "^1.2.27",
"@techpass/passport-openidconnect": "0.3.2",
"aws-sdk": "2.1030.0",
"bcrypt": "5.0.1",

27
packages/backend-core/src/events/emit/BBEventEmitter.ts Normal file
View File

@ -0,0 +1,27 @@
import { EventEmitter } from "events"
import * as context from "../../context"
import { Identity, Event } from "@budibase/types"
export interface EmittedEvent {
tenantId: string
identity: Identity
appId: string | undefined
properties: any
}
class BBEventEmitter extends EventEmitter {
emitEvent(event: Event, properties: any, identity: Identity) {
const tenantId = context.getTenantId()
const appId = context.getAppId()
const emittedEvent: EmittedEvent = {
tenantId,
identity,
appId,
properties,
}
this.emit(event, emittedEvent)
}
}
export const emitter = new BBEventEmitter()

1
packages/backend-core/src/events/emit/index.ts Normal file
View File

@ -0,0 +1 @@
export * from "./BBEventEmitter"

40
packages/backend-core/src/events/events.ts
View File

@ -2,6 +2,41 @@ import { Event } from "@budibase/types"
import { processors } from "./processors"
import * as identification from "./identification"
import * as backfill from "./backfill"
import { emitter, EmittedEvent } from "./emit"
import * as context from "../context"
import * as logging from "../logging"
const USE_EMITTER: any[] = [
Event.SERVED_BUILDER,
Event.SERVED_APP,
Event.SERVED_APP_PREVIEW,
]
for (let event of USE_EMITTER) {
emitter.on(event, async (props: EmittedEvent) => {
try {
await context.doInTenant(props.tenantId, async () => {
if (props.appId) {
await context.doInAppContext(props.appId, async () => {
await processors.processEvent(
event as Event,
props.identity,
props.properties
)
})
} else {
await processors.processEvent(
event as Event,
props.identity,
props.properties
)
}
})
} catch (e) {
logging.logAlert(`Unable to process async event ${event}`, e)
}
})
}
export const publishEvent = async (
event: Event,
@ -11,6 +46,11 @@ export const publishEvent = async (
// in future this should use async events via a distributed queue.
const identity = await identification.getCurrentIdentity()
if (USE_EMITTER.includes(event)) {
emitter.emitEvent(event, properties, identity)
return
}
const backfilling = await backfill.isBackfillingEvent(event)
// no backfill - send the event and exit
if (!backfilling) {

6
packages/backend-core/src/events/processors/AnalyticsProcessor.ts
View File

@ -32,7 +32,7 @@ export default class AnalyticsProcessor implements EventProcessor {
return
}
if (this.posthog) {
this.posthog.processEvent(event, identity, properties, timestamp)
await this.posthog.processEvent(event, identity, properties, timestamp)
}
}
@ -45,14 +45,14 @@ export default class AnalyticsProcessor implements EventProcessor {
return
}
if (this.posthog) {
this.posthog.identify(identity, timestamp)
await this.posthog.identify(identity, timestamp)
}
}
async identifyGroup(group: Group, timestamp?: string | number) {
// Group indentifications (tenant and installation) always on
if (this.posthog) {
this.posthog.identifyGroup(group, timestamp)
await this.posthog.identifyGroup(group, timestamp)
}
}

11
packages/backend-core/src/events/processors/posthog/rateLimiting.ts
View File

@ -45,7 +45,7 @@ export const limited = async (event: Event): Promise<boolean> => {
return false
}
const cachedEvent = (await readEvent(event)) as EventProperties
const cachedEvent = await readEvent(event)
if (cachedEvent) {
const timestamp = new Date(cachedEvent.timestamp)
const limit = RATE_LIMITS[event]
@ -76,14 +76,17 @@ export const limited = async (event: Event): Promise<boolean> => {
const eventKey = (event: RateLimitedEvent) => {
let key = `${CacheKeys.EVENTS_RATE_LIMIT}:${event}`
if (isPerApp(event)) {
key = key + context.getAppId()
key = key + ":" + context.getAppId()
}
return key
}
const readEvent = async (event: RateLimitedEvent) => {
const readEvent = async (
event: RateLimitedEvent
): Promise<EventProperties | undefined> => {
const key = eventKey(event)
return cache.get(key)
const result = await cache.get(key)
return result as EventProperties
}
const recordEvent = async (

4
packages/bbui/package.json
View File

@ -1,7 +1,7 @@
{
"name": "@budibase/bbui",
"description": "A UI solution used in the different Budibase projects.",
"version": "1.2.26-alpha.0",
"version": "1.2.27",
"license": "MPL-2.0",
"svelte": "src/index.js",
"module": "dist/bbui.es.js",
@ -38,7 +38,7 @@
],
"dependencies": {
"@adobe/spectrum-css-workflow-icons": "^1.2.1",
"@budibase/string-templates": "1.2.26-alpha.0",
"@budibase/string-templates": "^1.2.27",
"@spectrum-css/actionbutton": "^1.0.1",
"@spectrum-css/actiongroup": "^1.0.1",
"@spectrum-css/avatar": "^3.0.2",

10
packages/builder/package.json
View File

@ -1,6 +1,6 @@
{
"name": "@budibase/builder",
"version": "1.2.26-alpha.0",
"version": "1.2.27",
"license": "GPL-3.0",
"private": true,
"scripts": {
@ -69,10 +69,10 @@
}
},
"dependencies": {
"@budibase/bbui": "1.2.26-alpha.0",
"@budibase/client": "1.2.26-alpha.0",
"@budibase/frontend-core": "1.2.26-alpha.0",
"@budibase/string-templates": "1.2.26-alpha.0",
"@budibase/bbui": "^1.2.27",
"@budibase/client": "^1.2.27",
"@budibase/frontend-core": "^1.2.27",
"@budibase/string-templates": "^1.2.27",
"@sentry/browser": "5.19.1",
"@spectrum-css/page": "^3.0.1",
"@spectrum-css/vars": "^3.0.1",

2
packages/cli/package.json
View File

@ -1,6 +1,6 @@
{
"name": "@budibase/cli",
"version": "1.2.26-alpha.0",
"version": "1.2.27",
"description": "Budibase CLI, for developers, self hosting and migrations.",
"main": "src/index.js",
"bin": {

8
packages/client/package.json
View File

@ -1,6 +1,6 @@
{
"name": "@budibase/client",
"version": "1.2.26-alpha.0",
"version": "1.2.27",
"license": "MPL-2.0",
"module": "dist/budibase-client.js",
"main": "dist/budibase-client.js",
@ -19,9 +19,9 @@
"dev:builder": "rollup -cw"
},
"dependencies": {
"@budibase/bbui": "1.2.26-alpha.0",
"@budibase/frontend-core": "1.2.26-alpha.0",
"@budibase/string-templates": "1.2.26-alpha.0",
"@budibase/bbui": "^1.2.27",
"@budibase/frontend-core": "^1.2.27",
"@budibase/string-templates": "^1.2.27",
"@spectrum-css/button": "^3.0.3",
"@spectrum-css/card": "^3.0.3",
"@spectrum-css/divider": "^1.0.3",

4
packages/frontend-core/package.json
View File

@ -1,12 +1,12 @@
{
"name": "@budibase/frontend-core",
"version": "1.2.26-alpha.0",
"version": "1.2.27",
"description": "Budibase frontend core libraries used in builder and client",
"author": "Budibase",
"license": "MPL-2.0",
"svelte": "src/index.js",
"dependencies": {
"@budibase/bbui": "1.2.26-alpha.0",
"@budibase/bbui": "^1.2.27",
"lodash": "^4.17.21",
"svelte": "^3.46.2"
}

12
packages/server/package.json
View File

@ -1,7 +1,7 @@
{
"name": "@budibase/server",
"email": "hi@budibase.com",
"version": "1.2.26-alpha.0",
"version": "1.2.27",
"description": "Budibase Web Server",
"main": "src/index.ts",
"repository": {
@ -77,11 +77,11 @@
"license": "GPL-3.0",
"dependencies": {
"@apidevtools/swagger-parser": "10.0.3",
"@budibase/backend-core": "1.2.26-alpha.0",
"@budibase/client": "1.2.26-alpha.0",
"@budibase/pro": "1.2.26-alpha.0",
"@budibase/string-templates": "1.2.26-alpha.0",
"@budibase/types": "1.2.26-alpha.0",
"@budibase/backend-core": "^1.2.27",
"@budibase/client": "^1.2.27",
"@budibase/pro": "1.2.27",
"@budibase/string-templates": "^1.2.27",
"@budibase/types": "^1.2.27",
"@bull-board/api": "3.7.0",
"@bull-board/koa": "3.9.4",
"@elastic/elasticsearch": "7.10.0",

20
packages/server/src/api/controllers/static/index.js → packages/server/src/api/controllers/static/index.ts
View File

@ -18,14 +18,14 @@ const { DocumentTypes, isDevAppID } = require("../../../db/utils")
const { getAppDB, getAppId } = require("@budibase/backend-core/context")
const { setCookie, clearCookie } = require("@budibase/backend-core/utils")
const AWS = require("aws-sdk")
const { events } = require("@budibase/backend-core")
import { events } from "@budibase/backend-core"
const fs = require("fs")
const {
downloadTarballDirect,
} = require("../../../utilities/fileSystem/utilities")
async function prepareUpload({ s3Key, bucket, metadata, file }) {
async function prepareUpload({ s3Key, bucket, metadata, file }: any) {
const response = await upload({
bucket,
metadata,
@ -44,7 +44,7 @@ async function prepareUpload({ s3Key, bucket, metadata, file }) {
}
}
exports.toggleBetaUiFeature = async function (ctx) {
export const toggleBetaUiFeature = async function (ctx: any) {
const cookieName = `beta:${ctx.params.feature}`
if (ctx.cookies.get(cookieName)) {
@ -72,21 +72,21 @@ exports.toggleBetaUiFeature = async function (ctx) {
}
}
exports.serveBuilder = async function (ctx) {
export const serveBuilder = async function (ctx: any) {
const builderPath = resolve(TOP_LEVEL_PATH, "builder")
await send(ctx, ctx.file, { root: builderPath })
if (!ctx.file.includes("assets/")) {
if (ctx.file === "index.html") {
await events.serve.servedBuilder()
}
}
exports.uploadFile = async function (ctx) {
export const uploadFile = async function (ctx: any) {
let files =
ctx.request.files.file.length > 1
? Array.from(ctx.request.files.file)
: [ctx.request.files.file]
const uploads = files.map(async file => {
const uploads = files.map(async (file: any) => {
const fileExtension = [...file.name.split(".")].pop()
// filenames converted to UUIDs so they are unique
const processedFileName = `${uuid.v4()}.${fileExtension}`
@ -101,7 +101,7 @@ exports.uploadFile = async function (ctx) {
ctx.body = await Promise.all(uploads)
}
exports.serveApp = async function (ctx) {
export const serveApp = async function (ctx: any) {
const db = getAppDB({ skip_setup: true })
const appInfo = await db.get(DocumentTypes.APP_METADATA)
let appId = getAppId()
@ -134,13 +134,13 @@ exports.serveApp = async function (ctx) {
}
}
exports.serveClientLibrary = async function (ctx) {
export const serveClientLibrary = async function (ctx: any) {
return send(ctx, "budibase-client.js", {
root: join(NODE_MODULES_PATH, "@budibase", "client", "dist"),
})
}
exports.getSignedUploadURL = async function (ctx) {
export const getSignedUploadURL = async function (ctx: any) {
const database = getAppDB()
// Ensure datasource is valid

2
packages/string-templates/package.json
View File

@ -1,6 +1,6 @@
{
"name": "@budibase/string-templates",
"version": "1.2.26-alpha.0",
"version": "1.2.27",
"description": "Handlebars wrapper for Budibase templating.",
"main": "src/index.cjs",
"module": "dist/bundle.mjs",

2
packages/types/package.json
View File

@ -1,6 +1,6 @@
{
"name": "@budibase/types",
"version": "1.2.26-alpha.0",
"version": "1.2.27",
"description": "Budibase types",
"main": "dist/index.js",
"types": "dist/index.d.ts",

10
packages/worker/package.json
View File

@ -1,7 +1,7 @@
{
"name": "@budibase/worker",
"email": "hi@budibase.com",
"version": "1.2.26-alpha.0",
"version": "1.2.27",
"description": "Budibase background service",
"main": "src/index.ts",
"repository": {
@ -35,10 +35,10 @@
"author": "Budibase",
"license": "GPL-3.0",
"dependencies": {
"@budibase/backend-core": "1.2.26-alpha.0",
"@budibase/pro": "1.2.26-alpha.0",
"@budibase/string-templates": "1.2.26-alpha.0",
"@budibase/types": "1.2.26-alpha.0",
"@budibase/backend-core": "^1.2.27",
"@budibase/pro": "1.2.27",
"@budibase/string-templates": "^1.2.27",
"@budibase/types": "^1.2.27",
"@koa/router": "8.0.8",
"@sentry/node": "6.17.7",
"@techpass/passport-openidconnect": "0.3.2",