Merge branch 'develop' of github.com:Budibase/budibase into feature/budibase-api

This commit is contained in:
mike12345567 2022-02-10 17:42:28 +00:00
commit f031c34253
48 changed files with 490 additions and 586 deletions

View File

@ -137,7 +137,7 @@ If you wish to delete all the apps created in development and reset the environm
### Backend
For the backend we run [Redis](https://redis.io/), [CouchDB](https://couchdb.apache.org/), [MinIO](https://min.io/) and [Envoy](https://www.envoyproxy.io/) in Docker compose. This means that to develop Budibase you will need Docker and Docker compose installed. The backend services are then ran separately as Node services with nodemon so that they can be debugged outside of Docker.
For the backend we run [Redis](https://redis.io/), [CouchDB](https://couchdb.apache.org/), [MinIO](https://min.io/) and [NGINX](https://www.nginx.com/) in Docker compose. This means that to develop Budibase you will need Docker and Docker compose installed. The backend services are then ran separately as Node services with nodemon so that they can be debugged outside of Docker.
### Data Storage

View File

@ -38,7 +38,7 @@ jobs:
wc -l values.preprod.yaml
- name: Deploy to Preprod Environment
uses: deliverybot/helm@v1
uses: glopezep/helm@v1.7.1
with:
release: budibase-preprod
namespace: budibase

2
.gitignore vendored
View File

@ -64,7 +64,7 @@ typings/
# dotenv environment variables file
.env
!hosting/.env
hosting/.generated-envoy.dev.yaml
hosting/.generated-nginx.dev.conf
# parcel-bundler cache (https://parceljs.org/)
.cache

View File

@ -11,8 +11,8 @@ sources:
- https://github.com/Budibase/budibase
- https://budibase.com
type: application
version: 0.2.5
appVersion: 1.0.25
version: 0.2.6
appVersion: 1.0.48
dependencies:
- name: couchdb
version: 3.3.4

View File

@ -25,7 +25,7 @@ spec:
app.kubernetes.io/name: budibase-proxy
spec:
containers:
- image: budibase/proxy
- image: budibase/proxy:k8s
imagePullPolicy: Always
name: proxy-service
ports:

View File

@ -111,6 +111,10 @@ spec:
value: {{ .Values.globals.smtp.from | quote }}
- name: APPS_URL
value: http://app-service:{{ .Values.services.apps.port }}
- name: GOOGLE_CLIENT_ID
value: {{ .Values.globals.google.clientId | quote }}
- name: GOOGLE_CLIENT_SECRET
value: {{ .Values.globals.google.secret | quote }}
image: budibase/worker:{{ .Values.globals.appVersion }}
imagePullPolicy: Always
name: bbworker

View File

@ -1 +0,0 @@
hosting.properties

21
hosting/.env Normal file
View File

@ -0,0 +1,21 @@
# Use the main port in the builder for your self hosting URL, e.g. localhost:10000
MAIN_PORT=10000
# This section contains all secrets pertaining to the system
# These should be updated
JWT_SECRET=testsecret
MINIO_ACCESS_KEY=budibase
MINIO_SECRET_KEY=budibase
COUCH_DB_PASSWORD=budibase
COUCH_DB_USER=budibase
REDIS_PASSWORD=budibase
INTERNAL_API_KEY=budibase
# This section contains variables that do not need to be altered under normal circumstances
APP_PORT=4002
WORKER_PORT=4003
MINIO_PORT=4004
COUCH_DB_PORT=4005
REDIS_PORT=6379
WATCHTOWER_PORT=6161
BUDIBASE_ENVIRONMENT=PRODUCTION

View File

@ -3,9 +3,8 @@
# go into the app dir
cd /root
# fetch envoy and docker-compose files
# fetch nginx and docker-compose files
wget https://raw.githubusercontent.com/Budibase/budibase/master/hosting/docker-compose.yaml
wget https://raw.githubusercontent.com/Budibase/budibase/master/hosting/envoy.yaml
wget https://raw.githubusercontent.com/Budibase/budibase/master/hosting/hosting.properties
# Create .env file from hosting.properties using bash and then remove it

View File

@ -22,18 +22,21 @@ services:
retries: 3
proxy-service:
container_name: budi-envoy-dev
container_name: budi-nginx-dev
restart: always
image: envoyproxy/envoy:v1.16-latest
image: nginx:latest
volumes:
- ./.generated-envoy.dev.yaml:/etc/envoy/envoy.yaml
- ./.generated-nginx.dev.conf:/etc/nginx/nginx.conf
ports:
- "${MAIN_PORT}:10000"
depends_on:
- minio-service
- couchdb-service
extra_hosts:
- "host.docker.internal:host-gateway"
couchdb-service:
# platform: linux/amd64
container_name: budi-couchdb-dev
restart: always
image: ibmcom/couchdb3

View File

@ -80,9 +80,8 @@ services:
proxy-service:
restart: always
image: envoyproxy/envoy:v1.16-latest
volumes:
- ./envoy.yaml:/etc/envoy/envoy.yaml
container_name: bbproxy
image: budibase/proxy
ports:
- "${MAIN_PORT}:10000"
depends_on:
@ -125,7 +124,7 @@ services:
- "${WATCHTOWER_PORT}:8080"
volumes:
- /var/run/docker.sock:/var/run/docker.sock
command: --debug --http-api-update bbapps bbworker
command: --debug --http-api-update bbapps bbworker bbproxy
environment:
- WATCHTOWER_HTTP_API=true
- WATCHTOWER_HTTP_API_TOKEN=budibase

View File

@ -1,149 +0,0 @@
static_resources:
listeners:
- name: main_listener
address:
socket_address: { address: 0.0.0.0, port_value: 10000 }
filter_chains:
- filters:
- name: envoy.filters.network.http_connection_manager
typed_config:
"@type": type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager
stat_prefix: ingress
codec_type: auto
route_config:
name: local_route
virtual_hosts:
- name: local_services
domains: ["*"]
routes:
# special case to redirect specifically the route path
# to the builder, if this were a prefix then it would break minio
- match: { path: "/" }
redirect: { path_redirect: "/builder/" }
- match: { prefix: "/db/" }
route:
cluster: couchdb-service
prefix_rewrite: "/"
- match: { prefix: "/api/system/" }
route:
cluster: worker-dev
- match: { prefix: "/api/admin/" }
route:
cluster: worker-dev
- match: { prefix: "/api/global/" }
route:
cluster: worker-dev
- match: { prefix: "/api/" }
route:
cluster: server-dev
timeout: 120s
- match: { prefix: "/app_" }
route:
cluster: server-dev
- match: { prefix: "/app/" }
route:
cluster: server-dev
prefix_rewrite: "/"
# the below three cases are needed to make sure
# all traffic prefixed for the builder is passed through
# correctly.
- match: { path: "/" }
route:
cluster: builder-dev
- match: { prefix: "/builder/" }
route:
cluster: builder-dev
- match: { prefix: "/builder" }
route:
cluster: builder-dev
prefix_rewrite: "/builder/"
# minio is on the default route because this works
# best, minio + AWS SDK doesn't handle path proxy
- match: { prefix: "/" }
route:
cluster: minio-service
http_filters:
- name: envoy.filters.http.router
clusters:
- name: minio-service
connect_timeout: 0.25s
type: strict_dns
lb_policy: round_robin
load_assignment:
cluster_name: minio-service
endpoints:
- lb_endpoints:
- endpoint:
address:
socket_address:
address: minio-service
port_value: 9000
- name: couchdb-service
connect_timeout: 0.25s
type: strict_dns
lb_policy: round_robin
load_assignment:
cluster_name: couchdb-service
endpoints:
- lb_endpoints:
- endpoint:
address:
socket_address:
address: couchdb-service
port_value: 5984
- name: server-dev
connect_timeout: 0.25s
type: strict_dns
lb_policy: round_robin
load_assignment:
cluster_name: server-dev
endpoints:
- lb_endpoints:
- endpoint:
address:
socket_address:
address: {{ address }}
port_value: 4001
- name: builder-dev
connect_timeout: 15s
type: strict_dns
lb_policy: round_robin
load_assignment:
cluster_name: builder-dev
endpoints:
- lb_endpoints:
- endpoint:
address:
socket_address:
address: {{ address }}
port_value: 3000
- name: worker-dev
connect_timeout: 0.25s
type: strict_dns
lb_policy: round_robin
load_assignment:
cluster_name: worker-dev
endpoints:
- lb_endpoints:
- endpoint:
address:
socket_address:
address: {{ address }}
port_value: 4002

View File

@ -1,152 +0,0 @@
static_resources:
listeners:
- name: main_listener
address:
socket_address: { address: 0.0.0.0, port_value: 10000 }
filter_chains:
- filters:
- name: envoy.filters.network.http_connection_manager
typed_config:
"@type": type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager
stat_prefix: ingress
codec_type: auto
route_config:
name: local_route
virtual_hosts:
- name: local_services
domains: ["*"]
routes:
- match: { prefix: "/app/" }
route:
cluster: app-service
prefix_rewrite: "/"
- match: { path: "/v1/update" }
route:
cluster: watchtower-service
- match: { prefix: "/builder/" }
route:
cluster: app-service
- match: { prefix: "/builder" }
route:
cluster: app-service
- match: { prefix: "/app_" }
route:
cluster: app-service
# special cases for worker admin (deprecated), global and system API
- match: { prefix: "/api/global/" }
route:
cluster: worker-service
- match: { prefix: "/api/admin/" }
route:
cluster: worker-service
- match: { prefix: "/api/system/" }
route:
cluster: worker-service
- match: { path: "/" }
route:
cluster: app-service
# special case for when API requests are made, can just forward, not to minio
- match: { prefix: "/api/" }
route:
cluster: app-service
timeout: 120s
- match: { prefix: "/worker/" }
route:
cluster: worker-service
prefix_rewrite: "/"
- match: { prefix: "/db/" }
route:
cluster: couchdb-service
prefix_rewrite: "/"
# minio is on the default route because this works
# best, minio + AWS SDK doesn't handle path proxy
- match: { prefix: "/" }
route:
cluster: minio-service
http_filters:
- name: envoy.filters.http.router
clusters:
- name: app-service
connect_timeout: 0.25s
type: strict_dns
lb_policy: round_robin
load_assignment:
cluster_name: app-service
endpoints:
- lb_endpoints:
- endpoint:
address:
socket_address:
address: app-service
port_value: 4002
- name: minio-service
connect_timeout: 0.25s
type: strict_dns
lb_policy: round_robin
load_assignment:
cluster_name: minio-service
endpoints:
- lb_endpoints:
- endpoint:
address:
socket_address:
address: minio-service
port_value: 9000
- name: worker-service
connect_timeout: 0.25s
type: strict_dns
lb_policy: round_robin
load_assignment:
cluster_name: worker-service
endpoints:
- lb_endpoints:
- endpoint:
address:
socket_address:
address: worker-service
port_value: 4003
- name: couchdb-service
connect_timeout: 0.25s
type: strict_dns
lb_policy: round_robin
load_assignment:
cluster_name: couchdb-service
endpoints:
- lb_endpoints:
- endpoint:
address:
socket_address:
address: couchdb-service
port_value: 5984
- name: watchtower-service
connect_timeout: 0.25s
type: strict_dns
lb_policy: round_robin
load_assignment:
cluster_name: watchtower-service
endpoints:
- lb_endpoints:
- endpoint:
address:
socket_address:
address: watchtower-service
port_value: 8080

View File

@ -1,21 +0,0 @@
# Use the main port in the builder for your self hosting URL, e.g. localhost:10000
MAIN_PORT=10000
# This section contains all secrets pertaining to the system
# These should be updated
JWT_SECRET=testsecret
MINIO_ACCESS_KEY=budibase
MINIO_SECRET_KEY=budibase
COUCH_DB_PASSWORD=budibase
COUCH_DB_USER=budibase
REDIS_PASSWORD=budibase
INTERNAL_API_KEY=budibase
# This section contains variables that do not need to be altered under normal circumstances
APP_PORT=4002
WORKER_PORT=4003
MINIO_PORT=4004
COUCH_DB_PORT=4005
REDIS_PORT=6379
WATCHTOWER_PORT=6161
BUDIBASE_ENVIRONMENT=PRODUCTION

View File

@ -1,4 +0,0 @@
FROM envoyproxy/envoy:v1.16-latest
COPY envoy.yaml /etc/envoy/envoy.yaml
RUN chmod go+r /etc/envoy/envoy.yaml

View File

@ -1,146 +0,0 @@
static_resources:
listeners:
- name: main_listener
address:
socket_address: { address: 0.0.0.0, port_value: 10000 }
filter_chains:
- filters:
- name: envoy.filters.network.http_connection_manager
typed_config:
"@type": type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager
stat_prefix: ingress
codec_type: auto
route_config:
name: local_route
virtual_hosts:
- name: local_services
domains: ["*"]
routes:
- match: { prefix: "/app/" }
route:
cluster: app-service
prefix_rewrite: "/"
- match: { prefix: "/builder/" }
route:
cluster: app-service
- match: { prefix: "/builder" }
route:
cluster: app-service
- match: { prefix: "/app_" }
route:
cluster: app-service
# special cases for worker admin (deprecated), global and system API
- match: { prefix: "/api/global/" }
route:
cluster: worker-service
- match: { prefix: "/api/admin/" }
route:
cluster: worker-service
- match: { prefix: "/api/system/" }
route:
cluster: worker-service
- match: { path: "/" }
route:
cluster: app-service
- match:
safe_regex:
google_re2: {}
regex: "/api/.*/export"
route:
timeout: 0s
cluster: app-service
- match: { path: "/api/deploy" }
route:
timeout: 60s
cluster: app-service
# special case for when API requests are made, can just forward, not to minio
- match: { prefix: "/api/" }
route:
cluster: app-service
- match: { prefix: "/worker/" }
route:
cluster: worker-service
prefix_rewrite: "/"
- match: { prefix: "/db/" }
route:
cluster: couchdb-service
prefix_rewrite: "/"
# minio is on the default route because this works
# best, minio + AWS SDK doesn't handle path proxy
- match: { prefix: "/" }
route:
cluster: minio-service
http_filters:
- name: envoy.filters.http.router
clusters:
- name: app-service
connect_timeout: 0.25s
type: strict_dns
lb_policy: round_robin
load_assignment:
cluster_name: app-service
endpoints:
- lb_endpoints:
- endpoint:
address:
socket_address:
address: app-service.budibase.svc.cluster.local
port_value: 4002
- name: minio-service
connect_timeout: 0.25s
type: strict_dns
lb_policy: round_robin
load_assignment:
cluster_name: minio-service
endpoints:
- lb_endpoints:
- endpoint:
address:
socket_address:
address: minio-service.budibase.svc.cluster.local
port_value: 9000
- name: worker-service
connect_timeout: 0.25s
type: strict_dns
lb_policy: round_robin
load_assignment:
cluster_name: worker-service
endpoints:
- lb_endpoints:
- endpoint:
address:
socket_address:
address: worker-service.budibase.svc.cluster.local
port_value: 4001
- name: couchdb-service
connect_timeout: 0.25s
type: strict_dns
lb_policy: round_robin
load_assignment:
cluster_name: couchdb-service
endpoints:
- lb_endpoints:
- endpoint:
address:
socket_address:
address: budibase-prod-svc-couchdb
port_value: 5984

View File

@ -0,0 +1,2 @@
FROM nginx:latest
COPY nginx.conf /etc/nginx/nginx.conf

View File

@ -0,0 +1,127 @@
user nginx;
error_log /var/log/nginx/error.log debug;
pid /var/run/nginx.pid;
worker_processes auto;
worker_rlimit_nofile 33282;
events {
worker_connections 1024;
}
http {
limit_req_zone $binary_remote_addr zone=ratelimit:10m rate=10r/s;
include /etc/nginx/mime.types;
default_type application/octet-stream;
charset utf-8;
sendfile on;
tcp_nopush on;
tcp_nodelay on;
server_tokens off;
types_hash_max_size 2048;
# buffering
client_body_buffer_size 1K;
client_header_buffer_size 1k;
client_max_body_size 1k;
ignore_invalid_headers off;
proxy_buffering off;
log_format main '$remote_addr - $remote_user [$time_local] "$request" '
'$status $body_bytes_sent "$http_referer" '
'"$http_user_agent" "$http_x_forwarded_for"';
map $http_upgrade $connection_upgrade {
default "upgrade";
}
server {
listen 10000 default_server;
server_name _;
# Security Headers
add_header X-Frame-Options SAMEORIGIN always;
add_header X-Content-Type-Options nosniff always;
add_header X-XSS-Protection "1; mode=block" always;
add_header Content-Security-Policy "default-src 'self'; script-src 'self' 'unsafe-inline' 'unsafe-eval' https://cdn.budi.live https://js.intercomcdn.com https://widget.intercom.io; style-src 'self' 'unsafe-inline' https://cdn.jsdelivr.net https://fonts.googleapis.com https://rsms.me; object-src 'none'; base-uri 'self'; connect-src 'self' https://api-iam.intercom.io https://app.posthog.com wss://nexus-websocket-a.intercom.io; font-src 'self' https://cdn.jsdelivr.net https://fonts.gstatic.com https://rsms.me; frame-src 'self'; img-src https: data:; manifest-src 'self'; media-src 'self'; worker-src 'none';" always;
location /app {
proxy_pass http://app-service:4002;
rewrite ^/app/(.*)$ /$1 break;
}
location = / {
proxy_pass http://app-service.budibase.svc.cluster.local:4002;
}
location /builder/ {
proxy_http_version 1.1;
proxy_set_header Connection $connection_upgrade;
proxy_set_header Upgrade $http_upgrade;
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_pass http://app-service.budibase.svc.cluster.local:4002;
}
location ^/(builder|app_) {
proxy_pass http://app-service.budibase.svc.cluster.local:4002;
}
location ~ ^/api/(system|admin|global)/ {
proxy_pass http://worker-service.budibase.svc.cluster.local:4003;
}
location /worker/ {
proxy_pass http://worker-service.budibase.svc.cluster.local:4003;
rewrite ^/worker/(.*)$ /$1 break;
}
location /api/ {
# calls to the API are rate limited with bursting
limit_req zone=ratelimit burst=10 nodelay;
# 120s timeout on API requests
proxy_read_timeout 120s;
proxy_connect_timeout 120s;
proxy_send_timeout 120s;
proxy_http_version 1.1;
proxy_set_header Connection $connection_upgrade;
proxy_set_header Upgrade $http_upgrade;
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_pass http://app-service.budibase.svc.cluster.local:4002;
}
location /db/ {
proxy_pass http://budibase-prod-svc-couchdb:5984;
rewrite ^/db/(.*)$ /$1 break;
}
location / {
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto $scheme;
proxy_set_header Host $http_host;
proxy_connect_timeout 300;
proxy_http_version 1.1;
proxy_set_header Connection "";
chunked_transfer_encoding off;
proxy_pass http://minio-service.budibase.svc.cluster.local:9000;
}
client_header_timeout 60;
client_body_timeout 60;
keepalive_timeout 60;
# gzip
gzip on;
gzip_vary on;
gzip_proxied any;
gzip_comp_level 6;
gzip_types text/plain text/css text/xml application/json application/javascript application/rss+xml application/atom+xml image/svg+xml;
}
}

View File

@ -0,0 +1,91 @@
user nginx;
error_log /var/log/nginx/error.log debug;
pid /var/run/nginx.pid;
worker_processes auto;
worker_rlimit_nofile 33282;
events {
worker_connections 1024;
}
http {
include /etc/nginx/mime.types;
default_type application/octet-stream;
log_format main '$remote_addr - $remote_user [$time_local] "$request" '
'$status $body_bytes_sent "$http_referer" '
'"$http_user_agent" "$http_x_forwarded_for"';
map $http_upgrade $connection_upgrade {
default "upgrade";
}
server {
listen 10000 default_server;
server_name _;
client_max_body_size 1000m;
ignore_invalid_headers off;
proxy_buffering off;
location /db/ {
proxy_pass http://couchdb-service:5984;
rewrite ^/db/(.*)$ /$1 break;
}
location ~ ^/api/(system|admin|global)/ {
proxy_pass http://{{ address }}:4002;
}
location /api/ {
proxy_read_timeout 120s;
proxy_connect_timeout 120s;
proxy_send_timeout 120s;
proxy_pass http://{{ address }}:4001;
}
location /app_ {
proxy_pass http://{{ address }}:4001;
}
location /app/ {
proxy_pass http://{{ address }}:4001;
rewrite ^/app/(.*)$ /$1 break;
}
location /builder {
proxy_pass http://{{ address }}:3000;
rewrite ^/builder(.*)$ /builder/$1 break;
}
location /builder/ {
proxy_pass http://{{ address }}:3000;
proxy_http_version 1.1;
proxy_set_header Connection $connection_upgrade;
proxy_set_header Upgrade $http_upgrade;
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
}
location / {
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto $scheme;
proxy_set_header Host $http_host;
proxy_connect_timeout 300;
proxy_http_version 1.1;
proxy_set_header Connection "";
chunked_transfer_encoding off;
proxy_pass http://minio-service:9000;
}
client_header_timeout 60;
client_body_timeout 60;
keepalive_timeout 60;
gzip off;
gzip_comp_level 4;
}
}

2
hosting/proxy/Dockerfile Normal file
View File

@ -0,0 +1,2 @@
FROM nginx:latest
COPY nginx.conf /etc/nginx/nginx.conf

135
hosting/proxy/nginx.conf Normal file
View File

@ -0,0 +1,135 @@
user nginx;
error_log /var/log/nginx/error.log debug;
pid /var/run/nginx.pid;
worker_processes auto;
worker_rlimit_nofile 33282;
events {
worker_connections 1024;
}
http {
limit_req_zone $binary_remote_addr zone=ratelimit:10m rate=10r/s;
include /etc/nginx/mime.types;
default_type application/octet-stream;
charset utf-8;
sendfile on;
tcp_nopush on;
tcp_nodelay on;
server_tokens off;
types_hash_max_size 2048;
# buffering
client_body_buffer_size 1K;
client_header_buffer_size 1k;
client_max_body_size 1k;
ignore_invalid_headers off;
proxy_buffering off;
log_format main '$remote_addr - $remote_user [$time_local] "$request" '
'$status $body_bytes_sent "$http_referer" '
'"$http_user_agent" "$http_x_forwarded_for"';
map $http_upgrade $connection_upgrade {
default "upgrade";
}
server {
listen 10000 default_server;
server_name _;
client_max_body_size 1000m;
ignore_invalid_headers off;
proxy_buffering off;
# Security Headers
add_header X-Frame-Options SAMEORIGIN always;
add_header X-Content-Type-Options nosniff always;
add_header X-XSS-Protection "1; mode=block" always;
add_header Content-Security-Policy "default-src 'self'; script-src 'unsafe-inline' 'unsafe-eval' 'self' https://cdn.budi.live https://js.intercomcdn.com https://widget.intercom.io; style-src 'self' 'unsafe-inline' https://cdn.jsdelivr.net https://fonts.googleapis.com https://rsms.me; object-src 'none'; base-uri 'self'; connect-src 'self' https://api-iam.intercom.io https://app.posthog.com wss://nexus-websocket-a.intercom.io; font-src 'self' https://cdn.jsdelivr.net https://fonts.gstatic.com https://rsms.me; frame-src 'self'; img-src https: data:; manifest-src 'self'; media-src 'self'; worker-src 'none';" always;
location /app {
proxy_pass http://app-service:4002;
rewrite ^/app/(.*)$ /$1 break;
}
location = / {
proxy_pass http://app-service:4002;
}
location = /v1/update {
proxy_pass http://watchtower-service:8080;
}
location /builder/ {
proxy_http_version 1.1;
proxy_set_header Connection $connection_upgrade;
proxy_set_header Upgrade $http_upgrade;
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_pass http://app-service:4002;
}
location ^/(builder|app_) {
proxy_pass http://app-service:4002;
}
location ~ ^/api/(system|admin|global)/ {
proxy_pass http://worker-service:4003;
}
location /worker/ {
proxy_pass http://worker-service:4003;
rewrite ^/worker/(.*)$ /$1 break;
}
location /api/ {
# calls to the API are rate limited with bursting
limit_req zone=ratelimit burst=10 nodelay;
# 120s timeout on API requests
proxy_read_timeout 120s;
proxy_connect_timeout 120s;
proxy_send_timeout 120s;
proxy_http_version 1.1;
proxy_set_header Connection $connection_upgrade;
proxy_set_header Upgrade $http_upgrade;
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_pass http://app-service:4002;
}
location /db/ {
proxy_pass http://couchdb-service:5984;
rewrite ^/db/(.*)$ /$1 break;
}
location / {
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto $scheme;
proxy_set_header Host $http_host;
proxy_connect_timeout 300;
proxy_http_version 1.1;
proxy_set_header Connection "";
chunked_transfer_encoding off;
proxy_pass http://minio-service:9000;
}
client_header_timeout 60;
client_body_timeout 60;
keepalive_timeout 60;
# gzip
gzip on;
gzip_vary on;
gzip_proxied any;
gzip_comp_level 6;
gzip_types text/plain text/css text/xml application/json application/javascript application/rss+xml application/atom+xml image/svg+xml;
}
}

View File

@ -5,7 +5,7 @@ const path = require("path")
const IMAGES = {
worker: "budibase/worker",
apps: "budibase/apps",
proxy: "envoyproxy/envoy:v1.16-latest",
proxy: "budibase/proxy",
minio: "minio/minio",
couch: "ibmcom/couchdb3",
curl: "curlimages/curl",
@ -15,8 +15,7 @@ const IMAGES = {
const FILES = {
COMPOSE: "docker-compose.yaml",
ENVOY: "envoy.yaml",
PROPERTIES: "hosting.properties"
NGINX: "nginx.conf"
}
const OUTPUT_DIR = path.join(__dirname, "../", "bb-airgapped")

View File

@ -9,8 +9,10 @@ fi
echo "Tagging images with tag: $tag"
docker tag proxy-service budibase/proxy:$tag
docker tag app-service budibase/apps:$tag
docker tag worker-service budibase/worker:$tag
docker push --all-tags budibase/apps
docker push --all-tags budibase/worker
docker push --all-tags budibase/proxy

View File

@ -1,5 +1,5 @@
{
"version": "1.0.50-alpha.6",
"version": "1.0.57",
"npmClient": "yarn",
"packages": [
"packages/*"

View File

@ -44,9 +44,10 @@
"lint:fix": "yarn run lint:fix:ts && yarn run lint:fix:prettier && yarn run lint:fix:eslint",
"test:e2e": "lerna run cy:test",
"test:e2e:ci": "lerna run cy:ci",
"build:docker": "lerna run build:docker && cd hosting/scripts/linux/ && ./release-to-docker-hub.sh $BUDIBASE_RELEASE_VERSION && cd -",
"build:docker": "lerna run build:docker && npm run build:docker:proxy && cd hosting/scripts/linux/ && ./release-to-docker-hub.sh $BUDIBASE_RELEASE_VERSION && cd -",
"build:docker:proxy": "docker build hosting/proxy -t proxy-service",
"build:docker:selfhost": "lerna run build:docker && cd hosting/scripts/linux/ && ./release-to-docker-hub.sh latest && cd -",
"build:docker:develop": "node scripts/pinVersions && lerna run build:docker && cd hosting/scripts/linux/ && ./release-to-docker-hub.sh develop && cd -",
"build:docker:develop": "node scripts/pinVersions && lerna run build:docker && npm run build:docker:proxy && cd hosting/scripts/linux/ && ./release-to-docker-hub.sh develop && cd -",
"build:docker:airgap": "node hosting/scripts/airgapped/airgappedDockerBuild",
"build:digitalocean": "cd hosting/digitalocean && ./build.sh && cd -",
"build:docs": "lerna run build:docs",

View File

@ -1,6 +1,6 @@
{
"name": "@budibase/backend-core",
"version": "1.0.50-alpha.6",
"version": "1.0.57",
"description": "Budibase backend core libraries used in server and worker",
"main": "src/index.js",
"author": "Budibase",

View File

@ -11,6 +11,8 @@ module.exports = {
COUCH_DB_URL: process.env.COUCH_DB_URL,
COUCH_DB_USERNAME: process.env.COUCH_DB_USER,
COUCH_DB_PASSWORD: process.env.COUCH_DB_PASSWORD,
GOOGLE_CLIENT_ID: process.env.GOOGLE_CLIENT_ID,
GOOGLE_CLIENT_SECRET: process.env.GOOGLE_CLIENT_SECRET,
SALT_ROUNDS: process.env.SALT_ROUNDS,
REDIS_URL: process.env.REDIS_URL,
REDIS_PASSWORD: process.env.REDIS_PASSWORD,

View File

@ -1,22 +1,17 @@
const { getScopedConfig } = require("../../../db/utils")
const { getGlobalDB } = require("../../../tenancy")
const google = require("../google")
const { Configs, Cookies } = require("../../../constants")
const { Cookies } = require("../../../constants")
const { clearCookie, getCookie } = require("../../../utils")
const { getDB } = require("../../../db")
const environment = require("../../../environment")
async function preAuth(passport, ctx, next) {
const db = getGlobalDB()
// get the relevant config
const config = await getScopedConfig(db, {
type: Configs.GOOGLE,
workspace: ctx.query.workspace,
})
const publicConfig = await getScopedConfig(db, {
type: Configs.SETTINGS,
})
let callbackUrl = `${publicConfig.platformUrl}/api/global/auth/datasource/google/callback`
const strategy = await google.strategyFactory(config, callbackUrl)
const googleConfig = {
clientID: environment.GOOGLE_CLIENT_ID,
clientSecret: environment.GOOGLE_CLIENT_SECRET,
}
let callbackUrl = `${environment.PLATFORM_URL}/api/global/auth/datasource/google/callback`
const strategy = await google.strategyFactory(googleConfig, callbackUrl)
if (!ctx.query.appId || !ctx.query.datasourceId) {
ctx.throw(400, "appId and datasourceId query params not present.")
@ -30,18 +25,13 @@ async function preAuth(passport, ctx, next) {
}
async function postAuth(passport, ctx, next) {
const db = getGlobalDB()
// get the relevant config
const config = {
clientID: environment.GOOGLE_CLIENT_ID,
clientSecret: environment.GOOGLE_CLIENT_SECRET,
}
const config = await getScopedConfig(db, {
type: Configs.GOOGLE,
workspace: ctx.query.workspace,
})
const publicConfig = await getScopedConfig(db, {
type: Configs.SETTINGS,
})
let callbackUrl = `${publicConfig.platformUrl}/api/global/auth/datasource/google/callback`
let callbackUrl = `${environment.PLATFORM_URL}/api/global/auth/datasource/google/callback`
const strategy = await google.strategyFactory(
config,
callbackUrl,

View File

@ -1,7 +1,7 @@
{
"name": "@budibase/bbui",
"description": "A UI solution used in the different Budibase projects.",
"version": "1.0.50-alpha.6",
"version": "1.0.57",
"license": "MPL-2.0",
"svelte": "src/index.js",
"module": "dist/bbui.es.js",

View File

@ -15,7 +15,7 @@ process.env.COUCH_URL = `leveldb://${tmpdir}/.data/`
process.env.SELF_HOSTED = 1
process.env.WORKER_URL = "http://localhost:10002/"
process.env.APPS_URL = `http://localhost:${MAIN_PORT}/`
process.env.MINIO_URL = `http://localhost:${MAIN_PORT}/`
process.env.MINIO_URL = `http://localhost:4004`
process.env.MINIO_ACCESS_KEY = "budibase"
process.env.MINIO_SECRET_KEY = "budibase"
process.env.COUCH_DB_USER = "budibase"

View File

@ -1,6 +1,6 @@
{
"name": "@budibase/builder",
"version": "1.0.50-alpha.6",
"version": "1.0.57",
"license": "GPL-3.0",
"private": true,
"scripts": {
@ -64,10 +64,10 @@
}
},
"dependencies": {
"@budibase/bbui": "^1.0.50-alpha.6",
"@budibase/client": "^1.0.50-alpha.6",
"@budibase/frontend-core": "^1.0.50-alpha.6",
"@budibase/string-templates": "^1.0.50-alpha.6",
"@budibase/bbui": "^1.0.57",
"@budibase/client": "^1.0.57",
"@budibase/colorpicker": "1.1.2",
"@budibase/string-templates": "^1.0.57",
"@sentry/browser": "5.19.1",
"@spectrum-css/page": "^3.0.1",
"@spectrum-css/vars": "^3.0.1",

View File

@ -17,7 +17,7 @@
ds = await preAuthStep()
}
window.open(
`/api/global/auth/${tenantId}/datasource/google?datasourceId=${datasource._id}&appId=${$store.appId}`,
`/api/global/auth/${tenantId}/datasource/google?datasourceId=${ds._id}&appId=${$store.appId}`,
"_blank"
)
}}

View File

@ -1,6 +1,5 @@
node_modules/
docker-compose.yaml
envoy.yaml
hosting.properties
nginx.conf
build/
docker-error.log

View File

@ -1,6 +1,6 @@
{
"name": "@budibase/cli",
"version": "1.0.50-alpha.6",
"version": "1.0.57",
"description": "Budibase CLI, for developers, self hosting and migrations.",
"main": "src/index.js",
"bin": {

View File

@ -19,7 +19,6 @@ const BUDIBASE_SERVICES = ["app-service", "worker-service"]
const ERROR_FILE = "docker-error.log"
const FILE_URLS = [
"https://raw.githubusercontent.com/Budibase/budibase/master/hosting/docker-compose.yaml",
"https://raw.githubusercontent.com/Budibase/budibase/master/hosting/envoy.yaml",
]
const DO_USER_DATA_URL = "http://169.254.169.254/metadata/v1/user-data"
@ -141,11 +140,7 @@ async function stop() {
async function update() {
await checkDockerConfigured()
checkInitComplete()
if (
await confirmation(
"Do you wish to update you docker-compose.yaml and envoy.yaml?"
)
) {
if (await confirmation("Do you wish to update you docker-compose.yaml?")) {
await downloadFiles()
}
await handleError(async () => {

View File

@ -2946,7 +2946,7 @@
"settings": [
{
"type": "number",
"label": "Row Count",
"label": "Scroll Limit",
"key": "rowCount",
"defaultValue": 8
},

View File

@ -1,6 +1,6 @@
{
"name": "@budibase/client",
"version": "1.0.50-alpha.6",
"version": "1.0.57",
"license": "MPL-2.0",
"module": "dist/budibase-client.js",
"main": "dist/budibase-client.js",
@ -19,9 +19,9 @@
"dev:builder": "rollup -cw"
},
"dependencies": {
"@budibase/bbui": "^1.0.50-alpha.6",
"@budibase/frontend-core": "^1.0.50-alpha.6",
"@budibase/string-templates": "^1.0.50-alpha.6",
"@budibase/bbui": "^1.0.57",
"@budibase/standard-components": "^0.9.139",
"@budibase/string-templates": "^1.0.57",
"regexparam": "^1.3.0",
"rollup-plugin-polyfill-node": "^0.8.0",
"shortid": "^2.2.15",

View File

@ -1,12 +1,12 @@
{
"name": "@budibase/frontend-core",
"version": "1.0.50-alpha.6",
"version": "1.0.50-alpha.7",
"description": "Budibase frontend core libraries used in builder and client",
"author": "Budibase",
"license": "MPL-2.0",
"svelte": "src/index.js",
"dependencies": {
"@budibase/bbui": "^1.0.50-alpha.6",
"@budibase/bbui": "^1.0.50-alpha.7",
"lodash": "^4.17.21",
"svelte": "^3.46.2"
}

View File

@ -1,7 +1,7 @@
{
"name": "@budibase/server",
"email": "hi@budibase.com",
"version": "1.0.50-alpha.6",
"version": "1.0.57",
"description": "Budibase Web Server",
"main": "src/index.ts",
"repository": {
@ -70,9 +70,9 @@
"license": "GPL-3.0",
"dependencies": {
"@apidevtools/swagger-parser": "^10.0.3",
"@budibase/backend-core": "^1.0.50-alpha.6",
"@budibase/client": "^1.0.50-alpha.6",
"@budibase/string-templates": "^1.0.50-alpha.6",
"@budibase/backend-core": "^1.0.57",
"@budibase/client": "^1.0.57",
"@budibase/string-templates": "^1.0.57",
"@bull-board/api": "^3.7.0",
"@bull-board/koa": "^3.7.0",
"@elastic/elasticsearch": "7.10.0",

View File

@ -22,21 +22,21 @@ const Commands = {
}
async function init() {
// generate envoy file, always do this incase it has changed
// generate nginx file, always do this incase it has changed
const hostingPath = path.join(process.cwd(), "..", "..", "hosting")
const envoyHbsPath = path.join(hostingPath, "envoy.dev.yaml.hbs")
const envoyOutputPath = path.join(hostingPath, ".generated-envoy.dev.yaml")
const contents = fs.readFileSync(envoyHbsPath, "utf8")
const nginxHbsPath = path.join(hostingPath, "nginx.dev.conf.hbs")
const nginxOutputPath = path.join(hostingPath, ".generated-nginx.dev.conf")
const contents = fs.readFileSync(nginxHbsPath, "utf8")
const config = {
address: isLinux() ? "172.17.0.1" : "host.docker.internal",
}
fs.writeFileSync(envoyOutputPath, processStringSync(contents, config))
fs.writeFileSync(nginxOutputPath, processStringSync(contents, config))
const envFilePath = path.join(process.cwd(), ".env")
if (!fs.existsSync(envFilePath)) {
const envFileJson = {
PORT: 4001,
MINIO_URL: "http://localhost:10000/",
MINIO_URL: "http://localhost:4004",
COUCH_DB_URL: "http://budibase:budibase@localhost:10000/db/",
REDIS_URL: "localhost:6379",
WORKER_URL: "http://localhost:4002",

View File

@ -17,7 +17,7 @@ process.env.JWT_SECRET = "budibase"
process.env.COUCH_URL = `leveldb://${tmpdir}/.data/`
process.env.SELF_HOSTED = "1"
process.env.WORKER_URL = `http://localhost:${WORKER_PORT}/`
process.env.MINIO_URL = `http://localhost:${MAIN_PORT}/`
process.env.MINIO_URL = `http://localhost:4004`
process.env.MINIO_ACCESS_KEY = "budibase"
process.env.MINIO_SECRET_KEY = "budibase"
process.env.COUCH_DB_USER = "budibase"

View File

@ -1,7 +1,6 @@
const { definitions } = require("../../integrations")
exports.fetch = async function (ctx) {
// TODO: fetch these from a github repo etc
ctx.status = 200
ctx.body = definitions
}

View File

@ -169,13 +169,11 @@ module DynamoModule {
IndexName: query.index ? query.index : undefined,
...query.json,
}
if (query.index) {
const response = await this.client.query(params).promise()
if (response.Items) {
return response.Items
}
return response
const response = await this.client.query(params).promise()
if (response.Items) {
return response.Items
}
return response
}
async scan(query: { table: string; json: object; index: null | string }) {

View File

@ -11,6 +11,7 @@ const arangodb = require("./arangodb")
const rest = require("./rest")
const googlesheets = require("./googlesheets")
const { SourceNames } = require("../definitions/datasource")
const environment = require("../environment")
const DEFINITIONS = {
[SourceNames.POSTGRES]: postgres.schema,
@ -24,7 +25,6 @@ const DEFINITIONS = {
[SourceNames.MYSQL]: mysql.schema,
[SourceNames.ARANGODB]: arangodb.schema,
[SourceNames.REST]: rest.schema,
[SourceNames.GOOGLE_SHEETS]: googlesheets.schema,
}
const INTEGRATIONS = {
@ -39,7 +39,6 @@ const INTEGRATIONS = {
[SourceNames.MYSQL]: mysql.integration,
[SourceNames.ARANGODB]: arangodb.integration,
[SourceNames.REST]: rest.integration,
[SourceNames.GOOGLE_SHEETS]: googlesheets.integration,
}
// optionally add oracle integration if the oracle binary can be installed
@ -49,6 +48,11 @@ if (!(process.arch === "arm64" && process.platform === "darwin")) {
INTEGRATIONS[SourceNames.ORACLE] = oracle.integration
}
if (environment.SELF_HOSTED) {
DEFINITIONS[SourceNames.GOOGLE_SHEETS] = googlesheets.schema
INTEGRATIONS[SourceNames.GOOGLE_SHEETS] = googlesheets.integration
}
module.exports = {
definitions: DEFINITIONS,
integrations: INTEGRATIONS,

View File

@ -983,10 +983,10 @@
resolved "https://registry.yarnpkg.com/@bcoe/v8-coverage/-/v8-coverage-0.2.3.tgz#75a2e8b51cb758a7553d6804a5932d7aace75c39"
integrity sha512-0hYQ8SB4Db5zvZB4axdMHGwEaQjkZzFjQiN9LVYvIFB2nSUHW9tYpxWriPrWDASIxiaXax83REcLxuSdnGPZtw==
"@budibase/backend-core@^1.0.50-alpha.1":
version "1.0.50"
resolved "https://registry.yarnpkg.com/@budibase/backend-core/-/backend-core-1.0.50.tgz#f31bb365b938c6f889324be7033673d75ea5e127"
integrity sha512-C71kCWdFSrIIkrEuqgRKu2IKyPftK4Rq0HQLOHdnPfmYOGg8iJkeVAXeTQdowaoBXrfzUt/YZmA5ksGccxm3WQ==
"@budibase/backend-core@^1.0.50-alpha.6":
version "1.0.56"
resolved "https://registry.yarnpkg.com/@budibase/backend-core/-/backend-core-1.0.56.tgz#898ad4df1d923527cb340ae76ec548d1e5234755"
integrity sha512-Cos2TgI6grgSTiLPGlAZb53BEWPrzU3e9UXhhTtwiROraYxkVQob6kOQMK7Cvnm3IC+viXjDRijysvactlRwFA==
dependencies:
"@techpass/passport-openidconnect" "^0.3.0"
aws-sdk "^2.901.0"
@ -1056,10 +1056,10 @@
svelte-flatpickr "^3.2.3"
svelte-portal "^1.0.0"
"@budibase/bbui@^1.0.50":
version "1.0.50"
resolved "https://registry.yarnpkg.com/@budibase/bbui/-/bbui-1.0.50.tgz#f49f0e2255647ca36306ab44ed565d1962f1b27a"
integrity sha512-9k7SBKq5tIO0/w8uWjbFrLIeMhdgCMeEXNGS1gXON4pt3ydSblKuWdRSPwSs8BZNfUMid+UxPkkdtLgZIBKcPA==
"@budibase/bbui@^1.0.56":
version "1.0.56"
resolved "https://registry.yarnpkg.com/@budibase/bbui/-/bbui-1.0.56.tgz#3c8cd78c97a21a34fc4b9a45483c7eca739a1dd8"
integrity sha512-HNcQFSFyvtQBtDvqLhQwUCGVJsvNmRt5GZsgIZr73oKaowiC6cwBQm/uPg3I9GxqrTz6etPy0E5ELm/cZKptxA==
dependencies:
"@adobe/spectrum-css-workflow-icons" "^1.2.1"
"@spectrum-css/actionbutton" "^1.0.1"
@ -1107,14 +1107,14 @@
svelte-flatpickr "^3.2.3"
svelte-portal "^1.0.0"
"@budibase/client@^1.0.50-alpha.1":
version "1.0.50"
resolved "https://registry.yarnpkg.com/@budibase/client/-/client-1.0.50.tgz#513295389572c48269716c816204443aa215274e"
integrity sha512-pKLXPJLNQpKeoNixoQp1WsQWUTzucfNPuZNjCZyyCCavToj9kZArvgxAYRcJGgWAulqLDMAr+bC1L5UNJ25Tyw==
"@budibase/client@^1.0.50-alpha.6":
version "1.0.56"
resolved "https://registry.yarnpkg.com/@budibase/client/-/client-1.0.56.tgz#58376ee2d6f64d1a37e6cc8898006a74f5f810b2"
integrity sha512-a7D+AKzutYk3N5OlKndv4BYtt0vnkDysjNLRtkZnKj3ERwQzI3bERrs+Wp2o5KI3cbZFWYR8x179YYxjBTcnYg==
dependencies:
"@budibase/bbui" "^1.0.50"
"@budibase/bbui" "^1.0.56"
"@budibase/standard-components" "^0.9.139"
"@budibase/string-templates" "^1.0.50"
"@budibase/string-templates" "^1.0.56"
regexparam "^1.3.0"
rollup-plugin-polyfill-node "^0.8.0"
shortid "^2.2.15"
@ -1163,10 +1163,10 @@
svelte-apexcharts "^1.0.2"
svelte-flatpickr "^3.1.0"
"@budibase/string-templates@^1.0.50", "@budibase/string-templates@^1.0.50-alpha.1":
version "1.0.50"
resolved "https://registry.yarnpkg.com/@budibase/string-templates/-/string-templates-1.0.50.tgz#53386f3c09891ef21bd47870d25cf7e12f6fac86"
integrity sha512-jmqmikU3Xt0I0hY1S1QI8x5770Z+Tu6zGxb9iQsk3PUCkKCpHz70Rl3Q7x5nYsDfOlhT4YHaHKgYYQYJ6hRafw==
"@budibase/string-templates@^1.0.50-alpha.6", "@budibase/string-templates@^1.0.56":
version "1.0.56"
resolved "https://registry.yarnpkg.com/@budibase/string-templates/-/string-templates-1.0.56.tgz#13869566e344ec175904e7b535dba31f8ddd723d"
integrity sha512-lynmirU/3v+RPmuLGUG3SQUZ8EhjkaGsGTCtzzJq+59bR72zkpoxk+p2XbtJn5AWZvd4VN7d57w3+wGRMPTzoQ==
dependencies:
"@budibase/handlebars-helpers" "^0.11.7"
dayjs "^1.10.4"

View File

@ -1,6 +1,6 @@
{
"name": "@budibase/string-templates",
"version": "1.0.50-alpha.6",
"version": "1.0.57",
"description": "Handlebars wrapper for Budibase templating.",
"main": "src/index.cjs",
"module": "dist/bundle.mjs",

View File

@ -1,7 +1,7 @@
{
"name": "@budibase/worker",
"email": "hi@budibase.com",
"version": "1.0.50-alpha.6",
"version": "1.0.57",
"description": "Budibase background service",
"main": "src/index.ts",
"repository": {
@ -34,8 +34,8 @@
"author": "Budibase",
"license": "GPL-3.0",
"dependencies": {
"@budibase/backend-core": "^1.0.50-alpha.6",
"@budibase/string-templates": "^1.0.50-alpha.6",
"@budibase/backend-core": "^1.0.57",
"@budibase/string-templates": "^1.0.57",
"@koa/router": "^8.0.0",
"@sentry/node": "^6.0.0",
"@techpass/passport-openidconnect": "^0.3.0",

View File

@ -15,7 +15,7 @@ async function init() {
MINIO_SECRET_KEY: "budibase",
REDIS_URL: "localhost:6379",
REDIS_PASSWORD: "budibase",
MINIO_URL: "http://localhost:10000/",
MINIO_URL: "http://localhost:4004",
COUCH_DB_URL: "http://budibase:budibase@localhost:10000/db/",
COUCH_DB_USERNAME: "budibase",
COUCH_DB_PASSWORD: "budibase",

View File

@ -80,6 +80,11 @@ router
updateTenant,
authController.googleAuth
)
.get(
"/api/global/auth/:tenantId/datasource/:provider/callback",
updateTenant,
authController.datasourceAuth
)
.get(
"/api/global/auth/:tenantId/oidc/configs/:configId",
updateTenant,