Merge branch 'cypress-testing' of https://github.com/Budibase/budibase into cypress-testing

.github/CONTRIBUTING.md

@@ -137,7 +137,7 @@ If you wish to delete all the apps created in development and reset the environm
 
 ### Backend
 
-For the backend we run [Redis](https://redis.io/), [CouchDB](https://couchdb.apache.org/), [MinIO](https://min.io/) and [Envoy](https://www.envoyproxy.io/) in Docker compose. This means that to develop Budibase you will need Docker and Docker compose installed. The backend services are then ran separately as Node services with nodemon so that they can be debugged outside of Docker.
+For the backend we run [Redis](https://redis.io/), [CouchDB](https://couchdb.apache.org/), [MinIO](https://min.io/) and [NGINX](https://www.nginx.com/) in Docker compose. This means that to develop Budibase you will need Docker and Docker compose installed. The backend services are then ran separately as Node services with nodemon so that they can be debugged outside of Docker.
 
 ### Data Storage
 

.github/workflows/deploy-preprod.yml

@@ -38,7 +38,7 @@ jobs:
           wc -l values.preprod.yaml
 
       - name: Deploy to Preprod Environment
-        uses: deliverybot/helm@v1
+        uses: glopezep/helm@v1.7.1
         with:
           release: budibase-preprod
           namespace: budibase

.github/workflows/release-selfhost.yml

@@ -25,14 +25,17 @@ jobs:
           # Pull apps and worker images
           docker pull budibase/apps:$release_tag
           docker pull budibase/worker:$release_tag
+          docker pull budibase/proxy:$release_tag
 
           # Tag apps and worker images
           docker tag budibase/apps:$release_tag budibase/apps:$SELFHOST_TAG
           docker tag budibase/worker:$release_tag budibase/worker:$SELFHOST_TAG
+          docker tag budibase/proxy:$release_tag budibase/proxy:$SELFHOST_TAG
           
           # Push images
           docker push budibase/apps:$SELFHOST_TAG
           docker push budibase/worker:$SELFHOST_TAG
+          docker push budibase/proxy:$SELFHOST_TAG
         env:
           DOCKER_USER: ${{ secrets.DOCKER_USERNAME }}
           DOCKER_PASSWORD: ${{ secrets.DOCKER_API_KEY }}

.gitignore

@@ -64,7 +64,7 @@ typings/
 # dotenv environment variables file
 .env
 !hosting/.env
-hosting/.generated-envoy.dev.yaml
+hosting/.generated-nginx.dev.conf
 
 # parcel-bundler cache (https://parceljs.org/)
 .cache

charts/budibase/Chart.yaml

@@ -11,8 +11,8 @@ sources:
   - https://github.com/Budibase/budibase
   - https://budibase.com
 type: application
-version: 0.2.5
+version: 0.2.6
-appVersion: 1.0.25
+appVersion: 1.0.48
 dependencies:
   - name: couchdb
     version: 3.3.4

charts/budibase/templates/proxy-service-deployment.yaml

@@ -25,7 +25,7 @@ spec:
         app.kubernetes.io/name: budibase-proxy
     spec:
       containers:
-      - image: budibase/proxy
+      - image: budibase/proxy:k8s
         imagePullPolicy: Always
         name: proxy-service
         ports:

charts/budibase/templates/worker-service-deployment.yaml

@@ -111,6 +111,10 @@ spec:
           value: {{ .Values.globals.smtp.from | quote }}
         - name: APPS_URL
           value: http://app-service:{{ .Values.services.apps.port }}
+        - name: GOOGLE_CLIENT_ID
+          value: {{ .Values.globals.google.clientId | quote }}
+        - name: GOOGLE_CLIENT_SECRET
+          value: {{ .Values.globals.google.secret | quote }}
         image: budibase/worker:{{ .Values.globals.appVersion }}
         imagePullPolicy: Always
         name: bbworker

hosting/.env

@@ -1 +0,0 @@
-hosting.properties

hosting/.env

@@ -0,0 +1,21 @@
+# Use the main port in the builder for your self hosting URL, e.g. localhost:10000
+MAIN_PORT=10000
+
+# This section contains all secrets pertaining to the system
+# These should be updated
+JWT_SECRET=testsecret
+MINIO_ACCESS_KEY=budibase
+MINIO_SECRET_KEY=budibase
+COUCH_DB_PASSWORD=budibase
+COUCH_DB_USER=budibase
+REDIS_PASSWORD=budibase
+INTERNAL_API_KEY=budibase
+
+# This section contains variables that do not need to be altered under normal circumstances
+APP_PORT=4002
+WORKER_PORT=4003
+MINIO_PORT=4004
+COUCH_DB_PORT=4005
+REDIS_PORT=6379
+WATCHTOWER_PORT=6161
+BUDIBASE_ENVIRONMENT=PRODUCTION

hosting/digitalocean/files/var/lib/cloud/scripts/per-instance/001_onboot

@@ -3,9 +3,8 @@
 # go into the app dir
 cd /root
 
-# fetch envoy and docker-compose files
+# fetch nginx and docker-compose files
 wget https://raw.githubusercontent.com/Budibase/budibase/master/hosting/docker-compose.yaml
-wget https://raw.githubusercontent.com/Budibase/budibase/master/hosting/envoy.yaml
 wget https://raw.githubusercontent.com/Budibase/budibase/master/hosting/hosting.properties
 
 # Create .env file from hosting.properties using bash and then remove it

hosting/docker-compose.dev.yaml

@@ -22,18 +22,21 @@ services:
       retries: 3
 
   proxy-service:
-    container_name: budi-envoy-dev
+    container_name: budi-nginx-dev
     restart: always
-    image: envoyproxy/envoy:v1.16-latest
+    image: nginx:latest
     volumes:
-      - ./.generated-envoy.dev.yaml:/etc/envoy/envoy.yaml
+      - ./.generated-nginx.dev.conf:/etc/nginx/nginx.conf
     ports:
       - "${MAIN_PORT}:10000"
     depends_on:
       - minio-service
       - couchdb-service
+    extra_hosts:
+      - "host.docker.internal:host-gateway"
 
   couchdb-service:
+    # platform: linux/amd64
     container_name: budi-couchdb-dev
     restart: always
     image: ibmcom/couchdb3

hosting/docker-compose.yaml

@@ -80,9 +80,8 @@ services:
 
   proxy-service:
     restart: always
-    image: envoyproxy/envoy:v1.16-latest
+    container_name: bbproxy
-    volumes:
+    image: budibase/proxy
-      - ./envoy.yaml:/etc/envoy/envoy.yaml
     ports:
       - "${MAIN_PORT}:10000"
     depends_on:
@@ -125,7 +124,7 @@ services:
       - "${WATCHTOWER_PORT}:8080"
     volumes:
       - /var/run/docker.sock:/var/run/docker.sock
-    command: --debug --http-api-update bbapps bbworker
+    command: --debug --http-api-update bbapps bbworker bbproxy
     environment:
       - WATCHTOWER_HTTP_API=true
       - WATCHTOWER_HTTP_API_TOKEN=budibase

hosting/envoy.dev.yaml.hbs

@@ -1,149 +0,0 @@
-static_resources:
-  listeners:
-  - name: main_listener
-    address:
-      socket_address: { address: 0.0.0.0, port_value: 10000 }
-    filter_chains:
-    - filters:
-      - name: envoy.filters.network.http_connection_manager
-        typed_config:
-          "@type": type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager
-          stat_prefix: ingress
-          codec_type: auto
-          route_config:
-            name: local_route
-            virtual_hosts:
-            - name: local_services
-              domains: ["*"]
-              routes:
-              # special case to redirect specifically the route path
-              # to the builder, if this were a prefix then it would break minio
-              - match: { path: "/" }
-                redirect: { path_redirect: "/builder/" }
-
-              - match: { prefix: "/db/" }
-                route:
-                  cluster: couchdb-service
-                  prefix_rewrite: "/"
-
-              - match: { prefix: "/api/system/" }
-                route:
-                  cluster: worker-dev
-
-              - match: { prefix: "/api/admin/" }
-                route:
-                  cluster: worker-dev
-
-              - match: { prefix: "/api/global/" }
-                route:
-                  cluster: worker-dev
-
-              - match: { prefix: "/api/" }
-                route:
-                  cluster: server-dev
-                  timeout: 120s
-
-              - match: { prefix: "/app_" }
-                route:
-                  cluster: server-dev
-
-              - match: { prefix: "/app/" }
-                route:
-                  cluster: server-dev
-                  prefix_rewrite: "/"
-
-              # the below three cases are needed to make sure
-              # all traffic prefixed for the builder is passed through
-              # correctly.
-              - match: { path: "/" }
-                route:
-                  cluster: builder-dev
-
-              - match: { prefix: "/builder/" }
-                route:
-                  cluster: builder-dev
-
-              - match: { prefix: "/builder" }
-                route:
-                  cluster: builder-dev
-                  prefix_rewrite: "/builder/"
-
-              # minio is on the default route because this works
-              # best, minio + AWS SDK doesn't handle path proxy
-              - match: { prefix: "/" }
-                route:
-                  cluster: minio-service
-
-          http_filters:
-          - name: envoy.filters.http.router
-
-  clusters:
-  - name: minio-service
-    connect_timeout: 0.25s
-    type: strict_dns
-    lb_policy: round_robin
-    load_assignment:
-      cluster_name: minio-service
-      endpoints:
-      - lb_endpoints:
-        - endpoint:
-            address:
-              socket_address:
-                address: minio-service
-                port_value: 9000
-
-  - name: couchdb-service
-    connect_timeout: 0.25s
-    type: strict_dns
-    lb_policy: round_robin
-    load_assignment:
-      cluster_name: couchdb-service
-      endpoints:
-      - lb_endpoints:
-        - endpoint:
-            address:
-              socket_address:
-                address: couchdb-service
-                port_value: 5984
-
-  - name: server-dev
-    connect_timeout: 0.25s
-    type: strict_dns
-    lb_policy: round_robin
-    load_assignment:
-      cluster_name: server-dev
-      endpoints:
-      - lb_endpoints:
-        - endpoint:
-            address:
-              socket_address:
-                address: {{ address }}
-                port_value: 4001
-
-  - name: builder-dev
-    connect_timeout: 15s
-    type: strict_dns
-    lb_policy: round_robin
-    load_assignment:
-      cluster_name: builder-dev
-      endpoints:
-      - lb_endpoints:
-        - endpoint:
-            address:
-              socket_address:
-                address: {{ address }}
-                port_value: 3000
-
-  - name: worker-dev
-    connect_timeout: 0.25s
-    type: strict_dns
-    lb_policy: round_robin
-    load_assignment:
-      cluster_name: worker-dev
-      endpoints:
-      - lb_endpoints:
-        - endpoint:
-            address:
-              socket_address:
-                address: {{ address }}
-                port_value: 4002

hosting/hosting.properties

@@ -1,21 +0,0 @@
-# Use the main port in the builder for your self hosting URL, e.g. localhost:10000
-MAIN_PORT=10000
-
-# This section contains all secrets pertaining to the system
-# These should be updated
-JWT_SECRET=testsecret
-MINIO_ACCESS_KEY=budibase
-MINIO_SECRET_KEY=budibase
-COUCH_DB_PASSWORD=budibase
-COUCH_DB_USER=budibase
-REDIS_PASSWORD=budibase
-INTERNAL_API_KEY=budibase
-
-# This section contains variables that do not need to be altered under normal circumstances
-APP_PORT=4002
-WORKER_PORT=4003
-MINIO_PORT=4004
-COUCH_DB_PORT=4005
-REDIS_PORT=6379
-WATCHTOWER_PORT=6161
-BUDIBASE_ENVIRONMENT=PRODUCTION

hosting/kubernetes/envoy/Dockerfile

@@ -1,4 +0,0 @@
-FROM envoyproxy/envoy:v1.16-latest
-COPY envoy.yaml /etc/envoy/envoy.yaml
-RUN chmod go+r /etc/envoy/envoy.yaml
-

hosting/kubernetes/envoy/envoy.yaml

@@ -1,146 +0,0 @@
-static_resources:
-  listeners:
-  - name: main_listener
-    address:
-      socket_address: { address: 0.0.0.0, port_value: 10000 }
-    filter_chains:
-    - filters:
-      - name: envoy.filters.network.http_connection_manager
-        typed_config:
-          "@type": type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager
-          stat_prefix: ingress
-          codec_type: auto
-          route_config:
-            name: local_route
-            virtual_hosts:
-            - name: local_services
-              domains: ["*"]
-              routes:
-              - match: { prefix: "/app/" }
-                route:
-                  cluster: app-service
-                  prefix_rewrite: "/"
-
-              - match: { prefix: "/builder/" }
-                route:
-                  cluster: app-service
-
-              - match: { prefix: "/builder" }
-                route:
-                  cluster: app-service
-
-              - match: { prefix: "/app_" }
-                route:
-                  cluster: app-service
-
-              # special cases for worker admin (deprecated), global and system API
-              - match: { prefix: "/api/global/" }
-                route:
-                  cluster: worker-service
-
-              - match: { prefix: "/api/admin/" }
-                route:
-                  cluster: worker-service
-
-              - match: { prefix: "/api/system/" }
-                route:
-                  cluster: worker-service
-
-              - match: { path: "/" }
-                route:
-                  cluster: app-service
-
-              - match: 
-                  safe_regex: 
-                    google_re2: {}
-                    regex: "/api/.*/export"
-                route:
-                  timeout: 0s
-                  cluster: app-service
-
-              - match: { path: "/api/deploy" }
-                route:
-                  timeout: 60s
-                  cluster: app-service
-
-              # special case for when API requests are made, can just forward, not to minio
-              - match: { prefix: "/api/" }
-                route:
-                  cluster: app-service
-
-              - match: { prefix: "/worker/" }
-                route:
-                  cluster: worker-service
-                  prefix_rewrite: "/"
-
-              - match: { prefix: "/db/" }
-                route:
-                  cluster: couchdb-service
-                  prefix_rewrite: "/"
-
-              # minio is on the default route because this works
-              # best, minio + AWS SDK doesn't handle path proxy
-              - match: { prefix: "/" }
-                route:
-                  cluster: minio-service
-
-          http_filters:
-          - name: envoy.filters.http.router
-
-  clusters:
-  - name: app-service
-    connect_timeout: 0.25s
-    type: strict_dns
-    lb_policy: round_robin
-    load_assignment:
-      cluster_name: app-service
-      endpoints:
-      - lb_endpoints:
-        - endpoint:
-            address:
-              socket_address:
-                address: app-service.budibase.svc.cluster.local
-                port_value: 4002
-
-  - name: minio-service
-    connect_timeout: 0.25s
-    type: strict_dns
-    lb_policy: round_robin
-    load_assignment:
-      cluster_name: minio-service
-      endpoints:
-      - lb_endpoints:
-        - endpoint:
-            address:
-              socket_address:
-                address: minio-service.budibase.svc.cluster.local
-                port_value: 9000
-
-  - name: worker-service
-    connect_timeout: 0.25s
-    type: strict_dns
-    lb_policy: round_robin
-    load_assignment:
-      cluster_name: worker-service
-      endpoints:
-      - lb_endpoints:
-        - endpoint:
-            address:
-              socket_address:
-                address: worker-service.budibase.svc.cluster.local
-                port_value: 4001
-
-  - name: couchdb-service
-    connect_timeout: 0.25s
-    type: strict_dns
-    lb_policy: round_robin
-    load_assignment:
-      cluster_name: couchdb-service
-      endpoints:
-      - lb_endpoints:
-        - endpoint:
-            address:
-              socket_address:
-                address: budibase-prod-svc-couchdb 
-                port_value: 5984
-

hosting/kubernetes/nginx/Dockerfile

@@ -0,0 +1,2 @@
+FROM nginx:latest
+COPY nginx.conf /etc/nginx/nginx.conf

hosting/kubernetes/nginx/nginx.conf

@@ -0,0 +1,141 @@
+user                    nginx;
+error_log               /var/log/nginx/error.log debug;
+pid                     /var/run/nginx.pid;
+worker_processes        auto;
+worker_rlimit_nofile    33282;
+
+events {
+  worker_connections  1024;
+}
+
+http {
+  limit_req_zone $binary_remote_addr zone=ratelimit:10m rate=10r/s;
+  include       /etc/nginx/mime.types;
+  default_type  application/octet-stream;
+  charset utf-8;
+  sendfile on;
+  tcp_nopush on;
+  tcp_nodelay on;
+  server_tokens off;
+  types_hash_max_size 2048;
+
+  # buffering
+  client_body_buffer_size 1K;
+  client_header_buffer_size 1k;
+  client_max_body_size 10M;
+  ignore_invalid_headers off;
+  proxy_buffering off;
+
+  log_format  main  '$remote_addr - $remote_user [$time_local] "$request" '
+                    '$status $body_bytes_sent "$http_referer" '
+                    '"$http_user_agent" "$http_x_forwarded_for"';
+  
+  map $http_upgrade $connection_upgrade {
+    default     "upgrade";
+  }
+
+  server { 
+    listen       10000 default_server;
+    listen  [::]:10000 default_server;
+    server_name  _;
+    port_in_redirect off;
+
+    # Security Headers
+    add_header X-Frame-Options SAMEORIGIN always;
+    add_header X-Content-Type-Options nosniff always;
+    add_header X-XSS-Protection "1; mode=block" always;
+    add_header Content-Security-Policy  "default-src 'self'; script-src 'self' 'unsafe-inline' 'unsafe-eval' https://cdn.budi.live https://js.intercomcdn.com https://widget.intercom.io; style-src 'self' 'unsafe-inline' https://cdn.jsdelivr.net https://fonts.googleapis.com https://rsms.me; object-src 'none'; base-uri 'self'; connect-src 'self' https://api-iam.intercom.io https://app.posthog.com wss://nexus-websocket-a.intercom.io; font-src 'self' data: https://cdn.jsdelivr.net https://fonts.gstatic.com https://rsms.me; frame-src 'self'; img-src http: https: data:; manifest-src 'self'; media-src 'self'; worker-src 'none';" always;
+
+    location /app {
+      proxy_pass      http://app-service.budibase.svc.cluster.local:4002;
+      rewrite ^/app/(.*)$ /$1 break;
+    }
+
+    location = / {
+      proxy_http_version  1.1;
+      proxy_set_header    Connection          $connection_upgrade;
+      proxy_set_header    Upgrade             $http_upgrade;
+      proxy_set_header    Host                $host;
+      proxy_set_header    X-Real-IP           $remote_addr;
+      proxy_set_header    X-Forwarded-For     $proxy_add_x_forwarded_for;
+      proxy_pass      http://app-service.budibase.svc.cluster.local:4002;
+    }
+
+    location /builder/ {
+      proxy_http_version  1.1;
+      proxy_set_header    Connection          $connection_upgrade;
+      proxy_set_header    Upgrade             $http_upgrade;
+      proxy_set_header    Host                $host;
+      proxy_set_header    X-Real-IP           $remote_addr;
+      proxy_set_header    X-Forwarded-For     $proxy_add_x_forwarded_for;
+      proxy_pass      http://app-service.budibase.svc.cluster.local:4002;
+    }
+
+    location ~ ^/(builder|app_) {
+      proxy_http_version  1.1;
+      proxy_set_header    Connection          $connection_upgrade;
+      proxy_set_header    Upgrade             $http_upgrade;
+      proxy_set_header    Host                $host;
+      proxy_set_header    X-Real-IP           $remote_addr;
+      proxy_set_header    X-Forwarded-For     $proxy_add_x_forwarded_for;
+      proxy_pass      http://app-service.budibase.svc.cluster.local:4002;
+    }
+
+    location ~ ^/api/(system|admin|global)/ {
+      proxy_pass      http://worker-service.budibase.svc.cluster.local:4001;
+    }
+
+    location /worker/ {
+      proxy_pass      http://worker-service.budibase.svc.cluster.local:4001;
+      rewrite ^/worker/(.*)$ /$1 break;
+    }
+
+    location /api/ {
+      # calls to the API are rate limited with bursting
+      limit_req zone=ratelimit burst=10 nodelay;
+
+      # 120s timeout on API requests
+      proxy_read_timeout 120s;
+      proxy_connect_timeout 120s;
+      proxy_send_timeout 120s; 
+
+      proxy_http_version  1.1;
+      proxy_set_header    Connection          $connection_upgrade;
+      proxy_set_header    Upgrade             $http_upgrade;
+      proxy_set_header    Host                $host;
+      proxy_set_header    X-Real-IP           $remote_addr;
+      proxy_set_header    X-Forwarded-For     $proxy_add_x_forwarded_for;
+
+      proxy_pass      http://app-service.budibase.svc.cluster.local:4002;
+    }
+
+    location /db/ {
+      proxy_pass      http://budibase-prod-svc-couchdb:5984;
+      rewrite ^/db/(.*)$ /$1 break;
+    }
+
+    location / {
+      proxy_set_header X-Real-IP $remote_addr;
+      proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+      proxy_set_header X-Forwarded-Proto $scheme;
+      proxy_set_header Host $http_host;
+      proxy_set_header Connection "";
+      proxy_http_version 1.1;
+      chunked_transfer_encoding off;
+
+      proxy_connect_timeout 300;
+      proxy_pass      http://minio-service.budibase.svc.cluster.local:9000;
+    }
+
+    client_header_timeout 60;
+    client_body_timeout   60;
+    keepalive_timeout     60;
+
+    # gzip
+    gzip on;
+    gzip_vary on;
+    gzip_proxied any;
+    gzip_comp_level 6;
+    gzip_types text/plain text/css text/xml application/json application/javascript application/rss+xml application/atom+xml image/svg+xml;
+  }
+}

hosting/nginx.dev.conf.hbs

@@ -0,0 +1,91 @@
+user                    nginx;
+error_log               /var/log/nginx/error.log debug;
+pid                     /var/run/nginx.pid;
+worker_processes        auto;
+worker_rlimit_nofile    33282;
+
+events {
+  worker_connections  1024;
+}
+
+http {
+  include       /etc/nginx/mime.types;
+  default_type  application/octet-stream;
+
+  log_format  main  '$remote_addr - $remote_user [$time_local] "$request" '
+                    '$status $body_bytes_sent "$http_referer" '
+                    '"$http_user_agent" "$http_x_forwarded_for"';
+  
+  map $http_upgrade $connection_upgrade {
+    default     "upgrade";
+  }
+
+  server { 
+    listen       10000 default_server;
+    server_name  _;
+    client_max_body_size 1000m;
+    ignore_invalid_headers off;
+    proxy_buffering off;
+
+    location /db/ {
+      proxy_pass      http://couchdb-service:5984;
+      rewrite ^/db/(.*)$ /$1 break;
+    }
+
+    location ~ ^/api/(system|admin|global)/ {
+      proxy_pass      http://{{ address }}:4002;
+    }
+
+    location /api/ {
+      proxy_read_timeout 120s;
+      proxy_connect_timeout 120s;
+      proxy_send_timeout 120s; 
+      proxy_pass      http://{{ address }}:4001;
+    }
+
+    location /app_ {
+      proxy_pass      http://{{ address }}:4001;
+    }
+
+    location /app/ {
+      proxy_pass      http://{{ address }}:4001;
+      rewrite ^/app/(.*)$ /$1 break;
+    }
+
+    location /builder {
+      proxy_pass      http://{{ address }}:3000;
+      rewrite ^/builder(.*)$ /builder/$1 break;
+    }
+
+    location /builder/ {
+      proxy_pass      http://{{ address }}:3000;
+
+      proxy_http_version  1.1;
+      proxy_set_header    Connection          $connection_upgrade;
+      proxy_set_header    Upgrade             $http_upgrade;
+      proxy_set_header    Host                $host;
+      proxy_set_header    X-Real-IP           $remote_addr;
+      proxy_set_header    X-Forwarded-For     $proxy_add_x_forwarded_for;
+    }
+
+    location / {
+      proxy_set_header X-Real-IP $remote_addr;
+      proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+      proxy_set_header X-Forwarded-Proto $scheme;
+      proxy_set_header Host $http_host;
+
+      proxy_connect_timeout 300;
+      proxy_http_version 1.1;
+      proxy_set_header Connection "";
+      chunked_transfer_encoding off;
+
+      proxy_pass      http://minio-service:9000;
+    }
+
+    client_header_timeout 60;
+    client_body_timeout   60;
+    keepalive_timeout     60;
+    gzip                  off;
+    gzip_comp_level       4;
+  }
+}

hosting/proxy/Dockerfile

@@ -0,0 +1,2 @@
+FROM nginx:latest
+COPY nginx.conf /etc/nginx/nginx.conf

hosting/proxy/nginx.conf

@@ -0,0 +1,145 @@
+user                    nginx;
+error_log               /var/log/nginx/error.log debug;
+pid                     /var/run/nginx.pid;
+worker_processes        auto;
+worker_rlimit_nofile    33282;
+
+events {
+  worker_connections  1024;
+}
+
+http {
+  limit_req_zone $binary_remote_addr zone=ratelimit:10m rate=10r/s;
+  include       /etc/nginx/mime.types;
+  default_type  application/octet-stream;
+  charset utf-8;
+	sendfile on;
+	tcp_nopush on;
+	tcp_nodelay on;
+	server_tokens off;
+  types_hash_max_size 2048;
+
+  # buffering 
+  client_body_buffer_size 1K;
+  client_header_buffer_size 1k;
+  client_max_body_size 1k;
+  ignore_invalid_headers off;
+
+
+  log_format  main  '$remote_addr - $remote_user [$time_local] "$request" '
+                    '$status $body_bytes_sent "$http_referer" '
+                    '"$http_user_agent" "$http_x_forwarded_for"';
+  
+  map $http_upgrade $connection_upgrade {
+    default     "upgrade";
+  }
+
+  server { 
+    listen       10000 default_server;
+    listen  [::]:10000 default_server;
+    server_name  _;
+    client_max_body_size 1000m;
+    ignore_invalid_headers off;
+    proxy_buffering off;
+    port_in_redirect off;
+
+    # Security Headers
+    add_header X-Frame-Options SAMEORIGIN always;
+    add_header X-Content-Type-Options nosniff always;
+    add_header X-XSS-Protection "1; mode=block" always;
+    add_header Content-Security-Policy  "default-src 'self'; script-src 'self' 'unsafe-inline' 'unsafe-eval' https://cdn.budi.live https://js.intercomcdn.com https://widget.intercom.io; style-src 'self' 'unsafe-inline' https://cdn.jsdelivr.net https://fonts.googleapis.com https://rsms.me; object-src 'none'; base-uri 'self'; connect-src 'self' https://api-iam.intercom.io https://app.posthog.com wss://nexus-websocket-a.intercom.io; font-src 'self' data: https://cdn.jsdelivr.net https://fonts.gstatic.com https://rsms.me; frame-src 'self'; img-src http: https: data:; manifest-src 'self'; media-src 'self'; worker-src 'none';" always;
+
+    location /app {
+      proxy_pass      http://app-service:4002;
+      rewrite ^/app/(.*)$ /$1 break;
+    }
+
+    location = / {
+      port_in_redirect off;
+      proxy_pass      http://app-service:4002;
+    }
+
+    location = /v1/update {
+      proxy_pass      http://watchtower-service:8080;
+    }
+
+    location /builder/ {
+      port_in_redirect off;
+      proxy_http_version  1.1;
+      proxy_set_header    Connection          $connection_upgrade;
+      proxy_set_header    Upgrade             $http_upgrade;
+      proxy_set_header    Host                $host;
+      proxy_set_header    X-Real-IP           $remote_addr;
+      proxy_set_header    X-Forwarded-For     $proxy_add_x_forwarded_for;
+      proxy_pass      http://app-service:4002;
+    }
+
+    location ~ ^/(builder|app_) {
+      port_in_redirect off;
+      proxy_http_version  1.1;
+      proxy_set_header    Connection          $connection_upgrade;
+      proxy_set_header    Upgrade             $http_upgrade;
+      proxy_set_header    Host                $host;
+      proxy_set_header    X-Real-IP           $remote_addr;
+      proxy_set_header    X-Forwarded-For     $proxy_add_x_forwarded_for;
+      proxy_pass      http://app-service:4002;
+    }
+
+    location ~ ^/api/(system|admin|global)/ {
+      proxy_pass      http://worker-service:4003;
+    }
+
+    location /worker/ {
+      proxy_pass      http://worker-service:4003;
+      rewrite ^/worker/(.*)$ /$1 break;
+    }
+
+    location /api/ {
+      # calls to the API are rate limited with bursting
+      limit_req zone=ratelimit burst=10 nodelay;
+
+      # 120s timeout on API requests
+      proxy_read_timeout 120s;
+      proxy_connect_timeout 120s;
+      proxy_send_timeout 120s; 
+
+      proxy_http_version  1.1;
+      proxy_set_header    Connection          $connection_upgrade;
+      proxy_set_header    Upgrade             $http_upgrade;
+      proxy_set_header    Host                $host;
+      proxy_set_header    X-Real-IP           $remote_addr;
+      proxy_set_header    X-Forwarded-For     $proxy_add_x_forwarded_for;
+
+      proxy_pass      http://app-service:4002;
+    }
+
+    location /db/ {
+      proxy_pass      http://couchdb-service:5984;
+      rewrite ^/db/(.*)$ /$1 break;
+    }
+
+    location / {
+      proxy_set_header X-Real-IP $remote_addr;
+      proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+      proxy_set_header X-Forwarded-Proto $scheme;
+      proxy_set_header Host $http_host;
+
+      proxy_connect_timeout 300;
+      proxy_http_version 1.1;
+      proxy_set_header Connection "";
+      chunked_transfer_encoding off;
+      proxy_pass      http://minio-service:9000;
+    }
+
+    client_header_timeout 60;
+    client_body_timeout   60;
+    keepalive_timeout     60;
+
+    # gzip
+    gzip on;
+    gzip_vary on;
+    gzip_proxied any;
+    gzip_comp_level 6;
+    gzip_types text/plain text/css text/xml application/json application/javascript application/rss+xml application/atom+xml image/svg+xml;
+  }
+}

hosting/scripts/airgapped/airgappedDockerBuild.js

@@ -5,7 +5,7 @@ const path = require("path")
 const IMAGES = {
 	worker: "budibase/worker",
 	apps: "budibase/apps",
-	proxy: "envoyproxy/envoy:v1.16-latest",
+	proxy: "budibase/proxy",
 	minio: "minio/minio",
 	couch: "ibmcom/couchdb3",
 	curl: "curlimages/curl",
@@ -15,8 +15,7 @@ const IMAGES = {
 
 const FILES = {
 	COMPOSE: "docker-compose.yaml",
-	ENVOY: "envoy.yaml",
+	NGINX: "nginx.conf"
-	PROPERTIES: "hosting.properties"
 }
 
 const OUTPUT_DIR = path.join(__dirname, "../", "bb-airgapped")

hosting/scripts/linux/release-to-docker-hub.sh

@@ -9,8 +9,10 @@ fi
 
 echo "Tagging images with tag: $tag"
 
+docker tag proxy-service budibase/proxy:$tag
 docker tag app-service budibase/apps:$tag
 docker tag worker-service budibase/worker:$tag
 
 docker push --all-tags budibase/apps 
 docker push --all-tags budibase/worker
+docker push --all-tags budibase/proxy

lerna.json

@@ -1,5 +1,5 @@
 {
-  "version": "1.0.50-alpha.4",
+  "version": "1.0.66-alpha.0",
   "npmClient": "yarn",
   "packages": [
     "packages/*"

package.json

@@ -44,9 +44,10 @@
     "lint:fix": "yarn run lint:fix:ts && yarn run lint:fix:prettier && yarn run lint:fix:eslint",
     "test:e2e": "lerna run cy:test",
     "test:e2e:ci": "lerna run cy:ci",
-    "build:docker": "lerna run build:docker && cd hosting/scripts/linux/ && ./release-to-docker-hub.sh $BUDIBASE_RELEASE_VERSION && cd -",
+    "build:docker": "lerna run build:docker && npm run build:docker:proxy && cd hosting/scripts/linux/ && ./release-to-docker-hub.sh $BUDIBASE_RELEASE_VERSION && cd -",
+    "build:docker:proxy": "docker build hosting/proxy -t proxy-service",
     "build:docker:selfhost": "lerna run build:docker && cd hosting/scripts/linux/ && ./release-to-docker-hub.sh latest && cd -",
-    "build:docker:develop": "node scripts/pinVersions && lerna run build:docker && cd hosting/scripts/linux/ && ./release-to-docker-hub.sh develop && cd -",
+    "build:docker:develop": "node scripts/pinVersions && lerna run build:docker && npm run build:docker:proxy && cd hosting/scripts/linux/ && ./release-to-docker-hub.sh develop && cd -",
     "build:docker:airgap": "node hosting/scripts/airgapped/airgappedDockerBuild",
     "build:digitalocean": "cd hosting/digitalocean && ./build.sh && cd -",
     "build:docs": "lerna run build:docs",

packages/backend-core/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@budibase/backend-core",
-  "version": "1.0.50-alpha.4",
+  "version": "1.0.66-alpha.0",
   "description": "Budibase backend core libraries used in server and worker",
   "main": "src/index.js",
   "author": "Budibase",

packages/backend-core/src/environment.js

@@ -11,6 +11,8 @@ module.exports = {
   COUCH_DB_URL: process.env.COUCH_DB_URL,
   COUCH_DB_USERNAME: process.env.COUCH_DB_USER,
   COUCH_DB_PASSWORD: process.env.COUCH_DB_PASSWORD,
+  GOOGLE_CLIENT_ID: process.env.GOOGLE_CLIENT_ID,
+  GOOGLE_CLIENT_SECRET: process.env.GOOGLE_CLIENT_SECRET,
   SALT_ROUNDS: process.env.SALT_ROUNDS,
   REDIS_URL: process.env.REDIS_URL,
   REDIS_PASSWORD: process.env.REDIS_PASSWORD,

packages/backend-core/src/middleware/passport/datasource/google.js

@@ -1,22 +1,17 @@
-const { getScopedConfig } = require("../../../db/utils")
-const { getGlobalDB } = require("../../../tenancy")
 const google = require("../google")
-const { Configs, Cookies } = require("../../../constants")
+const { Cookies } = require("../../../constants")
 const { clearCookie, getCookie } = require("../../../utils")
 const { getDB } = require("../../../db")
+const environment = require("../../../environment")
 
 async function preAuth(passport, ctx, next) {
-  const db = getGlobalDB()
   // get the relevant config
-  const config = await getScopedConfig(db, {
+  const googleConfig = {
-    type: Configs.GOOGLE,
+    clientID: environment.GOOGLE_CLIENT_ID,
-    workspace: ctx.query.workspace,
+    clientSecret: environment.GOOGLE_CLIENT_SECRET,
-  })
+  }
-  const publicConfig = await getScopedConfig(db, {
+  let callbackUrl = `${environment.PLATFORM_URL}/api/global/auth/datasource/google/callback`
-    type: Configs.SETTINGS,
+  const strategy = await google.strategyFactory(googleConfig, callbackUrl)
-  })
-  let callbackUrl = `${publicConfig.platformUrl}/api/global/auth/datasource/google/callback`
-  const strategy = await google.strategyFactory(config, callbackUrl)
 
   if (!ctx.query.appId || !ctx.query.datasourceId) {
     ctx.throw(400, "appId and datasourceId query params not present.")
@@ -30,18 +25,13 @@ async function preAuth(passport, ctx, next) {
 }
 
 async function postAuth(passport, ctx, next) {
-  const db = getGlobalDB()
+  // get the relevant config
+  const config = {
+    clientID: environment.GOOGLE_CLIENT_ID,
+    clientSecret: environment.GOOGLE_CLIENT_SECRET,
+  }
 
-  const config = await getScopedConfig(db, {
+  let callbackUrl = `${environment.PLATFORM_URL}/api/global/auth/datasource/google/callback`
-    type: Configs.GOOGLE,
-    workspace: ctx.query.workspace,
-  })
-
-  const publicConfig = await getScopedConfig(db, {
-    type: Configs.SETTINGS,
-  })
-
-  let callbackUrl = `${publicConfig.platformUrl}/api/global/auth/datasource/google/callback`
   const strategy = await google.strategyFactory(
     config,
     callbackUrl,

packages/backend-core/src/middleware/passport/local.js

@@ -8,7 +8,7 @@ const { newid } = require("../../hashing")
 const { createASession } = require("../../security/sessions")
 const { getTenantId } = require("../../tenancy")
 
-const INVALID_ERR = "Invalid Credentials"
+const INVALID_ERR = "Invalid credentials"
 const SSO_NO_PASSWORD = "SSO user does not have a password set"
 const EXPIRED = "This account has expired. Please reset your password"
 

packages/bbui/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@budibase/bbui",
   "description": "A UI solution used in the different Budibase projects.",
-  "version": "1.0.50-alpha.4",
+  "version": "1.0.66-alpha.0",
   "license": "MPL-2.0",
   "svelte": "src/index.js",
   "module": "dist/bbui.es.js",

packages/bbui/src/Button/Button.svelte

@@ -1,5 +1,6 @@
 <script>
   import "@spectrum-css/button/dist/index-vars.css"
+  import Tooltip from "../Tooltip/Tooltip.svelte"
 
   export let disabled = false
   export let size = "M"
@@ -11,6 +12,9 @@
   export let quiet = false
   export let icon = undefined
   export let active = false
+  export let tooltip = undefined
+
+  let showTooltip = false
 </script>
 
 <button
@@ -24,6 +28,8 @@
   class="spectrum-Button spectrum-Button--size{size.toUpperCase()}"
   {disabled}
   on:click|preventDefault
+  on:mouseover={() => (showTooltip = true)}
+  on:mouseleave={() => (showTooltip = false)}
 >
   {#if icon}
     <svg
@@ -38,9 +44,29 @@
   {#if $$slots}
     <span class="spectrum-Button-label"><slot /></span>
   {/if}
+  {#if !disabled && tooltip}
+    <div class="tooltip-icon">
+      <svg
+        class="spectrum-Icon spectrum-Icon--size{size.toUpperCase()}"
+        focusable="false"
+        aria-hidden="true"
+        aria-label="Info"
+      >
+        <use xlink:href="#spectrum-icon-18-InfoOutline" />
+      </svg>
+    </div>
+  {/if}
+  {#if showTooltip && tooltip}
+    <div class="tooltip">
+      <Tooltip textWrapping={true} direction={"bottom"} text={tooltip} />
+    </div>
+  {/if}
 </button>
 
 <style>
+  button {
+    position: relative;
+  }
   .spectrum-Button-label {
     white-space: nowrap;
     overflow: hidden;
@@ -49,4 +75,19 @@
   .active {
     color: var(--spectrum-global-color-blue-600) !important;
   }
+  .tooltip {
+    position: absolute;
+    display: flex;
+    justify-content: center;
+    z-index: 100;
+    width: 160px;
+    text-align: center;
+    transform: translateX(-50%);
+    left: 50%;
+    top: calc(100% - 3px);
+  }
+  .tooltip-icon {
+    padding-left: var(--spacing-m);
+    line-height: 0;
+  }
 </style>

packages/bbui/src/ButtonGroup/ButtonGroup.svelte

@@ -1,6 +1,16 @@
 <script>
   import "@spectrum-css/buttongroup/dist/index-vars.css"
   export let vertical = false
+  export let gap = ""
+
+  $: gapStyle =
+    gap === "L"
+      ? "var(--spacing-l)"
+      : gap === "M"
+      ? "var(--spacing-m)"
+      : gap === "S"
+      ? "var(--spacing-s)"
+      : null
 
   function group(element) {
     const buttons = Array.from(element.getElementsByTagName("button"))
@@ -14,6 +24,7 @@
   use:group
   class="spectrum-ButtonGroup"
   class:spectrum-ButtonGroup--vertical={vertical}
+  style={gapStyle ? `gap: ${gapStyle};` : null}
 >
   <slot />
 </div>

packages/bbui/src/ColorPicker/ColorPicker.svelte

@@ -5,7 +5,7 @@
   import { fly } from "svelte/transition"
   import Icon from "../Icon/Icon.svelte"
   import Input from "../Form/Input.svelte"
-  import { capitalise } from "../utils/helpers"
+  import { capitalise } from "../helpers"
 
   export let value
   export let size = "M"

packages/bbui/src/Drawer/Drawer.svelte

@@ -44,6 +44,11 @@
 {/if}
 
 <style>
+  .buttons {
+    display: flex;
+    gap: var(--spacing-m);
+  }
+
   .drawer {
     position: absolute;
     bottom: 0;
@@ -74,4 +79,12 @@
     align-items: flex-start;
     gap: var(--spacing-xs);
   }
+
+  .buttons {
+    display: flex;
+    flex-direction: row;
+    justify-content: flex-start;
+    align-items: center;
+    gap: var(--spacing-m);
+  }
 </style>

packages/bbui/src/Form/Core/DatePicker.svelte

@@ -5,7 +5,7 @@
   import "@spectrum-css/textfield/dist/index-vars.css"
   import "@spectrum-css/picker/dist/index-vars.css"
   import { createEventDispatcher } from "svelte"
-  import { generateID } from "../../utils/helpers"
+  import { uuid } from "../../helpers"
 
   export let id = null
   export let disabled = false
@@ -14,16 +14,20 @@
   export let value = null
   export let placeholder = null
   export let appendTo = undefined
+  export let timeOnly = false
 
   const dispatch = createEventDispatcher()
-  const flatpickrId = `${generateID()}-wrapper`
+  const flatpickrId = `${uuid()}-wrapper`
   let open = false
-  let flatpickr
+  let flatpickr, flatpickrOptions, isTimeOnly
+
+  $: isTimeOnly = !timeOnly && value ? !isNaN(new Date(`0-${value}`)) : timeOnly
   $: flatpickrOptions = {
     element: `#${flatpickrId}`,
-    enableTime: enableTime || false,
+    enableTime: isTimeOnly || enableTime || false,
+    noCalendar: isTimeOnly || false,
     altInput: true,
-    altFormat: enableTime ? "F j Y, H:i" : "F j, Y",
+    altFormat: isTimeOnly ? "H:i" : enableTime ? "F j Y, H:i" : "F j, Y",
     wrap: true,
     appendTo,
     disableMobile: "true",
@@ -35,6 +39,11 @@
     if (newValue) {
       newValue = newValue.toISOString()
     }
+    // if time only set date component to today
+    if (timeOnly) {
+      const todayDate = new Date().toISOString().split("T")[0]
+      newValue = `${todayDate}T${newValue.split("T")[1]}`
+    }
     dispatch("change", newValue)
   }
 
@@ -67,7 +76,11 @@
       return null
     }
     let date
-    if (val instanceof Date) {
+    let time = new Date(`0-${val}`)
+    // it is a string like 00:00:00, just time
+    if (timeOnly || (typeof val === "string" && !isNaN(time))) {
+      date = time
+    } else if (val instanceof Date) {
       // Use real date obj if already parsed
       date = val
     } else if (isNaN(val)) {
@@ -77,7 +90,7 @@
       // Treat as numerical timestamp
       date = new Date(parseInt(val))
     }
-    const time = date.getTime()
+    time = date.getTime()
     if (isNaN(time)) {
       return null
     }
@@ -88,6 +101,7 @@
   }
 </script>
 
+{#key isTimeOnly}
   <Flatpickr
     bind:flatpickr
     value={parseDate(value)}
@@ -151,6 +165,7 @@
       </button>
     </div>
   </Flatpickr>
+{/key}
 {#if open}
   <div class="overlay" on:mousedown|self={flatpickr?.close} />
 {/if}

packages/bbui/src/Form/Core/Dropzone.svelte

@@ -3,7 +3,7 @@
   import "@spectrum-css/typography/dist/index-vars.css"
   import "@spectrum-css/illustratedmessage/dist/index-vars.css"
   import { createEventDispatcher } from "svelte"
-  import { generateID } from "../../utils/helpers"
+  import { uuid } from "../../helpers"
   import Icon from "../../Icon/Icon.svelte"
   import Link from "../../Link/Link.svelte"
   import Tag from "../../Tags/Tag.svelte"
@@ -37,7 +37,7 @@
     "jfif",
   ]
 
-  const fieldId = id || generateID()
+  const fieldId = id || uuid()
   let selectedImageIdx = 0
   let fileDragged = false
   let selectedUrl

packages/bbui/src/Form/DatePicker.svelte

@@ -9,6 +9,7 @@
   export let disabled = false
   export let error = null
   export let enableTime = true
+  export let timeOnly = false
   export let placeholder = null
   export let appendTo = undefined
 
@@ -27,6 +28,7 @@
     {value}
     {placeholder}
     {enableTime}
+    {timeOnly}
     {appendTo}
     on:change={onChange}
   />

packages/bbui/src/Table/DateTimeRenderer.svelte

@@ -2,9 +2,18 @@
   import dayjs from "dayjs"
 
   export let value
+
+  // adding the 0- will turn a string like 00:00:00 into a valid ISO
+  // date, but will make actual ISO dates invalid
+  $: time = new Date(`0-${value}`)
+  $: isTime = !isNaN(time)
 </script>
 
-<div>{dayjs(value).format("MMMM D YYYY, HH:mm")}</div>
+<div>
+  {dayjs(isTime ? time : value).format(
+    isTime ? "HH:mm:ss" : "MMMM D YYYY, HH:mm"
+  )}
+</div>
 
 <style>
   div {

packages/bbui/src/Table/Table.svelte

@@ -3,8 +3,7 @@
   import "@spectrum-css/table/dist/index-vars.css"
   import CellRenderer from "./CellRenderer.svelte"
   import SelectEditRenderer from "./SelectEditRenderer.svelte"
-  import { cloneDeep } from "lodash"
+  import { cloneDeep, deepGet } from "../helpers"
-  import { deepGet } from "../utils/helpers"
 
   /**
    * The expected schema is our normal couch schemas for our tables.

packages/bbui/src/Tooltip/TooltipWrapper.svelte

@@ -17,6 +17,7 @@
         class:icon-small={size === "M" || size === "S"}
         on:mouseover={() => (showTooltip = true)}
         on:mouseleave={() => (showTooltip = false)}
+        on:focus
       >
         <Icon name="InfoOutline" size="S" disabled={true} />
       </div>
@@ -47,7 +48,7 @@
     display: flex;
     justify-content: center;
     top: 15px;
-    z-index: 1;
+    z-index: 100;
     width: 160px;
   }
   .icon {

packages/bbui/src/helpers.js

@@ -1,11 +1,45 @@
-export const generateID = () => {
+/**
-  const rand = Math.random().toString(32).substring(2)
+ * Generates a DOM safe UUID.
-
+ * Starting with a letter is important to make it DOM safe.
-  // Starts with a letter so that its a valid DOM ID
+ * @return {string} a random DOM safe UUID
-  return `A${rand}`
+ */
+export function uuid() {
+  return "cxxxxxxxxxxxx4xxxyxxxxxxxxxxxxxxx".replace(/[xy]/g, c => {
+    const r = (Math.random() * 16) | 0
+    const v = c === "x" ? r : (r & 0x3) | 0x8
+    return v.toString(16)
+  })
 }
 
-export const capitalise = s => s.substring(0, 1).toUpperCase() + s.substring(1)
+/**
+ * Capitalises a string
+ * @param string the string to capitalise
+ * @return {string} the capitalised string
+ */
+export const capitalise = string => {
+  if (!string) {
+    return string
+  }
+  return string.substring(0, 1).toUpperCase() + string.substring(1)
+}
+
+/**
+ * Computes a short hash of a string
+ * @param string the string to compute a hash of
+ * @return {string} the hash string
+ */
+export const hashString = string => {
+  if (!string) {
+    return "0"
+  }
+  let hash = 0
+  for (let i = 0; i < string.length; i++) {
+    let char = string.charCodeAt(i)
+    hash = (hash << 5) - hash + char
+    hash = hash & hash // Convert to 32bit integer
+  }
+  return hash.toString()
+}
 
 /**
  * Gets a key within an object. The key supports dot syntax for retrieving deep
@@ -64,3 +98,11 @@ export const deepSet = (obj, key, value) => {
   }
   obj[split[split.length - 1]] = value
 }
+
+/**
+ * Deeply clones an object. Functions are not supported.
+ * @param obj the object to clone
+ */
+export const cloneDeep = obj => {
+  return JSON.parse(JSON.stringify(obj))
+}

packages/bbui/src/index.js

@@ -85,5 +85,5 @@ export { default as clickOutside } from "./Actions/click_outside"
 // Stores
 export { notifications, createNotificationStore } from "./Stores/notifications"
 
-// Utils
+// Helpers
-export * from "./utils/helpers"
+export * as Helpers from "./helpers"

packages/bbui/yarn.lock

@@ -1601,9 +1601,9 @@ ms@^2.1.1:
   integrity sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==
 
 nanoid@^3.1.22:
-  version "3.1.22"
+  version "3.3.0"
-  resolved "https://registry.yarnpkg.com/nanoid/-/nanoid-3.1.22.tgz#b35f8fb7d151990a8aebd5aa5015c03cf726f844"
+  resolved "https://registry.yarnpkg.com/nanoid/-/nanoid-3.3.0.tgz#5906f776fd886c66c24f3653e0c46fcb1d4ad6b0"
-  integrity sha512-/2ZUaJX2ANuLtTvqTlgqBQNJoQO398KyJgZloL0PZkC0dpysjncRUPsFe3DUPzz/y3h+u7C46np8RMuvF3jsSQ==
+  integrity sha512-JzxqqT5u/x+/KOFSd7JP15DOo9nOoHpx6DYatqIHUW2+flybkm+mdcraotSQR5WcnZr+qhGVh8Ted0KdfSMxlg==
 
 negotiator@0.6.2:
   version "0.6.2"

packages/builder/cypress/setup.js

@@ -15,7 +15,7 @@ process.env.COUCH_URL = `leveldb://${tmpdir}/.data/`
 process.env.SELF_HOSTED = 1
 process.env.WORKER_URL = "http://localhost:10002/"
 process.env.APPS_URL = `http://localhost:${MAIN_PORT}/`
-process.env.MINIO_URL = `http://localhost:${MAIN_PORT}/`
+process.env.MINIO_URL = `http://localhost:4004`
 process.env.MINIO_ACCESS_KEY = "budibase"
 process.env.MINIO_SECRET_KEY = "budibase"
 process.env.COUCH_DB_USER = "budibase"

packages/builder/package.json

@@ -1,13 +1,11 @@
 {
   "name": "@budibase/builder",
-  "version": "1.0.50-alpha.4",
+  "version": "1.0.66-alpha.0",
   "license": "GPL-3.0",
   "private": true,
   "scripts": {
     "build": "routify -b && vite build --emptyOutDir",
     "start": "routify -c rollup",
-    "test": "jest",
-    "test:watch": "jest --watchAll",
     "dev:builder": "routify -c dev:vite",
     "dev:vite": "vite --host 0.0.0.0",
     "rollup": "rollup -c -w",
@@ -66,10 +64,10 @@
     }
   },
   "dependencies": {
-    "@budibase/bbui": "^1.0.50-alpha.4",
+    "@budibase/bbui": "^1.0.66-alpha.0",
-    "@budibase/client": "^1.0.50-alpha.4",
+    "@budibase/client": "^1.0.66-alpha.0",
-    "@budibase/colorpicker": "1.1.2",
+    "@budibase/frontend-core": "^1.0.66-alpha.0",
-    "@budibase/string-templates": "^1.0.50-alpha.4",
+    "@budibase/string-templates": "^1.0.66-alpha.0",
     "@sentry/browser": "5.19.1",
     "@spectrum-css/page": "^3.0.1",
     "@spectrum-css/vars": "^3.0.1",

packages/builder/src/analytics/index.js

@@ -1,4 +1,4 @@
-import api from "builderStore/api"
+import { API } from "api"
 import PosthogClient from "./PosthogClient"
 import IntercomClient from "./IntercomClient"
 import SentryClient from "./SentryClient"
@@ -17,14 +17,12 @@ class AnalyticsHub {
   }
 
   async activate() {
-    const analyticsStatus = await api.get("/api/analytics")
+    // Check analytics are enabled
-    const json = await analyticsStatus.json()
+    const analyticsStatus = await API.getAnalyticsStatus()
-
+    if (analyticsStatus.enabled) {
-    // Analytics disabled
-    if (!json.enabled) return
-
       this.clients.forEach(client => client.init())
     }
+  }
 
   identify(id, metadata) {
     posthog.identify(id)

packages/builder/src/api.js

@@ -0,0 +1,48 @@
+import {
+  createAPIClient,
+  CookieUtils,
+  Constants,
+} from "@budibase/frontend-core"
+import { store } from "./builderStore"
+import { get } from "svelte/store"
+import { auth } from "./stores/portal"
+
+export const API = createAPIClient({
+  attachHeaders: headers => {
+    // Attach app ID header from store
+    headers["x-budibase-app-id"] = get(store).appId
+
+    // Add csrf token if authenticated
+    const user = get(auth).user
+    if (user?.csrfToken) {
+      headers["x-csrf-token"] = user.csrfToken
+    }
+  },
+
+  onError: error => {
+    const { url, message, status, method, handled } = error || {}
+
+    // Log all API errors to Sentry
+    // analytics.captureException(error)
+
+    // Log any errors that we haven't manually handled
+    if (!handled) {
+      console.error("Unhandled error from API client", error)
+      return
+    }
+
+    // Log all errors to console
+    console.warn(`[Builder] HTTP ${status} on ${method}:${url}\n\t${message}`)
+
+    // Logout on 403's
+    if (status === 403) {
+      // Remove cookies
+      CookieUtils.removeCookie(Constants.Cookies.Auth)
+
+      // Reload after removing cookie, go to login
+      if (!url.includes("self") && !url.includes("login")) {
+        location.reload()
+      }
+    }
+  },
+})

packages/builder/src/builderStore/api.js

@@ -1,49 +0,0 @@
-import { store } from "./index"
-import { get as svelteGet } from "svelte/store"
-import { removeCookie, Cookies } from "./cookies"
-import { auth } from "stores/portal"
-
-const apiCall =
-  method =>
-  async (url, body, headers = { "Content-Type": "application/json" }) => {
-    headers["x-budibase-app-id"] = svelteGet(store).appId
-    headers["x-budibase-api-version"] = "1"
-
-    // add csrf token if authenticated
-    const user = svelteGet(auth).user
-    if (user && user.csrfToken) {
-      headers["x-csrf-token"] = user.csrfToken
-    }
-
-    const json = headers["Content-Type"] === "application/json"
-    const resp = await fetch(url, {
-      method: method,
-      body: json ? JSON.stringify(body) : body,
-      headers,
-    })
-    if (resp.status === 403) {
-      if (url.includes("/api/templates")) {
-        return { json: () => [] }
-      }
-      removeCookie(Cookies.Auth)
-      // reload after removing cookie, go to login
-      if (!url.includes("self") && !url.includes("login")) {
-        location.reload()
-      }
-    }
-    return resp
-  }
-
-export const post = apiCall("POST")
-export const get = apiCall("GET")
-export const patch = apiCall("PATCH")
-export const del = apiCall("DELETE")
-export const put = apiCall("PUT")
-
-export default {
-  post: apiCall("POST"),
-  get: apiCall("GET"),
-  patch: apiCall("PATCH"),
-  delete: apiCall("DELETE"),
-  put: apiCall("PUT"),
-}

packages/builder/src/builderStore/dataBinding.js

@@ -15,10 +15,7 @@ import {
   encodeJSBinding,
 } from "@budibase/string-templates"
 import { TableNames } from "../constants"
-import {
+import { JSONUtils } from "@budibase/frontend-core"
-  convertJSONSchemaToTableSchema,
-  getJSONArrayDatasourceSchema,
-} from "./jsonUtils"
 import ActionDefinitions from "components/design/PropertiesPanel/PropertyControls/ButtonActionEditor/manifest.json"
 
 // Regex to match all instances of template strings
@@ -278,10 +275,7 @@ const getProviderContextBindings = (asset, dataProviders) => {
  */
 const getUserBindings = () => {
   let bindings = []
-  const { schema } = getSchemaForDatasource(null, {
+  const { schema } = getSchemaForTable(TableNames.USERS)
-    type: "table",
-    tableId: TableNames.USERS,
-  })
   const keys = Object.keys(schema).sort()
   const safeUser = makePropSafe("user")
   keys.forEach(key => {
@@ -388,9 +382,33 @@ export const getButtonContextBindings = (actions, actionId) => {
 }
 
 /**
- * Gets a schema for a datasource object.
+ * Gets the schema for a certain table ID.
+ * The options which can be passed in are:
+ *   formSchema: whether the schema is for a form
+ *   searchableSchema: whether to generate a searchable schema, which may have
+ *     fewer fields than a readable schema
+ * @param tableId the table ID to get the schema for
+ * @param options options for generating the schema
+ * @return {{schema: Object, table: Object}}
  */
-export const getSchemaForDatasource = (asset, datasource, isForm = false) => {
+export const getSchemaForTable = (tableId, options) => {
+  return getSchemaForDatasource(null, { type: "table", tableId }, options)
+}
+
+/**
+ * Gets a schema for a datasource object.
+ * The options which can be passed in are:
+ *   formSchema: whether the schema is for a form
+ *   searchableSchema: whether to generate a searchable schema, which may have
+ *     fewer fields than a readable schema
+ * @param asset the current root client app asset (layout or screen). This is
+ *   optional and only needed for "provider" datasource types.
+ * @param datasource the datasource definition
+ * @param options options for generating the schema
+ * @return {{schema: Object, table: Object}}
+ */
+export const getSchemaForDatasource = (asset, datasource, options) => {
+  options = options || {}
   let schema, table
 
   if (datasource) {
@@ -402,7 +420,7 @@ export const getSchemaForDatasource = (asset, datasource, isForm = false) => {
     if (type === "provider") {
       const component = findComponent(asset.props, datasource.providerId)
       const source = getDatasourceForProvider(asset, component)
-      return getSchemaForDatasource(asset, source, isForm)
+      return getSchemaForDatasource(asset, source, options)
     }
 
     // "query" datasources are those targeting non-plus datasources or
@@ -439,7 +457,7 @@ export const getSchemaForDatasource = (asset, datasource, isForm = false) => {
     else if (type === "jsonarray") {
       table = tables.find(table => table._id === datasource.tableId)
       let tableSchema = table?.schema
-      schema = getJSONArrayDatasourceSchema(tableSchema, datasource)
+      schema = JSONUtils.getJSONArrayDatasourceSchema(tableSchema, datasource)
     }
 
     // Otherwise we assume we're targeting an internal table or a plus
@@ -451,8 +469,16 @@ export const getSchemaForDatasource = (asset, datasource, isForm = false) => {
     // Determine the schema from the backing entity if not already determined
     if (table && !schema) {
       if (type === "view") {
+        // For views, the schema is pulled from the `views` property of the
+        // table
         schema = cloneDeep(table.views?.[datasource.name]?.schema)
-      } else if (type === "query" && isForm) {
+      } else if (
+        type === "query" &&
+        (options.formSchema || options.searchableSchema)
+      ) {
+        // For queries, if we are generating a schema for a form or a searchable
+        // schema then we want to use the query parameters rather than the
+        // query schema
         schema = {}
         const params = table.parameters || []
         params.forEach(param => {
@@ -461,6 +487,7 @@ export const getSchemaForDatasource = (asset, datasource, isForm = false) => {
           }
         })
       } else {
+        // Otherwise we just want the schema of the table
         schema = cloneDeep(table.schema)
       }
     }
@@ -471,9 +498,12 @@ export const getSchemaForDatasource = (asset, datasource, isForm = false) => {
       Object.keys(schema).forEach(fieldKey => {
         const fieldSchema = schema[fieldKey]
         if (fieldSchema?.type === "json") {
-          const jsonSchema = convertJSONSchemaToTableSchema(fieldSchema, {
+          const jsonSchema = JSONUtils.convertJSONSchemaToTableSchema(
+            fieldSchema,
+            {
               squashObjects: true,
-          })
+            }
+          )
           Object.keys(jsonSchema).forEach(jsonKey => {
             jsonAdditions[`${fieldKey}.${jsonKey}`] = {
               type: jsonSchema[jsonKey].type,
@@ -485,9 +515,31 @@ export const getSchemaForDatasource = (asset, datasource, isForm = false) => {
       schema = { ...schema, ...jsonAdditions }
     }
 
-    // Add _id and _rev fields for certain types
+    // Determine if we should add ID and rev to the schema
-    if (schema && !isForm && ["table", "link"].includes(datasource.type)) {
+    const isInternal = table && !table.sql
+    const isTable = ["table", "link"].includes(datasource.type)
+
+    // ID is part of the readable schema for all tables
+    // Rev is part of the readable schema for internal tables only
+    let addId = isTable
+    let addRev = isTable && isInternal
+
+    // Don't add ID or rev for form schemas
+    if (options.formSchema) {
+      addId = false
+      addRev = false
+    }
+
+    // ID is only searchable for internal tables
+    else if (options.searchableSchema) {
+      addId = isTable && isInternal
+    }
+
+    // Add schema properties if required
+    if (addId) {
       schema["_id"] = { type: "string" }
+    }
+    if (addRev) {
       schema["_rev"] = { type: "string" }
     }
 

packages/builder/src/builderStore/loadComponentLibraries.js

@@ -1,16 +0,0 @@
-import { get } from "builderStore/api"
-
-/**
- * Fetches the definitions for component library components. This includes
- * their props and other metadata from components.json.
- * @param {string} appId - ID of the currently running app
- */
-export const fetchComponentLibDefinitions = async appId => {
-  const LIB_DEFINITION_URL = `/api/${appId}/components/definitions`
-  try {
-    const libDefinitionResponse = await get(LIB_DEFINITION_URL)
-    return await libDefinitionResponse.json()
-  } catch (err) {
-    console.error(`Error fetching component definitions for ${appId}`, err)
-  }
-}

packages/builder/src/builderStore/store/automation/index.js

@@ -1,26 +1,40 @@
 import { writable } from "svelte/store"
-import api from "../../api"
+import { API } from "api"
 import Automation from "./Automation"
 import { cloneDeep } from "lodash/fp"
 import analytics, { Events } from "analytics"
 
+const initialAutomationState = {
+  automations: [],
+  blockDefinitions: {
+    TRIGGER: [],
+    ACTION: [],
+  },
+  selectedAutomation: null,
+}
+
+export const getAutomationStore = () => {
+  const store = writable(initialAutomationState)
+  store.actions = automationActions(store)
+  return store
+}
+
 const automationActions = store => ({
   fetch: async () => {
     const responses = await Promise.all([
-      api.get(`/api/automations`),
+      API.getAutomations(),
-      api.get(`/api/automations/definitions/list`),
+      API.getAutomationDefinitions(),
     ])
-    const jsonResponses = await Promise.all(responses.map(x => x.json()))
     store.update(state => {
       let selected = state.selectedAutomation?.automation
-      state.automations = jsonResponses[0]
+      state.automations = responses[0]
       state.blockDefinitions = {
-        TRIGGER: jsonResponses[1].trigger,
+        TRIGGER: responses[1].trigger,
-        ACTION: jsonResponses[1].action,
+        ACTION: responses[1].action,
       }
-      // if previously selected find the new obj and select it
+      // If previously selected find the new obj and select it
       if (selected) {
-        selected = jsonResponses[0].filter(
+        selected = responses[0].filter(
           automation => automation._id === selected._id
         )
         state.selectedAutomation = new Automation(selected[0])
@@ -36,40 +50,36 @@ const automationActions = store => ({
         steps: [],
       },
     }
-    const CREATE_AUTOMATION_URL = `/api/automations`
+    const response = await API.createAutomation(automation)
-    const response = await api.post(CREATE_AUTOMATION_URL, automation)
-    const json = await response.json()
     store.update(state => {
-      state.automations = [...state.automations, json.automation]
+      state.automations = [...state.automations, response.automation]
-      store.actions.select(json.automation)
+      store.actions.select(response.automation)
       return state
     })
   },
   save: async automation => {
-    const UPDATE_AUTOMATION_URL = `/api/automations`
+    const response = await API.updateAutomation(automation)
-    const response = await api.put(UPDATE_AUTOMATION_URL, automation)
-    const json = await response.json()
     store.update(state => {
-      const newAutomation = json.automation
+      const updatedAutomation = response.automation
       const existingIdx = state.automations.findIndex(
         existing => existing._id === automation._id
       )
       if (existingIdx !== -1) {
-        state.automations.splice(existingIdx, 1, newAutomation)
+        state.automations.splice(existingIdx, 1, updatedAutomation)
         state.automations = [...state.automations]
-        store.actions.select(newAutomation)
+        store.actions.select(updatedAutomation)
         return state
       }
     })
   },
   delete: async automation => {
-    const { _id, _rev } = automation
+    await API.deleteAutomation({
-    const DELETE_AUTOMATION_URL = `/api/automations/${_id}/${_rev}`
+      automationId: automation?._id,
-    await api.delete(DELETE_AUTOMATION_URL)
+      automationRev: automation?._rev,
-
+    })
     store.update(state => {
       const existingIdx = state.automations.findIndex(
-        existing => existing._id === _id
+        existing => existing._id === automation?._id
       )
       state.automations.splice(existingIdx, 1)
       state.automations = [...state.automations]
@@ -78,16 +88,17 @@ const automationActions = store => ({
       return state
     })
   },
-  trigger: async automation => {
-    const { _id } = automation
-    return await api.post(`/api/automations/${_id}/trigger`)
-  },
   test: async (automation, testData) => {
-    const { _id } = automation
-    const response = await api.post(`/api/automations/${_id}/test`, testData)
-    const json = await response.json()
     store.update(state => {
-      state.selectedAutomation.testResults = json
+      state.selectedAutomation.testResults = null
+      return state
+    })
+    const result = await API.testAutomation({
+      automationId: automation?._id,
+      testData,
+    })
+    store.update(state => {
+      state.selectedAutomation.testResults = result
       return state
     })
   },
@@ -143,17 +154,3 @@ const automationActions = store => ({
     })
   },
 })
-
-export const getAutomationStore = () => {
-  const INITIAL_AUTOMATION_STATE = {
-    automations: [],
-    blockDefinitions: {
-      TRIGGER: [],
-      ACTION: [],
-    },
-    selectedAutomation: null,
-  }
-  const store = writable(INITIAL_AUTOMATION_STATE)
-  store.actions = automationActions(store)
-  return store
-}

packages/builder/src/builderStore/store/frontend.js

@@ -14,8 +14,7 @@ import {
   database,
   tables,
 } from "stores/backend"
-import { fetchComponentLibDefinitions } from "../loadComponentLibraries"
+import { API } from "api"
-import api from "../api"
 import { FrontendTypes } from "constants"
 import analytics, { Events } from "analytics"
 import {
@@ -26,7 +25,7 @@ import {
   findComponent,
   getComponentSettings,
 } from "../componentUtils"
-import { uuid } from "../uuid"
+import { Helpers } from "@budibase/bbui"
 import { removeBindings } from "../dataBinding"
 
 const INITIAL_FRONTEND_STATE = {
@@ -70,15 +69,12 @@ export const getFrontendStore = () => {
     },
     initialise: async pkg => {
       const { layouts, screens, application, clientLibPath } = pkg
-      const components = await fetchComponentLibDefinitions(application.appId)
+
-      // make sure app isn't locked
+      // Fetch component definitions.
-      if (
+      // Allow errors to propagate.
-        components &&
+      let components = await API.fetchComponentLibDefinitions(application.appId)
-        components.status === 400 &&
+
-        components.message?.includes("lock")
+      // Reset store state
-      ) {
-        throw { ok: false, reason: "locked" }
-      }
       store.update(state => ({
         ...state,
         libraries: application.componentLibraries,
@@ -91,8 +87,8 @@ export const getFrontendStore = () => {
         description: application.description,
         appId: application.appId,
         url: application.url,
-        layouts,
+        layouts: layouts || [],
-        screens,
+        screens: screens || [],
         theme: application.theme || "spectrum--light",
         customTheme: application.customTheme,
         hasAppPackage: true,
@@ -104,51 +100,43 @@ export const getFrontendStore = () => {
       }))
 
       // Initialise backend stores
-      const [_integrations] = await Promise.all([
-        api.get("/api/integrations").then(r => r.json()),
-      ])
-      datasources.init()
-      integrations.set(_integrations)
-      queries.init()
       database.set(application.instance)
-      tables.init()
+      await datasources.init()
+      await integrations.init()
+      await queries.init()
+      await tables.init()
     },
     theme: {
       save: async theme => {
         const appId = get(store).appId
-        const response = await api.put(`/api/applications/${appId}`, { theme })
+        await API.saveAppMetadata({
-        if (response.status === 200) {
+          appId,
+          metadata: { theme },
+        })
         store.update(state => {
           state.theme = theme
           return state
         })
-        } else {
-          throw new Error("Error updating theme")
-        }
       },
     },
     customTheme: {
       save: async customTheme => {
         const appId = get(store).appId
-        const response = await api.put(`/api/applications/${appId}`, {
+        await API.saveAppMetadata({
-          customTheme,
+          appId,
+          metadata: { customTheme },
         })
-        if (response.status === 200) {
         store.update(state => {
           state.customTheme = customTheme
           return state
         })
-        } else {
-          throw new Error("Error updating theme")
-        }
       },
     },
     routing: {
       fetch: async () => {
-        const response = await api.get("/api/routing")
+        const response = await API.fetchAppRoutes()
-        const json = await response.json()
         store.update(state => {
-          state.routes = json.routes
+          state.routes = response.routes
           return state
         })
       },
@@ -172,82 +160,76 @@ export const getFrontendStore = () => {
           return state
         })
       },
-      create: async screen => {
-        screen = await store.actions.screens.save(screen)
-        store.update(state => {
-          state.selectedScreenId = screen._id
-          state.selectedComponentId = screen.props._id
-          state.currentFrontEndType = FrontendTypes.SCREEN
-          selectedAccessRole.set(screen.routing.roleId)
-          return state
-        })
-        return screen
-      },
       save: async screen => {
         const creatingNewScreen = screen._id === undefined
-        const response = await api.post(`/api/screens`, screen)
+        const savedScreen = await API.saveScreen(screen)
-        if (response.status !== 200) {
-          return
-        }
-        screen = await response.json()
-        await store.actions.routing.fetch()
-
         store.update(state => {
-          const foundScreen = state.screens.findIndex(
+          const idx = state.screens.findIndex(x => x._id === savedScreen._id)
-            el => el._id === screen._id
+          if (idx !== -1) {
-          )
+            state.screens.splice(idx, 1, savedScreen)
-          if (foundScreen !== -1) {
+          } else {
-            state.screens.splice(foundScreen, 1)
+            state.screens.push(savedScreen)
           }
-          state.screens.push(screen)
           return state
         })
 
-        if (creatingNewScreen) {
+        // Refresh routes
-          store.actions.screens.select(screen._id)
+        await store.actions.routing.fetch()
-        }
 
-        return screen
+        // Select the new screen if creating a new one
+        if (creatingNewScreen) {
+          store.actions.screens.select(savedScreen._id)
+        }
+        return savedScreen
       },
       delete: async screens => {
         const screensToDelete = Array.isArray(screens) ? screens : [screens]
 
-        const screenDeletePromises = []
+        // Build array of promises to speed up bulk deletions
-        store.update(state => {
+        const promises = []
-          for (let screenToDelete of screensToDelete) {
+        screensToDelete.forEach(screen => {
-            state.screens = state.screens.filter(
+          // Delete the screen
-              screen => screen._id !== screenToDelete._id
+          promises.push(
+            API.deleteScreen({
+              screenId: screen._id,
+              screenRev: screen._rev,
+            })
           )
-            screenDeletePromises.push(
+          // Remove links to this screen
-              api.delete(
+          promises.push(
-                `/api/screens/${screenToDelete._id}/${screenToDelete._rev}`
-              )
-            )
-            if (screenToDelete._id === state.selectedScreenId) {
-              state.selectedScreenId = null
-            }
-            //remove the link for this screen
-            screenDeletePromises.push(
             store.actions.components.links.delete(
-                screenToDelete.routing.route,
+              screen.routing.route,
-                screenToDelete.props._instanceName
+              screen.props._instanceName
             )
           )
+        })
+
+        await Promise.all(promises)
+        const deletedIds = screensToDelete.map(screen => screen._id)
+        store.update(state => {
+          // Remove deleted screens from state
+          state.screens = state.screens.filter(screen => {
+            return !deletedIds.includes(screen._id)
+          })
+          // Deselect the current screen if it was deleted
+          if (deletedIds.includes(state.selectedScreenId)) {
+            state.selectedScreenId = null
           }
           return state
         })
-        await Promise.all(screenDeletePromises)
+
+        // Refresh routes
+        await store.actions.routing.fetch()
       },
     },
     preview: {
       saveSelected: async () => {
         const state = get(store)
         const selectedAsset = get(currentAsset)
-
         if (state.currentFrontEndType !== FrontendTypes.LAYOUT) {
-          await store.actions.screens.save(selectedAsset)
+          return await store.actions.screens.save(selectedAsset)
         } else {
-          await store.actions.layouts.save(selectedAsset)
+          return await store.actions.layouts.save(selectedAsset)
         }
       },
       setDevice: device => {
@@ -271,25 +253,13 @@ export const getFrontendStore = () => {
         })
       },
       save: async layout => {
-        const layoutToSave = cloneDeep(layout)
+        const creatingNewLayout = layout._id === undefined
-        const creatingNewLayout = layoutToSave._id === undefined
+        const savedLayout = await API.saveLayout(layout)
-        const response = await api.post(`/api/layouts`, layoutToSave)
-        const savedLayout = await response.json()
-
-        // Abort if saving failed
-        if (response.status !== 200) {
-          return
-        }
-
         store.update(state => {
-          const layoutIdx = state.layouts.findIndex(
+          const idx = state.layouts.findIndex(x => x._id === savedLayout._id)
-            stateLayout => stateLayout._id === savedLayout._id
+          if (idx !== -1) {
-          )
+            state.layouts.splice(idx, 1, savedLayout)
-          if (layoutIdx >= 0) {
-            // update existing layout
-            state.layouts.splice(layoutIdx, 1, savedLayout)
           } else {
-            // save new layout
             state.layouts.push(savedLayout)
           }
           return state
@@ -299,7 +269,6 @@ export const getFrontendStore = () => {
         if (creatingNewLayout) {
           store.actions.layouts.select(savedLayout._id)
         }
-
         return savedLayout
       },
       find: layoutId => {
@@ -309,21 +278,20 @@ export const getFrontendStore = () => {
         const storeContents = get(store)
         return storeContents.layouts.find(layout => layout._id === layoutId)
       },
-      delete: async layoutToDelete => {
+      delete: async layout => {
-        const response = await api.delete(
+        if (!layout?._id) {
-          `/api/layouts/${layoutToDelete._id}/${layoutToDelete._rev}`
+          return
-        )
-        if (response.status !== 200) {
-          const json = await response.json()
-          throw new Error(json.message)
         }
+        await API.deleteLayout({
+          layoutId: layout._id,
+          layoutRev: layout._rev,
+        })
         store.update(state => {
-          state.layouts = state.layouts.filter(
+          // Select main layout if we deleted the selected layout
-            layout => layout._id !== layoutToDelete._id
+          if (layout._id === state.selectedLayoutId) {
-          )
-          if (layoutToDelete._id === state.selectedLayoutId) {
             state.selectedLayoutId = get(mainLayout)._id
           }
+          state.layouts = state.layouts.filter(x => x._id !== layout._id)
           return state
         })
       },
@@ -398,7 +366,7 @@ export const getFrontendStore = () => {
         }
 
         return {
-          _id: uuid(),
+          _id: Helpers.uuid(),
           _component: definition.component,
           _styles: { normal: {}, hover: {}, active: {} },
           _instanceName: `New ${definition.name}`,
@@ -415,16 +383,12 @@ export const getFrontendStore = () => {
           componentName,
           presetProps
         )
-        if (!componentInstance) {
+        if (!componentInstance || !asset) {
           return
         }
 
         // Find parent node to attach this component to
         let parentComponent
-
-        if (!asset) {
-          return
-        }
         if (selected) {
           // Use current screen or layout as parent if no component is selected
           const definition = store.actions.components.getDefinition(
@@ -552,7 +516,7 @@ export const getFrontendStore = () => {
               if (!component) {
                 return
               }
-              component._id = uuid()
+              component._id = Helpers.uuid()
               component._children?.forEach(randomizeIds)
             }
             randomizeIds(componentToPaste)
@@ -606,11 +570,6 @@ export const getFrontendStore = () => {
         selected._styles.custom = style
         await store.actions.preview.saveSelected()
       },
-      resetStyles: async () => {
-        const selected = get(selectedComponent)
-        selected._styles = { normal: {}, hover: {}, active: {} }
-        await store.actions.preview.saveSelected()
-      },
       updateConditions: async conditions => {
         const selected = get(selectedComponent)
         selected._conditions = conditions
@@ -665,7 +624,7 @@ export const getFrontendStore = () => {
               newLink = cloneDeep(nav._children[0])
 
               // Set our new props
-              newLink._id = uuid()
+              newLink._id = Helpers.uuid()
               newLink._instanceName = `${title} Link`
               newLink.url = url
               newLink.text = title

packages/builder/src/builderStore/store/notifications.js

@@ -1,23 +0,0 @@
-import { writable } from "svelte/store"
-import { generate } from "shortid"
-
-export const notificationStore = writable({
-  notifications: [],
-})
-
-export function send(message, type = "default") {
-  notificationStore.update(state => {
-    state.notifications = [
-      ...state.notifications,
-      { id: generate(), type, message },
-    ]
-    return state
-  })
-}
-
-export const notifier = {
-  danger: msg => send(msg, "danger"),
-  warning: msg => send(msg, "warning"),
-  info: msg => send(msg, "info"),
-  success: msg => send(msg, "success"),
-}

packages/builder/src/builderStore/store/screenTemplates/utils/Component.js

@@ -1,4 +1,4 @@
-import { uuid } from "builderStore/uuid"
+import { Helpers } from "@budibase/bbui"
 import { BaseStructure } from "./BaseStructure"
 
 export class Component extends BaseStructure {
@@ -6,7 +6,7 @@ export class Component extends BaseStructure {
     super(false)
     this._children = []
     this._json = {
-      _id: uuid(),
+      _id: Helpers.uuid(),
       _component: name,
       _styles: {
         normal: {},

packages/builder/src/builderStore/store/screenTemplates/utils/Screen.js

@@ -1,5 +1,5 @@
 import { BaseStructure } from "./BaseStructure"
-import { uuid } from "builderStore/uuid"
+import { Helpers } from "@budibase/bbui"
 
 export class Screen extends BaseStructure {
   constructor() {
@@ -7,7 +7,7 @@ export class Screen extends BaseStructure {
     this._json = {
       layoutId: "layout_private_master",
       props: {
-        _id: uuid(),
+        _id: Helpers.uuid(),
         _component: "@budibase/standard-components/container",
         _styles: {
           normal: {},

packages/builder/src/builderStore/store/screenTemplates/utils/commonComponents.js

@@ -141,7 +141,9 @@ const fieldTypeToComponentMap = {
 }
 
 export function makeDatasourceFormComponents(datasource) {
-  const { schema } = getSchemaForDatasource(null, datasource, true)
+  const { schema } = getSchemaForDatasource(null, datasource, {
+    formSchema: true,
+  })
   let components = []
   let fields = Object.keys(schema || {})
   fields.forEach(field => {

packages/builder/src/builderStore/store/theme.js

@@ -1,4 +1,4 @@
-import { localStorageStore } from "./localStorage"
+import { createLocalStorageStore } from "@budibase/frontend-core"
 
 export const getThemeStore = () => {
   const themeElement = document.documentElement
@@ -6,7 +6,7 @@ export const getThemeStore = () => {
     theme: "darkest",
     options: ["lightest", "light", "dark", "darkest"],
   }
-  const store = localStorageStore("bb-theme", initialValue)
+  const store = createLocalStorageStore("bb-theme", initialValue)
 
   // Update theme class when store changes
   store.subscribe(state => {

packages/builder/src/builderStore/uuid.js

@@ -1,9 +0,0 @@
-export function uuid() {
-  // always want to make this start with a letter, as this makes it
-  // easier to use with template string bindings in the client
-  return "cxxxxxxxxxxxx4xxxyxxxxxxxxxxxxxxx".replace(/[xy]/g, c => {
-    const r = (Math.random() * 16) | 0,
-      v = c == "x" ? r : (r & 0x3) | 0x8
-    return v.toString(16)
-  })
-}

packages/builder/src/components/automation/AutomationBuilder/FlowChart/ActionModal.svelte

@@ -6,6 +6,7 @@
     Body,
     Icon,
     Tooltip,
+    notifications,
   } from "@budibase/bbui"
   import { automationStore } from "builderStore"
   import { admin } from "stores/portal"
@@ -47,6 +48,7 @@
   }
 
   async function addBlockToAutomation() {
+    try {
       const newBlock = $automationStore.selectedAutomation.constructBlock(
         "ACTION",
         actionVal.stepId,
@@ -56,6 +58,9 @@
       await automationStore.actions.save(
         $automationStore.selectedAutomation?.automation
       )
+    } catch (error) {
+      notifications.error("Error saving automation")
+    }
   }
 </script>
 

packages/builder/src/components/automation/AutomationBuilder/FlowChart/FlowChart.svelte

@@ -30,26 +30,13 @@
   }
 
   async function deleteAutomation() {
+    try {
       await automationStore.actions.delete(
         $automationStore.selectedAutomation?.automation
       )
-    notifications.success("Automation deleted.")
+    } catch (error) {
+      notifications.error("Error deleting automation")
     }
-
-  async function testAutomation() {
-    const result = await automationStore.actions.trigger(
-      $automationStore.selectedAutomation.automation
-    )
-    if (result.status === 200) {
-      notifications.success(
-        `Automation ${$automationStore.selectedAutomation.automation.name} triggered successfully.`
-      )
-    } else {
-      notifications.error(
-        `Failed to trigger automation ${$automationStore.selectedAutomation.automation.name}.`
-      )
-    }
-    return result
   }
 </script>
 
@@ -85,7 +72,7 @@
         animate:flip={{ duration: 500 }}
         in:fly|local={{ x: 500, duration: 1500 }}
       >
-        <FlowItem {testDataModal} {testAutomation} {onSelect} {block} />
+        <FlowItem {testDataModal} {onSelect} {block} />
       </div>
     {/each}
   </div>
@@ -101,7 +88,7 @@
   </ConfirmDialog>
 
   <Modal bind:this={testDataModal} width="30%">
-    <TestDataModal {testAutomation} />
+    <TestDataModal />
   </Modal>
 </div>
 

packages/builder/src/components/automation/AutomationBuilder/FlowChart/FlowItem.svelte

@@ -10,6 +10,7 @@
     Button,
     StatusLight,
     ActionButton,
+    notifications,
   } from "@budibase/bbui"
   import AutomationBlockSetup from "../../SetupPanel/AutomationBlockSetup.svelte"
   import CreateWebhookModal from "components/automation/Shared/CreateWebhookModal.svelte"
@@ -54,10 +55,14 @@
   ).every(x => block?.inputs[x])
 
   async function deleteStep() {
+    try {
       automationStore.actions.deleteAutomationBlock(block)
       await automationStore.actions.save(
         $automationStore.selectedAutomation?.automation
       )
+    } catch (error) {
+      notifications.error("Error saving notification")
+    }
   }
 </script>
 

packages/builder/src/components/automation/AutomationBuilder/FlowChart/TestDataModal.svelte

@@ -1,5 +1,12 @@
 <script>
-  import { ModalContent, Tabs, Tab, TextArea, Label } from "@budibase/bbui"
+  import {
+    ModalContent,
+    Tabs,
+    Tab,
+    TextArea,
+    Label,
+    notifications,
+  } from "@budibase/bbui"
   import { automationStore } from "builderStore"
   import AutomationBlockSetup from "../../SetupPanel/AutomationBlockSetup.svelte"
   import { cloneDeep } from "lodash/fp"
@@ -37,6 +44,17 @@
       failedParse = "Invalid JSON"
     }
   }
+
+  const testAutomation = async () => {
+    try {
+      await automationStore.actions.test(
+        $automationStore.selectedAutomation?.automation,
+        testData
+      )
+    } catch (error) {
+      notifications.error("Error testing notification")
+    }
+  }
 </script>
 
 <ModalContent
@@ -44,12 +62,7 @@
   confirmText="Test"
   showConfirmButton={true}
   disabled={isError}
-  onConfirm={() => {
+  onConfirm={testAutomation}
-    automationStore.actions.test(
-      $automationStore.selectedAutomation?.automation,
-      testData
-    )
-  }}
   cancelText="Cancel"
 >
   <Tabs selected="Form" quiet

packages/builder/src/components/automation/AutomationPanel/AutomationList.svelte

@@ -4,10 +4,16 @@
   import { automationStore } from "builderStore"
   import NavItem from "components/common/NavItem.svelte"
   import EditAutomationPopover from "./EditAutomationPopover.svelte"
+  import { notifications } from "@budibase/bbui"
 
   $: selectedAutomationId = $automationStore.selectedAutomation?.automation?._id
-  onMount(() => {
+
-    automationStore.actions.fetch()
+  onMount(async () => {
+    try {
+      await automationStore.actions.fetch()
+    } catch (error) {
+      notifications.error("Error getting automations list")
+    }
   })
 
   function selectAutomation(automation) {

packages/builder/src/components/automation/AutomationPanel/CreateAutomationModal.svelte

@@ -24,6 +24,7 @@
     nameTouched && !name ? "Please specify a name for the automation." : null
 
   async function createAutomation() {
+    try {
       await automationStore.actions.create({
         name,
         instanceId,
@@ -43,10 +44,13 @@
         $automationStore.selectedAutomation?.automation
       )
 
-    notifications.success(`Automation ${name} created.`)
+      notifications.success(`Automation ${name} created`)
 
       $goto(`./${$automationStore.selectedAutomation.automation._id}`)
       analytics.captureEvent(Events.AUTOMATION.CREATED, { name })
+    } catch (error) {
+      notifications.error("Error creating automation")
+    }
   }
   $: triggers = Object.entries($automationStore.blockDefinitions.TRIGGER)
 

packages/builder/src/components/automation/AutomationPanel/EditAutomationPopover.svelte

@@ -11,9 +11,13 @@
   let updateAutomationDialog
 
   async function deleteAutomation() {
+    try {
       await automationStore.actions.delete(automation)
-    notifications.success("Automation deleted.")
+      notifications.success("Automation deleted successfully")
       $goto("../automate")
+    } catch (error) {
+      notifications.error("Error deleting automation")
+    }
   }
 </script>
 

packages/builder/src/components/automation/AutomationPanel/UpdateAutomationModal.svelte

@@ -20,14 +20,18 @@
   }
 
   async function saveAutomation() {
+    try {
       const updatedAutomation = {
         ...automation,
         name,
       }
       await automationStore.actions.save(updatedAutomation)
-    notifications.success(`Automation ${name} updated successfully.`)
+      notifications.success(`Automation ${name} updated successfully`)
       analytics.captureEvent(Events.AUTOMATION.SAVED, { name })
       hide()
+    } catch (error) {
+      notifications.error("Error saving automation")
+    }
   }
 
   function checkValid(evt) {

packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte

@@ -11,6 +11,7 @@
     Drawer,
     Modal,
     Detail,
+    notifications,
   } from "@budibase/bbui"
   import CreateWebhookModal from "components/automation/Shared/CreateWebhookModal.svelte"
 
@@ -27,8 +28,8 @@
   import { debounce } from "lodash"
   import ModalBindableInput from "components/common/bindings/ModalBindableInput.svelte"
   import FilterDrawer from "components/design/PropertiesPanel/PropertyControls/FilterEditor/FilterDrawer.svelte"
-  // need the client lucene builder to convert to the structure API expects
+  import { LuceneUtils } from "@budibase/frontend-core"
-  import { buildLuceneQuery } from "helpers/lucene"
+  import { getSchemaForTable } from "builderStore/dataBinding"
 
   export let block
   export let testData
@@ -45,15 +46,16 @@
     block || $automationStore.selectedBlock,
     $automationStore.selectedAutomation?.automation?.definition
   )
-
   $: inputData = testData ? testData : block.inputs
   $: tableId = inputData ? inputData.tableId : null
   $: table = tableId
     ? $tables.list.find(table => table._id === inputData.tableId)
     : { schema: {} }
-  $: schemaFields = table ? Object.values(table.schema) : []
+  $: schema = getSchemaForTable(tableId, { searchableSchema: true }).schema
+  $: schemaFields = Object.values(schema || {})
 
   const onChange = debounce(async function (e, key) {
+    try {
       if (isTestModal) {
         // Special case for webhook, as it requires a body, but the schema already brings back the body's contents
         if (stepId === "WEBHOOK") {
@@ -77,6 +79,9 @@
           $automationStore.selectedAutomation?.automation
         )
       }
+    } catch (error) {
+      notifications.error("Error saving automation")
+    }
   }, 800)
 
   function getAvailableBindings(block, automation) {
@@ -131,7 +136,7 @@
   }
 
   function saveFilters(key) {
-    const filters = buildLuceneQuery(tempFilters)
+    const filters = LuceneUtils.buildLuceneQuery(tempFilters)
     const defKey = `${key}-def`
     inputData[key] = filters
     inputData[defKey] = tempFilters

packages/builder/src/components/automation/Shared/CreateWebhookModal.svelte

@@ -1,5 +1,5 @@
 <script>
-  import { Icon } from "@budibase/bbui"
+  import { Icon, notifications } from "@budibase/bbui"
   import { automationStore } from "builderStore"
   import WebhookDisplay from "./WebhookDisplay.svelte"
   import { ModalContent } from "@budibase/bbui"
@@ -16,16 +16,25 @@
   onMount(async () => {
     if (!automation?.definition?.trigger?.inputs.schemaUrl) {
       // save the automation initially
+      try {
         await automationStore.actions.save(automation)
+      } catch (error) {
+        notifications.error("Error saving automation")
+      }
     }
     interval = setInterval(async () => {
+      try {
         await automationStore.actions.fetch()
-      const outputs = automation?.definition?.trigger.schema.outputs?.properties
+        const outputs =
+          automation?.definition?.trigger.schema.outputs?.properties
         // always one prop for the "body"
         if (Object.keys(outputs).length > 1) {
           propCount = Object.keys(outputs).length - 1
           finished = true
         }
+      } catch (error) {
+        notifications.error("Error getting automations list")
+      }
     }, POLL_RATE_MS)
     schemaURL = automation?.definition?.trigger?.inputs.schemaUrl
   })

packages/builder/src/components/backend/DataTable/DataTable.svelte

@@ -14,18 +14,19 @@
   import Table from "./Table.svelte"
   import { TableNames } from "constants"
   import CreateEditRow from "./modals/CreateEditRow.svelte"
-  import { fetchTableData } from "helpers/fetchTableData"
   import { Pagination } from "@budibase/bbui"
+  import { fetchData } from "@budibase/frontend-core"
+  import { API } from "api"
 
   let hideAutocolumns = true
+
   $: isUsersTable = $tables.selected?._id === TableNames.USERS
   $: type = $tables.selected?.type
   $: isInternal = type !== "external"
   $: schema = $tables.selected?.schema
   $: enrichedSchema = enrichSchema($tables.selected?.schema)
   $: id = $tables.selected?._id
-  $: search = searchTable(id)
+  $: fetch = createFetch(id)
-  $: columnOptions = Object.keys($search.schema || {})
 
   const enrichSchema = schema => {
     let tempSchema = { ...schema }
@@ -47,18 +48,24 @@
     return tempSchema
   }
   // Fetches new data whenever the table changes
-  const searchTable = tableId => {
+  const createFetch = tableId => {
-    return fetchTableData({
+    return fetchData({
+      API,
+      datasource: {
         tableId,
+        type: "table",
+      },
+      options: {
         schema,
         limit: 10,
         paginate: true,
+      },
     })
   }
 
   // Fetch data whenever sorting option changes
   const onSort = e => {
-    search.update({
+    fetch.update({
       sortColumn: e.detail.column,
       sortOrder: e.detail.order,
     })
@@ -66,22 +73,20 @@
 
   // Fetch data whenever filters change
   const onFilter = e => {
-    search.update({
+    fetch.update({
-      filters: e.detail,
+      filter: e.detail,
     })
   }
 
   // Fetch data whenever schema changes
   const onUpdateColumns = () => {
-    search.update({
+    fetch.refresh()
-      schema,
-    })
   }
 
   // Fetch data whenever rows are modified. Unfortunately we have to lose
   // our pagination place, as our bookmarks will have shifted.
   const onUpdateRows = () => {
-    search.update()
+    fetch.refresh()
   }
 </script>
 
@@ -91,9 +96,9 @@
     schema={enrichedSchema}
     {type}
     tableId={id}
-    data={$search.rows}
+    data={$fetch.rows}
     bind:hideAutocolumns
-    loading={$search.loading}
+    loading={$fetch.loading}
     on:sort={onSort}
     allowEditing
     disableSorting
@@ -138,11 +143,11 @@
     <div in:fade={{ delay: 200, duration: 100 }}>
       <div class="pagination">
         <Pagination
-          page={$search.pageNumber + 1}
+          page={$fetch.pageNumber + 1}
-          hasPrevPage={$search.hasPrevPage}
+          hasPrevPage={$fetch.hasPrevPage}
-          hasNextPage={$search.hasNextPage}
+          hasNextPage={$fetch.hasNextPage}
-          goToPrevPage={$search.loading ? null : search.prevPage}
+          goToPrevPage={$fetch.loading ? null : fetch.prevPage}
-          goToNextPage={$search.loading ? null : search.nextPage}
+          goToNextPage={$fetch.loading ? null : fetch.nextPage}
         />
       </div>
     </div>

packages/builder/src/components/backend/DataTable/RelationshipDataTable.svelte

@@ -1,7 +1,8 @@
 <script>
-  import api from "builderStore/api"
+  import { API } from "api"
   import Table from "./Table.svelte"
   import { tables } from "stores/backend"
+  import { notifications } from "@budibase/bbui"
 
   export let tableId
   export let rowId
@@ -27,9 +28,15 @@
   }
 
   async function fetchData(tableId, rowId) {
-    const QUERY_VIEW_URL = `/api/${tableId}/${rowId}/enrich`
+    try {
-    const response = await api.get(QUERY_VIEW_URL)
+      row = await API.fetchRelationshipData({
-    row = await response.json()
+        tableId,
+        rowId,
+      })
+    } catch (error) {
+      row = null
+      notifications.error("Error fetching relationship data")
+    }
   }
 </script>
 

packages/builder/src/components/backend/DataTable/Table.svelte

@@ -2,7 +2,7 @@
   import { fade } from "svelte/transition"
   import { goto, params } from "@roxi/routify"
   import { Table, Modal, Heading, notifications, Layout } from "@budibase/bbui"
-  import api from "builderStore/api"
+  import { API } from "api"
   import Spinner from "components/common/Spinner.svelte"
   import DeleteRowsButton from "./buttons/DeleteRowsButton.svelte"
   import CreateEditRow from "./modals/CreateEditRow.svelte"
@@ -88,12 +88,17 @@
   }
 
   const deleteRows = async () => {
-    await api.delete(`/api/${tableId}/rows`, {
+    try {
+      await API.deleteRows({
+        tableId,
         rows: selectedRows,
       })
       data = data.filter(row => !selectedRows.includes(row))
       notifications.success(`Successfully deleted ${selectedRows.length} rows`)
       selectedRows = []
+    } catch (error) {
+      notifications.error("Error deleting rows")
+    }
   }
 
   const editRow = row => {

packages/builder/src/components/backend/DataTable/ViewDataTable.svelte

@@ -1,5 +1,5 @@
 <script>
-  import api from "builderStore/api"
+  import { API } from "api"
   import { tables } from "stores/backend"
 
   import Table from "./Table.svelte"
@@ -9,6 +9,7 @@
   import ExportButton from "./buttons/ExportButton.svelte"
   import ManageAccessButton from "./buttons/ManageAccessButton.svelte"
   import HideAutocolumnButton from "./buttons/HideAutocolumnButton.svelte"
+  import { notifications } from "@budibase/bbui"
 
   export let view = {}
 
@@ -20,33 +21,31 @@
   $: name = view.name
 
   // Fetch rows for specified view
-  $: {
+  $: fetchViewData(name, view.field, view.groupBy, view.calculation)
-    loading = true
-    fetchViewData(name, view.field, view.groupBy, view.calculation)
-  }
 
   async function fetchViewData(name, field, groupBy, calculation) {
+    loading = true
     const _tables = $tables.list
     const allTableViews = _tables.map(table => table.views)
     const thisView = allTableViews.filter(
       views => views != null && views[name] != null
     )[0]
 
-    // don't fetch view data if the view no longer exists
+    // Don't fetch view data if the view no longer exists
     if (!thisView) {
+      loading = false
       return
     }
-    const params = new URLSearchParams()
+    try {
-    if (calculation) {
+      data = await API.fetchViewData({
-      params.set("field", field)
+        name,
-      params.set("calculation", calculation)
+        calculation,
+        field,
+        groupBy,
+      })
+    } catch (error) {
+      notifications.error("Error fetching view data")
     }
-    if (groupBy) {
-      params.set("group", groupBy)
-    }
-    const QUERY_VIEW_URL = `/api/views/${name}?${params}`
-    const response = await api.get(QUERY_VIEW_URL)
-    data = await response.json()
     loading = false
   }
 </script>

packages/builder/src/components/backend/DataTable/api.js

@@ -1,34 +0,0 @@
-import api from "builderStore/api"
-
-export async function createUser(user) {
-  const CREATE_USER_URL = `/api/users/metadata`
-  const response = await api.post(CREATE_USER_URL, user)
-  return await response.json()
-}
-
-export async function saveRow(row, tableId) {
-  const SAVE_ROW_URL = `/api/${tableId}/rows`
-  const response = await api.post(SAVE_ROW_URL, row)
-
-  return await response.json()
-}
-
-export async function deleteRow(row) {
-  const DELETE_ROWS_URL = `/api/${row.tableId}/rows`
-  return api.delete(DELETE_ROWS_URL, {
-    _id: row._id,
-    _rev: row._rev,
-  })
-}
-
-export async function fetchDataForTable(tableId) {
-  const FETCH_ROWS_URL = `/api/${tableId}/rows`
-
-  const response = await api.get(FETCH_ROWS_URL)
-  const json = await response.json()
-
-  if (response.status !== 200) {
-    throw new Error(json.message)
-  }
-  return json
-}

packages/builder/src/components/backend/DataTable/modals/CalculateModal.svelte

@@ -38,9 +38,13 @@
     })
 
   function saveView() {
+    try {
       views.save(view)
-    notifications.success(`View ${view.name} saved.`)
+      notifications.success(`View ${view.name} saved`)
       analytics.captureEvent(Events.VIEW.ADDED_CALCULATE, { field: view.field })
+    } catch (error) {
+      notifications.error("Error saving view")
+    }
   }
 </script>
 

packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte

@@ -124,7 +124,7 @@
       })
       dispatch("updatecolumns")
     } catch (err) {
-      notifications.error(err)
+      notifications.error("Error saving column")
     }
   }
 
@@ -133,6 +133,7 @@
   }
 
   function deleteColumn() {
+    try {
       field.name = deleteColName
       if (field.name === $tables.selected.primaryDisplay) {
         notifications.error("You cannot delete the display column")
@@ -142,9 +143,12 @@
         confirmDeleteDialog.hide()
         hide()
         deletion = false
-    }
         dispatch("updatecolumns")
       }
+    } catch (error) {
+      notifications.error("Error deleting column")
+    }
+  }
 
   function handleTypeChange(event) {
     // remove any extra fields that may not be related to this type

packages/builder/src/components/backend/DataTable/modals/CreateEditRow.svelte

@@ -3,7 +3,7 @@
   import { tables, rows } from "stores/backend"
   import { notifications } from "@budibase/bbui"
   import RowFieldControl from "../RowFieldControl.svelte"
-  import * as api from "../api"
+  import { API } from "api"
   import { ModalContent } from "@budibase/bbui"
   import ErrorsBox from "components/common/ErrorsBox.svelte"
   import { FIELDS } from "constants/backend"
@@ -22,30 +22,30 @@
   $: tableSchema = Object.entries(table?.schema ?? {})
 
   async function saveRow() {
-    const rowResponse = await api.saveRow(
+    errors = []
-      { ...row, tableId: table._id },
+    try {
-      table._id
+      await API.saveRow({ ...row, tableId: table._id })
-    )
+      notifications.success("Row saved successfully")
-
+      rows.save()
-    if (rowResponse.errors) {
+      dispatch("updaterows")
-      errors = Object.entries(rowResponse.errors)
+    } catch (error) {
+      if (error.handled) {
+        const response = error.json
+        if (response?.errors) {
+          errors = Object.entries(response.errors)
             .map(([key, error]) => ({ dataPath: key, message: error }))
             .flat()
+        } else if (error.status === 400 && response?.validationErrors) {
+          errors = Object.keys(response.validationErrors).map(field => ({
+            message: `${field} ${response.validationErrors[field][0]}`,
+          }))
+        }
+      } else {
+        notifications.error("Failed to save row")
+      }
       // Prevent modal closing if there were errors
       return false
-    } else if (rowResponse.status === 400 && rowResponse.validationErrors) {
-      errors = Object.keys(rowResponse.validationErrors).map(field => ({
-        message: `${field} ${rowResponse.validationErrors[field][0]}`,
-      }))
-      return false
-    } else if (rowResponse.status >= 400) {
-      errors = [{ message: rowResponse.message }]
-      return false
     }
-
-    notifications.success("Row saved successfully.")
-    rows.save(rowResponse)
-    dispatch("updaterows")
   }
 </script>
 

packages/builder/src/components/backend/DataTable/modals/CreateEditUser.svelte

@@ -4,7 +4,7 @@
   import { roles } from "stores/backend"
   import { notifications } from "@budibase/bbui"
   import RowFieldControl from "../RowFieldControl.svelte"
-  import * as backendApi from "../api"
+  import { API } from "api"
   import { ModalContent, Select } from "@budibase/bbui"
   import ErrorsBox from "components/common/ErrorsBox.svelte"
 
@@ -53,27 +53,31 @@
       return false
     }
 
-    const rowResponse = await backendApi.saveRow(
+    try {
-      { ...row, tableId: table._id },
+      await API.saveRow({ ...row, tableId: table._id })
-      table._id
+      notifications.success("User saved successfully")
-    )
+      rows.save()
-    if (rowResponse.errors) {
+      dispatch("updaterows")
-      if (Array.isArray(rowResponse.errors)) {
+    } catch (error) {
-        errors = rowResponse.errors.map(error => ({ message: error }))
+      if (error.handled) {
+        const response = error.json
+        if (response?.errors) {
+          if (Array.isArray(response.errors)) {
+            errors = response.errors.map(error => ({ message: error }))
           } else {
-        errors = Object.entries(rowResponse.errors)
+            errors = Object.entries(response.errors)
               .map(([key, error]) => ({ dataPath: key, message: error }))
               .flat()
           }
-      return false
+        } else if (error.status === 400) {
-    } else if (rowResponse.status === 400 || rowResponse.status === 500) {
+          errors = [{ message: response?.message || "Unknown error" }]
-      errors = [{ message: rowResponse.message }]
+        }
+      } else {
+        notifications.error("Error saving user")
+      }
+      // Prevent closing the modal on errors
       return false
     }
-
-    notifications.success("User saved successfully")
-    rows.save(rowResponse)
-    dispatch("updaterows")
   }
 </script>
 

packages/builder/src/components/backend/DataTable/modals/CreateViewModal.svelte

@@ -12,9 +12,10 @@
 
   function saveView() {
     if (views.includes(name)) {
-      notifications.error(`View exists with name ${name}.`)
+      notifications.error(`View exists with name ${name}`)
       return
     }
+    try {
       viewsStore.save({
         name,
         tableId: $tables.selected._id,
@@ -23,6 +24,9 @@
       notifications.success(`View ${name} created`)
       analytics.captureEvent(Events.VIEW.CREATED, { name })
       $goto(`../../view/${name}`)
+    } catch (error) {
+      notifications.error("Error creating view")
+    }
   }
 </script>
 

packages/builder/src/components/backend/DataTable/modals/EditRoles.svelte

@@ -1,7 +1,7 @@
 <script>
   import { ModalContent, Select, Input, Button } from "@budibase/bbui"
   import { onMount } from "svelte"
-  import api from "builderStore/api"
+  import { API } from "api"
   import { notifications } from "@budibase/bbui"
   import ErrorsBox from "components/common/ErrorsBox.svelte"
   import { roles } from "stores/backend"
@@ -24,8 +24,12 @@
     !builtInRoles.includes(selectedRole.name)
 
   const fetchBasePermissions = async () => {
-    const permissionsResponse = await api.get("/api/permission/builtin")
+    try {
-    basePermissions = await permissionsResponse.json()
+      basePermissions = await API.getBasePermissions()
+    } catch (error) {
+      notifications.error("Error fetching base permission options")
+      basePermissions = []
+    }
   }
 
   // Changes the selected role
@@ -68,23 +72,23 @@
     }
 
     // Save/create the role
-    const response = await roles.save(selectedRole)
+    try {
-    if (response.status === 200) {
+      await roles.save(selectedRole)
-      notifications.success("Role saved successfully.")
+      notifications.success("Role saved successfully")
-    } else {
+    } catch (error) {
-      notifications.error("Error saving role.")
+      notifications.error("Error saving role")
       return false
     }
   }
 
   // Deletes the selected role
   const deleteRole = async () => {
-    const response = await roles.delete(selectedRole)
+    try {
-    if (response.status === 200) {
+      await roles.delete(selectedRole)
       changeRole()
-      notifications.success("Role deleted successfully.")
+      notifications.success("Role deleted successfully")
-    } else {
+    } catch (error) {
-      notifications.error("Error deleting role.")
+      notifications.error("Error deleting role")
     }
   }
 

packages/builder/src/components/backend/DataTable/modals/ExportModal.svelte

@@ -1,7 +1,7 @@
 <script>
   import { Select, ModalContent, notifications } from "@budibase/bbui"
   import download from "downloadjs"
-  import { get } from "builderStore/api"
+  import { API } from "api"
 
   const FORMATS = [
     {
@@ -19,17 +19,14 @@
   let exportFormat = FORMATS[0].key
 
   async function exportView() {
-    const uri = encodeURIComponent(view)
+    try {
-    const response = await get(
+      const data = await API.exportView({
-      `/api/views/export?view=${uri}&format=${exportFormat}`
+        viewName: view,
-    )
+        format: exportFormat,
-    if (response.status === 200) {
+      })
-      const data = await response.text()
       download(data, `export.${exportFormat}`)
-    } else {
+    } catch (error) {
-      notifications.error(
+      notifications.error(`Unable to export ${exportFormat.toUpperCase()} data`)
-        `Unable to export ${exportFormat.toUpperCase()} data.`
-      )
     }
   }
 </script>

packages/builder/src/components/backend/DataTable/modals/FilterModal.svelte

@@ -72,11 +72,15 @@
   $: schema = viewTable && viewTable.schema ? viewTable.schema : {}
 
   function saveView() {
+    try {
       views.save(view)
-    notifications.success(`View ${view.name} saved.`)
+      notifications.success(`View ${view.name} saved`)
       analytics.captureEvent(Events.VIEW.ADDED_FILTER, {
         filters: JSON.stringify(view.filters),
       })
+    } catch (error) {
+      notifications.error("Error saving view")
+    }
   }
 
   function removeFilter(idx) {
@@ -158,7 +162,7 @@
             <Select
               bind:value={filter.value}
               options={fieldOptions(filter.key)}
-              getOptionLabel={x => x.toString()}
+              getOptionLabel={x => x?.toString() || ""}
             />
           {:else if filter.key && isDate(filter.key)}
             <DatePicker

packages/builder/src/components/backend/DataTable/modals/GroupByModal.svelte

@@ -19,8 +19,12 @@
       .map(([key]) => key)
 
   function saveView() {
+    try {
       views.save(view)
-    notifications.success(`View ${view.name} saved.`)
+      notifications.success(`View ${view.name} saved`)
+    } catch (error) {
+      notifications.error("Error saving view")
+    }
   }
 </script>
 

packages/builder/src/components/backend/DataTable/modals/ImportModal.svelte

@@ -1,7 +1,13 @@
 <script>
-  import { ModalContent, Label, notifications, Body } from "@budibase/bbui"
+  import {
+    ModalContent,
+    Label,
+    notifications,
+    Body,
+    Layout,
+  } from "@budibase/bbui"
   import TableDataImport from "../../TableNavigator/TableDataImport.svelte"
-  import api from "builderStore/api"
+  import { API } from "api"
   import { createEventDispatcher } from "svelte"
 
   const dispatch = createEventDispatcher()
@@ -12,15 +18,17 @@
   $: valid = dataImport?.csvString != null && dataImport?.valid
 
   async function importData() {
-    const response = await api.post(`/api/tables/${tableId}/import`, {
+    try {
-      dataImport,
+      await API.importTableData({
+        tableId,
+        data: dataImport,
       })
-    if (response.status !== 200) {
+      notifications.success("Rows successfully imported")
-      const error = await response.text()
+    } catch (error) {
-      notifications.error(`Unable to import data - ${error}`)
+      notifications.error("Unable to import data")
-    } else {
-      notifications.success("Rows successfully imported.")
     }
+
+    // Always refresh rows just to be sure
     dispatch("updaterows")
   }
 </script>
@@ -31,13 +39,12 @@
   onConfirm={importData}
   disabled={!valid}
 >
-  <Body
+  <Body size="S">
-    >Import rows to an existing table from a CSV. Only columns from the CSV
+    Import rows to an existing table from a CSV. Only columns from the CSV which
-    which exist in the table will be imported.</Body
+    exist in the table will be imported.
-  >
+  </Body>
+  <Layout gap="XS" noPadding>
     <Label grey extraSmall>CSV to import</Label>
     <TableDataImport bind:dataImport bind:existingTableId={tableId} />
+  </Layout>
 </ModalContent>
-
-<style>
-</style>

packages/builder/src/components/backend/DataTable/modals/ManageAccessModal.svelte

@@ -14,6 +14,7 @@
   export let permissions
 
   async function changePermission(level, role) {
+    try {
       await permissionsStore.save({
         level,
         role,
@@ -22,7 +23,10 @@
 
       // Show updated permissions in UI: REMOVE
       permissions = await permissionsStore.forResource(resourceId)
-    notifications.success("Updated permissions.")
+      notifications.success("Updated permissions")
+    } catch (error) {
+      notifications.error("Error updating permissions")
+    }
   }
 </script>
 

packages/builder/src/components/backend/DatasourceNavigator/DatasourceNavigator.svelte

@@ -10,6 +10,7 @@
   import TableNavigator from "components/backend/TableNavigator/TableNavigator.svelte"
   import { customQueryIconText, customQueryIconColor } from "helpers/data/utils"
   import ICONS from "./icons"
+  import { notifications } from "@budibase/bbui"
 
   let openDataSources = []
   $: enrichedDataSources = Array.isArray($datasources.list)
@@ -63,9 +64,13 @@
     }
   }
 
-  onMount(() => {
+  onMount(async () => {
-    datasources.fetch()
+    try {
-    queries.fetch()
+      await datasources.fetch()
+      await queries.fetch()
+    } catch (error) {
+      notifications.error("Error fetching datasources and queries")
+    }
   })
 
   const containsActiveEntity = datasource => {

packages/builder/src/components/backend/DatasourceNavigator/TableIntegrationMenu/CreateExternalTableModal.svelte

@@ -1,5 +1,5 @@
 <script>
-  import { ModalContent, Body, Input } from "@budibase/bbui"
+  import { ModalContent, Body, Input, notifications } from "@budibase/bbui"
   import { tables, datasources } from "stores/backend"
   import { goto } from "@roxi/routify"
 
@@ -29,10 +29,14 @@
   }
 
   async function saveTable() {
+    try {
       submitted = true
       const table = await tables.save(buildDefaultTable(name, datasource._id))
       await datasources.fetch()
       $goto(`../../table/${table._id}`)
+    } catch (error) {
+      notifications.error("Error saving table")
+    }
   }
 </script>
 

packages/builder/src/components/backend/DatasourceNavigator/TableIntegrationMenu/PlusConfigForm.svelte

@@ -90,8 +90,8 @@
       await datasources.updateSchema(datasource)
       notifications.success(`Datasource ${name} tables updated successfully.`)
       await tables.fetch()
-    } catch (err) {
+    } catch (error) {
-      notifications.error(`Error updating datasource schema: ${err}`)
+      notifications.error("Error updating datasource schema")
     }
   }
 

packages/builder/src/components/backend/DatasourceNavigator/TableIntegrationMenu/index.svelte

@@ -1,7 +1,7 @@
 <script>
-  import { Body } from "@budibase/bbui"
+  import { Body, notifications } from "@budibase/bbui"
   import { onMount } from "svelte"
-  import api from "builderStore/api"
+  import { API } from "api"
   import ICONS from "../icons"
 
   export let integration = {}
@@ -9,14 +9,17 @@
   const INTERNAL = "BUDIBASE"
 
   async function fetchIntegrations() {
-    const response = await api.get("/api/integrations")
+    let otherIntegrations
-    const json = await response.json()
+    try {
-
+      otherIntegrations = await API.getIntegrations()
+    } catch (error) {
+      otherIntegrations = {}
+      notifications.error("Error getting integrations")
+    }
     integrations = {
       [INTERNAL]: { datasource: {}, name: "INTERNAL/CSV" },
-      ...json,
+      ...otherIntegrations,
     }
-    return json
   }
 
   function selectIntegration(integrationType) {

packages/builder/src/components/backend/DatasourceNavigator/TableIntegrationMenu/rest/auth/RestAuthenticationBuilder.svelte

@@ -2,7 +2,7 @@
   import { Table, Modal, Layout, ActionButton } from "@budibase/bbui"
   import AuthTypeRenderer from "./AuthTypeRenderer.svelte"
   import RestAuthenticationModal from "./RestAuthenticationModal.svelte"
-  import { uuid } from "builderStore/uuid"
+  import { Helpers } from "@budibase/bbui"
 
   export let configs = []
 
@@ -29,7 +29,7 @@
         return c
       })
     } else {
-      config._id = uuid()
+      config._id = Helpers.uuid()
       configs = [...configs, config]
     }
   }

packages/builder/src/components/backend/DatasourceNavigator/TableIntegrationMenu/rest/auth/RestAuthenticationModal.svelte

@@ -213,6 +213,3 @@
     {/if}
   </Layout>
 </ModalContent>
-
-<style>
-</style>

packages/builder/src/components/backend/DatasourceNavigator/_components/GoogleButton.svelte

@@ -17,7 +17,7 @@
       ds = await preAuthStep()
     }
     window.open(
-      `/api/global/auth/${tenantId}/datasource/google?datasourceId=${datasource._id}&appId=${$store.appId}`,
+      `/api/global/auth/${tenantId}/datasource/google?datasourceId=${ds._id}&appId=${$store.appId}`,
       "_blank"
     )
   }}

packages/builder/src/components/backend/DatasourceNavigator/modals/CreateDatasourceModal.svelte

@@ -1,8 +1,15 @@
 <script>
-  import { ModalContent, Modal, Body, Layout, Detail } from "@budibase/bbui"
+  import {
+    ModalContent,
+    Modal,
+    Body,
+    Layout,
+    Detail,
+    notifications,
+  } from "@budibase/bbui"
   import { onMount } from "svelte"
   import ICONS from "../icons"
-  import api from "builderStore/api"
+  import { API } from "api"
   import { IntegrationNames, IntegrationTypes } from "constants/backend"
   import CreateTableModal from "components/backend/TableNavigator/modals/CreateTableModal.svelte"
   import DatasourceConfigModal from "components/backend/DatasourceNavigator/modals/DatasourceConfigModal.svelte"
@@ -12,7 +19,7 @@
   import ImportRestQueriesModal from "./ImportRestQueriesModal.svelte"
 
   export let modal
-  let integrations = []
+  let integrations = {}
   let integration = {}
   let internalTableModal
   let externalDatasourceModal
@@ -57,22 +64,32 @@
       externalDatasourceModal.hide()
       internalTableModal.show()
     } else if (integration.type === IntegrationTypes.REST) {
-      // skip modal for rest, create straight away
+      try {
+        // Skip modal for rest, create straight away
         const resp = await createRestDatasource(integration)
         $goto(`./datasource/${resp._id}`)
+      } catch (error) {
+        notifications.error("Error creating datasource")
+      }
     } else {
       externalDatasourceModal.show()
     }
   }
 
   async function fetchIntegrations() {
-    const response = await api.get("/api/integrations")
+    let newIntegrations = {
-    const json = await response.json()
-    integrations = {
       [IntegrationTypes.INTERNAL]: { datasource: {}, name: "INTERNAL/CSV" },
-      ...json,
     }
-    return json
+    try {
+      const integrationList = await API.getIntegrations()
+      newIntegrations = {
+        ...newIntegrations,
+        ...integrationList,
+      }
+    } catch (error) {
+      notifications.error("Error fetching integrations")
+    }
+    integrations = newIntegrations
   }
 </script>
 

packages/builder/src/components/backend/DatasourceNavigator/modals/DatasourceConfigModal.svelte

@@ -20,7 +20,7 @@
       $goto(`./datasource/${resp._id}`)
       notifications.success(`Datasource updated successfully.`)
     } catch (err) {
-      notifications.error(`Error saving datasource: ${err}`)
+      notifications.error("Error saving datasource")
     }
   }
 

packages/builder/src/components/backend/DatasourceNavigator/modals/ImportRestQueriesModal.svelte

@@ -79,8 +79,8 @@
       })
 
       return true
-    } catch (err) {
+    } catch (error) {
-      notifications.error(`Error importing: ${err}`)
+      notifications.error("Error importing queries")
       return false
     }
   }
@@ -130,6 +130,3 @@
     </Tabs>
   </Layout>
 </ModalContent>
-
-<style>
-</style>

packages/builder/src/components/backend/DatasourceNavigator/popovers/EditDatasourcePopover.svelte

@@ -12,6 +12,7 @@
   let updateDatasourceDialog
 
   async function deleteDatasource() {
+    try {
       let wasSelectedSource = $datasources.selected
       if (!wasSelectedSource && $queries.selected) {
         const queryId = $queries.selected
@@ -22,7 +23,7 @@
       const wasSelectedTable = $tables.selected
       await datasources.delete(datasource)
       notifications.success("Datasource deleted")
-    // navigate to first index page if the source you are deleting is selected
+      // Navigate to first index page if the source you are deleting is selected
       const entities = Object.values(datasource?.entities || {})
       if (
         wasSelectedSource === datasource._id ||
@@ -31,6 +32,9 @@
       ) {
         $goto("./datasource")
       }
+    } catch (error) {
+      notifications.error("Error deleting datasource")
+    }
   }
 </script>
 

packages/builder/src/components/backend/DatasourceNavigator/popovers/EditQueryPopover.svelte

@@ -10,6 +10,7 @@
   let confirmDeleteDialog
 
   async function deleteQuery() {
+    try {
       const wasSelectedQuery = $queries.selected
       // need to calculate this before the query is deleted
       const navigateToDatasource = wasSelectedQuery === query._id
@@ -22,14 +23,17 @@
         $goto(`./datasource/${query.datasourceId}`)
       }
       notifications.success("Query deleted")
+    } catch (error) {
+      notifications.error("Error deleting query")
+    }
   }
 
   async function duplicateQuery() {
     try {
       const newQuery = await queries.duplicate(query)
       onClickQuery(newQuery)
-    } catch (e) {
+    } catch (error) {
-      notifications.error(e.message)
+      notifications.error("Error duplicating query")
     }
   }
 </script>

packages/builder/src/components/backend/Datasources/CreateEditRelationship.svelte

@@ -9,7 +9,7 @@
     Body,
   } from "@budibase/bbui"
   import { tables } from "stores/backend"
-  import { uuid } from "builderStore/uuid"
+  import { Helpers } from "@budibase/bbui"
   import { writable } from "svelte/store"
 
   export let save
@@ -140,7 +140,7 @@
     const manyToMany =
       fromRelationship.relationshipType === RelationshipTypes.MANY_TO_MANY
     // main is simply used to know this is the side the user configured it from
-    const id = uuid()
+    const id = Helpers.uuid()
     if (!manyToMany) {
       delete fromRelationship.through
       delete toRelationship.through

packages/builder/src/components/backend/TableNavigator/TableDataImport.svelte

@@ -1,7 +1,7 @@
 <script>
   import { Select, InlineAlert, notifications } from "@budibase/bbui"
   import { FIELDS } from "constants/backend"
-  import api from "builderStore/api"
+  import { API } from "api"
 
   const BYTES_IN_MB = 1000000
   const FILE_SIZE_LIMIT = BYTES_IN_MB * 5
@@ -50,14 +50,13 @@
   }
 
   async function validateCSV() {
-    const response = await api.post("/api/tables/csv/validate", {
+    try {
+      const parseResult = await API.validateTableCSV({
         csvString,
         schema: schema || {},
         tableId: existingTableId,
       })
-
+      schema = parseResult?.schema
-    const parseResult = await response.json()
-    schema = parseResult && parseResult.schema
       fields = Object.keys(schema || {}).filter(
         key => schema[key].type !== "omit"
       )
@@ -67,11 +66,10 @@
         primaryDisplay = fields[0]
       }
 
-    if (response.status !== 200) {
-      notifications.error("CSV Invalid, please try another CSV file")
-      return []
-    }
       hasValidated = true
+    } catch (error) {
+      notifications.error("CSV Invalid, please try another CSV file")
+    }
   }
 
   async function handleFile(evt) {

packages/builder/src/components/backend/TableNavigator/popovers/EditTablePopover.svelte

@@ -49,8 +49,8 @@
       if (wasSelectedTable && wasSelectedTable._id === table._id) {
         $goto("./table")
       }
-    } catch (err) {
+    } catch (error) {
-      notifications.error(err)
+      notifications.error("Error deleting table")
     }
   }
 

packages/builder/src/components/backend/TableNavigator/popovers/EditViewPopover.svelte

@@ -27,11 +27,15 @@
   }
 
   async function deleteView() {
+    try {
       const name = view.name
       const id = view.tableId
       await views.delete(name)
       notifications.success("View deleted")
       $goto(`./table/${id}`)
+    } catch (error) {
+      notifications.error("Error deleting view")
+    }
   }
 </script>