Merge remote-tracking branch 'origin/develop' into feature/automation-data-types

.eslintignore

@@ -5,3 +5,5 @@ packages/server/builder
 packages/server/coverage
 packages/server/client
 packages/builder/.routify
+packages/builder/cypress/support/queryLevelTransformerFunction.js
+packages/builder/cypress/support/queryLevelTransformerFunctionWithData.js

.github/workflows/budibase_ci.yml

@@ -43,4 +43,8 @@ jobs:
         verbose: true 
 
     # TODO: parallelise this
-    - run: yarn test:e2e:ci
+    - name: Cypress run
+      uses: cypress-io/github-action@v2
+      with:
+        install: false
+        command: yarn test:e2e:ci

.github/workflows/release-selfhost.yml

@@ -25,14 +25,17 @@ jobs:
           # Pull apps and worker images
           docker pull budibase/apps:$release_tag
           docker pull budibase/worker:$release_tag
+          docker pull budibase/proxy:$release_tag
 
           # Tag apps and worker images
           docker tag budibase/apps:$release_tag budibase/apps:$SELFHOST_TAG
           docker tag budibase/worker:$release_tag budibase/worker:$SELFHOST_TAG
+          docker tag budibase/proxy:$release_tag budibase/proxy:$SELFHOST_TAG
           
           # Push images
           docker push budibase/apps:$SELFHOST_TAG
           docker push budibase/worker:$SELFHOST_TAG
+          docker push budibase/proxy:$SELFHOST_TAG
         env:
           DOCKER_USER: ${{ secrets.DOCKER_USERNAME }}
           DOCKER_PASSWORD: ${{ secrets.DOCKER_API_KEY }}

.gitignore

@@ -65,6 +65,7 @@ typings/
 .env
 !hosting/.env
 hosting/.generated-nginx.dev.conf
+hosting/proxy/.generated-nginx.prod.conf
 
 # parcel-bundler cache (https://parceljs.org/)
 .cache
@@ -95,3 +96,4 @@ hosting/.generated-nginx.dev.conf
 *.sublime-workspace
 
 bin/
+packages/builder/cypress.env.json

.prettierignore

@@ -7,3 +7,5 @@ packages/server/builder
 packages/server/coverage
 packages/server/client
 packages/builder/.routify
+packages/builder/cypress/support/queryLevelTransformerFunction.js
+packages/builder/cypress/support/queryLevelTransformerFunctionWithData.js

charts/budibase/Chart.yaml

@@ -11,7 +11,7 @@ sources:
   - https://github.com/Budibase/budibase
   - https://budibase.com
 type: application
-version: 0.2.6
+version: 0.2.8
 appVersion: 1.0.48
 dependencies:
   - name: couchdb

charts/budibase/templates/proxy-service-deployment.yaml

@@ -25,7 +25,7 @@ spec:
         app.kubernetes.io/name: budibase-proxy
     spec:
       containers:
-      - image: budibase/proxy:k8s
+      - image: budibase/proxy:{{ .Values.services.proxy.tag | default "k8s" }}
         imagePullPolicy: Always
         name: proxy-service
         ports:

charts/budibase/values.yaml

@@ -100,6 +100,9 @@ globals:
   cookieDomain: ""
   platformUrl: ""
   httpMigrations: "0"
+  google:
+    clientId: "" 
+    secret: ""
 
   createSecrets: true # creates an internal API key, JWT secrets and redis password for you
 
@@ -124,7 +127,7 @@ services:
     logLevel: info
 
   worker:
-    port: 4001
+    port: 4003
     replicaCount: 1
 
   couchdb:

hosting/docker-compose.yaml

@@ -7,8 +7,6 @@ services:
     restart: always
     image: budibase.docker.scarf.sh/budibase/apps
     container_name: bbapps
-    ports:
-      - "${APP_PORT}:4002"
     environment:
       SELF_HOSTED: 1
       COUCH_DB_URL: http://${COUCH_DB_USER}:${COUCH_DB_PASSWORD}@couchdb-service:5984
@@ -25,8 +23,6 @@ services:
       ENABLE_ANALYTICS: "true"
       REDIS_URL: redis-service:6379
       REDIS_PASSWORD: ${REDIS_PASSWORD}
-    volumes:
-      - ./logs:/logs
     depends_on:
       - worker-service
       - redis-service
@@ -35,8 +31,6 @@ services:
     restart: always
     image: budibase.docker.scarf.sh/budibase/worker
     container_name: bbworker
-    ports:
-      - "${WORKER_PORT}:4003"
     environment:
       SELF_HOSTED: 1
       PORT: 4003
@@ -53,8 +47,6 @@ services:
       INTERNAL_API_KEY: ${INTERNAL_API_KEY}
       REDIS_URL: redis-service:6379
       REDIS_PASSWORD: ${REDIS_PASSWORD}
-    volumes:
-      - ./logs:/logs
     depends_on:
       - redis-service
       - minio-service
@@ -65,8 +57,6 @@ services:
     image: minio/minio
     volumes:
       - minio_data:/data
-    ports:
-      - "${MINIO_PORT}:9000"
     environment:
       MINIO_ACCESS_KEY: ${MINIO_ACCESS_KEY}
       MINIO_SECRET_KEY: ${MINIO_SECRET_KEY}
@@ -80,10 +70,10 @@ services:
 
   proxy-service:
     restart: always
-    container_name: bbproxy
-    image: budibase/proxy
     ports:
       - "${MAIN_PORT}:10000"
+    container_name: bbproxy
+    image: budibase/proxy
     depends_on:
       - minio-service
       - worker-service
@@ -96,8 +86,6 @@ services:
     environment:
       - COUCHDB_PASSWORD=${COUCH_DB_PASSWORD}
       - COUCHDB_USER=${COUCH_DB_USER}
-    ports:
-      - "${COUCH_DB_PORT}:5984"
     volumes:
       - couchdb3_data:/opt/couchdb/data
 
@@ -113,15 +101,11 @@ services:
     restart: always
     image: redis
     command: redis-server --requirepass ${REDIS_PASSWORD}
-    ports:
-      - "${REDIS_PORT}:6379"
     volumes:
       - redis_data:/data
 
   watchtower-service:
     image: containrrr/watchtower
-    ports:
-      - "${WATCHTOWER_PORT}:8080"
     volumes:
       - /var/run/docker.sock:/var/run/docker.sock
     command: --debug --http-api-update bbapps bbworker bbproxy

hosting/envoy.yaml

@@ -0,0 +1,152 @@
+static_resources:
+  listeners:
+  - name: main_listener
+    address:
+      socket_address: { address: 0.0.0.0, port_value: 10000 }
+    filter_chains:
+    - filters:
+      - name: envoy.filters.network.http_connection_manager
+        typed_config:
+          "@type": type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager
+          stat_prefix: ingress
+          codec_type: auto
+          route_config:
+            name: local_route
+            virtual_hosts:
+            - name: local_services
+              domains: ["*"]
+              routes:
+              - match: { prefix: "/app/" }
+                route:
+                  cluster: app-service
+                  prefix_rewrite: "/"
+
+              - match: { path: "/v1/update" }
+                route:
+                  cluster: watchtower-service
+
+              - match: { prefix: "/builder/" }
+                route:
+                  cluster: app-service
+
+              - match: { prefix: "/builder" }
+                route:
+                  cluster: app-service
+              
+              - match: { prefix: "/app_" }
+                route:
+                  cluster: app-service
+
+              # special cases for worker admin (deprecated), global and system API
+              - match: { prefix: "/api/global/" }
+                route:
+                  cluster: worker-service
+
+              - match: { prefix: "/api/admin/" }
+                route:
+                  cluster: worker-service
+
+              - match: { prefix: "/api/system/" }
+                route:
+                  cluster: worker-service
+
+              - match: { path: "/" }
+                route:
+                  cluster: app-service
+
+              # special case for when API requests are made, can just forward, not to minio    
+              - match: { prefix: "/api/" }
+                route:
+                  cluster: app-service
+                  timeout: 120s
+
+              - match: { prefix: "/worker/" }
+                route: 
+                  cluster: worker-service
+                  prefix_rewrite: "/"
+
+              - match: { prefix: "/db/" }
+                route:
+                  cluster: couchdb-service
+                  prefix_rewrite: "/"
+
+              # minio is on the default route because this works
+              # best, minio + AWS SDK doesn't handle path proxy
+              - match: { prefix: "/" }
+                route:
+                  cluster: minio-service
+
+          http_filters:
+          - name: envoy.filters.http.router
+
+  clusters:
+  - name: app-service
+    connect_timeout: 0.25s
+    type: strict_dns
+    lb_policy: round_robin
+    load_assignment:
+      cluster_name: app-service
+      endpoints:
+      - lb_endpoints:
+        - endpoint:
+            address:
+              socket_address: 
+                address: app-service
+                port_value: 4002
+
+  - name: minio-service
+    connect_timeout: 0.25s
+    type: strict_dns
+    lb_policy: round_robin
+    load_assignment:
+      cluster_name: minio-service
+      endpoints:
+      - lb_endpoints:
+        - endpoint:
+            address:
+              socket_address:
+                address: minio-service
+                port_value: 9000
+
+  - name: worker-service
+    connect_timeout: 0.25s
+    type: strict_dns
+    lb_policy: round_robin
+    load_assignment:
+      cluster_name: worker-service
+      endpoints:
+      - lb_endpoints:
+        - endpoint:
+            address:
+              socket_address: 
+                address: worker-service
+                port_value: 4003
+
+  - name: couchdb-service
+    connect_timeout: 0.25s
+    type: strict_dns
+    lb_policy: round_robin
+    load_assignment:
+      cluster_name: couchdb-service
+      endpoints:
+      - lb_endpoints:
+        - endpoint:
+            address:
+              socket_address:
+                address: couchdb-service
+                port_value: 5984
+
+  - name: watchtower-service
+    connect_timeout: 0.25s
+    type: strict_dns
+    lb_policy: round_robin
+    load_assignment:
+      cluster_name: watchtower-service
+      endpoints:
+      - lb_endpoints:
+        - endpoint:
+            address:
+              socket_address: 
+                address: watchtower-service
+                port_value: 8080
+

hosting/hosting.properties

@@ -0,0 +1,21 @@
+# Use the main port in the builder for your self hosting URL, e.g. localhost:10000
+MAIN_PORT=10000
+
+# This section contains all secrets pertaining to the system
+# These should be updated
+JWT_SECRET=testsecret
+MINIO_ACCESS_KEY=budibase
+MINIO_SECRET_KEY=budibase
+COUCH_DB_PASSWORD=budibase
+COUCH_DB_USER=budibase
+REDIS_PASSWORD=budibase
+INTERNAL_API_KEY=budibase
+
+# This section contains variables that do not need to be altered under normal circumstances
+APP_PORT=4002
+WORKER_PORT=4003
+MINIO_PORT=4004
+COUCH_DB_PORT=4005
+REDIS_PORT=6379
+WATCHTOWER_PORT=6161
+BUDIBASE_ENVIRONMENT=PRODUCTION

hosting/kubernetes/nginx/Dockerfile

@@ -1,2 +0,0 @@
-FROM nginx:latest
-COPY nginx.conf /etc/nginx/nginx.conf

hosting/nginx.dev.conf.hbs

@@ -11,6 +11,7 @@ events {
 http {
   include       /etc/nginx/mime.types;
   default_type  application/octet-stream;
+  proxy_set_header Host $host;
 
   log_format  main  '$remote_addr - $remote_user [$time_local] "$request" '
                     '$status $body_bytes_sent "$http_referer" '
@@ -43,6 +44,10 @@ http {
       proxy_pass      http://{{ address }}:4001;
     }
 
+    location = / {
+      proxy_pass      http://{{ address }}:4001;
+    }
+
     location /app_ {
       proxy_pass      http://{{ address }}:4001;
     }
@@ -63,7 +68,6 @@ http {
       proxy_http_version  1.1;
       proxy_set_header    Connection          $connection_upgrade;
       proxy_set_header    Upgrade             $http_upgrade;
-      proxy_set_header    Host                $host;
       proxy_set_header    X-Real-IP           $remote_addr;
       proxy_set_header    X-Forwarded-For     $proxy_add_x_forwarded_for;
     }
@@ -72,7 +76,6 @@ http {
       proxy_set_header X-Real-IP $remote_addr;
       proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
       proxy_set_header X-Forwarded-Proto $scheme;
-      proxy_set_header Host $http_host;
 
       proxy_connect_timeout 300;
       proxy_http_version 1.1;

hosting/nginx.prod.conf.hbs

@@ -9,20 +9,28 @@ events {
 }
 
 http {
-  limit_req_zone $binary_remote_addr zone=ratelimit:10m rate=10r/s;
+  limit_req_zone $binary_remote_addr zone=ratelimit:10m rate=20r/s;
   include       /etc/nginx/mime.types;
   default_type  application/octet-stream;
+  proxy_set_header Host $host;
   charset utf-8;
   sendfile on;
   tcp_nopush on;
   tcp_nodelay on;
   server_tokens off;
   types_hash_max_size 2048;
+  {{#if compose}}
+  resolver 127.0.0.11 ipv6=off;
+  {{/if}}
+  {{#if k8s}}
+  resolver kube-dns.kube-system.svc.cluster.local valid=10s;
+  {{/if}}
+
 
   # buffering
   client_body_buffer_size 1K;
   client_header_buffer_size 1k;
-  client_max_body_size 1k;
+  client_max_body_size 10M;
   ignore_invalid_headers off;
   proxy_buffering off;
 
@@ -36,49 +44,63 @@ http {
 
   server {
     listen       10000 default_server;
+    listen  [::]:10000 default_server;
     server_name  _;
+    client_max_body_size 1000m;
+    ignore_invalid_headers off;
+    proxy_buffering off;
+    # port_in_redirect off;
 
     # Security Headers
     add_header X-Frame-Options SAMEORIGIN always;
     add_header X-Content-Type-Options nosniff always;
     add_header X-XSS-Protection "1; mode=block" always;
-    add_header Content-Security-Policy  "default-src 'self'; script-src 'self' 'unsafe-inline' 'unsafe-eval' https://cdn.budi.live https://js.intercomcdn.com https://widget.intercom.io; style-src 'self' 'unsafe-inline' https://cdn.jsdelivr.net https://fonts.googleapis.com https://rsms.me; object-src 'none'; base-uri 'self'; connect-src 'self' https://api-iam.intercom.io https://app.posthog.com wss://nexus-websocket-a.intercom.io; font-src 'self' https://cdn.jsdelivr.net https://fonts.gstatic.com https://rsms.me; frame-src 'self'; img-src https: data:; manifest-src 'self'; media-src 'self'; worker-src 'none';" always;
+    add_header Content-Security-Policy  "default-src 'self'; script-src 'self' 'unsafe-inline' 'unsafe-eval' https://cdn.budi.live https://js.intercomcdn.com https://widget.intercom.io; style-src 'self' 'unsafe-inline' https://cdn.jsdelivr.net https://fonts.googleapis.com https://rsms.me https://maxcdn.bootstrapcdn.com; object-src 'none'; base-uri 'self'; connect-src 'self' https://api-iam.intercom.io https://app.posthog.com wss://nexus-websocket-a.intercom.io; font-src 'self' data https://cdn.jsdelivr.net https://fonts.gstatic.com https://rsms.me https://maxcdn.bootstrapcdn.com; frame-src 'self'; img-src http: https: data; manifest-src 'self'; media-src 'self'; worker-src 'none';" always;
+
+    # upstreams
+    set $apps {{ apps }};
+    set $worker {{ worker }};
+    set $minio {{ minio }};
+    set $couchdb {{ couchdb }};
+    {{#if watchtower}}
+    set $watchtower {{ watchtower }};
+    {{/if}}
 
     location /app {
-      proxy_pass      http://app-service:4002;
+      proxy_pass      http://$apps:4002;
       rewrite ^/app/(.*)$ /$1 break;
     }
 
     location = / {
-      proxy_pass      http://app-service.budibase.svc.cluster.local:4002;
+      proxy_pass      http://$apps:4002;
     }
 
-    location /builder/ {
+    {{#if watchtower}}
+    location = /v1/update {
+      proxy_pass      http://$watchtower:8080;
+    }
+    {{/if}}
+    location ~ ^/(builder|app_) {
       proxy_http_version  1.1;
       proxy_set_header    Connection          $connection_upgrade;
       proxy_set_header    Upgrade             $http_upgrade;
-      proxy_set_header    Host                $host;
       proxy_set_header    X-Real-IP           $remote_addr;
       proxy_set_header    X-Forwarded-For     $proxy_add_x_forwarded_for;
-      proxy_pass      http://app-service.budibase.svc.cluster.local:4002;
+      proxy_pass      http://$apps:4002;
-    }
-
-    location ^/(builder|app_) {
-      proxy_pass      http://app-service.budibase.svc.cluster.local:4002;
     }
 
     location ~ ^/api/(system|admin|global)/ {
-      proxy_pass      http://worker-service.budibase.svc.cluster.local:4003;
+      proxy_pass      http://$worker:4003;
     }
 
     location /worker/ {
-      proxy_pass      http://worker-service.budibase.svc.cluster.local:4003;
+      proxy_pass      http://$worker:4003;
       rewrite ^/worker/(.*)$ /$1 break;
     }
 
     location /api/ {
       # calls to the API are rate limited with bursting
-      limit_req zone=ratelimit burst=10 nodelay;
+      limit_req zone=ratelimit burst=20 nodelay;
 
       # 120s timeout on API requests
       proxy_read_timeout 120s;
@@ -88,15 +110,14 @@ http {
       proxy_http_version  1.1;
       proxy_set_header    Connection          $connection_upgrade;
       proxy_set_header    Upgrade             $http_upgrade;
-      proxy_set_header    Host                $host;
       proxy_set_header    X-Real-IP           $remote_addr;
       proxy_set_header    X-Forwarded-For     $proxy_add_x_forwarded_for;
 
-      proxy_pass      http://app-service.budibase.svc.cluster.local:4002;
+      proxy_pass      http://$apps:4002;
     }
 
     location /db/ {
-      proxy_pass      http://budibase-prod-svc-couchdb:5984;
+      proxy_pass      http://$couchdb:5984;
       rewrite ^/db/(.*)$ /$1 break;
     }
 
@@ -104,13 +125,12 @@ http {
       proxy_set_header X-Real-IP $remote_addr;
       proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
       proxy_set_header X-Forwarded-Proto $scheme;
-      proxy_set_header Host $http_host;
 
       proxy_connect_timeout 300;
       proxy_http_version 1.1;
       proxy_set_header Connection "";
       chunked_transfer_encoding off;
-      proxy_pass      http://minio-service.budibase.svc.cluster.local:9000;
+      proxy_pass      http://$minio:9000;
     }
 
     client_header_timeout 60;

hosting/proxy/Dockerfile

@@ -1,2 +1,2 @@
 FROM nginx:latest
-COPY nginx.conf /etc/nginx/nginx.conf
+COPY .generated-nginx.prod.conf /etc/nginx/nginx.conf

hosting/proxy/nginx.conf

@@ -9,7 +9,7 @@ events {
 }
 
 http {
-  limit_req_zone $binary_remote_addr zone=ratelimit:10m rate=10r/s;
+  limit_req_zone $binary_remote_addr zone=ratelimit:10m rate=20r/s;
   include       /etc/nginx/mime.types;
   default_type  application/octet-stream;
   charset utf-8;
@@ -24,7 +24,6 @@ http {
   client_header_buffer_size 1k;
   client_max_body_size 1k;
   ignore_invalid_headers off;
-  proxy_buffering off;
 
 
   log_format  main  '$remote_addr - $remote_user [$time_local] "$request" '
@@ -37,16 +36,18 @@ http {
 
   server { 
     listen       10000 default_server;
+    listen  [::]:10000 default_server;
     server_name  _;
     client_max_body_size 1000m;
     ignore_invalid_headers off;
     proxy_buffering off;
+    port_in_redirect off;
 
     # Security Headers
     add_header X-Frame-Options SAMEORIGIN always;
     add_header X-Content-Type-Options nosniff always;
     add_header X-XSS-Protection "1; mode=block" always;
-    add_header Content-Security-Policy  "default-src 'self'; script-src 'unsafe-inline' 'unsafe-eval' 'self' https://cdn.budi.live https://js.intercomcdn.com https://widget.intercom.io; style-src 'self' 'unsafe-inline' https://cdn.jsdelivr.net https://fonts.googleapis.com https://rsms.me; object-src 'none'; base-uri 'self'; connect-src 'self' https://api-iam.intercom.io https://app.posthog.com wss://nexus-websocket-a.intercom.io; font-src 'self' https://cdn.jsdelivr.net https://fonts.gstatic.com https://rsms.me; frame-src 'self'; img-src https: data:; manifest-src 'self'; media-src 'self'; worker-src 'none';" always;
+    add_header Content-Security-Policy  "default-src 'self'; script-src 'self' 'unsafe-inline' 'unsafe-eval' https://cdn.budi.live https://js.intercomcdn.com https://widget.intercom.io; style-src 'self' 'unsafe-inline' https://cdn.jsdelivr.net https://fonts.googleapis.com https://rsms.me; object-src 'none'; base-uri 'self'; connect-src 'self' https://api-iam.intercom.io https://app.posthog.com wss://nexus-websocket-a.intercom.io; font-src 'self' data: https://cdn.jsdelivr.net https://fonts.gstatic.com https://rsms.me; frame-src 'self'; img-src http: https: data:; manifest-src 'self'; media-src 'self'; worker-src 'none';" always;
 
     location /app {
       proxy_pass      http://app-service:4002;
@@ -54,6 +55,7 @@ http {
     }
 
     location = / {
+      port_in_redirect off;
       proxy_pass      http://app-service:4002;
     }
 
@@ -62,6 +64,7 @@ http {
     }
 
     location /builder/ {
+      port_in_redirect off;
       proxy_http_version  1.1;
       proxy_set_header    Connection          $connection_upgrade;
       proxy_set_header    Upgrade             $http_upgrade;
@@ -71,7 +74,14 @@ http {
       proxy_pass      http://app-service:4002;
     }
 
-    location ^/(builder|app_) {
+    location ~ ^/(builder|app_) {
+      port_in_redirect off;
+      proxy_http_version  1.1;
+      proxy_set_header    Connection          $connection_upgrade;
+      proxy_set_header    Upgrade             $http_upgrade;
+      proxy_set_header    Host                $host;
+      proxy_set_header    X-Real-IP           $remote_addr;
+      proxy_set_header    X-Forwarded-For     $proxy_add_x_forwarded_for;
       proxy_pass      http://app-service:4002;
     }
 
@@ -86,7 +96,7 @@ http {
 
     location /api/ {
       # calls to the API are rate limited with bursting
-      limit_req zone=ratelimit burst=10 nodelay;
+      limit_req zone=ratelimit burst=20 nodelay;
 
       # 120s timeout on API requests
       proxy_read_timeout 120s;

hosting/scripts/airgapped/airgappedDockerBuild.js

@@ -15,7 +15,7 @@ const IMAGES = {
 
 const FILES = {
 	COMPOSE: "docker-compose.yaml",
-	NGINX: "nginx.conf"
+	ENV: ".env"
 }
 
 const OUTPUT_DIR = path.join(__dirname, "../", "bb-airgapped")
@@ -40,8 +40,7 @@ for (let image in IMAGES) {
 
 // copy config files
 copyFile(FILES.COMPOSE)
-copyFile(FILES.ENVOY)
+copyFile(FILES.ENV)
-copyFile(FILES.PROPERTIES)
 
 // compress
 execSync(`tar -czf bb-airgapped.tar.gz hosting/scripts/bb-airgapped`)

lerna.json

@@ -1,5 +1,5 @@
 {
-  "version": "1.0.58-alpha.7",
+  "version": "1.0.79-alpha.0",
   "npmClient": "yarn",
   "packages": [
     "packages/*"

package.json

@@ -42,12 +42,15 @@
     "lint:fix:prettier": "prettier --write \"packages/**/*.{js,ts,svelte}\"",
     "lint:fix:ts": "lerna run lint:fix",
     "lint:fix": "yarn run lint:fix:ts && yarn run lint:fix:prettier && yarn run lint:fix:eslint",
-    "test:e2e": "lerna run cy:test",
+    "test:e2e": "lerna run cy:test --stream",
-    "test:e2e:ci": "lerna run cy:ci",
+    "test:e2e:ci": "lerna run cy:ci --stream",
-    "build:docker": "lerna run build:docker && npm run build:docker:proxy && cd hosting/scripts/linux/ && ./release-to-docker-hub.sh $BUDIBASE_RELEASE_VERSION && cd -",
+    "build:docker": "lerna run build:docker && npm run build:docker:proxy:compose && cd hosting/scripts/linux/ && ./release-to-docker-hub.sh $BUDIBASE_RELEASE_VERSION && cd -",
     "build:docker:proxy": "docker build hosting/proxy -t proxy-service",
+    "build:docker:proxy:compose": "lerna run generate:proxy:compose && npm run build:docker:proxy",
+    "build:docker:proxy:preprod": "lerna run generate:proxy:preprod && npm run build:docker:proxy",
+    "build:docker:proxy:prod": "lerna run generate:proxy:prod && npm run build:docker:proxy",
     "build:docker:selfhost": "lerna run build:docker && cd hosting/scripts/linux/ && ./release-to-docker-hub.sh latest && cd -",
-    "build:docker:develop": "node scripts/pinVersions && lerna run build:docker && npm run build:docker:proxy && cd hosting/scripts/linux/ && ./release-to-docker-hub.sh develop && cd -",
+    "build:docker:develop": "node scripts/pinVersions && lerna run build:docker && npm run build:docker:proxy:compose && cd hosting/scripts/linux/ && ./release-to-docker-hub.sh develop && cd -",
     "build:docker:airgap": "node hosting/scripts/airgapped/airgappedDockerBuild",
     "build:digitalocean": "cd hosting/digitalocean && ./build.sh && cd -",
     "build:docs": "lerna run build:docs",

packages/backend-core/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@budibase/backend-core",
-  "version": "1.0.58-alpha.7",
+  "version": "1.0.79-alpha.0",
   "description": "Budibase backend core libraries used in server and worker",
   "main": "src/index.js",
   "author": "Budibase",

packages/backend-core/src/cloud/accounts.js

@@ -17,7 +17,7 @@ exports.getAccount = async email => {
   const json = await response.json()
 
   if (response.status !== 200) {
-    throw Error(`Error getting account by email ${email}`, json)
+    throw new Error(`Error getting account by email ${email}`, json)
   }
 
   return json[0]

packages/backend-core/src/context/deprovision.js

@@ -1,5 +1,5 @@
 const { getGlobalUserParams, getAllApps } = require("../db/utils")
-const { getDB, getCouch } = require("../db")
+const { getDB } = require("../db")
 const { getGlobalDB } = require("../tenancy")
 const { StaticDatabases } = require("../db/constants")
 
@@ -79,7 +79,7 @@ const removeGlobalDB = async tenantId => {
 
 const removeTenantApps = async tenantId => {
   try {
-    const apps = await getAllApps(getCouch(), { all: true })
+    const apps = await getAllApps({ all: true })
     const destroyPromises = apps.map(app => getDB(app.appId).destroy())
     await Promise.allSettled(destroyPromises)
   } catch (err) {

packages/backend-core/src/context/index.js

@@ -121,7 +121,7 @@ exports.getTenantId = () => {
   }
   const tenantId = cls.getFromContext(ContextKeys.TENANT_ID)
   if (!tenantId) {
-    throw Error("Tenant id not found")
+    throw new Error("Tenant id not found")
   }
   return tenantId
 }

packages/backend-core/src/db/utils.js

@@ -179,8 +179,11 @@ exports.getStartEndKeyURL = (base, baseKey, tenantId = null) => {
 /**
  * if in production this will use the CouchDB _all_dbs call to retrieve a list of databases. If testing
  * when using Pouch it will use the pouchdb-all-dbs package.
+ * opts.efficient can be provided to make sure this call is always quick in a multi-tenant environment,
+ * but it may not be 100% accurate in full efficiency mode (some tenantless apps may be missed).
  */
-exports.getAllDbs = async () => {
+exports.getAllDbs = async (opts = { efficient: false }) => {
+  const efficient = opts && opts.efficient
   // specifically for testing we use the pouch package for this
   if (env.isTest()) {
     return getCouch().allDbs()
@@ -197,7 +200,7 @@ exports.getAllDbs = async () => {
   }
   let couchUrl = `${exports.getCouchUrl()}/_all_dbs`
   let tenantId = getTenantId()
-  if (!env.MULTI_TENANCY || tenantId === DEFAULT_TENANT_ID) {
+  if (!env.MULTI_TENANCY || (!efficient && tenantId === DEFAULT_TENANT_ID)) {
     // just get all DBs when:
     // - single tenancy
     // - default tenant
@@ -225,13 +228,13 @@ exports.getAllDbs = async () => {
  *
  * @return {Promise<object[]>} returns the app information document stored in each app database.
  */
-exports.getAllApps = async ({ dev, all, idsOnly } = {}) => {
+exports.getAllApps = async ({ dev, all, idsOnly, efficient } = {}) => {
   const CouchDB = getCouch()
   let tenantId = getTenantId()
   if (!env.MULTI_TENANCY && !tenantId) {
     tenantId = DEFAULT_TENANT_ID
   }
-  let dbs = await exports.getAllDbs()
+  let dbs = await exports.getAllDbs({ efficient })
   const appDbNames = dbs.filter(dbName => {
     const split = dbName.split(SEPARATOR)
     // it is an app, check the tenantId

packages/backend-core/src/migrations/index.js

@@ -22,6 +22,7 @@ exports.getMigrationsDoc = async db => {
     if (err.status && err.status === 404) {
       return { _id: DocumentTypes.MIGRATIONS }
     }
+    console.error(err)
   }
 }
 
@@ -35,7 +36,7 @@ const runMigration = async (CouchDB, migration, options = {}) => {
   if (migrationType === exports.MIGRATION_TYPES.GLOBAL) {
     dbNames = [getGlobalDBName()]
   } else if (migrationType === exports.MIGRATION_TYPES.APP) {
-    const apps = await getAllApps(CouchDB, migration.opts)
+    const apps = await getAllApps(migration.opts)
     dbNames = apps.map(app => app.appId)
   } else {
     throw new Error(
@@ -94,6 +95,8 @@ exports.runMigrations = async (CouchDB, migrations, options = {}) => {
     if (!options.tenantIds || !options.tenantIds.length) {
       // run for all tenants
       tenantIds = await getTenantIds()
+    } else {
+      tenantIds = options.tenantIds
     }
   } else {
     // single tenancy

packages/bbui/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@budibase/bbui",
   "description": "A UI solution used in the different Budibase projects.",
-  "version": "1.0.58-alpha.7",
+  "version": "1.0.79-alpha.0",
   "license": "MPL-2.0",
   "svelte": "src/index.js",
   "module": "dist/bbui.es.js",
@@ -38,6 +38,7 @@
   ],
   "dependencies": {
     "@adobe/spectrum-css-workflow-icons": "^1.2.1",
+    "@budibase/string-templates": "^1.0.79-alpha.0",
     "@spectrum-css/actionbutton": "^1.0.1",
     "@spectrum-css/actiongroup": "^1.0.1",
     "@spectrum-css/avatar": "^3.0.2",

packages/bbui/src/Button/Button.svelte

@@ -17,8 +17,7 @@
   let showTooltip = false
 </script>
 
-<div class:container={!!tooltip}>
+<button
-  <button
   class:spectrum-Button--cta={cta}
   class:spectrum-Button--primary={primary}
   class:spectrum-Button--secondary={secondary}
@@ -32,7 +31,7 @@
   on:mouseover={() => (showTooltip = true)}
   on:focus={() => (showTooltip = true)}
   on:mouseleave={() => (showTooltip = false)}
-  >
+>
   {#if icon}
     <svg
       class="spectrum-Icon spectrum-Icon--size{size.toUpperCase()}"
@@ -58,21 +57,16 @@
       </svg>
     </div>
   {/if}
-  </button>
   {#if showTooltip && tooltip}
-    <div class="position">
     <div class="tooltip">
       <Tooltip textWrapping={true} direction={"bottom"} text={tooltip} />
     </div>
-    </div>
   {/if}
-</div>
+</button>
 
 <style>
-  .container {
+  button {
-    display: flex;
+    position: relative;
-    align-items: center;
-    flex-direction: column;
   }
   .spectrum-Button-label {
     white-space: nowrap;
@@ -90,12 +84,8 @@
     width: 160px;
     text-align: center;
     transform: translateX(-50%);
-    top: -5px;
+    left: 50%;
-  }
+    top: calc(100% - 3px);
-  .position {
-    position: relative;
-    width: 0;
-    height: 0;
   }
   .tooltip-icon {
     padding-left: var(--spacing-m);

packages/bbui/src/ButtonGroup/ButtonGroup.svelte

@@ -1,6 +1,16 @@
 <script>
   import "@spectrum-css/buttongroup/dist/index-vars.css"
   export let vertical = false
+  export let gap = ""
+
+  $: gapStyle =
+    gap === "L"
+      ? "var(--spacing-l)"
+      : gap === "M"
+      ? "var(--spacing-m)"
+      : gap === "S"
+      ? "var(--spacing-s)"
+      : null
 
   function group(element) {
     const buttons = Array.from(element.getElementsByTagName("button"))
@@ -14,6 +24,7 @@
   use:group
   class="spectrum-ButtonGroup"
   class:spectrum-ButtonGroup--vertical={vertical}
+  style={gapStyle ? `gap: ${gapStyle};` : null}
 >
   <slot />
 </div>

packages/bbui/src/ColorPicker/ColorPicker.svelte

@@ -10,6 +10,7 @@
   export let value
   export let size = "M"
   export let spectrumTheme
+  export let alignRight = false
 
   let open = false
 
@@ -133,6 +134,7 @@
       use:clickOutside={() => (open = false)}
       transition:fly={{ y: -20, duration: 200 }}
       class="spectrum-Popover spectrum-Popover--bottom spectrum-Picker-popover is-open"
+      class:spectrum-Popover--align-right={alignRight}
     >
       {#each categories as category}
         <div class="category">
@@ -250,6 +252,9 @@
     align-items: stretch;
     gap: var(--spacing-xl);
   }
+  .spectrum-Popover--align-right {
+    right: 0;
+  }
   .colors {
     display: grid;
     grid-template-columns: 1fr 1fr 1fr 1fr 1fr 1fr 1fr 1fr;

packages/bbui/src/Drawer/Drawer.svelte

@@ -1,5 +1,4 @@
 <script>
-  import { slide } from "svelte/transition"
   import Portal from "svelte-portal"
   import Button from "../Button/Button.svelte"
   import Body from "../Typography/Body.svelte"
@@ -7,7 +6,9 @@
 
   export let title
   export let fillWidth
+
   let visible = false
+
   export function show() {
     if (visible) {
       return
@@ -21,11 +22,27 @@
     }
     visible = false
   }
+
+  const easeInOutQuad = x => {
+    return x < 0.5 ? 2 * x * x : 1 - Math.pow(-2 * x + 2, 2) / 2
+  }
+
+  // Use a custom svelte transition here because the built-in slide
+  // transition has a horrible overshoot
+  const slide = () => {
+    return {
+      duration: 360,
+      css: t => {
+        const translation = 100 - Math.round(easeInOutQuad(t) * 100)
+        return `transform: translateY(${translation}%);`
+      },
+    }
+  }
 </script>
 
 {#if visible}
   <Portal>
-    <section class:fillWidth class="drawer" transition:slide>
+    <section class:fillWidth class="drawer" transition:slide|local>
       <header>
         <div class="text">
           <Heading size="XS">{title}</Heading>
@@ -44,6 +61,11 @@
 {/if}
 
 <style>
+  .buttons {
+    display: flex;
+    gap: var(--spacing-m);
+  }
+
   .drawer {
     position: absolute;
     bottom: 0;

packages/bbui/src/Modal/Modal.svelte

@@ -54,14 +54,22 @@
 
 <svelte:window on:keydown={handleKey} />
 
-<!-- These svelte if statements need to be defined like this. -->
+{#if inline}
-<!-- The modal transitions do not work if nested inside more than one "if" -->
+  {#if visible}
-{#if visible && inline}
     <div use:focusFirstInput class="spectrum-Modal inline is-open">
       <slot />
     </div>
-{:else if visible}
+  {/if}
+{:else}
+  <!--
+    We cannot conditionally render the portal as this leads to a missing
+    insertion point when using nested modals. Therefore we just conditionally
+    render the content of the portal.
+    It still breaks the modal animation, but its better than soft bricking the
+    screen.
+  -->
   <Portal target=".modal-container">
+    {#if visible}
       <div
         class="spectrum-Underlay is-open"
         in:fade={{ duration: 200 }}
@@ -82,6 +90,7 @@
           </div>
         </div>
       </div>
+    {/if}
   </Portal>
 {/if}
 

packages/bbui/src/Table/AttachmentRenderer.svelte

@@ -17,14 +17,16 @@
 {#each attachments as attachment}
   {#if isImage(attachment.extension)}
     <Link quiet target="_blank" href={attachment.url}>
+      <div class="center">
         <img src={attachment.url} alt={attachment.extension} />
+      </div>
     </Link>
   {:else}
     <Tooltip text={attachment.name} direction="right">
       <div class="file">
-        <Link quiet target="_blank" href={attachment.url}
+        <Link quiet target="_blank" href={attachment.url}>
-          >{attachment.extension}</Link
+          {attachment.extension}
-        >
+        </Link>
       </div>
     </Tooltip>
   {/if}
@@ -38,12 +40,15 @@
     height: 32px;
     max-width: 64px;
   }
+  .center,
   .file {
-    height: 32px;
     display: flex;
     flex-direction: row;
     justify-content: flex-start;
     align-items: center;
+  }
+  .file {
+    height: 32px;
     padding: 0 8px;
     color: var(--spectrum-global-color-gray-800);
     border: 1px solid var(--spectrum-global-color-gray-300);

packages/bbui/src/Table/BoldRenderer.svelte

@@ -7,5 +7,9 @@
 <style>
   .bold {
     font-weight: bold;
+    overflow: hidden;
+    text-overflow: ellipsis;
+    white-space: nowrap;
+    max-width: var(--max-cell-width);
   }
 </style>

packages/bbui/src/Table/CellRenderer.svelte

@@ -6,6 +6,7 @@
   import AttachmentRenderer from "./AttachmentRenderer.svelte"
   import ArrayRenderer from "./ArrayRenderer.svelte"
   import InternalRenderer from "./InternalRenderer.svelte"
+  import { processStringSync } from "@budibase/string-templates"
 
   export let row
   export let schema
@@ -28,10 +29,33 @@
   $: type = schema?.type ?? "string"
   $: customRenderer = customRenderers?.find(x => x.column === schema?.name)
   $: renderer = customRenderer?.component ?? typeMap[type] ?? StringRenderer
+  $: width = schema?.width || "150px"
+  $: cellValue = getCellValue(value, schema.template)
+
+  const getCellValue = (value, template) => {
+    if (!template) {
+      return value
+    }
+    return processStringSync(template, { value })
+  }
 </script>
 
-{#if renderer && (customRenderer || (value != null && value !== ""))}
+{#if renderer && (customRenderer || (cellValue != null && cellValue !== ""))}
-  <svelte:component this={renderer} {row} {schema} {value} on:clickrelationship>
+  <div style="--max-cell-width: {schema.width ? 'none' : '200px'};">
+    <svelte:component
+      this={renderer}
+      {row}
+      {schema}
+      value={cellValue}
+      on:clickrelationship
+    >
       <slot />
     </svelte:component>
+  </div>
 {/if}
+
+<style>
+  div {
+    display: contents;
+  }
+</style>

packages/bbui/src/Table/CodeRenderer.svelte

@@ -3,3 +3,12 @@
 </script>
 
 <code>{value}</code>
+
+<style>
+  code {
+    overflow: hidden;
+    text-overflow: ellipsis;
+    white-space: nowrap;
+    max-width: var(--max-cell-width);
+  }
+</style>

packages/bbui/src/Table/DateTimeRenderer.svelte

@@ -17,6 +17,8 @@
 
 <style>
   div {
-    width: 200px;
+    overflow: hidden;
+    text-overflow: ellipsis;
+    white-space: nowrap;
   }
 </style>

packages/bbui/src/Table/InternalRenderer.svelte

@@ -43,11 +43,3 @@
 <div on:click|stopPropagation={onClick}>
   <Icon size="S" name="Copy" />
 </div>
-
-<style>
-  div {
-    overflow: hidden;
-    text-overflow: ellipsis;
-    width: 150px;
-  }
-</style>

packages/bbui/src/Table/StringRenderer.svelte

@@ -8,6 +8,7 @@
   div {
     overflow: hidden;
     text-overflow: ellipsis;
-    width: 150px;
+    white-space: nowrap;
+    max-width: var(--max-cell-width);
   }
 </style>

packages/bbui/src/Table/Table.svelte

@@ -4,6 +4,7 @@
   import CellRenderer from "./CellRenderer.svelte"
   import SelectEditRenderer from "./SelectEditRenderer.svelte"
   import { cloneDeep, deepGet } from "../helpers"
+  import ProgressCircle from "../ProgressCircle/ProgressCircle.svelte"
 
   /**
    * The expected schema is our normal couch schemas for our tables.
@@ -14,6 +15,11 @@
    * sortable: Set to false to disable sorting data by a certain column
    * editable: Set to false to disable editing a certain column if the
    *  allowEditColumns prop is true
+   * width: the width of the column
+   * align: the alignment of the column
+   * template: a HBS or JS binding to use as the value
+   * background: the background color
+   * color: the text color
    */
   export let data = []
   export let schema = {}
@@ -28,13 +34,14 @@
   export let editColumnTitle = "Edit"
   export let customRenderers = []
   export let disableSorting = false
+  export let autoSortColumns = true
+  export let compact = false
 
   const dispatch = createEventDispatcher()
 
   // Config
-  const rowHeight = 55
   const headerHeight = 36
-  const rowPreload = 5
+  $: rowHeight = compact ? 46 : 55
 
   // Sorting state
   let sortColumn
@@ -45,32 +52,20 @@
   let loaded = false
   $: schema = fixSchema(schema)
   $: if (!loading) loaded = true
-  $: rows = data ?? []
+  $: fields = getFields(schema, showAutoColumns, autoSortColumns)
-  $: visibleRowCount = getVisibleRowCount(loaded, height, rows.length, rowCount)
+  $: rows = fields?.length ? data || [] : []
-  $: contentStyle = getContentStyle(visibleRowCount, rowCount)
+  $: visibleRowCount = getVisibleRowCount(
-  $: sortedRows = sortRows(rows, sortColumn, sortOrder)
+    loaded,
-  $: fields = getFields(schema, showAutoColumns)
+    height,
-  $: showEditColumn = allowEditRows || allowSelectRows
+    rows.length,
-
+    rowCount,
-  // Scrolling state
+    rowHeight
-  let timeout
-  let nextScrollTop = 0
-  let scrollTop = 0
-  $: firstVisibleRow = calculateFirstVisibleRow(scrollTop)
-  $: lastVisibleRow = calculateLastVisibleRow(
-    firstVisibleRow,
-    visibleRowCount,
-    rows.length
   )
-
+  $: contentStyle = getContentStyle(visibleRowCount, rowCount, rowHeight)
-  // Reset state when data changes
+  $: sortedRows = sortRows(rows, sortColumn, sortOrder)
-  $: rows.length, reset()
+  $: gridStyle = getGridStyle(fields, schema, showEditColumn)
-  const reset = () => {
+  $: showEditColumn = allowEditRows || allowSelectRows
-    nextScrollTop = 0
+  $: cellStyles = computeCellStyles(schema)
-    scrollTop = 0
-    clearTimeout(timeout)
-    timeout = null
-  }
 
   const fixSchema = schema => {
     let fixedSchema = {}
@@ -90,7 +85,7 @@
     return fixedSchema
   }
 
-  const getVisibleRowCount = (loaded, height, allRows, rowCount) => {
+  const getVisibleRowCount = (loaded, height, allRows, rowCount, rowHeight) => {
     if (!loaded) {
       return rowCount || 0
     }
@@ -100,11 +95,28 @@
     return Math.min(allRows, Math.ceil(height / rowHeight))
   }
 
-  const getContentStyle = (visibleRows, rowCount) => {
+  const getContentStyle = (visibleRows, rowCount, rowHeight) => {
     if (!rowCount || !visibleRows) {
       return ""
     }
-    return `height: ${headerHeight + visibleRows * (rowHeight + 1)}px;`
+    return `height: ${headerHeight + visibleRows * rowHeight}px;`
+  }
+
+  const getGridStyle = (fields, schema, showEditColumn) => {
+    let style = "grid-template-columns:"
+    if (showEditColumn) {
+      style += " auto"
+    }
+    fields?.forEach(field => {
+      const fieldSchema = schema[field]
+      if (fieldSchema.width) {
+        style += ` ${fieldSchema.width}`
+      } else {
+        style += " minmax(auto, 1fr)"
+      }
+    })
+    style += ";"
+    return style
   }
 
   const sortRows = (rows, sortColumn, sortOrder) => {
@@ -143,14 +155,14 @@
     return name || ""
   }
 
-  const getFields = (schema, showAutoColumns) => {
+  const getFields = (schema, showAutoColumns, autoSortColumns) => {
     let columns = []
     let autoColumns = []
     Object.entries(schema || {}).forEach(([field, fieldSchema]) => {
       if (!field || !fieldSchema) {
         return
       }
-      if (!fieldSchema?.autocolumn) {
+      if (!autoSortColumns || !fieldSchema?.autocolumn) {
         columns.push(fieldSchema)
       } else if (showAutoColumns) {
         autoColumns.push(fieldSchema)
@@ -171,28 +183,6 @@
       .map(column => column.name)
   }
 
-  const onScroll = event => {
-    nextScrollTop = event.target.scrollTop
-    if (timeout) {
-      return
-    }
-    timeout = setTimeout(() => {
-      scrollTop = nextScrollTop
-      timeout = null
-    }, 50)
-  }
-
-  const calculateFirstVisibleRow = scrollTop => {
-    return Math.max(Math.floor(scrollTop / (rowHeight + 1)) - rowPreload, 0)
-  }
-
-  const calculateLastVisibleRow = (firstRow, visibleRowCount, allRowCount) => {
-    if (visibleRowCount === 0) {
-      return -1
-    }
-    return Math.min(firstRow + visibleRowCount + 2 * rowPreload, allRowCount)
-  }
-
   const editColumn = (e, field) => {
     e.stopPropagation()
     dispatch("editcolumn", field)
@@ -213,33 +203,57 @@
       selectedRows = [...selectedRows, row]
     }
   }
+
+  const computeCellStyles = schema => {
+    let styles = {}
+    Object.keys(schema || {}).forEach(field => {
+      styles[field] = ""
+      if (schema[field].color) {
+        styles[field] += `color: ${schema[field].color};`
+      }
+      if (schema[field].background) {
+        styles[field] += `background-color: ${schema[field].background};`
+      }
+      if (schema[field].align === "Center") {
+        styles[field] += "justify-content: center; text-align: center;"
+      }
+      if (schema[field].align === "Right") {
+        styles[field] += "justify-content: flex-end; text-align: right;"
+      }
+    })
+    return styles
+  }
 </script>
 
-<div class="wrapper" bind:offsetHeight={height}>
+<div
-  {#if !loaded}
+  class="wrapper"
-    <div class="loading" style={contentStyle} />
+  class:wrapper--quiet={quiet}
-  {:else}
+  class:wrapper--compact={compact}
-    <div
+  bind:offsetHeight={height}
-      on:scroll={onScroll}
-      class:quiet
   style={`--row-height: ${rowHeight}px; --header-height: ${headerHeight}px;`}
-      class="container"
+>
-    >
+  {#if !loaded}
-      <div style={contentStyle}>
+    <div class="loading" style={contentStyle}>
-        <table class="spectrum-Table" class:spectrum-Table--quiet={quiet}>
+      <ProgressCircle />
+    </div>
+  {:else}
+    <div class="spectrum-Table" style={`${contentStyle}${gridStyle}`}>
       {#if fields.length}
-            <thead class="spectrum-Table-head">
+        <div class="spectrum-Table-head">
-              <tr>
           {#if showEditColumn}
-                  <th class="spectrum-Table-headCell">
+            <div
-                    <div class="spectrum-Table-headCell-content">
+              class="spectrum-Table-headCell spectrum-Table-headCell--divider spectrum-Table-headCell--edit"
+            >
               {editColumnTitle || ""}
             </div>
-                  </th>
           {/if}
           {#each fields as field}
-                  <th
+            <div
               class="spectrum-Table-headCell"
+              class:spectrum-Table-headCell--alignCenter={schema[field]
+                .align === "Center"}
+              class:spectrum-Table-headCell--alignRight={schema[field].align ===
+                "Right"}
               class:is-sortable={schema[field].sortable !== false}
               class:is-sorted-desc={sortColumn === field &&
                 sortOrder === "Descending"}
@@ -247,7 +261,6 @@
                 sortOrder === "Ascending"}
               on:click={() => sortBy(schema[field])}
             >
-                    <div class="spectrum-Table-headCell-content">
               <div class="title">{getDisplayName(schema[field])}</div>
               {#if schema[field]?.autocolumn}
                 <svg
@@ -276,26 +289,20 @@
                 </svg>
               {/if}
             </div>
-                  </th>
           {/each}
-              </tr>
+        </div>
-            </thead>
       {/if}
-          <tbody class="spectrum-Table-body">
+      {#if sortedRows?.length}
-            {#if sortedRows?.length && fields.length}
         {#each sortedRows as row, idx}
-                <tr
+          <div
+            class="spectrum-Table-row"
             on:click={() => dispatch("click", row)}
             on:click={() => toggleSelectRow(row)}
-                  class="spectrum-Table-row"
-                  class:hidden={idx < firstVisibleRow || idx > lastVisibleRow}
           >
-                  {#if idx >= firstVisibleRow && idx <= lastVisibleRow}
             {#if showEditColumn}
-                      <td
+              <div
-                        class="spectrum-Table-cell spectrum-Table-cell--divider"
+                class="spectrum-Table-cell spectrum-Table-cell--divider spectrum-Table-cell--edit"
               >
-                        <div class="spectrum-Table-cell-content">
                 <SelectEditRenderer
                   data={row}
                   selected={selectedRows.includes(row)}
@@ -305,15 +312,13 @@
                   {allowEditRows}
                 />
               </div>
-                      </td>
             {/if}
             {#each fields as field}
-                      <td
+              <div
                 class="spectrum-Table-cell"
-                        class:spectrum-Table-cell--divider={!!schema[field]
+                class:spectrum-Table-cell--divider={!!schema[field].divider}
-                          .divider}
+                style={cellStyles[field]}
               >
-                        <div class="spectrum-Table-cell-content">
                 <CellRenderer
                   {customRenderers}
                   {row}
@@ -324,59 +329,107 @@
                   <slot />
                 </CellRenderer>
               </div>
-                      </td>
             {/each}
-                  {/if}
+          </div>
-                </tr>
         {/each}
       {:else}
-              <tr class="placeholder-row">
+        <div class="placeholder" class:placeholder--no-fields={!fields?.length}>
-                {#if showEditColumn}
-                  <td class="placeholder-offset" />
-                {/if}
-                {#each fields as field}
-                  <td />
-                {/each}
-                <div class="placeholder" class:has-fields={fields.length > 0}>
           <div class="placeholder-content">
-                    <svg
+            <svg class="spectrum-Icon spectrum-Icon--sizeXXL" focusable="false">
-                      class="spectrum-Icon spectrum-Icon--sizeXXL"
-                      focusable="false"
-                    >
               <use xlink:href="#spectrum-icon-18-Table" />
             </svg>
             <div>No rows found</div>
           </div>
         </div>
-              </tr>
       {/if}
-          </tbody>
-        </table>
-      </div>
     </div>
   {/if}
 </div>
 
 <style>
+  /* Wrapper */
   .wrapper {
-    background-color: var(--spectrum-alias-background-color-secondary);
-    overflow: hidden;
     position: relative;
     z-index: 0;
+    --table-bg: var(--spectrum-global-color-gray-50);
+    --table-border: 1px solid var(--spectrum-alias-border-color-mid);
+    --cell-padding: var(--spectrum-global-dimension-size-250);
+  }
+  .wrapper--quiet {
+    --table-bg: var(--spectrum-alias-background-color-transparent);
+  }
+  .wrapper--compact {
+    --cell-padding: var(--spectrum-global-dimension-size-150);
   }
 
-  .container {
+  /* Loading */
-    height: 100%;
+  .loading {
-    position: relative;
+    display: grid;
+    place-items: center;
+    min-height: 100px;
+  }
+
+  /* Table */
+  .spectrum-Table {
+    width: 100%;
+    border-radius: 0;
+    display: grid;
     overflow: auto;
   }
-  .container.quiet {
-    border: none;
-  }
-  table {
-    width: 100%;
-  }
 
+  /* Header */
+  .spectrum-Table-head {
+    display: contents;
+  }
+  .spectrum-Table-head > :first-child {
+    border-left: 1px solid transparent;
+    padding-left: var(--cell-padding);
+  }
+  .spectrum-Table-head > :last-child {
+    border-right: 1px solid transparent;
+    padding-right: var(--cell-padding);
+  }
+  .spectrum-Table-headCell {
+    height: var(--header-height);
+    position: sticky;
+    top: 0;
+    text-overflow: ellipsis;
+    white-space: nowrap;
+    background-color: var(--spectrum-alias-background-color-secondary);
+    z-index: 2;
+    border-bottom: var(--table-border);
+    padding: 0 calc(var(--cell-padding) / 1.33);
+    display: flex;
+    flex-direction: row;
+    justify-content: flex-start;
+    align-items: center;
+    user-select: none;
+  }
+  .spectrum-Table-headCell--alignCenter {
+    justify-content: center;
+  }
+  .spectrum-Table-headCell--alignRight {
+    justify-content: flex-end;
+  }
+  .spectrum-Table-headCell--divider {
+    padding-right: var(--cell-padding);
+  }
+  .spectrum-Table-headCell--divider + .spectrum-Table-headCell {
+    padding-left: var(--cell-padding);
+  }
+  .spectrum-Table-headCell--edit {
+    position: sticky;
+    left: 0;
+    z-index: 3;
+  }
+  .spectrum-Table-headCell .title {
+    overflow: hidden;
+    text-overflow: ellipsis;
+  }
+  .spectrum-Table-headCell:hover .spectrum-Table-editIcon {
+    opacity: 1;
+    transition: opacity 0.2s ease;
+  }
   .spectrum-Table-headCell .spectrum-Icon {
     pointer-events: all;
     margin-left: var(
@@ -392,63 +445,93 @@
   .spectrum-Table-editIcon {
     opacity: 0;
   }
-  .spectrum-Table-headCell:hover .spectrum-Table-editIcon {
+
-    opacity: 1;
+  /* Table rows */
-    transition: opacity 0.2s ease;
+  .spectrum-Table-row {
+    display: contents;
+  }
+  .spectrum-Table-row:hover .spectrum-Table-cell {
+    /*background-color: var(--hover-bg) !important;*/
+  }
+  .spectrum-Table-row:hover .spectrum-Table-cell:after {
+    background-color: var(--spectrum-alias-highlight-hover);
+  }
+  .wrapper--quiet .spectrum-Table-row {
+    border-left: none;
+    border-right: none;
+  }
+  .spectrum-Table-row > :first-child {
+    border-left: var(--table-border);
+    padding-left: var(--cell-padding);
+  }
+  .spectrum-Table-row > :last-child {
+    border-right: var(--table-border);
+    padding-right: var(--cell-padding);
   }
 
-  th {
+  /* Table cells */
-    vertical-align: middle;
+  .spectrum-Table-cell {
-    height: var(--header-height);
+    flex: 1 1 auto;
-    position: sticky;
+    padding: 0 calc(var(--cell-padding) / 1.33);
-    top: 0;
+    border-top: none;
-    z-index: 2;
+    border-bottom: none;
-    background-color: var(--spectrum-alias-background-color-secondary);
+    border-radius: 0;
-    border-bottom: 1px solid
+    text-overflow: ellipsis;
-      var(--spectrum-table-border-color, var(--spectrum-alias-border-color-mid));
-  }
-  .spectrum-Table-headCell-content {
     white-space: nowrap;
+    height: var(--row-height);
     display: flex;
     flex-direction: row;
     justify-content: flex-start;
     align-items: center;
-    user-select: none;
+    gap: 4px;
+    border-bottom: 1px solid var(--spectrum-alias-border-color-mid);
+    background-color: var(--table-bg);
+    z-index: 1;
   }
-  .spectrum-Table-headCell-content .title {
+  .spectrum-Table-cell--divider {
-    overflow: hidden;
+    padding-right: var(--cell-padding);
-    text-overflow: ellipsis;
+  }
+  .spectrum-Table-cell--divider + .spectrum-Table-cell {
+    padding-left: var(--cell-padding);
+  }
+  .spectrum-Table-cell--edit {
+    position: sticky;
+    left: 0;
+    z-index: 2;
+  }
+  .spectrum-Table-cell:after {
+    content: "";
+    position: absolute;
+    width: 100%;
+    height: 100%;
+    background-color: transparent;
+    top: 0;
+    left: 0;
+    pointer-events: none;
+    transition: background-color
+      var(--spectrum-global-animation-duration-100, 0.13s) ease-in-out;
   }
 
-  .placeholder-row {
+  /* Placeholder  */
-    position: relative;
-    height: 150px;
-  }
-  .placeholder-row td {
-    border-top: none !important;
-    border-bottom: none !important;
-  }
-  .placeholder-offset {
-    width: 1px;
-  }
   .placeholder {
-    top: 0;
-    height: 100%;
-    left: 0;
-    width: 100%;
-    position: absolute;
     display: flex;
     flex-direction: row;
     justify-content: center;
     align-items: center;
+    border: var(--table-border);
+    border-top: none;
+    grid-column: 1 / -1;
+    background-color: var(--table-bg);
   }
-  .placeholder.has-fields {
+  .placeholder--no-fields {
-    top: var(--header-height);
+    border-top: var(--table-border);
-    height: calc(100% - var(--header-height));
+  }
+  .wrapper--quiet .placeholder {
+    border-left: none;
+    border-right: none;
   }
-
   .placeholder-content {
-    padding: 20px;
+    padding: 40px;
     display: flex;
     flex-direction: column;
     justify-content: center;
@@ -466,41 +549,4 @@
     );
     text-align: center;
   }
-
-  tbody {
-    z-index: 1;
-  }
-  tbody tr {
-    height: var(--row-height);
-  }
-  tbody tr.hidden {
-    height: calc(var(--row-height) + 1px);
-  }
-  td {
-    padding-top: 0;
-    padding-bottom: 0;
-    border-bottom: none;
-    border-top: 1px solid
-      var(--spectrum-table-border-color, var(--spectrum-alias-border-color-mid));
-    border-radius: 0;
-  }
-  tr:first-child td {
-    border-top: none;
-  }
-  tr:last-child td {
-    border-bottom: 1px solid
-      var(--spectrum-table-border-color, var(--spectrum-alias-border-color-mid));
-  }
-  td.spectrum-Table-cell--divider {
-    width: 1px;
-  }
-  .spectrum-Table-cell-content {
-    height: var(--row-height);
-    white-space: nowrap;
-    display: flex;
-    flex-direction: row;
-    justify-content: flex-start;
-    align-items: center;
-    gap: 4px;
-  }
 </style>

packages/builder/cypress.json

@@ -1,9 +1,11 @@
 {
-  "baseUrl": "http://localhost:10001/builder/",
+  "baseUrl": "http://localhost:4100",
-  "video": true,
+  "video": false,
   "projectId": "bmbemn",
   "env": {
-    "PORT": "10001",
+    "PORT": "4100",
-    "JWT_SECRET": "test"
+    "WORKER_PORT": "4200",
+    "JWT_SECRET": "test",
+    "HOST_IP": ""
   }
 }

packages/builder/cypress/integration/addMultiOptionDatatype.spec.js

@@ -1,4 +1,7 @@
-context("Add Multi-Option Datatype", () => {
+import filterTests from "../support/filterTests"
+
+filterTests(['all'], () => {
+  context("Add Multi-Option Datatype", () => {
     before(() => {
       cy.login()
       cy.createTestApp()
@@ -41,4 +44,5 @@ context("Add Multi-Option Datatype", () => {
         .contains("(5)")
       })
     })
+  })
 })

packages/builder/cypress/integration/addRadioButtons.spec.js

@@ -1,10 +1,13 @@
-context("Add Radio Buttons", () => {
+import filterTests from "../support/filterTests"
+
+filterTests(['all'], () => {
+    context("Add Radio Buttons", () => {
         before(() => {
         cy.login()
         cy.createTestApp()
         })
 
-it("should add Radio Buttons options picker on form, add data, and confirm", () => {
+    it("should add Radio Buttons options picker on form, add data, and confirm", () => {
         cy.navigateToFrontend()
         cy.addComponent("Form", "Form")
         cy.addComponent("Form", "Options Picker").then((componentId) => {
@@ -32,4 +35,5 @@ it("should add Radio Buttons options picker on form, add data, and confirm", ()
             })
             cy.addCustomSourceOptions(totalRadioButtons)
         }
+    })
 })

packages/builder/cypress/integration/autoScreensUI.spec.js

@@ -0,0 +1,51 @@
+import filterTests from "../support/filterTests"
+
+filterTests(['smoke', 'all'], () => {
+    context("Auto Screens UI", () => {
+        before(() => {
+        cy.login()
+        cy.createTestApp()
+        })
+        
+        it("should generate internal table screens", () => {
+            // Create autogenerated screens from the internal table
+            cy.createAutogeneratedScreens(["Cypress Tests"])
+            // Confirm screens have been auto generated
+            cy.get(".nav-items-container").contains("cypress-tests").click({ force: true })
+            cy.get(".nav-items-container").should('contain', 'cypress-tests/:id')
+            .and('contain', 'cypress-tests/new/row')
+        })
+        
+        it("should generate multiple internal table screens at once", () => {
+            // Create a second internal table
+            const initialTable = "Cypress Tests"
+            const secondTable = "Table Two"
+            cy.createTable(secondTable)
+            // Create autogenerated screens from the internal tables
+            cy.createAutogeneratedScreens([initialTable, secondTable])
+            // Confirm screens have been auto generated
+            cy.get(".nav-items-container").contains("cypress-tests").click({ force: true })
+            // Previously generated tables are suffixed with numbers - as expected
+            cy.get(".nav-items-container").should('contain', 'cypress-tests-2/:id')
+            .and('contain', 'cypress-tests-2/new/row')
+            cy.get(".nav-items-container").contains("table-two").click()
+            cy.get(".nav-items-container").should('contain', 'table-two/:id')
+            .and('contain', 'table-two/new/row')
+        })
+        
+        if (Cypress.env("TEST_ENV")) {
+            it("should generate data source screens", () => {
+                // Using MySQL data source for testing this
+                const datasource = "MySQL"
+                // Select & configure MySQL data source
+                cy.selectExternalDatasource(datasource)
+                cy.addDatasourceConfig(datasource)
+                // Create autogenerated screens from a MySQL table - MySQL contains books table
+                cy.createAutogeneratedScreens(["books"])
+                cy.get(".nav-items-container").contains("books").click()
+                cy.get(".nav-items-container").should('contain', 'books/:id')
+                .and('contain', 'books/new/row')
+            })
+        }
+    })
+})

packages/builder/cypress/integration/changeAppIconAndColour.spec.js

@@ -0,0 +1,43 @@
+import filterTests from "../support/filterTests"
+
+filterTests(['all'], () => {
+  context("Change Application Icon and Colour", () => {
+      before(() => {
+        cy.login()
+      })
+      
+      it("should change the icon and colour for an application", () => {
+          // Search for test application
+          cy.searchForApplication("Cypress Tests")
+          cy.get(".appTable")
+            .within(() => {
+              cy.get(".spectrum-Icon").eq(1).click()
+            })
+          cy.get(".spectrum-Menu").contains("Edit icon").click()
+          // Select random icon
+          cy.get(".grid").within(() => {
+              cy.get(".icon-item").eq(Math.floor(Math.random() * 23) + 1).click()
+          })
+          // Select random colour
+          cy.get(".fill").click()
+          cy.get(".colors").within(() => {
+              cy.get(".color").eq(Math.floor(Math.random() * 33) + 1).click()
+          })
+          cy.intercept('**/applications/**').as('iconChange')
+          cy.get(".spectrum-Button").contains("Save").click({ force: true })
+          cy.wait("@iconChange")
+          cy.get("@iconChange").its('response.statusCode')
+          .should('eq', 200)
+          cy.wait(1000)
+          // Confirm icon has changed from default
+          // Confirm colour has been applied - There is no default colour
+          cy.get(".appTable")
+            .within(() => {
+              cy.get('[aria-label]').eq(0).children()
+              .should('have.attr', 'xlink:href').and('not.contain', '#spectrum-icon-18-Apps')
+              cy.get(".title").children().children()
+              .should('have.attr', 'style').and('contains', 'color')
+            })
+      })
+  })
+})

packages/builder/cypress/integration/createApp.spec.js

@@ -1,8 +1,12 @@
-context("Create an Application", () => {
+import filterTests from '../support/filterTests'
+
+filterTests(['smoke', 'all'], () => {
+  context("Create an Application", () => {
     it("should create a new application", () => {
       cy.login()
       cy.createTestApp()
-    cy.visit(`localhost:${Cypress.env("PORT")}/builder`)
+      cy.visit(`${Cypress.config().baseUrl}/builder`)
       cy.contains("Cypress Tests").should("exist")
     })
+  })  
 })

packages/builder/cypress/integration/createAutomation.spec.js

@@ -1,10 +1,12 @@
-context("Create a automation", () => {
+import filterTests from "../support/filterTests"
+
+filterTests(['smoke', 'all'], () => {
+  context("Create a automation", () => {
     before(() => {
       cy.login()
       cy.createTestApp()
     })
 
-  // https://on.cypress.io/interacting-with-elements
     it("should create a automation", () => {
       cy.createTestTableWithData()
       cy.wait(2000)
@@ -24,14 +26,14 @@ context("Create a automation", () => {
       cy.contains("dog").click()
       cy.wait(2000)
       // Create action
-    cy.get(".block > .spectrum-Icon").click()
+      cy.get('[aria-label="AddCircle"]').eq(1).click()
       cy.get(".modal-inner-wrapper").within(() => {
         cy.wait(1000)
         cy.contains("Create Row").trigger('mouseover').click().click()
         cy.get(".spectrum-Button--cta").click()
       })
       cy.contains("Setup").click()
-    cy.get(".spectrum-Picker-label").eq(1).click()
+      cy.get(".spectrum-Picker-label").click()
       cy.contains("dog").click()
       cy.get(".spectrum-Textfield-input")
       .first()
@@ -63,4 +65,5 @@ context("Create a automation", () => {
       cy.contains("Data").click()
       cy.contains("automationGoodboy")
     })
+  })
 })

packages/builder/cypress/integration/createBinding.spec.js

@@ -1,4 +1,7 @@
-context("Create Bindings", () => {
+import filterTests from "../support/filterTests"
+
+filterTests(['smoke', 'all'], () => {
+  context("Create Bindings", () => {
     before(() => {
       cy.login()
       cy.createTestApp()
@@ -42,9 +45,9 @@ context("Create Bindings", () => {
         cy.getComponent(componentId).should("have.text", "3")
       })
     })
-})
+  })
 
-const addSettingBinding = (setting, bindingText, clickOption = true) => {
+  const addSettingBinding = (setting, bindingText, clickOption = true) => {
     cy.get(`[data-cy="setting-${setting}"] [data-cy=text-binding-button]`).click()
     cy.get(".drawer").within(() => {
       if (clickOption) {
@@ -55,4 +58,5 @@ const addSettingBinding = (setting, bindingText, clickOption = true) => {
       }
       cy.contains("Save").click()
     })
-}
+  }
+})

packages/builder/cypress/integration/createComponents.spec.js

@@ -1,5 +1,9 @@
 // TODO for now components are skipped, might not be good to keep doing this
-xcontext("Create Components", () => {
+
+import filterTests from "../support/filterTests"
+
+filterTests(['all'], () => {
+  xcontext("Create Components", () => {
     let headlineId
 
     before(() => {
@@ -89,4 +93,5 @@ xcontext("Create Components", () => {
         cy.getComponent(componentId).should("not.exist")
       })
     })
+  })
 })

packages/builder/cypress/integration/createScreen.js

@@ -1,4 +1,7 @@
-context("Screen Tests", () => {
+import filterTests from "../support/filterTests"
+
+filterTests(["smoke", "all"], () => {
+  context("Screen Tests", () => {
     before(() => {
       cy.login()
       cy.createTestApp()
@@ -18,4 +21,5 @@ context("Screen Tests", () => {
         cy.contains("/test-with-spaces").should("exist")
       })
     })
+  })
 })

packages/builder/cypress/integration/createTable.spec.js

@@ -1,4 +1,7 @@
-context("Create a Table", () => {
+import filterTests from "../support/filterTests"
+
+filterTests(["smoke", "all"], () => {
+  context("Create a Table", () => {
     before(() => {
       cy.login()
       cy.createTestApp()
@@ -31,7 +34,6 @@ context("Create a Table", () => {
       cy.contains("nameupdated ").should("contain", "nameupdated")
     })
 
-  
     it("edits a row", () => {
       cy.contains("button", "Edit").click({ force: true })
       cy.wait(1000)
@@ -48,12 +50,45 @@ context("Create a Table", () => {
       cy.contains("RoverUpdated").should("not.exist")
     })
 
+    if (Cypress.env("TEST_ENV")) {
+      // No Pagination in CI - Test env only for the next two tests
+      it("Adds 15 rows and checks pagination", () => {
+        // 10 rows per page, 15 rows should create 2 pages within table
+        const totalRows = 16
+        for (let i = 1; i < totalRows; i++) {
+          cy.addRow([i])
+        }
+        cy.wait(1000)
+        cy.get(".spectrum-Pagination").within(() => {
+          cy.get(".spectrum-ActionButton").eq(1).click()
+        })
+        cy.get(".spectrum-Pagination").within(() => {
+          cy.get(".spectrum-Body--secondary").contains("Page 2")
+        })
+      })
+
+      it("Deletes rows and checks pagination", () => {
+        // Delete rows, removing second page of rows from table
+        const deleteRows = 5
+        cy.get(".spectrum-Checkbox-input").check({ force: true })
+        cy.get(".spectrum-Table")
+        cy.contains("Delete 5 row(s)").click()
+        cy.get(".spectrum-Modal").contains("Delete").click()
+        cy.wait(1000)
+
+        // Confirm table only has one page
+        cy.get(".spectrum-Pagination").within(() => {
+          cy.get(".spectrum-ActionButton").eq(1).should("not.be.enabled")
+        })
+      })
+    }
+
     it("deletes a column", () => {
+      const columnName = "nameupdated"
       cy.get(".title").click()
       cy.get(".spectrum-Table-editIcon > use").click()
       cy.contains("Delete").click()
-    cy.wait(50)
+      cy.get('[data-cy="delete-column-confirm"]').type(columnName)
-    cy.get(`[data-cy="delete-column-confirm"]`).type("nameupdated")
       cy.contains("Delete Column").click()
       cy.contains("nameupdated").should("not.exist")
     })
@@ -67,8 +102,9 @@ context("Create a Table", () => {
           cy.get(".actions .spectrum-Icon").click({ force: true })
         })
       cy.get(".spectrum-Menu > :nth-child(2)").click()
-    cy.get(`[data-cy="delete-table-confirm"]`).type("dog")
+      cy.get('[data-cy="delete-table-confirm"]').type("dog")
       cy.contains("Delete Table").click()
       cy.contains("dog").should("not.exist")
     })
+  })
 })

packages/builder/cypress/integration/createUser.spec.js

@@ -1,10 +0,0 @@
-context("Create a User", () => {
-  before(() => {
-    cy.login()
-  })
-
-  it("should create a user", () => {
-    cy.createUser("bbuser@test.com")
-    cy.contains("bbuser").should("be.visible")
-  })
-})

packages/builder/cypress/integration/createUserAndRoles.spec.js

@@ -0,0 +1,180 @@
+import filterTests from "../support/filterTests"
+
+filterTests(["smoke", "all"], () => {
+  context("Create a User and Assign Roles", () => {
+    before(() => {
+      cy.login()
+    })
+
+    it("should create a user", () => {
+      cy.createUser("bbuser@test.com")
+      cy.get(".spectrum-Table").should("contain", "bbuser")
+    })
+
+    it("should confirm there is No Access for a New User", () => {
+      // Click into the user
+      cy.contains("bbuser").click()
+      cy.wait(500)
+      // Get No Access table - Confirm it has apps in it
+      cy.get(".spectrum-Table").eq(1).should("not.contain", "No rows found")
+      // Get Configure Roles table - Confirm it has no apps
+      cy.get(".spectrum-Table").eq(0).contains("No rows found")
+    })
+
+    it("should assign role types", () => {
+      // 3 apps minimum required - to assign an app to each role type
+      cy.request(`${Cypress.config().baseUrl}/api/applications?status=all`)
+        .its("body")
+        .then(val => {
+          if (val.length < 3) {
+            for (let i = 1; i < 3; i++) {
+              const uuid = () => Cypress._.random(0, 1e6)
+              const name = uuid()
+              cy.createApp(name)
+            }
+          }
+        })
+      // Navigate back to the user
+      cy.visit(`${Cypress.config().baseUrl}/builder`)
+      cy.wait(500)
+      cy.get(".spectrum-SideNav").contains("Users").click()
+      cy.wait(500)
+      cy.get(".spectrum-Table").contains("bbuser").click()
+      cy.wait(1000)
+      for (let i = 0; i < 3; i++) {
+        cy.get(".spectrum-Table")
+          .eq(1)
+          .find(".spectrum-Table-row")
+          .eq(0)
+          .find(".spectrum-Table-cell")
+          .eq(0)
+          .click()
+        cy.wait(500)
+        cy.get(".spectrum-Dialog-grid")
+          .contains("Choose an option")
+          .click()
+          .then(() => {
+            cy.wait(1000)
+            if (i == 0) {
+              cy.get(".spectrum-Popover").contains("Admin").click()
+            }
+            if (i == 1) {
+              cy.get(".spectrum-Popover").contains("Power").click()
+            }
+            if (i == 2) {
+              cy.get(".spectrum-Popover").contains("Basic").click()
+            }
+            cy.wait(1000)
+            cy.get(".spectrum-Button")
+              .contains("Update role")
+              .click({ force: true })
+          })
+      }
+      // Confirm roles exist within Configure roles table
+      cy.wait(2000)
+      cy.get(".spectrum-Table")
+        .eq(0)
+        .within(assginedRoles => {
+          expect(assginedRoles).to.contain("Admin")
+          expect(assginedRoles).to.contain("Power")
+          expect(assginedRoles).to.contain("Basic")
+        })
+    })
+
+    it("should unassign role types", () => {
+      // Set each app within Configure roles table to 'No Access'
+      cy.get(".spectrum-Table")
+        .eq(0)
+        .find(".spectrum-Table-row")
+        .its("length")
+        .then(len => {
+          for (let i = 0; i < len; i++) {
+            cy.get(".spectrum-Table")
+              .eq(0)
+              .find(".spectrum-Table-row")
+              .eq(0)
+              .find(".spectrum-Table-cell")
+              .eq(0)
+              .click()
+              .then(() => {
+                cy.get(".spectrum-Picker").eq(1).click({ force: true })
+                cy.wait(500)
+                cy.get(".spectrum-Popover").contains("No Access").click()
+              })
+            cy.get(".spectrum-Button")
+              .contains("Update role")
+              .click({ force: true })
+            cy.wait(1000)
+          }
+        })
+      // Confirm Configure roles table no longer has any apps in it
+      cy.get(".spectrum-Table").eq(0).contains("No rows found")
+    })
+
+    it("should enable Developer access", () => {
+      // Enable Developer access
+      cy.get(".field")
+        .eq(4)
+        .within(() => {
+          cy.get(".spectrum-Switch-input").click({ force: true })
+        })
+      // No Access table should now be empty
+      cy.get(".container")
+        .contains("No Access")
+        .parent()
+        .within(() => {
+          cy.get(".spectrum-Table").contains("No rows found")
+        })
+
+      // Each app within Configure roles should have Admin access
+      cy.get(".spectrum-Table")
+        .eq(0)
+        .find(".spectrum-Table-row")
+        .its("length")
+        .then(len => {
+          for (let i = 0; i < len; i++) {
+            cy.get(".spectrum-Table")
+              .eq(0)
+              .find(".spectrum-Table-row")
+              .eq(i)
+              .contains("Admin")
+            cy.wait(500)
+          }
+        })
+    })
+
+    it("should disable Developer access", () => {
+      // Disable Developer access
+      cy.get(".field")
+        .eq(4)
+        .within(() => {
+          cy.get(".spectrum-Switch-input").click({ force: true })
+        })
+      // Configure roles table should now be empty
+      cy.get(".container")
+        .contains("Configure roles")
+        .parent()
+        .within(() => {
+          cy.get(".spectrum-Table").contains("No rows found")
+        })
+    })
+
+    it("should delete a user", () => {
+      // Click Delete user button
+      cy.get(".spectrum-Button")
+        .contains("Delete user")
+        .click({ force: true })
+        .then(() => {
+          // Confirm deletion within modal
+          cy.wait(500)
+          cy.get(".spectrum-Dialog-grid").within(() => {
+            cy.get(".spectrum-Button")
+              .contains("Delete user")
+              .click({ force: true })
+            cy.wait(4000)
+          })
+        })
+      cy.get(".spectrum-Table").should("not.have.text", "bbuser")
+    })
+  })
+})

packages/builder/cypress/integration/createView.spec.js

@@ -1,4 +1,7 @@
-context("Create a View", () => {
+import filterTests from "../support/filterTests"
+
+filterTests(['smoke', 'all'], () => {
+  context("Create a View", () => {
     before(() => {
       cy.login()
       cy.createTestApp()
@@ -141,12 +144,13 @@ context("Create a View", () => {
       cy.wait(500)
       cy.contains("TestView Updated").should("not.exist")
     })
-})
+  })
 
-function removeSpacing(headers) {
+  function removeSpacing(headers) {
     let newHeaders = []
     for (let header of headers) {
       newHeaders.push(header.replace(/\s\s+/g, " "))
     }
     return newHeaders
-}
+  }
+})

packages/builder/cypress/integration/customThemingProperties.spec.js

@@ -1,4 +1,7 @@
-xcontext("Custom Theming Properties", () => {
+import filterTests from "../support/filterTests"
+
+filterTests(['all'], () => {
+  xcontext("Custom Theming Properties", () => {
       before(() => {
         cy.login()
         cy.createTestApp()
@@ -80,5 +83,5 @@ xcontext("Custom Theming Properties", () => {
           .parent().find(".container.svelte-z3cm5a").click()
           .get('[title="Gray 800"]').children().find('[aria-label="Checkmark"]')
       }
-
+  })
 })

packages/builder/cypress/integration/datasources/datasourceWizard.spec.js

@@ -0,0 +1,43 @@
+import filterTests from "../../support/filterTests"
+
+filterTests(['all'], () => {
+  context("Datasource Wizard", () => {
+    if (Cypress.env("TEST_ENV")) {
+      before(() => {
+        cy.login()
+        cy.createTestApp()
+      })
+
+    it("should navigate in and out of a datasource via wizard", () => {
+      // Select PostgreSQL and add config (without fetch)
+      const datasource = "Oracle"
+      cy.selectExternalDatasource(datasource)
+      cy.addDatasourceConfig(datasource, true)
+      
+      // Navigate back within datasource wizard
+      cy.get(".spectrum-Dialog-grid").within(() => {
+        cy.get(".spectrum-Button").contains("Back").click({ force: true })
+        cy.wait(1000)
+      })
+    
+      // Select PostgreSQL datasource again
+      cy.get(".item-list").contains(datasource).click()
+      cy.get(".spectrum-Dialog-grid").within(() => {
+        cy.get(".spectrum-Button").contains("Continue").click({ force: true })
+      })
+      
+      // Fetch tables after selection
+      // Previously entered config should not have been saved
+      // Config is back to default values
+      // Modal will close and provide 500 error
+      cy.intercept('**/datasources').as('datasourceConnection')
+      cy.get(".spectrum-Dialog-grid").within(() => {
+        cy.get(".spectrum-Button").contains("Save and fetch tables").click({ force: true })
+      })
+      cy.wait("@datasourceConnection")
+      cy.get("@datasourceConnection").its('response.body')
+          .should('have.property', 'status', 500)
+      })
+    }
+  })
+})

packages/builder/cypress/integration/datasources/mySql.spec.js

@@ -0,0 +1,222 @@
+import filterTests from "../../support/filterTests"
+
+filterTests(["all"], () => {
+  context("MySQL Datasource Testing", () => {
+    if (Cypress.env("TEST_ENV")) {
+      before(() => {
+        cy.login()
+        cy.createTestApp()
+      })
+      const datasource = "MySQL"
+      const queryName = "Cypress Test Query"
+      const queryRename = "CT Query Rename"
+
+      it("Should add MySQL data source without configuration", () => {
+        // Select MySQL data source
+        cy.selectExternalDatasource(datasource)
+        // Attempt to fetch tables without applying configuration
+        cy.intercept("**/datasources").as("datasource")
+        cy.get(".spectrum-Button")
+          .contains("Save and fetch tables")
+          .click({ force: true })
+        // Intercept Request after button click & apply assertions
+        cy.wait("@datasource")
+        cy.get("@datasource")
+          .its("response.body")
+          .should(
+            "have.property",
+            "message",
+            "connect ECONNREFUSED 127.0.0.1:3306"
+          )
+        cy.get("@datasource")
+          .its("response.body")
+          .should("have.property", "status", 500)
+      })
+
+      it("should add MySQL data source and fetch tables", () => {
+        // Add & configure MySQL data source
+        cy.selectExternalDatasource(datasource)
+        cy.intercept("**/datasources").as("datasource")
+        cy.addDatasourceConfig(datasource)
+        // Check response from datasource after adding configuration
+        cy.wait("@datasource")
+        cy.get("@datasource").its("response.statusCode").should("eq", 200)
+        // Confirm fetch tables was successful
+        cy.get(".spectrum-Table")
+          .eq(0)
+          .find(".spectrum-Table-row")
+          .its("length")
+          .should("be.gt", 0)
+      })
+
+      it("should check table fetching error", () => {
+        // MySQL test data source contains tables without primary keys
+        cy.get(".spectrum-InLineAlert")
+          .should("contain", "Error fetching tables")
+          .and("contain", "No primary key constraint found")
+      })
+
+      it("should define a One relationship type", () => {
+        // Select relationship type & configure
+        cy.get(".spectrum-Button")
+          .contains("Define relationship")
+          .click({ force: true })
+        cy.get(".spectrum-Dialog-grid").within(() => {
+          cy.get(".spectrum-Picker").eq(0).click()
+          cy.get(".spectrum-Popover").contains("One").click()
+          cy.get(".spectrum-Picker").eq(1).click()
+          cy.get(".spectrum-Popover").contains("REGIONS").click()
+          cy.get(".spectrum-Picker").eq(2).click()
+          cy.get(".spectrum-Popover").contains("REGION_ID").click()
+          cy.get(".spectrum-Picker").eq(3).click()
+          cy.get(".spectrum-Popover").contains("COUNTRIES").click()
+          cy.get(".spectrum-Picker").eq(4).click()
+          cy.get(".spectrum-Popover").contains("REGION_ID").click()
+          // Save relationship & reload page
+          cy.get(".spectrum-Button").contains("Save").click({ force: true })
+          cy.reload()
+        })
+        // Confirm table length & column name
+        cy.get(".spectrum-Table")
+          .eq(1)
+          .find(".spectrum-Table-row")
+          .its("length")
+          .should("eq", 1)
+        cy.get(".spectrum-Table-cell").should("contain", "COUNTRIES to REGIONS")
+      })
+
+      it("should define a Many relationship type", () => {
+        // Select relationship type & configure
+        cy.get(".spectrum-Button")
+          .contains("Define relationship")
+          .click({ force: true })
+        cy.get(".spectrum-Dialog-grid").within(() => {
+          cy.get(".spectrum-Picker").eq(0).click()
+          cy.get(".spectrum-Popover").contains("Many").click()
+          cy.get(".spectrum-Picker").eq(1).click()
+          cy.get(".spectrum-Popover").contains("LOCATIONS").click()
+          cy.get(".spectrum-Picker").eq(2).click()
+          cy.get(".spectrum-Popover").contains("REGIONS").click()
+          cy.get(".spectrum-Picker").eq(3).click()
+          cy.get(".spectrum-Popover").contains("COUNTRIES").click()
+          cy.get(".spectrum-Picker").eq(4).click()
+          cy.get(".spectrum-Popover").contains("COUNTRY_ID").click()
+          cy.get(".spectrum-Picker").eq(5).click()
+          cy.get(".spectrum-Popover").contains("REGION_ID").click()
+          // Save relationship & reload page
+          cy.get(".spectrum-Button").contains("Save").click({ force: true })
+          cy.reload()
+          cy.wait(1000)
+        })
+        // Confirm table length & relationship name
+        cy.get(".spectrum-Table")
+          .eq(1)
+          .find(".spectrum-Table-row")
+          .its("length")
+          .should("eq", 2)
+        cy.get(".spectrum-Table-cell").should(
+          "contain",
+          "LOCATIONS through COUNTRIES → REGIONS"
+        )
+      })
+
+      it("should delete relationships", () => {
+        // Delete both relationships
+        cy.get(".spectrum-Table")
+          .eq(1)
+          .find(".spectrum-Table-row")
+          .its("length")
+          .then(len => {
+            for (let i = 0; i < len; i++) {
+              cy.get(".spectrum-Table")
+                .eq(1)
+                .within(() => {
+                  cy.get(".spectrum-Table-row").eq(0).click()
+                  cy.wait(500)
+                })
+              cy.get(".spectrum-Dialog-grid").within(() => {
+                cy.get(".spectrum-Button")
+                  .contains("Delete")
+                  .click({ force: true })
+              })
+              cy.reload()
+            }
+            // Confirm relationships no longer exist
+            cy.get(".spectrum-Body").should(
+              "contain",
+              "No relationships configured"
+            )
+          })
+      })
+
+      it("should add a query", () => {
+        // Add query
+        cy.get(".spectrum-Button").contains("Add query").click({ force: true })
+        cy.get(".spectrum-Form-item")
+          .eq(0)
+          .within(() => {
+            cy.get("input").type(queryName)
+          })
+        // Insert Query within Fields section
+        cy.get(".CodeMirror textarea")
+          .eq(0)
+          .type("SELECT * FROM books", { force: true })
+        // Intercept query execution
+        cy.intercept("**/queries/preview").as("query")
+        cy.get(".spectrum-Button").contains("Run Query").click({ force: true })
+        cy.wait(500)
+        cy.wait("@query")
+        // Assert against Status Code & Body
+        cy.get("@query").its("response.statusCode").should("eq", 200)
+        cy.get("@query").its("response.body").should("not.be.empty")
+        // Save query
+        cy.get(".spectrum-Button").contains("Save Query").click({ force: true })
+        cy.get(".nav-item").should("contain", queryName)
+      })
+
+      it("should duplicate a query", () => {
+        // Get last nav item - The query
+        cy.get(".nav-item")
+          .last()
+          .within(() => {
+            cy.get(".icon").eq(1).click({ force: true })
+          })
+        // Select and confirm duplication
+        cy.get(".spectrum-Menu").contains("Duplicate").click()
+        cy.get(".nav-item").should("contain", queryName + " (1)")
+      })
+
+      it("should edit a query name", () => {
+        // Rename query
+        cy.get(".spectrum-Form-item")
+          .eq(0)
+          .within(() => {
+            cy.get("input").clear().type(queryRename)
+          })
+        // Save query
+        cy.get(".spectrum-Button").contains("Save Query").click({ force: true })
+        cy.get(".nav-item").should("contain", queryRename)
+      })
+
+      it("should delete a query", () => {
+        // Get last nav item - The query
+        for (let i = 0; i < 2; i++) {
+          cy.get(".nav-item")
+            .last()
+            .within(() => {
+              cy.get(".icon").eq(1).click({ force: true })
+            })
+          // Select Delete
+          cy.get(".spectrum-Menu").contains("Delete").click()
+          cy.get(".spectrum-Button")
+            .contains("Delete Query")
+            .click({ force: true })
+          cy.wait(1000)
+        }
+        // Confirm deletion
+        cy.get(".nav-item").should("not.contain", queryName)
+        cy.get(".nav-item").should("not.contain", queryRename)
+      })
+    }
+  })
+})

packages/builder/cypress/integration/datasources/oracle.spec.js

@@ -0,0 +1,230 @@
+import filterTests from "../../support/filterTests"
+
+filterTests(["all"], () => {
+  context("Oracle Datasource Testing", () => {
+    if (Cypress.env("TEST_ENV")) {
+      before(() => {
+        cy.login()
+        cy.createTestApp()
+      })
+      const datasource = "Oracle"
+      const queryName = "Cypress Test Query"
+      const queryRename = "CT Query Rename"
+
+      it("Should add Oracle data source and skip table fetch", () => {
+        // Select Oracle data source
+        cy.selectExternalDatasource(datasource)
+        // Skip table fetch - no config added
+        cy.get(".spectrum-Button")
+          .contains("Skip table fetch")
+          .click({ force: true })
+        cy.wait(500)
+        // Confirm config contains localhost
+        cy.get(".spectrum-Textfield-input")
+          .eq(1)
+          .should("have.value", "localhost")
+        // Add another Oracle data source, configure & skip table fetch
+        cy.selectExternalDatasource(datasource)
+        cy.addDatasourceConfig(datasource, true)
+        // Confirm config and no tables
+        cy.get(".spectrum-Textfield-input")
+          .eq(1)
+          .should("have.value", Cypress.env("oracle").HOST)
+        cy.get(".spectrum-Body").eq(2).should("contain", "No tables found.")
+      })
+
+      it("Should add Oracle data source and fetch tables without configuration", () => {
+        // Select Oracle data source
+        cy.selectExternalDatasource(datasource)
+        // Attempt to fetch tables without applying configuration
+        cy.intercept("**/datasources").as("datasource")
+        cy.get(".spectrum-Button")
+          .contains("Save and fetch tables")
+          .click({ force: true })
+        // Intercept Request after button click & apply assertions
+        cy.wait("@datasource")
+        cy.get("@datasource")
+          .its("response.body")
+          .should("have.property", "status", 500)
+      })
+
+      it("should add Oracle data source and fetch tables", () => {
+        // Add & configure Oracle data source
+        cy.selectExternalDatasource(datasource)
+        cy.intercept("**/datasources").as("datasource")
+        cy.addDatasourceConfig(datasource)
+        // Check response from datasource after adding configuration
+        cy.wait("@datasource")
+        cy.get("@datasource").its("response.statusCode").should("eq", 200)
+        // Confirm fetch tables was successful
+        cy.get(".spectrum-Table")
+          .eq(0)
+          .find(".spectrum-Table-row")
+          .its("length")
+          .should("be.gt", 0)
+      })
+
+      it("should define a One relationship type", () => {
+        // Select relationship type & configure
+        cy.get(".spectrum-Button")
+          .contains("Define relationship")
+          .click({ force: true })
+        cy.get(".spectrum-Dialog-grid").within(() => {
+          cy.get(".spectrum-Picker").eq(0).click()
+          cy.get(".spectrum-Popover").contains("One").click()
+          cy.get(".spectrum-Picker").eq(1).click()
+          cy.get(".spectrum-Popover").contains("REGIONS").click()
+          cy.get(".spectrum-Picker").eq(2).click()
+          cy.get(".spectrum-Popover").contains("REGION_ID").click()
+          cy.get(".spectrum-Picker").eq(3).click()
+          cy.get(".spectrum-Popover").contains("COUNTRIES").click()
+          cy.get(".spectrum-Picker").eq(4).click()
+          cy.get(".spectrum-Popover").contains("REGION_ID").click()
+          // Save relationship & reload page
+          cy.get(".spectrum-Button").contains("Save").click({ force: true })
+          cy.reload()
+        })
+        // Confirm table length & column name
+        cy.get(".spectrum-Table")
+          .eq(1)
+          .find(".spectrum-Table-row")
+          .its("length")
+          .should("eq", 1)
+        cy.get(".spectrum-Table-cell").should("contain", "COUNTRIES to REGIONS")
+      })
+
+      it("should define a Many relationship type", () => {
+        // Select relationship type & configure
+        cy.get(".spectrum-Button")
+          .contains("Define relationship")
+          .click({ force: true })
+        cy.get(".spectrum-Dialog-grid").within(() => {
+          cy.get(".spectrum-Picker").eq(0).click()
+          cy.get(".spectrum-Popover").contains("Many").click()
+          cy.get(".spectrum-Picker").eq(1).click()
+          cy.get(".spectrum-Popover").contains("LOCATIONS").click()
+          cy.get(".spectrum-Picker").eq(2).click()
+          cy.get(".spectrum-Popover").contains("REGIONS").click()
+          cy.get(".spectrum-Picker").eq(3).click()
+          cy.get(".spectrum-Popover").contains("COUNTRIES").click()
+          cy.get(".spectrum-Picker").eq(4).click()
+          cy.get(".spectrum-Popover").contains("COUNTRY_ID").click()
+          cy.get(".spectrum-Picker").eq(5).click()
+          cy.get(".spectrum-Popover").contains("REGION_ID").click()
+          // Save relationship & reload page
+          cy.get(".spectrum-Button").contains("Save").click({ force: true })
+          cy.reload()
+        })
+        // Confirm table length & relationship name
+        cy.get(".spectrum-Table")
+          .eq(1)
+          .find(".spectrum-Table-row")
+          .its("length")
+          .should("eq", 2)
+        cy.get(".spectrum-Table-cell").should(
+          "contain",
+          "LOCATIONS through COUNTRIES → REGIONS"
+        )
+      })
+
+      it("should delete relationships", () => {
+        // Delete both relationships
+        cy.get(".spectrum-Table")
+          .eq(1)
+          .find(".spectrum-Table-row")
+          .its("length")
+          .then(len => {
+            for (let i = 0; i < len; i++) {
+              cy.get(".spectrum-Table")
+                .eq(1)
+                .within(() => {
+                  cy.get(".spectrum-Table-row").eq(0).click()
+                  cy.wait(500)
+                })
+              cy.get(".spectrum-Dialog-grid").within(() => {
+                cy.get(".spectrum-Button")
+                  .contains("Delete")
+                  .click({ force: true })
+              })
+              cy.reload()
+            }
+            // Confirm relationships no longer exist
+            cy.get(".spectrum-Body").should(
+              "contain",
+              "No relationships configured"
+            )
+          })
+      })
+
+      it("should add a query", () => {
+        // Add query
+        cy.get(".spectrum-Button").contains("Add query").click({ force: true })
+        cy.get(".spectrum-Form-item")
+          .eq(0)
+          .within(() => {
+            cy.get("input").type(queryName)
+          })
+        // Insert Query within Fields section
+        cy.get(".CodeMirror textarea")
+          .eq(0)
+          .type("SELECT * FROM JOBS", { force: true })
+        // Intercept query execution
+        cy.intercept("**/queries/preview").as("query")
+        cy.get(".spectrum-Button").contains("Run Query").click({ force: true })
+        cy.wait(500)
+        cy.wait("@query")
+        // Assert against Status Code & Body
+        cy.get("@query").its("response.statusCode").should("eq", 200)
+        cy.get("@query").its("response.body").should("not.be.empty")
+        // Save query
+        cy.get(".spectrum-Button").contains("Save Query").click({ force: true })
+        cy.get(".nav-item").should("contain", queryName)
+      })
+
+      it("should duplicate a query", () => {
+        // Get query nav item
+        cy.get(".nav-item")
+          .contains(queryName)
+          .parent()
+          .within(() => {
+            cy.get(".spectrum-Icon").eq(1).click({ force: true })
+          })
+        // Select and confirm duplication
+        cy.get(".spectrum-Menu").contains("Duplicate").click()
+        cy.get(".nav-item").should("contain", queryName + " (1)")
+      })
+
+      it("should edit a query name", () => {
+        // Rename query
+        cy.get(".spectrum-Form-item")
+          .eq(0)
+          .within(() => {
+            cy.get("input").clear().type(queryRename)
+          })
+        // Save query
+        cy.get(".spectrum-Button").contains("Save Query").click({ force: true })
+        cy.get(".nav-item").should("contain", queryRename)
+      })
+
+      it("should delete a query", () => {
+        // Get query nav item - QueryName
+        cy.get(".nav-item")
+          .contains(queryName)
+          .parent()
+          .within(() => {
+            cy.get(".spectrum-Icon").eq(1).click({ force: true })
+          })
+
+        // Select Delete
+        cy.get(".spectrum-Menu").contains("Delete").click()
+        cy.get(".spectrum-Button")
+          .contains("Delete Query")
+          .click({ force: true })
+        cy.wait(1000)
+
+        // Confirm deletion
+        cy.get(".nav-item").should("not.contain", queryName)
+      })
+    }
+  })
+})

packages/builder/cypress/integration/datasources/postgreSql.spec.js

@@ -0,0 +1,285 @@
+import filterTests from "../../support/filterTests"
+
+filterTests(["all"], () => {
+  context("PostgreSQL Datasource Testing", () => {
+    if (Cypress.env("TEST_ENV")) {
+      before(() => {
+        cy.login()
+        cy.createTestApp()
+      })
+      const datasource = "PostgreSQL"
+      const queryName = "Cypress Test Query"
+      const queryRename = "CT Query Rename"
+
+      it("Should add PostgreSQL data source without configuration", () => {
+        // Select PostgreSQL data source
+        cy.selectExternalDatasource(datasource)
+        // Attempt to fetch tables without applying configuration
+        cy.intercept("**/datasources").as("datasource")
+        cy.get(".spectrum-Button")
+          .contains("Save and fetch tables")
+          .click({ force: true })
+        // Intercept Request after button click & apply assertions
+        cy.wait("@datasource")
+        cy.get("@datasource")
+          .its("response.body")
+          .should(
+            "have.property",
+            "message",
+            "connect ECONNREFUSED 127.0.0.1:5432"
+          )
+        cy.get("@datasource")
+          .its("response.body")
+          .should("have.property", "status", 500)
+      })
+
+      it("should add PostgreSQL data source and fetch tables", () => {
+        // Add & configure PostgreSQL data source
+        cy.selectExternalDatasource(datasource)
+        cy.intercept("**/datasources").as("datasource")
+        cy.addDatasourceConfig(datasource)
+        // Check response from datasource after adding configuration
+        cy.wait("@datasource")
+        cy.get("@datasource").its("response.statusCode").should("eq", 200)
+        // Confirm fetch tables was successful
+        cy.get(".spectrum-Table")
+          .eq(0)
+          .find(".spectrum-Table-row")
+          .its("length")
+          .should("be.gt", 0)
+      })
+
+      it("should define a One relationship type", () => {
+        // Select relationship type & configure
+        cy.get(".spectrum-Button")
+          .contains("Define relationship")
+          .click({ force: true })
+        cy.get(".spectrum-Dialog-grid").within(() => {
+          cy.get(".spectrum-Picker").eq(0).click()
+          cy.get(".spectrum-Popover").contains("One").click()
+          cy.get(".spectrum-Picker").eq(1).click()
+          cy.get(".spectrum-Popover").contains("REGIONS").click()
+          cy.get(".spectrum-Picker").eq(2).click()
+          cy.get(".spectrum-Popover").contains("REGION_ID").click()
+          cy.get(".spectrum-Picker").eq(3).click()
+          cy.get(".spectrum-Popover").contains("COUNTRIES").click()
+          cy.get(".spectrum-Picker").eq(4).click()
+          cy.get(".spectrum-Popover").contains("REGION_ID").click()
+          // Save relationship & reload page
+          cy.get(".spectrum-Button").contains("Save").click({ force: true })
+          cy.reload()
+        })
+        // Confirm table length & column name
+        cy.get(".spectrum-Table")
+          .eq(1)
+          .find(".spectrum-Table-row")
+          .its("length")
+          .should("eq", 1)
+        cy.get(".spectrum-Table-cell").should("contain", "COUNTRIES to REGIONS")
+      })
+
+      it("should define a Many relationship type", () => {
+        // Select relationship type & configure
+        cy.get(".spectrum-Button")
+          .contains("Define relationship")
+          .click({ force: true })
+        cy.get(".spectrum-Dialog-grid").within(() => {
+          cy.get(".spectrum-Picker").eq(0).click()
+          cy.get(".spectrum-Popover").contains("Many").click()
+          cy.get(".spectrum-Picker").eq(1).click()
+          cy.get(".spectrum-Popover").contains("LOCATIONS").click()
+          cy.get(".spectrum-Picker").eq(2).click()
+          cy.get(".spectrum-Popover").contains("REGIONS").click()
+          cy.get(".spectrum-Picker").eq(3).click()
+          cy.get(".spectrum-Popover").contains("COUNTRIES").click()
+          cy.get(".spectrum-Picker").eq(4).click()
+          cy.get(".spectrum-Popover").contains("COUNTRY_ID").click()
+          cy.get(".spectrum-Picker").eq(5).click()
+          cy.get(".spectrum-Popover").contains("REGION_ID").click()
+          // Save relationship & reload page
+          cy.get(".spectrum-Button").contains("Save").click({ force: true })
+          cy.reload()
+        })
+        // Confirm table length & relationship name
+        cy.get(".spectrum-Table")
+          .eq(1)
+          .find(".spectrum-Table-row")
+          .its("length")
+          .should("eq", 2)
+        cy.get(".spectrum-Table-cell").should(
+          "contain",
+          "LOCATIONS through COUNTRIES → REGIONS"
+        )
+      })
+
+      it("should delete a relationship", () => {
+        cy.get(".hierarchy-items-container").contains(datasource).click()
+        cy.reload()
+        // Delete one relationship
+        cy.get(".spectrum-Table")
+          .eq(1)
+          .within(() => {
+            cy.get(".spectrum-Table-row").eq(0).click()
+            cy.wait(500)
+          })
+        cy.get(".spectrum-Dialog-grid").within(() => {
+          cy.get(".spectrum-Button").contains("Delete").click({ force: true })
+        })
+        cy.reload()
+        // Confirm relationship was deleted
+        cy.get(".spectrum-Table")
+          .eq(1)
+          .find(".spectrum-Table-row")
+          .its("length")
+          .should("eq", 1)
+      })
+
+      it("should add a query", () => {
+        // Add query
+        cy.get(".spectrum-Button").contains("Add query").click({ force: true })
+        cy.get(".spectrum-Form-item")
+          .eq(0)
+          .within(() => {
+            cy.get("input").type(queryName)
+          })
+        // Insert Query within Fields section
+        cy.get(".CodeMirror textarea")
+          .eq(0)
+          .type("SELECT * FROM books", { force: true })
+        // Intercept query execution
+        cy.intercept("**/queries/preview").as("query")
+        cy.get(".spectrum-Button").contains("Run Query").click({ force: true })
+        cy.wait(500)
+        cy.wait("@query")
+        // Assert against Status Code & Body
+        cy.get("@query").its("response.statusCode").should("eq", 200)
+        cy.get("@query").its("response.body").should("not.be.empty")
+        // Save query
+        cy.get(".spectrum-Button").contains("Save Query").click({ force: true })
+        cy.get(".hierarchy-items-container").should("contain", queryName)
+      })
+
+      it("should switch to schema with no tables", () => {
+        // Switch Schema - To one without any tables
+        cy.get(".hierarchy-items-container").contains(datasource).click()
+        switchSchema("randomText")
+
+        // No tables displayed
+        cy.get(".spectrum-Body").eq(2).should("contain", "No tables found")
+
+        // Previously created query should be visible
+        cy.get(".spectrum-Table").should("contain", queryName)
+      })
+
+      it("should switch schemas", () => {
+        // Switch schema - To one with tables
+        switchSchema("1")
+
+        // Confirm tables exist - Check for specific one
+        cy.get(".spectrum-Table").eq(0).should("contain", "test")
+        cy.get(".spectrum-Table")
+          .eq(0)
+          .find(".spectrum-Table-row")
+          .its("length")
+          .should("eq", 1)
+
+        // Confirm specific table visible within left nav bar
+        cy.get(".hierarchy-items-container").should("contain", "test")
+
+        // Switch back to public schema
+        switchSchema("public")
+
+        // Confirm tables exist - again
+        cy.get(".spectrum-Table").eq(0).should("contain", "REGIONS")
+        cy.get(".spectrum-Table")
+          .eq(0)
+          .find(".spectrum-Table-row")
+          .its("length")
+          .should("be.gt", 1)
+
+        // Confirm specific table visible within left nav bar
+        cy.get(".hierarchy-items-container").should("contain", "REGIONS")
+
+        // No relationships and one query
+        cy.get(".spectrum-Body")
+          .eq(3)
+          .should("contain", "No relationships configured.")
+        cy.get(".spectrum-Table").eq(1).should("contain", queryName)
+      })
+
+      it("should duplicate a query", () => {
+        // Get last nav item - The query
+        cy.get(".nav-item")
+          .last()
+          .within(() => {
+            cy.get(".icon").eq(1).click({ force: true })
+          })
+        // Select and confirm duplication
+        cy.get(".spectrum-Menu").contains("Duplicate").click()
+        cy.get(".nav-item").should("contain", queryName + " (1)")
+      })
+
+      it("should edit a query name", () => {
+        // Access query
+        cy.get(".hierarchy-items-container")
+          .contains(queryName + " (1)")
+          .click()
+
+        // Rename query
+        cy.get(".spectrum-Form-item")
+          .eq(0)
+          .within(() => {
+            cy.get("input").clear().type(queryRename)
+          })
+
+        // Run and Save query
+        cy.get(".spectrum-Button").contains("Run Query").click({ force: true })
+        cy.wait(500)
+        cy.get(".spectrum-Button").contains("Save Query").click({ force: true })
+        cy.get(".nav-item").should("contain", queryRename)
+      })
+
+      it("should delete a query", () => {
+        // Get last nav item - The query
+        for (let i = 0; i < 2; i++) {
+          cy.get(".nav-item")
+            .last()
+            .within(() => {
+              cy.get(".icon").eq(1).click({ force: true })
+            })
+          // Select Delete
+          cy.get(".spectrum-Menu").contains("Delete").click()
+          cy.get(".spectrum-Button")
+            .contains("Delete Query")
+            .click({ force: true })
+          cy.wait(1000)
+        }
+        // Confirm deletion
+        cy.get(".nav-item").should("not.contain", queryName)
+        cy.get(".nav-item").should("not.contain", queryRename)
+      })
+
+      const switchSchema = schema => {
+        // Edit configuration - Change Schema
+        cy.get(".spectrum-Textfield")
+          .eq(6)
+          .within(() => {
+            cy.get("input").clear().type(schema)
+          })
+        // Save configuration & fetch
+        cy.get(".spectrum-Button").contains("Save").click({ force: true })
+        cy.get(".spectrum-Button")
+          .contains("Fetch tables")
+          .click({ force: true })
+        // Click fetch tables again within modal
+        cy.get(".spectrum-Dialog-grid").within(() => {
+          cy.get(".spectrum-Button")
+            .contains("Fetch tables")
+            .click({ force: true })
+        })
+        cy.reload()
+        cy.wait(5000)
+      }
+    }
+  })
+})

packages/builder/cypress/integration/datasources/rest.spec.js

@@ -0,0 +1,43 @@
+import filterTests from "../../support/filterTests"
+
+filterTests(['smoke', 'all'], () => {
+    context("REST Datasource Testing", () => {
+        before(() => {
+          cy.login()
+          cy.createTestApp()
+        })
+        
+        const datasource = "REST"
+        const restUrl = "https://api.openbrewerydb.org/breweries"
+        
+        it("Should add REST data source with incorrect API", () => {
+            // Select REST data source
+            cy.selectExternalDatasource(datasource)
+            // Enter incorrect api & attempt to send query
+            cy.wait(500)
+            cy.get(".spectrum-Button").contains("Add query").click({ force: true })
+            cy.intercept('**/preview').as('queryError')
+            cy.get("input").clear().type("random text")
+            cy.get(".spectrum-Button").contains("Send").click({ force: true })
+            // Intercept Request after button click & apply assertions
+            cy.wait("@queryError")
+            cy.get("@queryError").its('response.body')
+            .should('have.property', 'message', 'Invalid URL: http://random text?')
+            cy.get("@queryError").its('response.body')
+            .should('have.property', 'status', 400)
+        })
+        
+        it("should add and configure a REST datasource", () => {
+            // Select REST datasource and create query
+            cy.selectExternalDatasource(datasource)
+            cy.wait(500)
+            // createRestQuery confirms query creation
+            cy.createRestQuery("GET", restUrl)
+            // Confirm status code response within REST datasource
+            cy.get(".spectrum-FieldLabel")
+            .contains("Status")
+            .children()
+            .should('contain', 200)
+        })
+    })
+})

packages/builder/cypress/integration/queryLevelTransformers.spec.js

@@ -0,0 +1,116 @@
+import filterTests from "../support/filterTests"
+
+filterTests(['smoke', 'all'], () => {
+  context("Query Level Transformers", () => {
+    before(() => {
+      cy.login()
+      cy.deleteApp("Cypress Tests")
+      cy.createApp("Cypress Tests")
+    })
+      
+    it("should write a transformer function", () => {
+        // Add REST datasource - contains API for breweries
+        const datasource = "REST"
+        const restUrl = "https://api.openbrewerydb.org/breweries"
+        cy.selectExternalDatasource(datasource)
+        cy.createRestQuery("GET", restUrl)
+        cy.get(".spectrum-Tabs-itemLabel").contains("Transformer").click()
+        // Get Transformer Function from file
+        cy.readFile("cypress/support/queryLevelTransformerFunction.js").then((transformerFunction) => {
+          cy.get(".CodeMirror textarea")
+          // Highlight current text and overwrite with file contents
+          .type(Cypress.platform === 'darwin' ? '{cmd}a' : '{ctrl}a', { force: true })
+          .type(transformerFunction, { parseSpecialCharSequences: false })
+        })
+        // Send Query
+        cy.intercept('**/queries/preview').as('query')
+        cy.get(".spectrum-Button").contains("Send").click({ force: true })
+        cy.wait("@query")
+        // Assert against Status Code, body, & body rows
+        cy.get("@query").its('response.statusCode')
+        .should('eq', 200)
+        cy.get("@query").its('response.body').should('not.be.empty')
+        cy.get("@query").its('response.body.rows').should('not.be.empty')
+      })
+        
+    it("should add data to the previous query", () => {
+      // Add REST datasource - contains API for breweries
+      const datasource = "REST"
+      const restUrl = "https://api.openbrewerydb.org/breweries"
+      cy.selectExternalDatasource(datasource)
+      cy.createRestQuery("GET", restUrl)
+      cy.get(".spectrum-Tabs-itemLabel").contains("Transformer").click()
+      // Get Transformer Function with Data from file
+      cy.readFile("cypress/support/queryLevelTransformerFunctionWithData.js").then((transformerFunction) => {
+        //console.log(transformerFunction[1])
+        cy.get(".CodeMirror textarea")
+        // Highlight current text and overwrite with file contents
+        .type(Cypress.platform === 'darwin' ? '{cmd}a' : '{ctrl}a', { force: true })
+        .type(transformerFunction, { parseSpecialCharSequences: false })
+      })
+      // Send Query
+      cy.intercept('**/queries/preview').as('query')
+      cy.get(".spectrum-Button").contains("Send").click({ force: true })
+      cy.wait("@query")
+      // Assert against Status Code, body, & body rows
+      cy.get("@query").its('response.statusCode')
+      .should('eq', 200)
+      cy.get("@query").its('response.body').should('not.be.empty')
+      cy.get("@query").its('response.body.rows').should('not.be.empty')
+    })
+    
+    it("should run an invalid query within the transformer section", () => {
+      // Add REST datasource - contains API for breweries
+      const datasource = "REST"
+      const restUrl = "https://api.openbrewerydb.org/breweries"
+      cy.selectExternalDatasource(datasource)
+      cy.createRestQuery("GET", restUrl)
+      cy.get(".spectrum-Tabs-itemLabel").contains("Transformer").click()
+      // Clear the code box and add "test"
+      cy.get(".CodeMirror textarea")
+      .type(Cypress.platform === 'darwin' ? '{cmd}a' : '{ctrl}a', { force: true })
+      .type("test")
+      // Run Query and intercept
+      cy.intercept('**/preview').as('queryError')
+      cy.get(".spectrum-Button").contains("Send").click({ force: true })
+      cy.wait("@queryError")
+      cy.wait(500)
+      // Assert against message and status for the query error
+      cy.get("@queryError").its('response.body').should('have.property', 'message', "test is not defined")
+      cy.get("@queryError").its('response.body').should('have.property', 'status', 400)
+    })
+    
+    xit("should run an invalid query via POST request", () => {
+      // POST request with transformer as null
+      cy.request({method: 'POST',
+      url: `${Cypress.config().baseUrl}/api/queries/`,
+      body: {fields : {"headers":{},"queryString":null,"path":null},
+      parameters : [],
+      schema : {},
+      name : "test",
+      queryVerb : "read",
+      transformer : null,
+      datasourceId: "test"},
+      // Expected 400 error - Transformer must be a string
+      failOnStatusCode: false}).then((response) => {
+        expect(response.status).to.equal(400)
+        expect(response.body.message).to.include('Invalid body - "transformer" must be a string')
+      })
+    })
+    
+    xit("should run an empty query", () => {
+      // POST request with Transformer as an empty string
+      cy.request({method: 'POST',
+      url: `${Cypress.config().baseUrl}/api/queries/preview`,
+      body: {fields : {"headers":{},"queryString":null,"path":null},
+      queryVerb : "read",
+      transformer : "",
+      datasourceId: "test"},
+      // Expected 400 error - Transformer is not allowed to be empty
+      failOnStatusCode: false}).then((response) => {
+        expect(response.status).to.equal(400)
+        expect(response.body.message).to.include('Invalid body - "transformer" is not allowed to be empty')
+      })
+    })
+  })
+})

packages/builder/cypress/integration/renameAnApplication.spec.js

@@ -1,21 +1,31 @@
-context("Rename an App", () => {
+import filterTests from "../support/filterTests"
+
+filterTests(['all'], () => {
+    context("Rename an App", () => {
         beforeEach(() => {
         cy.login()
         cy.createTestApp()
         })
 
-it("should rename an unpublished application", () => {
+    it("should rename an unpublished application", () => {
+        const appName = "Cypress Tests"
         const appRename = "Cypress Renamed"
         // Rename app, Search for app, Confirm name was changed
         cy.get(".home-logo").click()
-    renameApp(appRename)
+        renameApp(appName, appRename)
+        cy.reload()
+        cy.wait(1000)
         cy.searchForApplication(appRename)
         cy.get(".appTable").find(".title").should("have.length", 1)
-    cy.deleteApp(appRename)
+        // Set app name back to Cypress Tests
-})
+        cy.reload()
+        cy.wait(1000)
+        renameApp(appRename, appName)
+    })
         
-xit("Should rename a published application", () => {
+    xit("Should rename a published application", () => {
         // It is not possible to rename a published application
+        const appName = "Cypress Tests"
         const appRename = "Cypress Renamed"
         // Publish the app
         cy.get(".toprightnav")
@@ -27,24 +37,29 @@ xit("Should rename a published application", () => {
             })
         // Rename app, Search for app, Confirm name was changed
         cy.get(".home-logo").click()
-    renameApp(appRename, true)
+        renameApp(appName, appRename, true)
         cy.searchForApplication(appRename)
-    cy.get(".appTable").find(".title").should("have.length", 1)
+        cy.get(".appTable").find(".wrapper").should("have.length", 1)
-})
+    })
 
-it("Should try to rename an application to have no name", () => {
+    it("Should try to rename an application to have no name", () => {
+        const appName = "Cypress Tests"
         cy.get(".home-logo").click()
-    renameApp(" ", false, true)
+        renameApp(appName, " ", false, true)
+        cy.wait(500)
         // Close modal and confirm name has not been changed
         cy.get(".spectrum-Dialog-grid").contains("Cancel").click()
-    cy.searchForApplication("Cypress Tests")
+        cy.reload() 
+        cy.wait(1000)
+        cy.searchForApplication(appName)
         cy.get(".appTable").find(".title").should("have.length", 1)
-})
 
-xit("Should create two applications with the same name", () => {
+    })
+
+    xit("Should create two applications with the same name", () => {
         // It is not possible to have applications with the same name
         const appName = "Cypress Tests"
-    cy.visit(`localhost:${Cypress.env("PORT")}/builder`)
+        cy.visit(`${Cypress.config().baseUrl}/builder`)
         cy.wait(500)
         cy.get(".spectrum-Button").contains("Create app").click({force: true})
         cy.contains(/Start from scratch/).click()
@@ -54,27 +69,40 @@ xit("Should create two applications with the same name", () => {
         cy.get(".spectrum-ButtonGroup").contains("Create app").click({force: true})
         cy.get(".error").should("have.text", "Another app with the same name already exists")      
         })
-})
+    })
 
-it("should validate application names", () => {
+    it("should validate application names", () => {
         // App name must be letters, numbers and spaces only
         // This test checks numbers and special characters specifically
+        const appName = "Cypress Tests"
         const numberName = 12345
         const specialCharName = "£$%^"
         cy.get(".home-logo").click()
-    renameApp(numberName)
+        renameApp(appName, numberName)
+        cy.reload()
+        cy.wait(1000)
         cy.searchForApplication(numberName)
         cy.get(".appTable").find(".title").should("have.length", 1)
-    renameApp(specialCharName)
+        cy.reload()
+        cy.wait(1000)
+        renameApp(numberName, specialCharName)
         cy.get(".error").should("have.text", "App name must be letters, numbers and spaces only")
-})
+        // Set app name back to Cypress Tests
+        cy.reload()
+        cy.wait(1000)
+        renameApp(numberName, appName)
+    })
         
-    const renameApp = (appName, published, noName) => {
+        const renameApp = (originalName, changedName, published, noName) => {
-    cy.request(`localhost:${Cypress.env("PORT")}/api/applications?status=all`)
+            cy.searchForApplication(originalName)
+            cy.request(`${Cypress.config().baseUrl}/api/applications?status=all`)
         .its("body")
         .then(val => {
             if (val.length > 0) {
-            cy.get(".appTable > :nth-child(5) > :nth-child(2) > .spectrum-Icon").click()
+                cy.get(".appTable")
+                .within(() => {
+                    cy.get(".spectrum-Icon").eq(1).click()
+                    })
                 // Check for when an app is published
                 if (published == true){
                     // Should not have Edit as option, will unpublish app
@@ -93,11 +121,13 @@ it("should validate application names", () => {
                         return cy
                     }
                     cy.get("input").clear()
-                cy.get("input").eq(0).type(appName).should("have.value", appName).blur()
+                    cy.get("input").eq(0).type(changedName).should("have.value", changedName).blur()
                     cy.get(".spectrum-ButtonGroup").contains("Save").click({force: true})
                     cy.wait(500)
         })
         }
         })
-}
+
+    }
+    })
 })

packages/builder/cypress/integration/revertApp.spec.js

@@ -0,0 +1,67 @@
+import filterTests from "../support/filterTests"
+
+filterTests(['smoke', 'all'], () => {
+    context("Revert apps", () => {
+        before(() => {
+        cy.login()
+        cy.createTestApp()
+        })
+        
+        it("should try to revert an unpublished app", () => {
+            // Click revert icon
+            cy.get(".toprightnav").within(() => {
+                cy.get(".spectrum-Icon").eq(1).click()
+            })
+            cy.get(".spectrum-Dialog-grid").within(() => {
+                // Enter app name before revert
+                cy.get("input").type("Cypress Tests")
+                cy.intercept('**/revert').as('revertApp')
+                // Click Revert
+                cy.get(".spectrum-Button").contains("Revert").click({ force: true })
+                // Intercept Request after button click & apply assertions
+                cy.wait("@revertApp")
+                cy.get("@revertApp").its('response.body').should('have.property', 'message', "App has not yet been deployed")
+                cy.get("@revertApp").its('response.body').should('have.property', 'status', 400)
+            })
+        })
+        
+        it("should revert a published app", () => {
+            // Add initial component - Paragraph
+            cy.addComponent("Elements", "Paragraph")
+            // Publish app
+            cy.get(".spectrum-Button").contains("Publish").click({ force: true })
+            cy.get(".spectrum-ButtonGroup").within(() => {
+                cy.get(".spectrum-Button").contains("Publish").click({ force: true })
+            })
+            // Add second component - Button
+            cy.addComponent("Elements", "Button")
+            // Click Revert
+            cy.get(".toprightnav").within(() => {
+                cy.get(".spectrum-Icon").eq(1).click()
+            })
+            cy.get(".spectrum-Dialog-grid").within(() => {
+                // Click Revert
+                cy.get(".spectrum-Button").contains("Revert").click({ force: true })
+                cy.wait(1000)
+            })
+            // Confirm Paragraph component is still visible
+            cy.get(".root").contains("New Paragraph")
+            // Confirm Button component is not visible
+            cy.get(".root").should("not.have.text", "New Button")
+            cy.wait(500)
+        })
+        
+        it("should enter incorrect app name when reverting", () => {
+            // Click Revert
+            cy.get(".toprightnav").within(() => {
+                cy.get(".spectrum-Icon").eq(1).click({ force: true })
+            })
+            // Enter incorrect app name
+            cy.get(".spectrum-Dialog-grid").within(() => {
+                cy.get("input").type("Cypress Tests")
+                // Revert button within modal should be disabled
+                cy.get(".spectrum-Button").eq(1).should('be.disabled')
+            })
+        })
+    })
+})

packages/builder/cypress/setup.js

@@ -4,17 +4,17 @@ const path = require("path")
 const tmpdir = path.join(require("os").tmpdir(), ".budibase")
 
 // normal development system
-const WORKER_PORT = "10002"
+const SERVER_PORT = cypressConfig.env.PORT
-const MAIN_PORT = cypressConfig.env.PORT
+const WORKER_PORT = cypressConfig.env.WORKER_PORT
+
 process.env.BUDIBASE_API_KEY = "6BE826CB-6B30-4AEC-8777-2E90464633DE"
 process.env.NODE_ENV = "cypress"
 process.env.ENABLE_ANALYTICS = "false"
-process.env.PORT = MAIN_PORT
 process.env.JWT_SECRET = cypressConfig.env.JWT_SECRET
 process.env.COUCH_URL = `leveldb://${tmpdir}/.data/`
 process.env.SELF_HOSTED = 1
-process.env.WORKER_URL = "http://localhost:10002/"
+process.env.WORKER_URL = `http://localhost:${WORKER_PORT}/`
-process.env.APPS_URL = `http://localhost:${MAIN_PORT}/`
+process.env.APPS_URL = `http://localhost:${SERVER_PORT}/`
 process.env.MINIO_URL = `http://localhost:4004`
 process.env.MINIO_ACCESS_KEY = "budibase"
 process.env.MINIO_SECRET_KEY = "budibase"
@@ -33,11 +33,14 @@ exports.run = (
   // require("dotenv").config({ path: resolve(dir, ".env") })
   // don't make this a variable or top level require
   // it will cause environment module to be loaded prematurely
-  require(serverLoc)
+
+  // override the port with the worker port temporarily
   process.env.PORT = WORKER_PORT
   require(workerLoc)
-  // reload main port for rest of system
+
-  process.env.PORT = MAIN_PORT
+  // override the port with the server port
+  process.env.PORT = SERVER_PORT
+  require(serverLoc)
 }
 
 if (require.main === module) {

packages/builder/cypress/support/commands.js

@@ -10,7 +10,7 @@ Cypress.on("uncaught:exception", () => {
 })
 
 Cypress.Commands.add("login", () => {
-  cy.visit(`localhost:${Cypress.env("PORT")}/builder`)
+  cy.visit(`${Cypress.config().baseUrl}/builder`)
   cy.wait(2000)
   cy.url().then(url => {
     if (url.includes("builder/admin")) {
@@ -33,36 +33,68 @@ Cypress.Commands.add("login", () => {
 })
 
 Cypress.Commands.add("createApp", name => {
-  cy.visit(`localhost:${Cypress.env("PORT")}/builder`)
+  cy.visit(`${Cypress.config().baseUrl}/builder`)
   cy.wait(500)
-  cy.request(`${Cypress.config().baseUrl}api/applications?status=all`)
-    .its("body")
-    .then(body => {
-      if (body.length > 0) {
   cy.get(".spectrum-Button").contains("Create app").click({ force: true })
-      }
-    })
   cy.get(".spectrum-Modal").within(() => {
     cy.get("input").eq(0).type(name).should("have.value", name).blur()
     cy.get(".spectrum-ButtonGroup").contains("Create app").click()
-    cy.wait(7000)
+    cy.wait(5000)
   })
+  cy.createTable("Cypress Tests", true)
 })
 
-Cypress.Commands.add("deleteApp", appName => {
+Cypress.Commands.add("deleteApp", name => {
-  cy.visit(`localhost:${Cypress.env("PORT")}/builder`)
+  cy.visit(`${Cypress.config().baseUrl}/builder`)
-  cy.wait(1000)
+  cy.wait(2000)
-  cy.request(`localhost:${Cypress.env("PORT")}/api/applications?status=all`)
+  cy.request(`${Cypress.config().baseUrl}/api/applications?status=all`)
     .its("body")
     .then(val => {
       if (val.length > 0) {
-        cy.get(
+        cy.searchForApplication(name)
-          ".appTable > :nth-child(5) > :nth-child(2) > .spectrum-Icon"
+        cy.get(".appTable").within(() => {
-        ).click()
+          cy.get(".spectrum-Icon").eq(1).click()
-        cy.contains("Delete").click()
+        })
-        cy.get(".spectrum-Modal").within(() => {
+        cy.get(".spectrum-Menu").then($menu => {
-          cy.get("input").type(appName)
+          if ($menu.text().includes("Unpublish")) {
+            cy.get(".spectrum-Menu").contains("Unpublish").click()
+            cy.get(".spectrum-Dialog-grid").contains("Unpublish app").click()
+          } else {
+            cy.get(".spectrum-Menu").contains("Delete").click()
+            cy.get(".spectrum-Dialog-grid").within(() => {
+              cy.get("input").type(name)
+            })
             cy.get(".spectrum-Button--warning").click()
+          }
+        })
+      } else {
+        return
+      }
+    })
+})
+
+Cypress.Commands.add("deleteAllApps", () => {
+  cy.visit(`${Cypress.config().baseUrl}/builder`)
+  cy.wait(500)
+  cy.request(`${Cypress.config().baseUrl}/api/applications?status=all`)
+    .its("body")
+    .then(val => {
+      for (let i = 0; i < val.length; i++) {
+        cy.get(".spectrum-Heading")
+          .eq(1)
+          .then(app => {
+            const name = app.text()
+            cy.get(".title")
+              .children()
+              .within(() => {
+                cy.get(".spectrum-Icon").eq(0).click()
+              })
+            cy.get(".spectrum-Menu").contains("Delete").click()
+            cy.get(".spectrum-Dialog-grid").within(() => {
+              cy.get("input").type(name)
+              cy.get(".spectrum-Button--warning").click()
+            })
+            cy.reload()
           })
       }
     })
@@ -72,6 +104,7 @@ Cypress.Commands.add("createTestApp", () => {
   const appName = "Cypress Tests"
   cy.deleteApp(appName)
   cy.createApp(appName, "This app is used for Cypress testing.")
+  cy.createScreen("home", "home")
 })
 
 Cypress.Commands.add("createTestTableWithData", () => {
@@ -80,10 +113,18 @@ Cypress.Commands.add("createTestTableWithData", () => {
   cy.addColumn("dog", "age", "Number")
 })
 
-Cypress.Commands.add("createTable", tableName => {
+Cypress.Commands.add("createTable", (tableName, initialTable) => {
-  cy.contains("Budibase DB").click()
+  if (!initialTable) {
-  cy.contains("Create new table").click()
+    cy.navigateToDataSection()
-
+    cy.get(".add-button").click()
+  }
+  cy.wait(7000)
+  cy.get(".spectrum-Modal")
+    .contains("Budibase DB")
+    .click({ force: true })
+    .then(() => {
+      cy.get(".spectrum-Button").contains("Continue").click({ force: true })
+    })
   cy.get(".spectrum-Modal").within(() => {
     cy.wait(1000)
     cy.get("input").first().type(tableName).blur()
@@ -190,22 +231,49 @@ Cypress.Commands.add("navigateToFrontend", () => {
   cy.wait(1000)
   cy.contains("Design").click()
   cy.get(".spectrum-Search").type("/")
-  cy.createScreen("home", "home")
-  cy.addComponent("Elements", "Headline")
   cy.get(".nav-item").contains("home").click()
 })
 
+Cypress.Commands.add("navigateToDataSection", () => {
+  // Clicks on the Data tab
+  cy.wait(500)
+  cy.contains("Data").click()
+})
+
 Cypress.Commands.add("createScreen", (screenName, route) => {
+  cy.contains("Design").click()
   cy.get("[aria-label=AddCircle]").click()
   cy.get(".spectrum-Modal").within(() => {
-    cy.get(".item").first().click()
+    cy.get(".item").contains("Blank").click()
-    cy.get(".spectrum-Button--cta").click()
+    cy.get(".spectrum-Button").contains("Add Screens").click({ force: true })
+    cy.wait(500)
   })
+  cy.get(".spectrum-Dialog-grid").within(() => {
+    cy.get(".spectrum-Form-itemField").eq(0).type(screenName)
+    cy.get(".spectrum-Form-itemField").eq(1).type(route)
+    cy.get(".spectrum-Button").contains("Continue").click({ force: true })
+    cy.wait(1000)
+  })
+})
+
+Cypress.Commands.add("createAutogeneratedScreens", screenNames => {
+  // Screen name must already exist within data source
+  cy.contains("Design").click()
+  cy.get("[aria-label=AddCircle]").click()
+  for (let i = 0; i < screenNames.length; i++) {
+    cy.get(".item").contains(screenNames[i]).click()
+  }
+  cy.get(".spectrum-Button").contains("Add Screens").click({ force: true })
+  cy.wait(4000)
+})
+
+Cypress.Commands.add("addRow", values => {
+  cy.contains("Create row").click()
   cy.get(".spectrum-Modal").within(() => {
-    cy.get("input").first().clear().type(screenName)
+    for (let i = 0; i < values.length; i++) {
-    cy.get("input").eq(1).clear().type(route)
+      cy.get("input").eq(i).type(values[i]).blur()
-    cy.get(".spectrum-Button--cta").click()
+    }
-    cy.wait(2000)
+    cy.get(".spectrum-ButtonGroup").contains("Create").click()
   })
 })
 
@@ -243,7 +311,144 @@ Cypress.Commands.add("addCustomSourceOptions", totalOptions => {
 })
 
 Cypress.Commands.add("searchForApplication", appName => {
+  cy.wait(1000)
+  // Searches for the app
+  cy.get(".filter").then(() => {
     cy.get(".spectrum-Textfield").within(() => {
       cy.get("input").eq(0).type(appName)
     })
+  })
+  // Confirms app exists after search
+  cy.get(".appTable").contains(appName)
+})
+
+Cypress.Commands.add("selectExternalDatasource", datasourceName => {
+  // Navigates to Data Section
+  cy.navigateToDataSection()
+  // Open Data Source modal
+  cy.get(".nav").within(() => {
+    cy.get(".add-button").click()
+  })
+  // Clicks specified datasource & continue
+  cy.get(".item-list").contains(datasourceName).click()
+  cy.get(".spectrum-Dialog-grid").within(() => {
+    cy.get(".spectrum-Button").contains("Continue").click({ force: true })
+  })
+})
+
+Cypress.Commands.add("addDatasourceConfig", (datasource, skipFetch) => {
+  // selectExternalDatasource should be called prior to this
+  // Adds the config for specified datasource & fetches tables
+  // Currently supports MySQL, PostgreSQL, Oracle
+  // Host IP Address
+  cy.wait(500)
+  cy.get(".spectrum-Dialog-grid").within(() => {
+    cy.get(".form-row")
+      .eq(0)
+      .within(() => {
+        cy.get(".spectrum-Textfield").within(() => {
+          if (datasource == "Oracle") {
+            cy.get("input").clear().type(Cypress.env("oracle").HOST)
+          } else {
+            cy.get("input").clear().type(Cypress.env("HOST_IP"))
+          }
+        })
+      })
+  })
+  // Database Name
+  cy.get(".spectrum-Dialog-grid").within(() => {
+    if (datasource == "MySQL") {
+      cy.get(".form-row")
+        .eq(4)
+        .within(() => {
+          cy.get("input").clear().type(Cypress.env("mysql").DATABASE)
+        })
+    } else {
+      cy.get(".form-row")
+        .eq(2)
+        .within(() => {
+          if (datasource == "PostgreSQL") {
+            cy.get("input").clear().type(Cypress.env("postgresql").DATABASE)
+          }
+          if (datasource == "Oracle") {
+            cy.get("input").clear().type(Cypress.env("oracle").DATABASE)
+          }
+        })
+    }
+  })
+  // User
+  cy.get(".spectrum-Dialog-grid").within(() => {
+    if (datasource == "MySQL") {
+      cy.get(".form-row")
+        .eq(2)
+        .within(() => {
+          cy.get("input").clear().type(Cypress.env("mysql").USER)
+        })
+    } else {
+      cy.get(".form-row")
+        .eq(3)
+        .within(() => {
+          if (datasource == "PostgreSQL") {
+            cy.get("input").clear().type(Cypress.env("postgresql").USER)
+          }
+          if (datasource == "Oracle") {
+            cy.get("input").clear().type(Cypress.env("oracle").USER)
+          }
+        })
+    }
+  })
+  // Password
+  cy.get(".spectrum-Dialog-grid").within(() => {
+    if (datasource == "MySQL") {
+      cy.get(".form-row")
+        .eq(3)
+        .within(() => {
+          cy.get("input").clear().type(Cypress.env("mysql").PASSWORD)
+        })
+    } else {
+      cy.get(".form-row")
+        .eq(4)
+        .within(() => {
+          if (datasource == "PostgreSQL") {
+            cy.get("input").clear().type(Cypress.env("postgresql").PASSWORD)
+          }
+          if (datasource == "Oracle") {
+            cy.get("input").clear().type(Cypress.env("oracle").PASSWORD)
+          }
+        })
+    }
+  })
+  // Click to fetch tables
+  if (skipFetch) {
+    cy.get(".spectrum-Dialog-grid").within(() => {
+      cy.get(".spectrum-Button")
+        .contains("Skip table fetch")
+        .click({ force: true })
+    })
+  } else {
+    cy.get(".spectrum-Dialog-grid").within(() => {
+      cy.get(".spectrum-Button")
+        .contains("Save and fetch tables")
+        .click({ force: true })
+      cy.wait(1000)
+    })
+  }
+})
+
+Cypress.Commands.add("createRestQuery", (method, restUrl) => {
+  // addExternalDatasource should be called prior to this
+  // Configures REST datasource & sends query
+  cy.wait(1000)
+  cy.get(".spectrum-Button").contains("Add query").click({ force: true })
+  // Select Method & add Rest URL
+  cy.get(".spectrum-Picker-label").eq(1).click()
+  cy.get(".spectrum-Menu").contains(method).click()
+  cy.get("input").clear().type(restUrl)
+  // Send query
+  cy.get(".spectrum-Button").contains("Send").click({ force: true })
+  cy.wait(500)
+  cy.get(".spectrum-Button").contains("Save").click({ force: true })
+  cy.get(".hierarchy-items-container")
+    .should("contain", method)
+    .and("contain", restUrl)
 })

packages/builder/cypress/support/filterTests.js

@@ -0,0 +1,16 @@
+const filterTests = (testTags, runTest) => {
+  // testTags is an array of tags
+  // runTest is all tests
+  if (Cypress.env("TEST_TAGS")) {
+    const tags = Cypress.env("TEST_TAGS").split("/")
+    const found = testTags.some($testTags => tags.includes($testTags))
+
+    if (found) {
+      runTest()
+    }
+  } else {
+    runTest()
+  }
+}
+
+export default filterTests

packages/builder/cypress/support/queryLevelTransformerFunction.js

@@ -0,0 +1,14 @@
+/* eslint-disable */
+const breweries = data
+const totals = {}
+
+for (let brewery of breweries) 
+  {const state = brewery.state
+  if (totals[state] == null) 
+    {totals[state] = 1
+  } else 
+    {totals[state]++
+  }
+}
+const entries = Object.entries(totals)
+return entries.map(([state, count]) => ({ state, count }))

packages/builder/cypress/support/queryLevelTransformerFunctionWithData.js

@@ -0,0 +1,31 @@
+/* eslint-disable */
+const breweries = data
+const totals = {}
+for (let brewery of breweries) 
+  {const state = brewery.state
+  if (totals[state] == null) 
+    {totals[state] = 1
+  } else 
+    {totals[state]++
+  }
+}
+const stateCodes = 
+  {texas: "tx",
+  colorado: "co",
+  florida: "fl",
+  iwoa: "ia",
+  louisiana: "la",
+  california: "ca",
+  pennsylvania: "pa",
+  georgia: "ga",
+  "new hampshire": "nh",
+  virginia: "va",
+  michigan: "mi",
+  maryland: "md",
+  ohio: "oh",
+}
+const entries = Object.entries(totals)
+return entries.map(([state, count]) => 
+  {stateCodes[state.toLowerCase()]
+  return { state, count, flag: "http://flags.ox3.in/svg/us/${stateCode}.svg" }
+})

packages/builder/nuxt.config.js

@@ -0,0 +1,4 @@
+export default {
+  ssr: false,
+  target: "static",
+}

packages/builder/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@budibase/builder",
-  "version": "1.0.58-alpha.7",
+  "version": "1.0.79-alpha.0",
   "license": "GPL-3.0",
   "private": true,
   "scripts": {
@@ -11,12 +11,13 @@
     "rollup": "rollup -c -w",
     "cy:setup": "ts-node ./cypress/ts/setup.ts",
     "cy:setup:ci": "node ./cypress/setup.js",
-    "cy:run": "cypress run",
     "cy:open": "cypress open",
-    "cy:run:ci": "cypress run --record",
+    "cy:run": "cypress run",
-    "cy:test": "start-server-and-test cy:setup http://localhost:10001/builder cy:run",
+    "cy:run:ci": "xvfb-run cypress run --headed --browser chrome --record",
-    "cy:ci": "start-server-and-test cy:setup:ci http://localhost:10001/builder cy:run",
+    "cy:test": "start-server-and-test cy:setup http://localhost:4100/builder cy:run",
-    "cy:debug": "start-server-and-test cy:setup http://localhost:10001/builder cy:open"
+    "cy:ci": "start-server-and-test cy:setup:ci http://localhost:4100/builder cy:run:ci",
+    "cy:debug": "start-server-and-test cy:setup http://localhost:4100/builder cy:open",
+    "cy:debug:ci": "start-server-and-test cy:setup:ci http://localhost:4100/builder cy:open"
   },
   "jest": {
     "globals": {
@@ -64,10 +65,10 @@
     }
   },
   "dependencies": {
-    "@budibase/bbui": "^1.0.58-alpha.7",
+    "@budibase/bbui": "^1.0.79-alpha.0",
-    "@budibase/client": "^1.0.58-alpha.7",
+    "@budibase/client": "^1.0.79-alpha.0",
-    "@budibase/frontend-core": "^1.0.58-alpha.7",
+    "@budibase/frontend-core": "^1.0.79-alpha.0",
-    "@budibase/string-templates": "^1.0.58-alpha.7",
+    "@budibase/string-templates": "^1.0.79-alpha.0",
     "@sentry/browser": "5.19.1",
     "@spectrum-css/page": "^3.0.1",
     "@spectrum-css/vars": "^3.0.1",
@@ -94,7 +95,7 @@
     "@testing-library/jest-dom": "^5.11.10",
     "@testing-library/svelte": "^3.0.0",
     "babel-jest": "^26.6.3",
-    "cypress": "^5.1.0",
+    "cypress": "^9.3.1",
     "cypress-terminal-report": "^1.4.1",
     "identity-obj-proxy": "^3.0.0",
     "jest": "^26.6.3",

packages/builder/src/builderStore/dataBinding.js

@@ -275,10 +275,7 @@ const getProviderContextBindings = (asset, dataProviders) => {
  */
 const getUserBindings = () => {
   let bindings = []
-  const { schema } = getSchemaForDatasource(null, {
+  const { schema } = getSchemaForTable(TableNames.USERS)
-    type: "table",
-    tableId: TableNames.USERS,
-  })
   const keys = Object.keys(schema).sort()
   const safeUser = makePropSafe("user")
   keys.forEach(key => {
@@ -385,9 +382,33 @@ export const getButtonContextBindings = (actions, actionId) => {
 }
 
 /**
- * Gets a schema for a datasource object.
+ * Gets the schema for a certain table ID.
+ * The options which can be passed in are:
+ *   formSchema: whether the schema is for a form
+ *   searchableSchema: whether to generate a searchable schema, which may have
+ *     fewer fields than a readable schema
+ * @param tableId the table ID to get the schema for
+ * @param options options for generating the schema
+ * @return {{schema: Object, table: Object}}
  */
-export const getSchemaForDatasource = (asset, datasource, isForm = false) => {
+export const getSchemaForTable = (tableId, options) => {
+  return getSchemaForDatasource(null, { type: "table", tableId }, options)
+}
+
+/**
+ * Gets a schema for a datasource object.
+ * The options which can be passed in are:
+ *   formSchema: whether the schema is for a form
+ *   searchableSchema: whether to generate a searchable schema, which may have
+ *     fewer fields than a readable schema
+ * @param asset the current root client app asset (layout or screen). This is
+ *   optional and only needed for "provider" datasource types.
+ * @param datasource the datasource definition
+ * @param options options for generating the schema
+ * @return {{schema: Object, table: Object}}
+ */
+export const getSchemaForDatasource = (asset, datasource, options) => {
+  options = options || {}
   let schema, table
 
   if (datasource) {
@@ -399,7 +420,7 @@ export const getSchemaForDatasource = (asset, datasource, isForm = false) => {
     if (type === "provider") {
       const component = findComponent(asset.props, datasource.providerId)
       const source = getDatasourceForProvider(asset, component)
-      return getSchemaForDatasource(asset, source, isForm)
+      return getSchemaForDatasource(asset, source, options)
     }
 
     // "query" datasources are those targeting non-plus datasources or
@@ -448,8 +469,16 @@ export const getSchemaForDatasource = (asset, datasource, isForm = false) => {
     // Determine the schema from the backing entity if not already determined
     if (table && !schema) {
       if (type === "view") {
+        // For views, the schema is pulled from the `views` property of the
+        // table
         schema = cloneDeep(table.views?.[datasource.name]?.schema)
-      } else if (type === "query" && isForm) {
+      } else if (
+        type === "query" &&
+        (options.formSchema || options.searchableSchema)
+      ) {
+        // For queries, if we are generating a schema for a form or a searchable
+        // schema then we want to use the query parameters rather than the
+        // query schema
         schema = {}
         const params = table.parameters || []
         params.forEach(param => {
@@ -458,6 +487,7 @@ export const getSchemaForDatasource = (asset, datasource, isForm = false) => {
           }
         })
       } else {
+        // Otherwise we just want the schema of the table
         schema = cloneDeep(table.schema)
       }
     }
@@ -485,11 +515,35 @@ export const getSchemaForDatasource = (asset, datasource, isForm = false) => {
       schema = { ...schema, ...jsonAdditions }
     }
 
-    // Add _id and _rev fields for certain types
+    // Determine if we should add ID and rev to the schema
-    if (schema && !isForm && ["table", "link"].includes(datasource.type)) {
+    const isInternal = table && !table.sql
+    const isTable = ["table", "link"].includes(datasource.type)
+
+    // ID is part of the readable schema for all tables
+    // Rev is part of the readable schema for internal tables only
+    let addId = isTable
+    let addRev = isTable && isInternal
+
+    // Don't add ID or rev for form schemas
+    if (options.formSchema) {
+      addId = false
+      addRev = false
+    }
+
+    // ID is only searchable for internal tables
+    else if (options.searchableSchema) {
+      addId = isTable && isInternal
+    }
+
+    // Add schema properties if required
+    if (schema) {
+      if (addId) {
         schema["_id"] = { type: "string" }
+      }
+      if (addRev) {
         schema["_rev"] = { type: "string" }
       }
+    }
 
     // Ensure there are "name" properties for all fields and that field schema
     // are objects

packages/builder/src/builderStore/store/screenTemplates/utils/commonComponents.js

@@ -141,7 +141,9 @@ const fieldTypeToComponentMap = {
 }
 
 export function makeDatasourceFormComponents(datasource) {
-  const { schema } = getSchemaForDatasource(null, datasource, true)
+  const { schema } = getSchemaForDatasource(null, datasource, {
+    formSchema: true,
+  })
   let components = []
   let fields = Object.keys(schema || {})
   fields.forEach(field => {

packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte

@@ -28,8 +28,8 @@
   import { debounce } from "lodash"
   import ModalBindableInput from "components/common/bindings/ModalBindableInput.svelte"
   import FilterDrawer from "components/design/PropertiesPanel/PropertyControls/FilterEditor/FilterDrawer.svelte"
-  // need the client lucene builder to convert to the structure API expects
   import { LuceneUtils } from "@budibase/frontend-core"
+  import { getSchemaForTable } from "builderStore/dataBinding"
 
   export let block
   export let testData
@@ -46,13 +46,13 @@
     block || $automationStore.selectedBlock,
     $automationStore.selectedAutomation?.automation?.definition
   )
-
   $: inputData = testData ? testData : block.inputs
   $: tableId = inputData ? inputData.tableId : null
   $: table = tableId
     ? $tables.list.find(table => table._id === inputData.tableId)
     : { schema: {} }
-  $: schemaFields = table ? Object.values(table.schema) : []
+  $: schema = getSchemaForTable(tableId, { searchableSchema: true }).schema
+  $: schemaFields = Object.values(schema || {})
 
   const onChange = debounce(async function (e, key) {
     try {

packages/builder/src/components/backend/DataTable/DataTable.svelte

@@ -98,7 +98,7 @@
     tableId={id}
     data={$fetch.rows}
     bind:hideAutocolumns
-    loading={$fetch.loading}
+    loading={!$fetch.loaded}
     on:sort={onSort}
     allowEditing
     disableSorting

packages/builder/src/components/backend/DataTable/Table.svelte

@@ -136,7 +136,7 @@
     </div>
   </div>
   {#key tableId}
-    <div class="table-wrapper" in:fade={{ delay: 200, duration: 100 }}>
+    <div class="table-wrapper">
       <Table
         {data}
         {schema}

packages/builder/src/components/backend/DataTable/ViewDataTable.svelte

@@ -57,7 +57,8 @@
   {data}
   {loading}
   {type}
-  allowEditing={!view?.calculation}
+  allowEditing={false}
+  rowCount={10}
   bind:hideAutocolumns
 >
   <ViewFilterButton {view} />

packages/builder/src/components/common/bindings/BindingPanel.svelte

@@ -23,6 +23,10 @@
   const dispatch = createEventDispatcher()
 
   export let bindings
+  // jsValue/hbsValue are the state of the value that is being built
+  // within this binding panel - the value should not be updated until
+  // the binding panel is saved. This is the default value of the
+  // expression when the binding panel is opened, but shouldn't be updated.
   export let value = ""
   export let valid
   export let allowJS = false
@@ -51,16 +55,16 @@
   })
   $: codeMirrorHints = bindings?.map(x => `$("${x.readableBinding}")`)
 
-  const updateValue = value => {
+  const updateValue = val => {
-    valid = isValid(readableToRuntimeBinding(bindings, value))
+    valid = isValid(readableToRuntimeBinding(bindings, val))
     if (valid) {
-      dispatch("change", value)
+      dispatch("change", val)
     }
   }
 
   // Adds a HBS helper to the expression
   const addHelper = helper => {
-    hbsValue = addHBSBinding(value, getCaretPosition(), helper.text)
+    hbsValue = addHBSBinding(hbsValue, getCaretPosition(), helper.text)
     updateValue(hbsValue)
   }
 
@@ -142,7 +146,7 @@
                   <div class="helper__description">
                     {@html helper.description}
                   </div>
-                  <pre class="helper__example">{helper.example || ''}</pre>
+                  <pre class="helper__example">{helper.example || ""}</pre>
                 </div>
               </li>
             {/each}

packages/builder/src/components/common/bindings/utils.js

@@ -1,7 +1,10 @@
 export function addHBSBinding(value, caretPos, binding) {
   binding = typeof binding === "string" ? binding : binding.path
   value = value == null ? "" : value
-  if (!value.includes("{{") && !value.includes("}}")) {
+
+  const left = caretPos?.start ? value.substring(0, caretPos.start) : ""
+  const right = caretPos?.end ? value.substring(caretPos.end) : ""
+  if (!left.includes("{{") || !right.includes("}}")) {
     binding = `{{ ${binding} }}`
   }
   if (caretPos.start) {

packages/builder/src/components/design/NavigationPanel/ScreenWizard.svelte

@@ -12,11 +12,12 @@
   let screenName = ""
   let url = ""
   let selectedScreens = []
-  let roleId = $selectedAccessRole || "BASIC"
   let showProgressCircle = false
   let routeError
   let createdScreens = []
 
+  $: roleId = $selectedAccessRole || "BASIC"
+
   const createScreens = async () => {
     for (let screen of selectedScreens) {
       let test = screen.create()
@@ -74,6 +75,7 @@
       }
 
       draftScreen.routing.route = route
+      draftScreen.routing.roleId = roleId
 
       await store.actions.screens.save(draftScreen)
       if (draftScreen.props._instanceName.endsWith("List")) {

packages/builder/src/components/design/PropertiesPanel/PropertyControls/ButtonActionEditor/ButtonActionDrawer.svelte

@@ -126,6 +126,7 @@
   </Layout>
   <Layout noPadding>
     {#if selectedActionComponent}
+      {#key selectedAction.id}
         <div class="selected-action-container">
           <svelte:component
             this={selectedActionComponent}
@@ -133,6 +134,7 @@
             bindings={allBindings}
           />
         </div>
+      {/key}
     {/if}
   </Layout>
 </DrawerContent>

packages/builder/src/components/design/PropertiesPanel/PropertyControls/ButtonActionEditor/actions/DuplicateRow.svelte

@@ -4,7 +4,7 @@
   import { tables } from "stores/backend"
   import {
     getContextProviderComponents,
-    getSchemaForDatasource,
+    getSchemaForTable,
   } from "builderStore/dataBinding"
   import SaveFields from "./SaveFields.svelte"
 
@@ -60,7 +60,7 @@
   }
 
   const getSchemaFields = (asset, tableId) => {
-    const { schema } = getSchemaForDatasource(asset, { type: "table", tableId })
+    const { schema } = getSchemaForTable(tableId)
     delete schema._id
     delete schema._rev
     return Object.values(schema || {})

packages/builder/src/components/design/PropertiesPanel/PropertyControls/ButtonActionEditor/actions/SaveRow.svelte

@@ -4,7 +4,7 @@
   import { tables } from "stores/backend"
   import {
     getContextProviderComponents,
-    getSchemaForDatasource,
+    getSchemaForTable,
   } from "builderStore/dataBinding"
   import SaveFields from "./SaveFields.svelte"
 
@@ -60,7 +60,7 @@
   }
 
   const getSchemaFields = (asset, tableId) => {
-    const { schema } = getSchemaForDatasource(asset, { type: "table", tableId })
+    const { schema } = getSchemaForTable(tableId)
     return Object.values(schema || {})
   }
 

packages/builder/src/components/design/PropertiesPanel/PropertyControls/ColumnEditor/CellDrawer.svelte

@@ -0,0 +1,66 @@
+<script>
+  import {
+    Input,
+    Select,
+    ColorPicker,
+    DrawerContent,
+    Layout,
+    Label,
+  } from "@budibase/bbui"
+  import { store } from "builderStore"
+  import DrawerBindableInput from "components/common/bindings/DrawerBindableInput.svelte"
+
+  export let column
+</script>
+
+<DrawerContent>
+  <div class="container">
+    <Layout noPadding gap="S">
+      <Input bind:value={column.width} label="Width" placeholder="Auto" />
+      <Select
+        label="Alignment"
+        bind:value={column.align}
+        options={["Left", "Center", "Right"]}
+        placeholder="Default"
+      />
+      <DrawerBindableInput
+        label="Value"
+        value={column.template}
+        on:change={e => (column.template = e.detail)}
+        placeholder={`{{ Value }}`}
+        bindings={[
+          {
+            readableBinding: "Value",
+            runtimeBinding: "[value]",
+          },
+        ]}
+      />
+      <Layout noPadding gap="XS">
+        <Label>Background color</Label>
+        <ColorPicker
+          value={column.background}
+          on:change={e => (column.background = e.detail)}
+          alignRight
+          spectrumTheme={$store.theme}
+        />
+      </Layout>
+      <Layout noPadding gap="XS">
+        <Label>Text color</Label>
+        <ColorPicker
+          value={column.color}
+          on:change={e => (column.color = e.detail)}
+          alignRight
+          spectrumTheme={$store.theme}
+        />
+      </Layout>
+    </Layout>
+  </div>
+</DrawerContent>
+
+<style>
+  .container {
+    width: 100%;
+    max-width: 240px;
+    margin: 0 auto;
+  }
+</style>

packages/builder/src/components/design/PropertiesPanel/PropertyControls/ColumnEditor/CellEditor.svelte

@@ -0,0 +1,34 @@
+<script>
+  import { Drawer, Button, Icon } from "@budibase/bbui"
+  import CellDrawer from "./CellDrawer.svelte"
+
+  export let column
+
+  let boundValue
+  let drawer
+
+  $: updateBoundValue(column)
+
+  const updateBoundValue = value => {
+    boundValue = { ...value }
+  }
+
+  const open = () => {
+    updateBoundValue(column)
+    drawer.show()
+  }
+
+  const save = () => {
+    column = boundValue
+    drawer.hide()
+  }
+</script>
+
+<Icon name="Settings" hoverable size="S" on:click={open} />
+<Drawer bind:this={drawer} title="Table Columns">
+  <svelte:fragment slot="description">
+    "{column.name}" column settings
+  </svelte:fragment>
+  <Button cta slot="buttons" on:click={save}>Save</Button>
+  <CellDrawer slot="body" bind:column={boundValue} />
+</Drawer>

packages/builder/src/components/design/PropertiesPanel/PropertyControls/ColumnEditor/ColumnDrawer.svelte

@@ -4,17 +4,19 @@
     Icon,
     DrawerContent,
     Layout,
-    Input,
     Select,
     Label,
     Body,
+    Input,
   } from "@budibase/bbui"
   import { flip } from "svelte/animate"
   import { dndzone } from "svelte-dnd-action"
   import { generate } from "shortid"
+  import CellEditor from "./CellEditor.svelte"
 
   export let columns = []
   export let options = []
+  export let schema = {}
 
   const flipDurationMs = 150
   let dragDisabled = true
@@ -61,11 +63,23 @@
     dragDisabled = true
   }
 
+  const addAllColumns = () => {
+    let newColumns = columns || []
+    options.forEach(field => {
+      const fieldSchema = schema[field]
+      const hasCol = columns && columns.findIndex(x => x.name === field) !== -1
+      if (!fieldSchema?.autocolumn && !hasCol) {
+        newColumns.push({
+          name: field,
+          displayName: field,
+        })
+      }
+    })
+    columns = newColumns
+  }
+
   const reset = () => {
-    columns = options.map(col => ({
+    columns = []
-      name: col,
-      displayName: col,
-    }))
   }
 </script>
 
@@ -79,6 +93,7 @@
             <Label size="L">Column</Label>
             <Label size="L">Label</Label>
             <div />
+            <div />
           </div>
           <div
             class="columns"
@@ -108,6 +123,7 @@
                   on:change={e => (column.displayName = e.detail)}
                 />
                 <Input bind:value={column.displayName} placeholder="Label" />
+                <CellEditor bind:column />
                 <Icon
                   name="Close"
                   hoverable
@@ -121,19 +137,25 @@
         </Layout>
       {:else}
         <div class="column">
-          <div />
+          <div class="wide">
-          <Body size="S">Add the first column to your table.</Body>
+            <Body size="S">
+              By default, all table columns will automatically be shown.
+              <br />
+              You can manually control which columns are included in your table,
+              and their appearance, by adding them below.
+            </Body>
+          </div>
         </div>
       {/if}
-      <div class="columns">
       <div class="column">
-          <div />
+        <div class="buttons wide">
-          <div class="buttons">
+          <Button secondary icon="Add" on:click={addColumn}>Add column</Button>
-            <Button secondary icon="Add" on:click={addColumn}>
+          <Button secondary quiet on:click={addAllColumns}>
-              Add column
+            Add all columns
           </Button>
+          {#if columns?.length}
             <Button secondary quiet on:click={reset}>Reset columns</Button>
-          </div>
+          {/if}
         </div>
       </div>
     </Layout>
@@ -156,7 +178,7 @@
   .column {
     gap: var(--spacing-l);
     display: grid;
-    grid-template-columns: 20px 1fr 1fr 20px;
+    grid-template-columns: 20px 1fr 1fr auto auto;
     align-items: center;
     border-radius: var(--border-radius-s);
     transition: background-color ease-in-out 130ms;
@@ -168,6 +190,9 @@
     display: grid;
     place-items: center;
   }
+  .wide {
+    grid-column: 2 / -1;
+  }
   .buttons {
     display: flex;
     flex-direction: row;

packages/builder/src/components/design/PropertiesPanel/PropertyControls/ColumnEditor/ColumnEditor.svelte

@@ -18,22 +18,30 @@
   let boundValue
 
   $: datasource = getDatasourceForProvider($currentAsset, componentInstance)
-  $: schema = getSchemaForDatasource($currentAsset, datasource).schema
+  $: schema = getSchema($currentAsset, datasource)
   $: options = Object.keys(schema || {})
   $: sanitisedValue = getValidColumns(value, options)
   $: updateBoundValue(sanitisedValue)
 
+  const getSchema = (asset, datasource) => {
+    const schema = getSchemaForDatasource(asset, datasource).schema
+
+    // Don't show ID and rev in tables
+    if (schema) {
+      delete schema._id
+      delete schema._rev
+    }
+
+    return schema
+  }
+
   const updateBoundValue = value => {
     boundValue = cloneDeep(value)
   }
 
   const getValidColumns = (columns, options) => {
-    // If no columns then default to all columns
     if (!Array.isArray(columns) || !columns.length) {
-      return options.map(col => ({
+      return []
-        name: col,
-        displayName: col,
-      }))
     }
     // We need to account for legacy configs which would just be an array
     // of strings
@@ -48,17 +56,22 @@
     })
   }
 
+  const open = () => {
+    updateBoundValue(sanitisedValue)
+    drawer.show()
+  }
+
   const save = () => {
     dispatch("change", getValidColumns(boundValue, options))
     drawer.hide()
   }
 </script>
 
-<ActionButton on:click={drawer.show}>Configure columns</ActionButton>
+<ActionButton on:click={open}>Configure columns</ActionButton>
 <Drawer bind:this={drawer} title="Table Columns">
   <svelte:fragment slot="description">
     Configure the columns in your table.
   </svelte:fragment>
   <Button cta slot="buttons" on:click={save}>Save</Button>
-  <ColumnDrawer slot="body" bind:columns={boundValue} {options} />
+  <ColumnDrawer slot="body" bind:columns={boundValue} {options} {schema} />
 </Drawer>

packages/builder/src/components/design/PropertiesPanel/PropertyControls/FilterEditor/FilterDrawer.svelte

@@ -184,6 +184,7 @@
     max-width: 1000px;
     margin: 0 auto;
   }
+
   .fields {
     display: grid;
     column-gap: var(--spacing-l);

packages/builder/src/components/design/PropertiesPanel/PropertyControls/FilterEditor/FilterEditor.svelte

@@ -18,7 +18,9 @@
   let tempValue = value || []
 
   $: dataSource = getDatasourceForProvider($currentAsset, componentInstance)
-  $: schema = getSchemaForDatasource($currentAsset, dataSource)?.schema
+  $: schema = getSchemaForDatasource($currentAsset, dataSource, {
+    searchableSchema: true,
+  })?.schema
   $: schemaFields = Object.values(schema || {})
 
   const saveFilter = async () => {

packages/builder/src/components/design/PropertiesPanel/PropertyControls/FormFieldSelect.svelte

@@ -17,7 +17,9 @@
     component => component._component === "@budibase/standard-components/form"
   )
   $: datasource = getDatasourceForProvider($currentAsset, form)
-  $: schema = getSchemaForDatasource($currentAsset, datasource, true).schema
+  $: schema = getSchemaForDatasource($currentAsset, datasource, {
+    formSchema: true,
+  }).schema
   $: options = getOptions(schema, type)
 
   const getOptions = (schema, type) => {

packages/builder/src/components/design/PropertiesPanel/PropertyControls/SearchFieldSelect.svelte

@@ -15,7 +15,9 @@
 
   const dispatch = createEventDispatcher()
   $: datasource = getDatasourceForProvider($currentAsset, componentInstance)
-  $: schema = getSchemaForDatasource($currentAsset, datasource).schema
+  $: schema = getSchemaForDatasource($currentAsset, datasource, {
+    searchableSchema: true,
+  }).schema
   $: options = getOptions(datasource, schema || {})
   $: boundValue = getSelectedOption(value, options)
 

packages/builder/src/components/design/PropertiesPanel/PropertyControls/ValidationEditor/ValidationDrawer.svelte

@@ -116,6 +116,7 @@
   $: schemaRules = parseRulesFromSchema(field, dataSourceSchema || {})
   $: fieldType = type?.split("/")[1] || "string"
   $: constraintOptions = getConstraintsForType(fieldType)
+
   const getConstraintsForType = type => {
     return ConstraintMap[type]
   }

packages/builder/src/components/integration/QueryViewer.svelte

@@ -39,7 +39,7 @@
   $: datasource = $datasources.list.find(ds => ds._id === query.datasourceId)
   $: query.schema = fieldsToSchema(fields)
   $: datasourceType = datasource?.source
-  $: integrationInfo = $integrations[datasourceType]
+  $: integrationInfo = datasourceType ? $integrations[datasourceType] : null
   $: queryConfig = integrationInfo?.query
   $: shouldShowQueryConfig = queryConfig && query.queryVerb
   $: readQuery = query.queryVerb === "read" || query.readable
@@ -160,7 +160,7 @@
     </div>
     <div class="viewer-controls">
       <Heading size="S">Results</Heading>
-      <ButtonGroup>
+      <ButtonGroup gap="M">
         <Button cta disabled={queryInvalid} on:click={saveQuery}>
           Save Query
         </Button>

packages/builder/src/constants/backend/index.js

@@ -84,10 +84,7 @@ export const FIELDS = {
   FORMULA: {
     name: "Formula",
     type: "formula",
-    constraints: {
+    constraints: {},
-      type: "string",
-      presence: false,
-    },
   },
   JSON: {
     name: "JSON",

packages/builder/src/helpers/data/utils.js

@@ -30,8 +30,8 @@ export function breakQueryString(qs) {
   const params = qs.split("&")
   let paramObj = {}
   for (let param of params) {
-    const [key, value] = param.split("=")
+    const split = param.split("=")
-    paramObj[key] = value
+    paramObj[split[0]] = split.slice(1).join("=")
   }
   return paramObj
 }

packages/builder/src/pages/builder/app/[application]/design/[assetType]/index.svelte

@@ -45,6 +45,14 @@
         store.actions.screens.select(id)
       }
     }
+
+    // If we didn't find a valid asset, just update the preview type
+    if (!id) {
+      store.update(state => {
+        state.currentFrontEndType = assetType
+        return state
+      })
+    }
   }
 </script>
 

packages/builder/src/pages/builder/portal/manage/users/index.svelte

@@ -43,7 +43,7 @@
   let createUserModal
   let basicOnboardingModal
 
-  function openBasicOnoboardingModal() {
+  function openBasicOnboardingModal() {
     createUserModal.hide()
     basicOnboardingModal.show()
   }
@@ -91,7 +91,7 @@
 </Layout>
 
 <Modal bind:this={createUserModal}>
-  <AddUserModal on:change={openBasicOnoboardingModal} />
+  <AddUserModal on:change={openBasicOnboardingModal} />
 </Modal>
 <Modal bind:this={basicOnboardingModal}><BasicOnboardingModal {email} /></Modal>
 

packages/builder/src/stores/backend/queries.js

@@ -60,7 +60,7 @@ export function createQueriesStore() {
       })
       return savedQuery
     },
-    import: async (data, datasourceId) => {
+    import: async ({ data, datasourceId }) => {
       return await API.importQueries({
         datasourceId,
         data,

packages/builder/src/stores/portal/users.js

@@ -11,7 +11,7 @@ export function createUsersStore() {
   }
 
   async function invite({ email, builder, admin }) {
-    await API.inviteUser({
+    return API.inviteUser({
       email,
       builder,
       admin,
@@ -19,7 +19,7 @@ export function createUsersStore() {
   }
 
   async function acceptInvite(inviteCode, password) {
-    await API.acceptInvite({
+    return API.acceptInvite({
       inviteCode,
       password,
     })

packages/cli/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@budibase/cli",
-  "version": "1.0.58-alpha.7",
+  "version": "1.0.79-alpha.0",
   "description": "Budibase CLI, for developers, self hosting and migrations.",
   "main": "src/index.js",
   "bin": {