Merge branch 'master' of github.com:Budibase/budibase into chore/refactor-finalizerow

.eslintignore

@@ -9,8 +9,5 @@ packages/server/client
 packages/server/coverage
 packages/builder/.routify
 packages/sdk/sdk
-packages/account-portal/packages/server/build
-packages/account-portal/packages/ui/.routify
-packages/account-portal/packages/ui/build
 **/*.ivm.bundle.js
 packages/server/build/oldClientVersions/**/**

.github/workflows/budibase_ci.yml

@@ -64,18 +64,15 @@ jobs:
       - run: yarn --frozen-lockfile
 
       # Run build all the projects
-      - name: Build OSS
-        run: yarn build:oss
-      - name: Build account portal
-        run: yarn build:account-portal
-        if: ${{ env.IS_OSS_CONTRIBUTOR == 'false' }}
+      - name: Build
+        run: yarn build
       # Check the types of the projects built via esbuild
       - name: Check types
         run: |
           if ${{ env.ONLY_AFFECTED_TASKS }}; then
-            yarn check:types --since=${{ env.NX_BASE_BRANCH }} --ignore @budibase/account-portal-server
+            yarn check:types --since=${{ env.NX_BASE_BRANCH }}
           else
-            yarn check:types --ignore @budibase/account-portal-server
+            yarn check:types
           fi
 
   helm-lint:
@@ -117,9 +114,11 @@ jobs:
       - name: Test
         run: |
           if ${{ env.ONLY_AFFECTED_TASKS }}; then
-            yarn test --ignore=@budibase/worker --ignore=@budibase/server --since=${{ env.NX_BASE_BRANCH }}
+            yarn test -- --ignore=@budibase/worker --ignore=@budibase/server --ignore=@budibase/builder --no-prefix --since=${{ env.NX_BASE_BRANCH }} -- --verbose --reporters=default --reporters=github-actions
+            yarn test -- --scope=@budibase/builder --since=${{ env.NX_BASE_BRANCH }}
           else
-            yarn test --ignore=@budibase/worker --ignore=@budibase/server
+            yarn test -- --ignore=@budibase/worker --ignore=@budibase/server --ignore=@budibase/builder --no-prefix -- --verbose --reporters=default --reporters=github-actions
+            yarn test -- --scope=@budibase/builder --no-prefix
           fi
 
   test-worker:
@@ -141,13 +140,22 @@ jobs:
       - name: Test worker
         run: |
           if ${{ env.ONLY_AFFECTED_TASKS }}; then
-            node scripts/run-affected.js --task=test --scope=@budibase/worker --since=${{ env.NX_BASE_BRANCH }}
-          else
-            yarn test --scope=@budibase/worker
+            AFFECTED=$(yarn --silent nx show projects --affected -t test --base=${{ env.NX_BASE_BRANCH }} -p @budibase/worker)
+            if [ -z "$AFFECTED" ]; then
+              echo "No affected tests to run"
+              exit 0
+            fi
           fi
 
+          cd packages/worker
+          yarn test --verbose --reporters=default --reporters=github-actions
+
   test-server:
-    runs-on: budi-tubby-tornado-quad-core-300gb
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        datasource:
+          [mssql, mysql, postgres, mongodb, mariadb, oracle, sqs, none]
     steps:
       - name: Checkout repo
         uses: actions/checkout@v4
@@ -170,12 +178,19 @@ jobs:
 
       - name: Pull testcontainers images
         run: |
-          docker pull mcr.microsoft.com/mssql/server@${{ steps.dotenv.outputs.MSSQL_SHA }} &
-          docker pull mysql@${{ steps.dotenv.outputs.MYSQL_SHA }} &
-          docker pull postgres@${{ steps.dotenv.outputs.POSTGRES_SHA }} &
-          docker pull mongo@${{ steps.dotenv.outputs.MONGODB_SHA }} &
-          docker pull mariadb@${{ steps.dotenv.outputs.MARIADB_SHA }} &
-          docker pull budibase/oracle-database:23.2-slim-faststart &
+          if [ "${{ matrix.datasource }}" == "mssql" ]; then
+            docker pull mcr.microsoft.com/mssql/server@${{ steps.dotenv.outputs.MSSQL_SHA }}
+          elif [ "${{ matrix.datasource }}" == "mysql" ]; then
+            docker pull mysql@${{ steps.dotenv.outputs.MYSQL_SHA }}
+          elif [ "${{ matrix.datasource }}" == "postgres" ]; then
+            docker pull postgres@${{ steps.dotenv.outputs.POSTGRES_SHA }}
+          elif [ "${{ matrix.datasource }}" == "mongodb" ]; then
+            docker pull mongo@${{ steps.dotenv.outputs.MONGODB_SHA }}
+          elif [ "${{ matrix.datasource }}" == "mariadb" ]; then
+            docker pull mariadb@${{ steps.dotenv.outputs.MARIADB_SHA }}
+          elif [ "${{ matrix.datasource }}" == "oracle" ]; then
+            docker pull budibase/oracle-database:23.2-slim-faststart
+          fi
           docker pull minio/minio &
           docker pull redis &
           docker pull testcontainers/ryuk:0.5.1 &
@@ -186,13 +201,25 @@ jobs:
       - run: yarn --frozen-lockfile
 
       - name: Test server
+        env:
+          DATASOURCE: ${{ matrix.datasource }}
         run: |
           if ${{ env.ONLY_AFFECTED_TASKS }}; then
-            node scripts/run-affected.js --task=test --scope=@budibase/server --since=${{ env.NX_BASE_BRANCH }}
-          else
-            yarn test --scope=@budibase/server
+            AFFECTED=$(yarn --silent nx show projects --affected -t test --base=${{ env.NX_BASE_BRANCH }} -p @budibase/server)
+            if [ -z "$AFFECTED" ]; then
+              echo "No affected tests to run"
+              exit 0
+            fi
           fi
 
+          FILTER="./src/tests/filters/datasource-tests.js"
+          if [ "${{ matrix.datasource }}" == "none" ]; then
+            FILTER="./src/tests/filters/non-datasource-tests.js"
+          fi
+
+          cd packages/server
+          yarn test --filter $FILTER --verbose --reporters=default --reporters=github-actions
+
   check-pro-submodule:
     runs-on: ubuntu-latest
     if: inputs.run_as_oss != true  && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == 'Budibase/budibase')
@@ -252,64 +279,6 @@ jobs:
             echo 'All good, the submodule had been merged and setup correctly!'
           fi
 
-  check-accountportal-submodule:
-    runs-on: ubuntu-latest
-    if: inputs.run_as_oss != true  && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == 'Budibase/budibase')
-    steps:
-      - name: Checkout repo and submodules
-        uses: actions/checkout@v4
-        with:
-          submodules: true
-          token: ${{ secrets.PERSONAL_ACCESS_TOKEN || github.token }}
-          fetch-depth: 0
-
-      - uses: dorny/paths-filter@v3
-        id: changes
-        with:
-          filters: |
-            src:
-              - packages/account-portal/**
-
-      - if: steps.changes.outputs.src == 'true'
-        name: Check account portal commit
-        id: get_accountportal_commits
-        run: |
-          cd packages/account-portal
-          accountportal_commit=$(git rev-parse HEAD)
-
-          branch="${{ github.base_ref || github.ref_name  }}"
-          echo "Running on branch '$branch' (base_ref=${{ github.base_ref }}, ref_name=${{ github.head_ref }})"
-
-          base_commit=$(git rev-parse origin/master)
-
-          if [[ ! -z $base_commit ]]; then
-            echo "target_branch=$branch"
-            echo "target_branch=$branch" >> "$GITHUB_OUTPUT"
-            echo "accountportal_commit=$accountportal_commit"
-            echo "accountportal_commit=$accountportal_commit" >> "$GITHUB_OUTPUT"
-            echo "base_commit=$base_commit"
-            echo "base_commit=$base_commit" >> "$GITHUB_OUTPUT"
-          else
-            echo "Nothing to do - branch to branch merge."
-          fi
-
-      - name: Check submodule merged to base branch
-        if: ${{ steps.get_accountportal_commits.outputs.base_commit != '' }}
-        uses: actions/github-script@v7
-        with:
-          github-token: ${{ secrets.GITHUB_TOKEN }}
-          script: |
-            const submoduleCommit = '${{ steps.get_accountportal_commits.outputs.accountportal_commit }}';
-            const baseCommit = '${{ steps.get_accountportal_commits.outputs.base_commit }}';
-
-            if (submoduleCommit !== baseCommit) {
-              console.error('Submodule commit does not match the latest commit on the "${{ steps.get_accountportal_commits.outputs.target_branch }}" branch.');
-              console.error('Refer to the account portal repo to merge your changes: https://github.com/Budibase/account-portal/blob/master/docs/index.md')
-              process.exit(1);
-            } else {
-              console.log('All good, the submodule had been merged and setup correctly!')
-            }
-
   check-lockfile:
     runs-on: ubuntu-latest
     steps:

.gitignore

@@ -8,6 +8,7 @@ packages/server/build/oldClientVersions/**/*
 packages/builder/src/components/deploy/clientVersions.json
 packages/server/src/integrations/tests/utils/*.lock
 packages/builder/vite.config.mjs.timestamp*
+packages/account-portal
 
 # Logs
 logs
@@ -110,4 +111,4 @@ budibase-component
 budibase-datasource
 
 *.iml
-.nx
+.nx

.gitmodules

@@ -1,6 +1,3 @@
 [submodule "packages/pro"]
 	path = packages/pro
 	url = git@github.com:Budibase/budibase-pro.git
-[submodule "packages/account-portal"]
-	path = packages/account-portal
-	url = git@github.com:Budibase/account-portal.git

.prettierignore

@@ -9,8 +9,4 @@ packages/backend-core/coverage
 packages/builder/.routify
 packages/sdk/sdk
 packages/pro/coverage
-packages/account-portal/packages/ui/build
-packages/account-portal/packages/ui/.routify
-packages/account-portal/packages/server/build
-packages/account-portal/packages/server/coverage
 **/*.ivm.bundle.js

.vscode/launch.json

@@ -20,16 +20,6 @@
       "args": ["${workspaceFolder}/packages/worker/src/index.ts"],
       "cwd": "${workspaceFolder}/packages/worker"
     },
-    {
-      "name": "Camunda Worker",
-      "type": "node",
-      "request": "launch",
-      "runtimeArgs": ["--nolazy", "-r", "ts-node/register/transpile-only"],
-      "args": [
-        "${workspaceFolder}/packages/account-portal/packages/server/src/v2/run.ts"
-      ],
-      "cwd": "${workspaceFolder}/packages/account-portal/packages/server"
-    },
     {
       "type": "chrome",
       "request": "launch",

charts/budibase/templates/secrets.yaml

@@ -12,12 +12,12 @@ metadata:
 type: Opaque
 data:
   {{- if $existingSecret }}
-  internalApiKey: {{ index $existingSecret.data "internalApiKey" }}
-  jwtSecret: {{ index $existingSecret.data "jwtSecret" }}
-  objectStoreAccess: {{ index $existingSecret.data "objectStoreAccess" }}
-  objectStoreSecret: {{ index $existingSecret.data "objectStoreSecret" }}
-  bbEncryptionKey: {{ index $existingSecret.data "bbEncryptionKey" }}
-  apiEncryptionKey: {{ index $existingSecret.data "apiEncryptionKey" }}
+  internalApiKey: {{ index $existingSecret.data "internalApiKey" | quote }}
+  jwtSecret: {{ index $existingSecret.data "jwtSecret" | quote }}
+  objectStoreAccess: {{ index $existingSecret.data "objectStoreAccess" | quote }}
+  objectStoreSecret: {{ index $existingSecret.data "objectStoreSecret" | quote }}
+  bbEncryptionKey: {{ index $existingSecret.data "bbEncryptionKey" | quote }}
+  apiEncryptionKey: {{ index $existingSecret.data "apiEncryptionKey" | quote }}
   {{- else }}
   internalApiKey: {{ template "budibase.defaultsecret" .Values.globals.internalApiKey }}
   jwtSecret: {{ template "budibase.defaultsecret" .Values.globals.jwtSecret }}

examples/nextjs-api-sales/yarn.lock

@@ -423,9 +423,9 @@ core-js-pure@^3.20.2:
   integrity sha512-12VZfFIu+wyVbBebyHmRTuEE/tZrB4tJToWcwAMcsp3h4+sHR+fMJWbKpYiCRWlhFBq+KNyO8rIV9rTkeVmznQ==
 
 cross-spawn@^7.0.2:
-  version "7.0.3"
-  resolved "https://registry.yarnpkg.com/cross-spawn/-/cross-spawn-7.0.3.tgz#f73a85b9d5d41d045551c177e2882d4ac85728a6"
-  integrity sha512-iRDPJKUPVEND7dHPO8rkbOnPpyDygcDFtWjpeWNCgy8WP2rXcxXL8TskReQl6OrB2G7+UJrags1q15Fudc7G6w==
+  version "7.0.6"
+  resolved "https://registry.yarnpkg.com/cross-spawn/-/cross-spawn-7.0.6.tgz#8a58fe78f00dcd70c370451759dfbfaf03e8ee9f"
+  integrity sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA==
   dependencies:
     path-key "^3.1.0"
     shebang-command "^2.0.0"

globalSetup.ts

@@ -62,6 +62,7 @@ export default async function setup() {
         },
       ])
       .withLabels({ "com.budibase": "true" })
+      .withTmpFs({ "/data": "rw" })
       .withReuse()
       .withWaitStrategy(
         Wait.forSuccessfulCommand(
@@ -72,6 +73,7 @@ export default async function setup() {
     const minio = new GenericContainer("minio/minio")
       .withExposedPorts(9000)
       .withCommand(["server", "/data"])
+      .withTmpFs({ "/data": "rw" })
       .withEnvironment({
         MINIO_ACCESS_KEY: "budibase",
         MINIO_SECRET_KEY: "budibase",

hosting/.env

@@ -19,7 +19,6 @@ MINIO_PORT=4004
 COUCH_DB_PORT=4005
 COUCH_DB_SQS_PORT=4006
 REDIS_PORT=6379
-WATCHTOWER_PORT=6161
 BUDIBASE_ENVIRONMENT=PRODUCTION
 SQL_MAX_ROWS=
 

hosting/docker-compose.build.yaml

@@ -74,7 +74,6 @@ services:
       - WORKER_UPSTREAM_URL=http://worker-service:4003
       - MINIO_UPSTREAM_URL=http://minio-service:9000
       - COUCHDB_UPSTREAM_URL=http://couchdb-service:5984
-      - WATCHTOWER_UPSTREAM_URL=http://watchtower-service:8080
       - RESOLVER=127.0.0.11
     depends_on:
       - minio-service

hosting/docker-compose.yaml

@@ -87,7 +87,6 @@ services:
       - WORKER_UPSTREAM_URL=http://worker-service:4003
       - MINIO_UPSTREAM_URL=http://minio-service:9000
       - COUCHDB_UPSTREAM_URL=http://couchdb-service:5984
-      - WATCHTOWER_UPSTREAM_URL=http://watchtower-service:8080
       - RESOLVER=127.0.0.11
     depends_on:
       - minio-service
@@ -112,19 +111,6 @@ services:
     volumes:
       - redis_data:/data
 
-  watchtower-service:
-    restart: always
-    image: containrrr/watchtower
-    volumes:
-      - /var/run/docker.sock:/var/run/docker.sock
-    command: --debug --http-api-update bbapps bbworker bbproxy
-    environment:
-      - WATCHTOWER_HTTP_API=true
-      - WATCHTOWER_HTTP_API_TOKEN=budibase
-      - WATCHTOWER_CLEANUP=true
-    labels:
-      - "com.centurylinklabs.watchtower.enable=false"
-
 volumes:
   couchdb3_data:
     driver: local

hosting/envoy.yaml

@@ -1,152 +0,0 @@
-static_resources:
-  listeners:
-  - name: main_listener
-    address:
-      socket_address: { address: 0.0.0.0, port_value: 10000 }
-    filter_chains:
-    - filters:
-      - name: envoy.filters.network.http_connection_manager
-        typed_config:
-          "@type": type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager
-          stat_prefix: ingress
-          codec_type: auto
-          route_config:
-            name: local_route
-            virtual_hosts:
-            - name: local_services
-              domains: ["*"]
-              routes:
-              - match: { prefix: "/app/" }
-                route:
-                  cluster: app-service
-                  prefix_rewrite: "/"
-
-              - match: { path: "/v1/update" }
-                route:
-                  cluster: watchtower-service
-
-              - match: { prefix: "/builder/" }
-                route:
-                  cluster: app-service
-
-              - match: { prefix: "/builder" }
-                route:
-                  cluster: app-service
-              
-              - match: { prefix: "/app_" }
-                route:
-                  cluster: app-service
-
-              # special cases for worker admin (deprecated), global and system API
-              - match: { prefix: "/api/global/" }
-                route:
-                  cluster: worker-service
-
-              - match: { prefix: "/api/admin/" }
-                route:
-                  cluster: worker-service
-
-              - match: { prefix: "/api/system/" }
-                route:
-                  cluster: worker-service
-
-              - match: { path: "/" }
-                route:
-                  cluster: app-service
-
-              # special case for when API requests are made, can just forward, not to minio    
-              - match: { prefix: "/api/" }
-                route:
-                  cluster: app-service
-                  timeout: 120s
-
-              - match: { prefix: "/worker/" }
-                route: 
-                  cluster: worker-service
-                  prefix_rewrite: "/"
-
-              - match: { prefix: "/db/" }
-                route:
-                  cluster: couchdb-service
-                  prefix_rewrite: "/"
-
-              # minio is on the default route because this works
-              # best, minio + AWS SDK doesn't handle path proxy
-              - match: { prefix: "/" }
-                route:
-                  cluster: minio-service
-
-          http_filters:
-          - name: envoy.filters.http.router
-
-  clusters:
-  - name: app-service
-    connect_timeout: 0.25s
-    type: strict_dns
-    lb_policy: round_robin
-    load_assignment:
-      cluster_name: app-service
-      endpoints:
-      - lb_endpoints:
-        - endpoint:
-            address:
-              socket_address: 
-                address: app-service
-                port_value: 4002
-
-  - name: minio-service
-    connect_timeout: 0.25s
-    type: strict_dns
-    lb_policy: round_robin
-    load_assignment:
-      cluster_name: minio-service
-      endpoints:
-      - lb_endpoints:
-        - endpoint:
-            address:
-              socket_address:
-                address: minio-service
-                port_value: 9000
-
-  - name: worker-service
-    connect_timeout: 0.25s
-    type: strict_dns
-    lb_policy: round_robin
-    load_assignment:
-      cluster_name: worker-service
-      endpoints:
-      - lb_endpoints:
-        - endpoint:
-            address:
-              socket_address: 
-                address: worker-service
-                port_value: 4003
-
-  - name: couchdb-service
-    connect_timeout: 0.25s
-    type: strict_dns
-    lb_policy: round_robin
-    load_assignment:
-      cluster_name: couchdb-service
-      endpoints:
-      - lb_endpoints:
-        - endpoint:
-            address:
-              socket_address:
-                address: couchdb-service
-                port_value: 5984
-
-  - name: watchtower-service
-    connect_timeout: 0.25s
-    type: strict_dns
-    lb_policy: round_robin
-    load_assignment:
-      cluster_name: watchtower-service
-      endpoints:
-      - lb_endpoints:
-        - endpoint:
-            address:
-              socket_address: 
-                address: watchtower-service
-                port_value: 8080
-

hosting/hosting.properties

@@ -18,7 +18,6 @@ WORKER_PORT=4003
 MINIO_PORT=4004
 COUCH_DB_PORT=4005
 REDIS_PORT=6379
-WATCHTOWER_PORT=6161
 BUDIBASE_ENVIRONMENT=PRODUCTION
 
 # An admin user can be automatically created initially if these are set
@@ -26,4 +25,4 @@ BB_ADMIN_USER_EMAIL=
 BB_ADMIN_USER_PASSWORD=
 
 # A path that is watched for plugin bundles. Any bundles found are imported automatically/
-PLUGINS_DIR=
+PLUGINS_DIR=

hosting/portainer/template.json

@@ -78,11 +78,6 @@
           "default": "6379",
           "preset": true
         },
-        {
-          "name": "WATCHTOWER_PORT",
-          "default": "6161",
-          "preset": true
-        },
         {
           "name": "BUDIBASE_ENVIRONMENT",
           "default": "PRODUCTION",

hosting/proxy/Dockerfile

@@ -22,5 +22,4 @@ ENV APPS_UPSTREAM_URL=http://app-service:4002
 ENV WORKER_UPSTREAM_URL=http://worker-service:4003
 ENV MINIO_UPSTREAM_URL=http://minio-service:9000
 ENV COUCHDB_UPSTREAM_URL=http://couchdb-service:5984
-ENV WATCHTOWER_UPSTREAM_URL=http://watchtower-service:8080
 ENV RESOLVER=127.0.0.11

hosting/proxy/nginx.prod.conf

@@ -50,19 +50,6 @@ http {
     ignore_invalid_headers off;
     proxy_buffering off;
 
-    set $csp_default "default-src 'self'";
-    set $csp_script "script-src 'self' 'unsafe-inline' 'unsafe-eval' https://*.budibase.net https://cdn.budi.live https://js.intercomcdn.com https://widget.intercom.io https://d2l5prqdbvm3op.cloudfront.net https://us-assets.i.posthog.com";
-    set $csp_style "style-src 'self' 'unsafe-inline' https://cdn.jsdelivr.net https://fonts.googleapis.com https://rsms.me https://maxcdn.bootstrapcdn.com";
-    set $csp_object "object-src 'none'";
-    set $csp_base_uri "base-uri 'self'";
-    set $csp_connect "connect-src 'self' https://*.budibase.app https://*.budibaseqa.app https://*.budibase.net https://api-iam.intercom.io https://api-iam.intercom.io  https://api-ping.intercom.io https://app.posthog.com https://us.i.posthog.com wss://nexus-websocket-a.intercom.io  wss://nexus-websocket-b.intercom.io https://nexus-websocket-a.intercom.io https://nexus-websocket-b.intercom.io  https://uploads.intercomcdn.com  https://uploads.intercomusercontent.com https://*.amazonaws.com https://*.s3.amazonaws.com https://*.s3.us-east-2.amazonaws.com https://*.s3.us-east-1.amazonaws.com https://*.s3.us-west-1.amazonaws.com https://*.s3.us-west-2.amazonaws.com https://*.s3.af-south-1.amazonaws.com https://*.s3.ap-east-1.amazonaws.com https://*.s3.ap-southeast-3.amazonaws.com https://*.s3.ap-south-1.amazonaws.com https://*.s3.ap-northeast-3.amazonaws.com https://*.s3.ap-northeast-2.amazonaws.com https://*.s3.ap-southeast-1.amazonaws.com https://*.s3.ap-southeast-2.amazonaws.com https://*.s3.ap-northeast-1.amazonaws.com https://*.s3.ca-central-1.amazonaws.com https://*.s3.cn-north-1.amazonaws.com https://*.s3.cn-northwest-1.amazonaws.com https://*.s3.eu-central-1.amazonaws.com https://*.s3.eu-west-1.amazonaws.com https://*.s3.eu-west-2.amazonaws.com https://*.s3.eu-south-1.amazonaws.com https://*.s3.eu-west-3.amazonaws.com https://*.s3.eu-north-1.amazonaws.com https://*.s3.sa-east-1.amazonaws.com https://*.s3.me-south-1.amazonaws.com https://*.s3.us-gov-east-1.amazonaws.com https://*.s3.us-gov-west-1.amazonaws.com https://api.github.com";    
-    set $csp_font "font-src 'self' data: https://cdn.jsdelivr.net https://fonts.gstatic.com https://rsms.me https://maxcdn.bootstrapcdn.com  https://js.intercomcdn.com  https://fonts.intercomcdn.com";
-    set $csp_frame "frame-src 'self' https:";
-    set $csp_img "img-src http: https: data: blob:";
-    set $csp_manifest "manifest-src 'self'";
-    set $csp_media "media-src 'self' https://js.intercomcdn.com https://cdn.budi.live"; 
-    set $csp_worker "worker-src blob:";
-
     error_page 502 503 504 /error.html;
     location = /error.html {
       root /usr/share/nginx/html;
@@ -73,7 +60,6 @@ http {
     add_header X-Frame-Options SAMEORIGIN always;
     add_header X-Content-Type-Options nosniff always;
     add_header X-XSS-Protection "1; mode=block" always;
-    add_header Content-Security-Policy "${csp_default}; ${csp_script}; ${csp_style}; ${csp_object}; ${csp_base_uri}; ${csp_connect}; ${csp_font}; ${csp_frame}; ${csp_img}; ${csp_manifest}; ${csp_media}; ${csp_worker};" always;
     add_header Strict-Transport-Security "max-age=31536000; includeSubDomains; preload" always;
 
     # upstreams
@@ -81,7 +67,6 @@ http {
     set $worker ${WORKER_UPSTREAM_URL};
     set $minio ${MINIO_UPSTREAM_URL};
     set $couchdb ${COUCHDB_UPSTREAM_URL};
-    set $watchtower ${WATCHTOWER_UPSTREAM_URL};
 
     location /health {
       access_log off;
@@ -107,10 +92,6 @@ http {
       proxy_pass      $apps;
     }
 
-    location = /v1/update {
-      proxy_pass      $watchtower;
-    }
-
     location ~ ^/(builder|app_) {
       proxy_http_version  1.1;
 
@@ -125,6 +106,12 @@ http {
 
     location ~ ^/api/(system|admin|global)/ {
       proxy_set_header    Host                $host;
+
+      # Enable buffering for potentially large OIDC configs
+      proxy_buffering on;
+      proxy_buffer_size 16k;
+      proxy_buffers 4 32k;
+
       proxy_pass      $worker;
     }
 

hosting/scripts/airgapped/airgappedDockerBuild.js

@@ -12,7 +12,6 @@ let IMAGES = {
 	couch: "ibmcom/couchdb3",
 	curl: "curlimages/curl",
 	redis: "redis",
-	watchtower: "containrrr/watchtower",
 }
 
 if (IS_SINGLE_IMAGE) {
@@ -53,4 +52,4 @@ if (!IS_SINGLE_IMAGE) {
 copyFile(FILES.ENV)
 
 // compress
-execSync(`tar -czf bb-airgapped.tar.gz hosting/scripts/bb-airgapped`)
+execSync(`tar -czf bb-airgapped.tar.gz hosting/scripts/bb-airgapped`)

lerna.json

@@ -1,12 +1,7 @@
 {
   "$schema": "node_modules/lerna/schemas/lerna-schema.json",
-  "version": "3.0.3",
+  "version": "3.2.12",
   "npmClient": "yarn",
-  "packages": [
-    "packages/*",
-    "!packages/account-portal",
-    "packages/account-portal/packages/*"
-  ],
   "concurrency": 20,
   "command": {
     "publish": {

nx.json

@@ -2,7 +2,6 @@
   "$schema": "./node_modules/nx/schemas/nx-schema.json",
   "tasksRunnerOptions": {
     "default": {
-      "runner": "nx-cloud",
       "options": {
         "cacheableOperations": ["build", "test", "check:types"]
       }

package.json

@@ -9,6 +9,7 @@
     "@types/node": "20.10.0",
     "@types/proper-lockfile": "^4.1.4",
     "@typescript-eslint/parser": "6.9.0",
+    "depcheck": "^1.4.7",
     "esbuild": "^0.18.17",
     "esbuild-node-externals": "^1.14.0",
     "eslint": "^8.52.0",
@@ -24,22 +25,22 @@
     "prettier": "2.8.8",
     "prettier-plugin-svelte": "^2.3.0",
     "proper-lockfile": "^4.1.2",
-    "svelte": "^4.2.10",
+    "svelte": "4.2.19",
     "svelte-eslint-parser": "^0.33.1",
     "typescript": "5.5.2",
     "typescript-eslint": "^7.3.1",
-    "yargs": "^17.7.2"
+    "yargs": "^17.7.2",
+    "cross-spawn": "7.0.6"
   },
   "scripts": {
     "get-past-client-version": "node scripts/getPastClientVersion.js",
     "setup": "git config submodule.recurse true && git submodule update && node ./hosting/scripts/setup.js && yarn && yarn build && yarn dev",
     "build": "DISABLE_V8_COMPILE_CACHE=1 NODE_OPTIONS=--max-old-space-size=1500 lerna run build --stream",
     "build:apps": "DISABLE_V8_COMPILE_CACHE=1 yarn build --scope @budibase/server --scope @budibase/worker",
-    "build:oss": "DISABLE_V8_COMPILE_CACHE=1 NODE_OPTIONS=--max-old-space-size=1500 lerna run build --stream --ignore @budibase/account-portal-server --ignore @budibase/account-portal-ui",
     "build:cli": "yarn build --scope @budibase/cli",
-    "build:account-portal": "NODE_OPTIONS=--max-old-space-size=1500 lerna run build --stream --scope @budibase/account-portal-server --scope @budibase/account-portal-ui",
     "build:dev": "lerna run --stream prebuild && yarn nx run-many --target=build --output-style=dynamic --watch --preserveWatchOutput",
-    "check:types": "lerna run --concurrency 2 check:types --ignore @budibase/account-portal-server",
+    "check:types": "yarn check:dependencies && lerna run --concurrency 2 check:types",
+    "check:dependencies": "lerna run --concurrency 2 check:dependencies",
     "build:sdk": "lerna run --stream build:sdk",
     "deps:circular": "madge packages/server/dist/index.js packages/worker/src/index.ts packages/backend-core/dist/src/index.js packages/cli/src/index.js --circular",
     "release": "lerna publish from-package --yes --force-publish --no-git-tag-version --no-push --no-git-reset",
@@ -52,15 +53,12 @@
     "kill-server": "kill-port 4001 4002",
     "kill-accountportal": "kill-port 3001 4003",
     "kill-all": "yarn run kill-builder && yarn run kill-server && yarn kill-accountportal",
-    "dev": "yarn run kill-all && lerna run --parallel prebuild && lerna run --stream dev --ignore=@budibase/account-portal-ui --ignore @budibase/account-portal-server",
-    "dev:noserver": "yarn run kill-builder && lerna run --stream dev:stack:up --ignore @budibase/account-portal-server && lerna run --stream dev --ignore @budibase/backend-core --ignore @budibase/server --ignore @budibase/worker --ignore=@budibase/account-portal-ui --ignore @budibase/account-portal-server",
+    "dev": "yarn run kill-all && lerna run --parallel prebuild && lerna run --stream dev",
+    "dev:noserver": "yarn run kill-builder && lerna run --stream dev:stack:up && lerna run --stream dev --ignore @budibase/backend-core --ignore @budibase/server --ignore @budibase/worker",
     "dev:server": "yarn run kill-server && lerna run --stream dev --scope @budibase/worker --scope @budibase/server",
-    "dev:accountportal": "yarn kill-accountportal && lerna run dev --stream --scope @budibase/account-portal-ui --scope @budibase/account-portal-server",
-    "dev:camunda": "./scripts/deploy-camunda.sh",
-    "dev:all": "yarn run kill-all && lerna run --stream dev",
     "dev:built": "yarn run kill-all && cd packages/server && yarn dev:stack:up && cd ../../ && lerna run --stream dev:built",
     "dev:docker": "./scripts/devDocker.sh",
-    "test": "lerna run --concurrency 1 --stream test --stream",
+    "test": "lerna run --concurrency 1 --stream test",
     "test:containers:kill": "./scripts/killTestcontainers.sh",
     "lint:eslint": "eslint packages --max-warnings=0",
     "lint:prettier": "prettier --check \"packages/**/*.{js,ts,svelte}\" && prettier --write \"examples/**/*.{js,ts,svelte}\"",
@@ -98,9 +96,7 @@
   },
   "workspaces": {
     "packages": [
-      "packages/*",
-      "!packages/account-portal",
-      "packages/account-portal/packages/*"
+      "packages/*"
     ]
   },
   "resolutions": {
@@ -114,7 +110,7 @@
     "semver": "7.5.3",
     "http-cache-semantics": "4.1.1",
     "msgpackr": "1.10.1",
-    "axios": "1.6.3",
+    "axios": "1.7.7",
     "xml2js": "0.6.2",
     "unset-value": "2.0.1",
     "passport": "0.6.0",
@@ -124,6 +120,5 @@
   },
   "engines": {
     "node": ">=20.0.0 <21.0.0"
-  },
-  "dependencies": {}
+  }
 }

packages/account-portal

@@ -1 +0,0 @@
-Subproject commit 9bef5d1656b4f3c991447ded6d65b0eba393a140

packages/backend-core/.npmignore

@@ -1,6 +1,4 @@
 *
 !dist/**/*
 dist/tsconfig.build.tsbuildinfo
-!package.json
-!src/**
-!tests/**
+!package.json

packages/backend-core/package.json

@@ -9,6 +9,13 @@
     "./tests": "./dist/tests/index.js",
     "./*": "./dist/*.js"
   },
+  "typesVersions": {
+    "*": {
+      "tests": [
+        "dist/tests/index.d.ts"
+      ]
+    }
+  },
   "author": "Budibase",
   "license": "GPL-3.0",
   "scripts": {
@@ -17,6 +24,7 @@
     "build": "tsc -p tsconfig.build.json --paths null && node ./scripts/build.js",
     "build:dev": "yarn prebuild && tsc --build --watch --preserveWatchOutput",
     "check:types": "tsc -p tsconfig.json --noEmit --paths null --target es2020",
+    "check:dependencies": "node ../../scripts/depcheck.js",
     "test": "bash scripts/test.sh",
     "test:watch": "jest --watchAll"
   },
@@ -25,17 +33,21 @@
     "@budibase/pouchdb-replication-stream": "1.2.11",
     "@budibase/shared-core": "0.0.0",
     "@budibase/types": "0.0.0",
+    "@techpass/passport-openidconnect": "0.3.3",
     "aws-cloudfront-sign": "3.0.2",
-    "aws-sdk": "2.1030.0",
+    "aws-sdk": "2.1692.0",
     "bcrypt": "5.1.0",
     "bcryptjs": "2.4.3",
     "bull": "4.10.1",
     "correlation-id": "4.0.0",
-    "dd-trace": "5.2.0",
+    "dd-trace": "5.26.0",
     "dotenv": "16.0.1",
+    "google-auth-library": "^8.0.1",
+    "google-spreadsheet": "npm:@budibase/google-spreadsheet@4.1.5",
     "ioredis": "5.3.2",
     "joi": "17.6.0",
     "jsonwebtoken": "9.0.2",
+    "knex": "2.4.2",
     "koa-passport": "^6.0.0",
     "koa-pino-logger": "4.0.0",
     "lodash": "4.17.21",
@@ -46,17 +58,17 @@
     "pino": "8.11.0",
     "pino-http": "8.3.3",
     "posthog-node": "4.0.1",
-    "pouchdb": "7.3.0",
-    "pouchdb-find": "7.2.2",
+    "pouchdb": "9.0.0",
+    "pouchdb-find": "9.0.0",
     "redlock": "4.2.0",
     "rotating-file-stream": "3.1.0",
     "sanitize-s3-objectkey": "0.0.1",
     "semver": "^7.5.4",
     "tar-fs": "2.1.1",
-    "uuid": "^8.3.2",
-    "knex": "2.4.2"
+    "uuid": "^8.3.2"
   },
   "devDependencies": {
+    "@jest/types": "^29.6.3",
     "@shopify/jest-koa-mocks": "5.1.1",
     "@swc/core": "1.3.71",
     "@swc/jest": "0.2.27",
@@ -64,8 +76,9 @@
     "@types/cookies": "0.7.8",
     "@types/jest": "29.5.5",
     "@types/lodash": "4.14.200",
+    "@types/node": "^22.9.0",
     "@types/node-fetch": "2.6.4",
-    "@types/pouchdb": "6.4.0",
+    "@types/pouchdb": "6.4.2",
     "@types/redlock": "4.0.7",
     "@types/semver": "7.3.7",
     "@types/tar-fs": "2.0.1",
@@ -74,6 +87,7 @@
     "ioredis-mock": "8.9.0",
     "jest": "29.7.0",
     "jest-serial-runner": "1.2.1",
+    "nock": "^13.5.6",
     "pino-pretty": "10.0.0",
     "pouchdb-adapter-memory": "7.2.2",
     "testcontainers": "^10.7.2",

packages/backend-core/src/cache/tests/docWritethrough.spec.ts

@@ -1,7 +1,12 @@
 import tk from "timekeeper"
 
 import _ from "lodash"
-import { DBTestConfiguration, generator, structures } from "../../../tests"
+import {
+  DBTestConfiguration,
+  generator,
+  structures,
+  utils,
+} from "../../../tests"
 import { getDB } from "../../db"
 
 import {
@@ -10,15 +15,14 @@ import {
   init,
 } from "../docWritethrough"
 
-import InMemoryQueue from "../../queue/inMemoryQueue"
-
 const initialTime = Date.now()
 
 async function waitForQueueCompletion() {
-  const queue: InMemoryQueue = DocWritethroughProcessor.queue as never
-  await queue.waitForCompletion()
+  await utils.queue.processMessages(DocWritethroughProcessor.queue)
 }
 
+beforeAll(() => utils.queue.useRealQueues())
+
 describe("docWritethrough", () => {
   beforeAll(() => {
     init()
@@ -67,7 +71,7 @@ describe("docWritethrough", () => {
         const patch3 = generatePatchObject(3)
         await docWritethrough.patch(patch3)
 
-        expect(await db.get(documentId)).toEqual({
+        expect(await db.tryGet(documentId)).toEqual({
           _id: documentId,
           ...patch1,
           ...patch2,
@@ -92,7 +96,7 @@ describe("docWritethrough", () => {
 
         await waitForQueueCompletion()
 
-        expect(await db.get(documentId)).toEqual(
+        expect(await db.tryGet(documentId)).toEqual(
           expect.objectContaining({
             _id: documentId,
             ...patch1,
@@ -117,7 +121,7 @@ describe("docWritethrough", () => {
         await waitForQueueCompletion()
 
         expect(date1).not.toEqual(date2)
-        expect(await db.get(documentId)).toEqual(
+        expect(await db.tryGet(documentId)).toEqual(
           expect.objectContaining({
             createdAt: date1.toISOString(),
             updatedAt: date2.toISOString(),
@@ -135,7 +139,7 @@ describe("docWritethrough", () => {
         await docWritethrough.patch(patch2)
 
         const keyToOverride = _.sample(Object.keys(patch1))!
-        expect(await db.get(documentId)).toEqual(
+        expect(await db.tryGet(documentId)).toEqual(
           expect.objectContaining({
             [keyToOverride]: patch1[keyToOverride],
           })
@@ -150,7 +154,7 @@ describe("docWritethrough", () => {
         await docWritethrough.patch(patch3)
         await waitForQueueCompletion()
 
-        expect(await db.get(documentId)).toEqual(
+        expect(await db.tryGet(documentId)).toEqual(
           expect.objectContaining({
             ...patch1,
             ...patch2,
@@ -180,14 +184,14 @@ describe("docWritethrough", () => {
         await secondDocWritethrough.patch(doc2Patch2)
         await waitForQueueCompletion()
 
-        expect(await db.get(docWritethrough.docId)).toEqual(
+        expect(await db.tryGet(docWritethrough.docId)).toEqual(
           expect.objectContaining({
             ...doc1Patch,
             ...doc1Patch2,
           })
         )
 
-        expect(await db.get(secondDocWritethrough.docId)).toEqual(
+        expect(await db.tryGet(secondDocWritethrough.docId)).toEqual(
           expect.objectContaining({
             ...doc2Patch,
             ...doc2Patch2,
@@ -203,7 +207,7 @@ describe("docWritethrough", () => {
         await docWritethrough.patch(initialPatch)
         await waitForQueueCompletion()
 
-        expect(await db.get(documentId)).toEqual(
+        expect(await db.tryGet(documentId)).toEqual(
           expect.objectContaining(initialPatch)
         )
 
@@ -214,10 +218,10 @@ describe("docWritethrough", () => {
         await docWritethrough.patch(extraPatch)
         await waitForQueueCompletion()
 
-        expect(await db.get(documentId)).toEqual(
+        expect(await db.tryGet(documentId)).toEqual(
           expect.objectContaining(extraPatch)
         )
-        expect(await db.get(documentId)).not.toEqual(
+        expect(await db.tryGet(documentId)).not.toEqual(
           expect.objectContaining(initialPatch)
         )
       })
@@ -242,7 +246,7 @@ describe("docWritethrough", () => {
         expect(queueMessageSpy).toHaveBeenCalledTimes(5)
 
         await waitForQueueCompletion()
-        expect(await db.get(documentId)).toEqual(
+        expect(await db.tryGet(documentId)).toEqual(
           expect.objectContaining(patches)
         )
 
@@ -250,7 +254,7 @@ describe("docWritethrough", () => {
         expect(queueMessageSpy).toHaveBeenCalledTimes(45)
 
         await waitForQueueCompletion()
-        expect(await db.get(documentId)).toEqual(
+        expect(await db.tryGet(documentId)).toEqual(
           expect.objectContaining(patches)
         )
 
@@ -258,20 +262,18 @@ describe("docWritethrough", () => {
         expect(queueMessageSpy).toHaveBeenCalledTimes(55)
 
         await waitForQueueCompletion()
-        expect(await db.get(documentId)).toEqual(
+        expect(await db.tryGet(documentId)).toEqual(
           expect.objectContaining(patches)
         )
       })
     })
 
-    // This is not yet supported
-    // eslint-disable-next-line jest/no-disabled-tests
-    it.skip("patches will execute in order", async () => {
+    it("patches will execute in order", async () => {
       let incrementalValue = 0
       const keyToOverride = generator.word()
       async function incrementalPatches(count: number) {
         for (let i = 0; i < count; i++) {
-          await docWritethrough.patch({ [keyToOverride]: incrementalValue++ })
+          await docWritethrough.patch({ [keyToOverride]: ++incrementalValue })
         }
       }
 
@@ -279,13 +281,13 @@ describe("docWritethrough", () => {
         await incrementalPatches(5)
 
         await waitForQueueCompletion()
-        expect(await db.get(documentId)).toEqual(
+        expect(await db.tryGet(documentId)).toEqual(
           expect.objectContaining({ [keyToOverride]: 5 })
         )
 
         await incrementalPatches(40)
         await waitForQueueCompletion()
-        expect(await db.get(documentId)).toEqual(
+        expect(await db.tryGet(documentId)).toEqual(
           expect.objectContaining({ [keyToOverride]: 45 })
         )
       })

packages/backend-core/src/db/couch/DatabaseImpl.ts

@@ -10,7 +10,6 @@ import {
   DatabaseQueryOpts,
   DBError,
   Document,
-  FeatureFlag,
   isDocument,
   RowResponse,
   RowValue,
@@ -27,7 +26,6 @@ import { SQLITE_DESIGN_DOC_ID } from "../../constants"
 import { DDInstrumentedDatabase } from "../instrumentation"
 import { checkSlashesInUrl } from "../../helpers"
 import { sqlLog } from "../../sql/utils"
-import { flags } from "../../features"
 
 const DATABASE_NOT_FOUND = "Database does not exist."
 
@@ -192,7 +190,7 @@ export class DatabaseImpl implements Database {
     }
   }
 
-  private async performCall<T>(call: DBCallback<T>): Promise<any> {
+  private async performCall<T>(call: DBCallback<T>): Promise<T> {
     const db = this.getDb()
     const fnc = await call(db)
     try {
@@ -456,10 +454,7 @@ export class DatabaseImpl implements Database {
   }
 
   async destroy() {
-    if (
-      (await flags.isEnabled(FeatureFlag.SQS)) &&
-      (await this.exists(SQLITE_DESIGN_DOC_ID))
-    ) {
+    if (await this.exists(SQLITE_DESIGN_DOC_ID)) {
       // delete the design document, then run the cleanup operation
       const definition = await this.get<SQLiteDefinition>(SQLITE_DESIGN_DOC_ID)
       // remove all tables - save the definition then trigger a cleanup
@@ -472,7 +467,7 @@ export class DatabaseImpl implements Database {
     } catch (err: any) {
       // didn't exist, don't worry
       if (err.statusCode === 404) {
-        return
+        return { ok: true }
       } else {
         throw new CouchDBError(err.message, err)
       }

packages/backend-core/src/db/instrumentation.ts

@@ -27,7 +27,7 @@ export class DDInstrumentedDatabase implements Database {
 
   exists(docId?: string): Promise<boolean> {
     return tracer.trace("db.exists", span => {
-      span?.addTags({ db_name: this.name, doc_id: docId })
+      span.addTags({ db_name: this.name, doc_id: docId })
       if (docId) {
         return this.db.exists(docId)
       }
@@ -37,15 +37,17 @@ export class DDInstrumentedDatabase implements Database {
 
   get<T extends Document>(id?: string | undefined): Promise<T> {
     return tracer.trace("db.get", span => {
-      span?.addTags({ db_name: this.name, doc_id: id })
+      span.addTags({ db_name: this.name, doc_id: id })
       return this.db.get(id)
     })
   }
 
   tryGet<T extends Document>(id?: string | undefined): Promise<T | undefined> {
-    return tracer.trace("db.tryGet", span => {
-      span?.addTags({ db_name: this.name, doc_id: id })
-      return this.db.tryGet(id)
+    return tracer.trace("db.tryGet", async span => {
+      span.addTags({ db_name: this.name, doc_id: id })
+      const doc = await this.db.tryGet<T>(id)
+      span.addTags({ doc_found: doc !== undefined })
+      return doc
     })
   }
 
@@ -53,13 +55,15 @@ export class DDInstrumentedDatabase implements Database {
     ids: string[],
     opts?: { allowMissing?: boolean | undefined } | undefined
   ): Promise<T[]> {
-    return tracer.trace("db.getMultiple", span => {
-      span?.addTags({
+    return tracer.trace("db.getMultiple", async span => {
+      span.addTags({
         db_name: this.name,
         num_docs: ids.length,
         allow_missing: opts?.allowMissing,
       })
-      return this.db.getMultiple(ids, opts)
+      const docs = await this.db.getMultiple<T>(ids, opts)
+      span.addTags({ num_docs_found: docs.length })
+      return docs
     })
   }
 
@@ -69,12 +73,14 @@ export class DDInstrumentedDatabase implements Database {
     idOrDoc: string | Document,
     rev?: string
   ): Promise<DocumentDestroyResponse> {
-    return tracer.trace("db.remove", span => {
-      span?.addTags({ db_name: this.name, doc_id: idOrDoc })
+    return tracer.trace("db.remove", async span => {
+      span.addTags({ db_name: this.name, doc_id: idOrDoc, rev })
       const isDocument = typeof idOrDoc === "object"
       const id = isDocument ? idOrDoc._id! : idOrDoc
       rev = isDocument ? idOrDoc._rev : rev
-      return this.db.remove(id, rev)
+      const resp = await this.db.remove(id, rev)
+      span.addTags({ ok: resp.ok })
+      return resp
     })
   }
 
@@ -83,7 +89,11 @@ export class DDInstrumentedDatabase implements Database {
     opts?: { silenceErrors?: boolean }
   ): Promise<void> {
     return tracer.trace("db.bulkRemove", span => {
-      span?.addTags({ db_name: this.name, num_docs: documents.length })
+      span.addTags({
+        db_name: this.name,
+        num_docs: documents.length,
+        silence_errors: opts?.silenceErrors,
+      })
       return this.db.bulkRemove(documents, opts)
     })
   }
@@ -92,15 +102,21 @@ export class DDInstrumentedDatabase implements Database {
     document: AnyDocument,
     opts?: DatabasePutOpts | undefined
   ): Promise<DocumentInsertResponse> {
-    return tracer.trace("db.put", span => {
-      span?.addTags({ db_name: this.name, doc_id: document._id })
-      return this.db.put(document, opts)
+    return tracer.trace("db.put", async span => {
+      span.addTags({
+        db_name: this.name,
+        doc_id: document._id,
+        force: opts?.force,
+      })
+      const resp = await this.db.put(document, opts)
+      span.addTags({ ok: resp.ok })
+      return resp
     })
   }
 
   bulkDocs(documents: AnyDocument[]): Promise<DocumentBulkResponse[]> {
     return tracer.trace("db.bulkDocs", span => {
-      span?.addTags({ db_name: this.name, num_docs: documents.length })
+      span.addTags({ db_name: this.name, num_docs: documents.length })
       return this.db.bulkDocs(documents)
     })
   }
@@ -108,9 +124,15 @@ export class DDInstrumentedDatabase implements Database {
   allDocs<T extends Document | RowValue>(
     params: DatabaseQueryOpts
   ): Promise<AllDocsResponse<T>> {
-    return tracer.trace("db.allDocs", span => {
-      span?.addTags({ db_name: this.name })
-      return this.db.allDocs(params)
+    return tracer.trace("db.allDocs", async span => {
+      span.addTags({ db_name: this.name, ...params })
+      const resp = await this.db.allDocs<T>(params)
+      span.addTags({
+        total_rows: resp.total_rows,
+        rows_length: resp.rows.length,
+        offset: resp.offset,
+      })
+      return resp
     })
   }
 
@@ -118,57 +140,75 @@ export class DDInstrumentedDatabase implements Database {
     viewName: string,
     params: DatabaseQueryOpts
   ): Promise<AllDocsResponse<T>> {
-    return tracer.trace("db.query", span => {
-      span?.addTags({ db_name: this.name, view_name: viewName })
-      return this.db.query(viewName, params)
+    return tracer.trace("db.query", async span => {
+      span.addTags({ db_name: this.name, view_name: viewName, ...params })
+      const resp = await this.db.query<T>(viewName, params)
+      span.addTags({
+        total_rows: resp.total_rows,
+        rows_length: resp.rows.length,
+        offset: resp.offset,
+      })
+      return resp
     })
   }
 
-  destroy(): Promise<void | OkResponse> {
-    return tracer.trace("db.destroy", span => {
-      span?.addTags({ db_name: this.name })
-      return this.db.destroy()
+  destroy(): Promise<OkResponse> {
+    return tracer.trace("db.destroy", async span => {
+      span.addTags({ db_name: this.name })
+      const resp = await this.db.destroy()
+      span.addTags({ ok: resp.ok })
+      return resp
     })
   }
 
-  compact(): Promise<void | OkResponse> {
-    return tracer.trace("db.compact", span => {
-      span?.addTags({ db_name: this.name })
-      return this.db.compact()
+  compact(): Promise<OkResponse> {
+    return tracer.trace("db.compact", async span => {
+      span.addTags({ db_name: this.name })
+      const resp = await this.db.compact()
+      span.addTags({ ok: resp.ok })
+      return resp
     })
   }
 
   dump(stream: Writable, opts?: DatabaseDumpOpts | undefined): Promise<any> {
     return tracer.trace("db.dump", span => {
-      span?.addTags({ db_name: this.name })
+      span.addTags({
+        db_name: this.name,
+        batch_limit: opts?.batch_limit,
+        batch_size: opts?.batch_size,
+        style: opts?.style,
+        timeout: opts?.timeout,
+        num_doc_ids: opts?.doc_ids?.length,
+        view: opts?.view,
+      })
       return this.db.dump(stream, opts)
     })
   }
 
   load(...args: any[]): Promise<any> {
     return tracer.trace("db.load", span => {
-      span?.addTags({ db_name: this.name })
+      span.addTags({ db_name: this.name, num_args: args.length })
       return this.db.load(...args)
     })
   }
 
   createIndex(...args: any[]): Promise<any> {
     return tracer.trace("db.createIndex", span => {
-      span?.addTags({ db_name: this.name })
+      span.addTags({ db_name: this.name, num_args: args.length })
       return this.db.createIndex(...args)
     })
   }
 
   deleteIndex(...args: any[]): Promise<any> {
     return tracer.trace("db.deleteIndex", span => {
-      span?.addTags({ db_name: this.name })
+      span.addTags({ db_name: this.name, num_args: args.length })
       return this.db.deleteIndex(...args)
     })
   }
 
   getIndexes(...args: any[]): Promise<any> {
     return tracer.trace("db.getIndexes", span => {
-      span?.addTags({ db_name: this.name })
+      span.addTags({ db_name: this.name, num_args: args.length })
       return this.db.getIndexes(...args)
     })
   }
@@ -177,22 +217,27 @@ export class DDInstrumentedDatabase implements Database {
     sql: string,
     parameters?: SqlQueryBinding
   ): Promise<T[]> {
-    return tracer.trace("db.sql", span => {
-      span?.addTags({ db_name: this.name })
-      return this.db.sql(sql, parameters)
+    return tracer.trace("db.sql", async span => {
+      span.addTags({ db_name: this.name, num_bindings: parameters?.length })
+      const resp = await this.db.sql<T>(sql, parameters)
+      span.addTags({ num_rows: resp.length })
+      return resp
     })
   }
 
   sqlPurgeDocument(docIds: string[] | string): Promise<void> {
     return tracer.trace("db.sqlPurgeDocument", span => {
-      span?.addTags({ db_name: this.name })
+      span.addTags({
+        db_name: this.name,
+        num_docs: Array.isArray(docIds) ? docIds.length : 1,
+      })
       return this.db.sqlPurgeDocument(docIds)
     })
   }
 
   sqlDiskCleanup(): Promise<void> {
     return tracer.trace("db.sqlDiskCleanup", span => {
-      span?.addTags({ db_name: this.name })
+      span.addTags({ db_name: this.name })
       return this.db.sqlDiskCleanup()
     })
   }

packages/backend-core/src/environment.ts

@@ -1,6 +1,7 @@
 import { existsSync, readFileSync } from "fs"
 import { ServiceType } from "@budibase/types"
 import { cloneDeep } from "lodash"
+import { createSecretKey } from "crypto"
 
 function isTest() {
   return isJest()
@@ -18,6 +19,12 @@ function isDev() {
   return process.env.NODE_ENV !== "production"
 }
 
+function parseIntSafe(number?: string) {
+  if (number) {
+    return parseInt(number)
+  }
+}
+
 let LOADED = false
 if (!LOADED && isDev() && !isTest()) {
   require("dotenv").config()
@@ -126,8 +133,12 @@ const environment = {
   },
   BUDIBASE_ENVIRONMENT: process.env.BUDIBASE_ENVIRONMENT,
   JS_BCRYPT: process.env.JS_BCRYPT,
-  JWT_SECRET: process.env.JWT_SECRET,
-  JWT_SECRET_FALLBACK: process.env.JWT_SECRET_FALLBACK,
+  JWT_SECRET: process.env.JWT_SECRET
+    ? createSecretKey(Buffer.from(process.env.JWT_SECRET))
+    : undefined,
+  JWT_SECRET_FALLBACK: process.env.JWT_SECRET_FALLBACK
+    ? createSecretKey(Buffer.from(process.env.JWT_SECRET_FALLBACK))
+    : undefined,
   ENCRYPTION_KEY: process.env.ENCRYPTION_KEY,
   API_ENCRYPTION_KEY: getAPIEncryptionKey(),
   COUCH_DB_URL: process.env.COUCH_DB_URL || "http://localhost:4005",
@@ -225,6 +236,8 @@ const environment = {
   OPENAI_API_KEY: process.env.OPENAI_API_KEY,
   MIN_VERSION_WITHOUT_POWER_ROLE:
     process.env.MIN_VERSION_WITHOUT_POWER_ROLE || "3.0.0",
+  DISABLE_CONTENT_SECURITY_POLICY: process.env.DISABLE_CONTENT_SECURITY_POLICY,
+  BSON_BUFFER_SIZE: parseIntSafe(process.env.BSON_BUFFER_SIZE),
 }
 
 export function setEnv(newEnvVars: Partial<typeof environment>): () => void {

packages/backend-core/src/features/features.ts

@@ -267,12 +267,10 @@ export class FlagSet<V extends Flag<any>, T extends { [key: string]: V }> {
 // All of the machinery in this file is to make sure that flags have their
 // default values set correctly and their types flow through the system.
 export const flags = new FlagSet({
-  [FeatureFlag.DEFAULT_VALUES]: Flag.boolean(env.isDev()),
-  [FeatureFlag.AUTOMATION_BRANCHING]: Flag.boolean(env.isDev()),
-  [FeatureFlag.SQS]: Flag.boolean(true),
-  [FeatureFlag.AI_CUSTOM_CONFIGS]: Flag.boolean(env.isDev()),
-  [FeatureFlag.ENRICHED_RELATIONSHIPS]: Flag.boolean(env.isDev()),
-  [FeatureFlag.BUDIBASE_AI]: Flag.boolean(env.isDev()),
+  [FeatureFlag.DEFAULT_VALUES]: Flag.boolean(true),
+  [FeatureFlag.AUTOMATION_BRANCHING]: Flag.boolean(true),
+  [FeatureFlag.AI_CUSTOM_CONFIGS]: Flag.boolean(true),
+  [FeatureFlag.BUDIBASE_AI]: Flag.boolean(true),
 })
 
 type UnwrapPromise<T> = T extends Promise<infer U> ? U : T

packages/backend-core/src/middleware/contentSecurityPolicy.ts

@@ -0,0 +1,113 @@
+import crypto from "crypto"
+
+const CSP_DIRECTIVES = {
+  "default-src": ["'self'"],
+  "script-src": [
+    "'self'",
+    "'unsafe-eval'",
+    "https://*.budibase.net",
+    "https://cdn.budi.live",
+    "https://js.intercomcdn.com",
+    "https://widget.intercom.io",
+    "https://d2l5prqdbvm3op.cloudfront.net",
+    "https://us-assets.i.posthog.com",
+  ],
+  "style-src": [
+    "'self'",
+    "'unsafe-inline'",
+    "https://cdn.jsdelivr.net",
+    "https://fonts.googleapis.com",
+    "https://rsms.me",
+    "https://maxcdn.bootstrapcdn.com",
+  ],
+  "object-src": ["'none'"],
+  "base-uri": ["'self'"],
+  "connect-src": [
+    "'self'",
+    "https://*.budibase.app",
+    "https://*.budibaseqa.app",
+    "https://*.budibase.net",
+    "https://api-iam.intercom.io",
+    "https://api-ping.intercom.io",
+    "https://app.posthog.com",
+    "https://us.i.posthog.com",
+    "wss://nexus-websocket-a.intercom.io",
+    "wss://nexus-websocket-b.intercom.io",
+    "https://nexus-websocket-a.intercom.io",
+    "https://nexus-websocket-b.intercom.io",
+    "https://uploads.intercomcdn.com",
+    "https://uploads.intercomusercontent.com",
+    "https://*.amazonaws.com",
+    "https://*.s3.amazonaws.com",
+    "https://*.s3.us-east-2.amazonaws.com",
+    "https://*.s3.us-east-1.amazonaws.com",
+    "https://*.s3.us-west-1.amazonaws.com",
+    "https://*.s3.us-west-2.amazonaws.com",
+    "https://*.s3.af-south-1.amazonaws.com",
+    "https://*.s3.ap-east-1.amazonaws.com",
+    "https://*.s3.ap-south-1.amazonaws.com",
+    "https://*.s3.ap-northeast-2.amazonaws.com",
+    "https://*.s3.ap-southeast-1.amazonaws.com",
+    "https://*.s3.ap-southeast-2.amazonaws.com",
+    "https://*.s3.ap-northeast-1.amazonaws.com",
+    "https://*.s3.ca-central-1.amazonaws.com",
+    "https://*.s3.cn-north-1.amazonaws.com",
+    "https://*.s3.cn-northwest-1.amazonaws.com",
+    "https://*.s3.eu-central-1.amazonaws.com",
+    "https://*.s3.eu-west-1.amazonaws.com",
+    "https://*.s3.eu-west-2.amazonaws.com",
+    "https://*.s3.eu-south-1.amazonaws.com",
+    "https://*.s3.eu-west-3.amazonaws.com",
+    "https://*.s3.eu-north-1.amazonaws.com",
+    "https://*.s3.sa-east-1.amazonaws.com",
+    "https://*.s3.me-south-1.amazonaws.com",
+    "https://*.s3.us-gov-east-1.amazonaws.com",
+    "https://*.s3.us-gov-west-1.amazonaws.com",
+    "https://api.github.com",
+  ],
+  "font-src": [
+    "'self'",
+    "data:",
+    "https://cdn.jsdelivr.net",
+    "https://fonts.gstatic.com",
+    "https://rsms.me",
+    "https://maxcdn.bootstrapcdn.com",
+    "https://js.intercomcdn.com",
+    "https://fonts.intercomcdn.com",
+  ],
+  "frame-src": ["'self'", "https:"],
+  "img-src": ["http:", "https:", "data:", "blob:"],
+  "manifest-src": ["'self'"],
+  "media-src": [
+    "'self'",
+    "https://js.intercomcdn.com",
+    "https://cdn.budi.live",
+  ],
+  "worker-src": ["blob:"],
+}
+
+export async function contentSecurityPolicy(ctx: any, next: any) {
+  try {
+    const nonce = crypto.randomBytes(16).toString("base64")
+
+    const directives = { ...CSP_DIRECTIVES }
+    directives["script-src"] = [
+      ...CSP_DIRECTIVES["script-src"],
+      `'nonce-${nonce}'`,
+    ]
+
+    ctx.state.nonce = nonce
+
+    const cspHeader = Object.entries(directives)
+      .map(([key, sources]) => `${key} ${sources.join(" ")}`)
+      .join("; ")
+    ctx.set("Content-Security-Policy", cspHeader)
+    await next()
+  } catch (err: any) {
+    console.error(
+      `Error occurred in Content-Security-Policy middleware: ${err}`
+    )
+  }
+}
+
+export default contentSecurityPolicy

packages/backend-core/src/middleware/index.ts

@@ -19,5 +19,6 @@ export { default as pino } from "../logging/pino/middleware"
 export { default as correlation } from "../logging/correlation/middleware"
 export { default as errorHandling } from "./errorHandling"
 export { default as querystringToBody } from "./querystringToBody"
+export { default as csp } from "./contentSecurityPolicy"
 export * as joiValidator from "./joi-validator"
 export { default as ip } from "./ip"

packages/backend-core/src/middleware/tests/contentSecurityPolicy.spec.ts

@@ -0,0 +1,75 @@
+import crypto from "crypto"
+import contentSecurityPolicy from "../contentSecurityPolicy"
+
+jest.mock("crypto", () => ({
+  randomBytes: jest.fn(),
+  randomUUID: jest.fn(),
+}))
+
+describe("contentSecurityPolicy middleware", () => {
+  let ctx: any
+  let next: any
+  const mockNonce = "mocked/nonce"
+
+  beforeEach(() => {
+    ctx = {
+      state: {},
+      set: jest.fn(),
+    }
+    next = jest.fn()
+    // @ts-ignore
+    crypto.randomBytes.mockReturnValue(Buffer.from(mockNonce, "base64"))
+  })
+
+  afterEach(() => {
+    jest.clearAllMocks()
+  })
+
+  it("should generate a nonce and set it in the script-src directive", async () => {
+    await contentSecurityPolicy(ctx, next)
+
+    expect(ctx.state.nonce).toBe(mockNonce)
+    expect(ctx.set).toHaveBeenCalledWith(
+      "Content-Security-Policy",
+      expect.stringContaining(
+        `script-src 'self' 'unsafe-eval' https://*.budibase.net https://cdn.budi.live https://js.intercomcdn.com https://widget.intercom.io https://d2l5prqdbvm3op.cloudfront.net https://us-assets.i.posthog.com 'nonce-${mockNonce}'`
+      )
+    )
+    expect(next).toHaveBeenCalled()
+  })
+
+  it("should include all CSP directives in the header", async () => {
+    await contentSecurityPolicy(ctx, next)
+
+    const cspHeader = ctx.set.mock.calls[0][1]
+    expect(cspHeader).toContain("default-src 'self'")
+    expect(cspHeader).toContain("script-src 'self' 'unsafe-eval'")
+    expect(cspHeader).toContain("style-src 'self' 'unsafe-inline'")
+    expect(cspHeader).toContain("object-src 'none'")
+    expect(cspHeader).toContain("base-uri 'self'")
+    expect(cspHeader).toContain("connect-src 'self'")
+    expect(cspHeader).toContain("font-src 'self'")
+    expect(cspHeader).toContain("frame-src 'self'")
+    expect(cspHeader).toContain("img-src http: https: data: blob:")
+    expect(cspHeader).toContain("manifest-src 'self'")
+    expect(cspHeader).toContain("media-src 'self'")
+    expect(cspHeader).toContain("worker-src blob:")
+  })
+
+  it("should handle errors and log an error message", async () => {
+    const consoleSpy = jest.spyOn(console, "error").mockImplementation()
+    const error = new Error("Test error")
+    // @ts-ignore
+    crypto.randomBytes.mockImplementation(() => {
+      throw error
+    })
+
+    await contentSecurityPolicy(ctx, next)
+
+    expect(consoleSpy).toHaveBeenCalledWith(
+      `Error occurred in Content-Security-Policy middleware: ${error}`
+    )
+    expect(next).not.toHaveBeenCalled()
+    consoleSpy.mockRestore()
+  })
+})

packages/backend-core/src/queue/inMemoryQueue.ts

@@ -1,5 +1,5 @@
 import events from "events"
-import { newid, timeout } from "../utils"
+import { newid } from "../utils"
 import { Queue, QueueOptions, JobOptions } from "./queue"
 
 interface JobMessage {
@@ -141,7 +141,7 @@ class InMemoryQueue implements Partial<Queue> {
     } else {
       pushMessage()
     }
-    return {} as any
+    return { id: jobId } as any
   }
 
   /**
@@ -184,16 +184,6 @@ class InMemoryQueue implements Partial<Queue> {
     // do nothing
     return this as any
   }
-
-  async waitForCompletion() {
-    do {
-      await timeout(50)
-    } while (this.hasRunningJobs())
-  }
-
-  hasRunningJobs() {
-    return this._addCount > this._runCount
-  }
 }
 
 export default InMemoryQueue

packages/backend-core/src/queue/queue.ts

@@ -15,7 +15,7 @@ const QUEUE_LOCK_MS = Duration.fromMinutes(5).toMs()
 const QUEUE_LOCK_RENEW_INTERNAL_MS = Duration.fromSeconds(30).toMs()
 // cleanup the queue every 60 seconds
 const CLEANUP_PERIOD_MS = Duration.fromSeconds(60).toMs()
-let QUEUES: BullQueue.Queue[] | InMemoryQueue[] = []
+let QUEUES: BullQueue.Queue[] = []
 let cleanupInterval: NodeJS.Timeout
 
 async function cleanup() {
@@ -45,11 +45,18 @@ export function createQueue<T>(
   if (opts.jobOptions) {
     queueConfig.defaultJobOptions = opts.jobOptions
   }
-  let queue: any
+  let queue: BullQueue.Queue<T>
   if (!env.isTest()) {
     queue = new BullQueue(jobQueue, queueConfig)
+  } else if (
+    process.env.BULL_TEST_REDIS_PORT &&
+    !isNaN(+process.env.BULL_TEST_REDIS_PORT)
+  ) {
+    queue = new BullQueue(jobQueue, {
+      redis: { host: "localhost", port: +process.env.BULL_TEST_REDIS_PORT },
+    })
   } else {
-    queue = new InMemoryQueue(jobQueue, queueConfig)
+    queue = new InMemoryQueue(jobQueue, queueConfig) as any
   }
   addListeners(queue, jobQueue, opts?.removeStalledCb)
   QUEUES.push(queue)

packages/backend-core/src/security/tests/encryption.spec.ts

@@ -4,7 +4,7 @@ import env from "../../environment"
 describe("encryption", () => {
   it("should throw an error if API encryption key is not set", () => {
     const jwt = getSecret(SecretOption.API)
-    expect(jwt).toBe(env.JWT_SECRET)
+    expect(jwt).toBe(env.JWT_SECRET?.export().toString())
   })
 
   it("should throw an error if encryption key is not set", () => {

packages/backend-core/tests/core/utilities/index.ts

@@ -4,3 +4,4 @@ export { generator } from "./structures"
 export * as testContainerUtils from "./testContainerUtils"
 export * as utils from "./utils"
 export * from "./jestUtils"
+export * as queue from "./queue"

packages/backend-core/tests/core/utilities/queue.ts

@@ -0,0 +1,9 @@
+import { Queue } from "bull"
+
+export async function processMessages(queue: Queue) {
+  do {
+    await queue.whenCurrentJobsFinished()
+  } while (await queue.count())
+
+  await queue.whenCurrentJobsFinished()
+}

packages/backend-core/tests/core/utilities/testContainerUtils.ts

@@ -1,4 +1,6 @@
 import { execSync } from "child_process"
+import { cloneDeep } from "lodash"
+import { GenericContainer, StartedTestContainer } from "testcontainers"
 
 const IPV4_PORT_REGEX = new RegExp(`0\\.0\\.0\\.0:(\\d+)->(\\d+)/tcp`, "g")
 
@@ -106,3 +108,58 @@ export function setupEnv(...envs: any[]) {
     }
   }
 }
+
+export async function startContainer(container: GenericContainer) {
+  const imageName = (container as any).imageName.string as string
+  let key: string = imageName
+  if (imageName.includes("@sha256")) {
+    key = imageName.split("@")[0]
+  }
+  key = key.replace(/\//g, "-").replace(/:/g, "-")
+
+  container = container
+    .withReuse()
+    .withLabels({ "com.budibase": "true" })
+    .withName(`${key}_testcontainer`)
+
+  let startedContainer: StartedTestContainer | undefined = undefined
+  let lastError = undefined
+  for (let i = 0; i < 10; i++) {
+    try {
+      // container.start() is not an idempotent operation, calling `start`
+      // modifies the internal state of a GenericContainer instance such that
+      // the hash it uses to determine reuse changes. We need to clone the
+      // container before calling start to ensure that we're using the same
+      // reuse hash every time.
+      const containerCopy = cloneDeep(container)
+      startedContainer = await containerCopy.start()
+      lastError = undefined
+      break
+    } catch (e: any) {
+      lastError = e
+      await new Promise(resolve => setTimeout(resolve, 1000))
+    }
+  }
+
+  if (!startedContainer) {
+    if (lastError) {
+      throw lastError
+    }
+    throw new Error(`failed to start container: ${imageName}`)
+  }
+
+  const info = getContainerById(startedContainer.getId())
+  if (!info) {
+    throw new Error("Container not found")
+  }
+
+  // Some Docker runtimes, when you expose a port, will bind it to both
+  // 127.0.0.1 and ::1, so ipv4 and ipv6. The port spaces of ipv4 and ipv6
+  // addresses are not shared, and testcontainers will sometimes give you back
+  // the ipv6 port. There's no way to know that this has happened, and if you
+  // try to then connect to `localhost:port` you may attempt to bind to the v4
+  // address which could be unbound or even an entirely different container. For
+  // that reason, we don't use testcontainers' `getExposedPort` function,
+  // preferring instead our own method that guaranteed v4 ports.
+  return getExposedV4Ports(info)
+}

packages/backend-core/tests/core/utilities/utils/index.ts

@@ -1 +1,2 @@
 export * as time from "./time"
+export * as queue from "./queue"

packages/backend-core/tests/core/utilities/utils/queue.ts

@@ -0,0 +1,27 @@
+import { Queue } from "bull"
+import { GenericContainer, Wait } from "testcontainers"
+import { startContainer } from "../testContainerUtils"
+
+export async function useRealQueues() {
+  const ports = await startContainer(
+    new GenericContainer("redis")
+      .withExposedPorts(6379)
+      .withWaitStrategy(
+        Wait.forSuccessfulCommand(`redis-cli`).withStartupTimeout(10000)
+      )
+  )
+
+  const port = ports.find(x => x.container === 6379)?.host
+  if (!port) {
+    throw new Error("Redis port not found")
+  }
+  process.env.BULL_TEST_REDIS_PORT = port.toString()
+}
+
+export async function processMessages(queue: Queue) {
+  do {
+    await queue.whenCurrentJobsFinished()
+  } while (await queue.count())
+
+  await queue.whenCurrentJobsFinished()
+}

packages/bbui/package.json

@@ -81,6 +81,7 @@
     "@spectrum-css/typography": "3.0.1",
     "@spectrum-css/underlay": "2.0.9",
     "@spectrum-css/vars": "3.0.1",
+    "atrament": "^4.3.0",
     "dayjs": "^1.10.8",
     "easymde": "^2.16.1",
     "svelte-dnd-action": "^0.9.8",

packages/bbui/src/Form/Core/Dropzone.svelte

@@ -8,6 +8,7 @@
   import Link from "../../Link/Link.svelte"
   import Tag from "../../Tags/Tag.svelte"
   import Tags from "../../Tags/Tags.svelte"
+  import ProgressCircle from "../../ProgressCircle/ProgressCircle.svelte"
 
   const BYTES_IN_KB = 1000
   const BYTES_IN_MB = 1000000
@@ -39,12 +40,14 @@
     "jfif",
     "webp",
   ]
-
   const fieldId = id || uuid()
+
   let selectedImageIdx = 0
   let fileDragged = false
   let selectedUrl
   let fileInput
+  let loading = false
+
   $: selectedImage = value?.[selectedImageIdx] ?? null
   $: fileCount = value?.length ?? 0
   $: isImage =
@@ -86,10 +89,15 @@
     }
 
     if (processFiles) {
-      const processedFiles = await processFiles(fileList)
-      const newValue = [...value, ...processedFiles]
-      dispatch("change", newValue)
-      selectedImageIdx = newValue.length - 1
+      loading = true
+      try {
+        const processedFiles = await processFiles(fileList)
+        const newValue = [...value, ...processedFiles]
+        dispatch("change", newValue)
+        selectedImageIdx = newValue.length - 1
+      } finally {
+        loading = false
+      }
     } else {
       dispatch("change", fileList)
     }
@@ -227,7 +235,7 @@
   {#if showDropzone}
     <div
       class="spectrum-Dropzone"
-      class:disabled
+      class:disabled={disabled || loading}
       role="region"
       tabindex="0"
       on:dragover={handleDragOver}
@@ -241,7 +249,7 @@
           id={fieldId}
           {disabled}
           type="file"
-          multiple
+          multiple={maximum !== 1}
           accept={extensions}
           bind:this={fileInput}
           on:change={handleFile}
@@ -339,6 +347,12 @@
           {/if}
         {/if}
       </div>
+
+      {#if loading}
+        <div class="loading">
+          <ProgressCircle size="M" />
+        </div>
+      {/if}
     </div>
   {/if}
 </div>
@@ -464,6 +478,7 @@
 
   .spectrum-Dropzone {
     height: 220px;
+    position: relative;
   }
   .compact .spectrum-Dropzone {
     height: 40px;
@@ -488,4 +503,14 @@
   .tag {
     margin-top: 8px;
   }
+
+  .loading {
+    position: absolute;
+    display: grid;
+    place-items: center;
+    height: 100%;
+    width: 100%;
+    top: 0;
+    left: 0;
+  }
 </style>

packages/builder/src/components/automation/AutomationBuilder/AutomationBuilder.svelte

@@ -3,8 +3,8 @@
   import Flowchart from "./FlowChart/FlowChart.svelte"
 </script>
 
-{#if $selectedAutomation}
-  {#key $selectedAutomation._id}
-    <Flowchart automation={$selectedAutomation} />
+{#if $selectedAutomation?.data}
+  {#key $selectedAutomation.data._id}
+    <Flowchart automation={$selectedAutomation.data} />
   {/key}
 {/if}

packages/builder/src/components/automation/AutomationBuilder/DraggableCanvas.svelte

@@ -0,0 +1,497 @@
+<script>
+  import { writable } from "svelte/store"
+  import {
+    setContext,
+    onMount,
+    createEventDispatcher,
+    onDestroy,
+    tick,
+  } from "svelte"
+  import { Utils } from "@budibase/frontend-core"
+  import { selectedAutomation, automationStore } from "stores/builder"
+
+  export function zoomIn() {
+    const scale = Number(Math.min($view.scale + 0.1, 1.5).toFixed(2))
+    view.update(state => ({
+      ...state,
+      scale,
+    }))
+  }
+
+  export function zoomOut() {
+    const scale = Number(Math.max($view.scale - 0.1, 0).toFixed(2))
+    view.update(state => ({
+      ...state,
+      scale,
+    }))
+  }
+
+  export async function reset() {
+    contentDims = {
+      ...contentDims,
+      w: contentDims.original.w,
+      h: contentDims.original.h,
+    }
+    dragOffset = []
+    contentPos.update(state => ({
+      ...state,
+      x: 0,
+      y: 0,
+    }))
+    view.update(state => ({
+      ...state,
+      scale: 1,
+    }))
+  }
+
+  export async function zoomToFit() {
+    const { width: wViewPort, height: hViewPort } =
+      viewPort.getBoundingClientRect()
+
+    const scaleTarget = Math.min(
+      wViewPort / contentDims.original.w,
+      hViewPort / contentDims.original.h
+    )
+
+    // Smallest ratio determines which dimension needs squeezed
+    view.update(state => ({
+      ...state,
+      scale: scaleTarget,
+    }))
+
+    await tick()
+
+    const adjustedY = (hViewPort - contentDims.original.h) / 2
+
+    contentPos.update(state => ({
+      ...state,
+      x: 0,
+      y: parseInt(0 + adjustedY),
+    }))
+  }
+
+  const dispatch = createEventDispatcher()
+
+  // View State
+  const view = writable({
+    dragging: false,
+    moveStep: null,
+    dragSpot: null,
+    scale: 1,
+    dropzones: {},
+    //focus - node to center on?
+  })
+
+  setContext("draggableView", view)
+
+  // View internal pos tracking
+  const internalPos = writable({ x: 0, y: 0 })
+  setContext("viewPos", internalPos)
+
+  // Content pos tracking
+  const contentPos = writable({ x: 0, y: 0, scrollX: 0, scrollY: 0 })
+  setContext("contentPos", contentPos)
+
+  // Elements
+  let mainContent
+  let viewPort
+  let contentWrap
+
+  // Mouse down
+  let down = false
+
+  // Monitor the size of the viewPort
+  let observer
+
+  // Size of the core display content
+  let contentDims = {}
+
+  // Size of the view port
+  let viewDims = {}
+
+  // When dragging the content, maintain the drag start offset
+  let dragOffset
+
+  // Used when focusing the UI on trigger
+  let loaded = false
+
+  // Edge around the draggable content
+  let contentDragPadding = 200
+
+  const onScale = async () => {
+    dispatch("zoom", $view.scale)
+    await getDims()
+  }
+
+  const getDims = async () => {
+    if (!mainContent) return
+
+    if (!contentDims.original) {
+      contentDims.original = {
+        w: parseInt(mainContent.getBoundingClientRect().width),
+        h: parseInt(mainContent.getBoundingClientRect().height),
+      }
+    }
+
+    viewDims = viewPort.getBoundingClientRect()
+
+    contentDims = {
+      ...contentDims,
+      w: contentDims.original.w * $view.scale,
+      h: contentDims.original.h * $view.scale,
+    }
+  }
+
+  const eleXY = (coords, ele) => {
+    const { clientX, clientY } = coords
+    const rect = ele.getBoundingClientRect()
+    const x = Math.round(clientX - rect.left)
+    const y = Math.round(clientY - rect.top)
+    return { x: Math.max(x, 0), y: Math.max(y, 0) }
+  }
+
+  const buildWrapStyles = (pos, scale, dims) => {
+    const { x, y } = pos
+    const { w, h } = dims
+    return `--posX: ${x}px; --posY: ${y}px; --scale: ${scale}; --wrapH: ${h}px; --wrapW: ${w}px`
+  }
+
+  const onViewScroll = e => {
+    e.preventDefault()
+    let currentScale = $view.scale
+    let scrollIncrement = 35
+    let xBump = 0
+    let yBump = 0
+
+    if (e.shiftKey) {
+      // Scroll horizontal - Needs Limits
+      xBump = scrollIncrement * (e.deltaX < 0 ? -1 : 1)
+      contentPos.update(state => ({
+        ...state,
+        x: state.x - xBump,
+        y: state.y,
+        // If scrolling *and* dragging, maintain a record of the scroll offset
+        ...($view.dragging
+          ? {
+              scrollX: state.scrollX - xBump,
+            }
+          : {}),
+      }))
+    } else if (e.ctrlKey || e.metaKey) {
+      // Scale the content on scrolling
+      let updatedScale
+      if (e.deltaY < 0) {
+        updatedScale = Math.min(1, currentScale + 0.05)
+      } else if (e.deltaY > 0) {
+        updatedScale = Math.max(0, currentScale - 0.05)
+      }
+
+      view.update(state => ({
+        ...state,
+        scale: Number(updatedScale.toFixed(2)),
+      }))
+    } else {
+      yBump = scrollIncrement * (e.deltaY < 0 ? -1 : 1)
+      contentPos.update(state => ({
+        ...state,
+        x: state.x,
+        y: state.y - yBump,
+        // If scrolling *and* dragging, maintain a record of the scroll offset
+        ...($view.dragging
+          ? {
+              scrollY: state.scrollY - yBump,
+            }
+          : {}),
+      }))
+    }
+  }
+
+  // Optimization options
+  const onViewMouseMove = async e => {
+    if (!viewPort) {
+      return
+    }
+
+    const { x, y } = eleXY(e, viewPort)
+
+    internalPos.update(() => ({
+      x,
+      y,
+    }))
+
+    if (down && !$view.dragging && dragOffset) {
+      contentPos.update(state => ({
+        ...state,
+        x: x - dragOffset[0],
+        y: y - dragOffset[1],
+      }))
+    }
+  }
+
+  const onViewDragEnd = () => {
+    down = false
+    dragOffset = [0, 0]
+  }
+
+  const handleDragDrop = () => {
+    const sourceBlock = $selectedAutomation.blockRefs[$view.moveStep.id]
+    const sourcePath = sourceBlock.pathTo
+
+    const dropZone = $view.dropzones[$view.droptarget]
+    const destPath = dropZone?.path
+
+    automationStore.actions.moveBlock(
+      sourcePath,
+      destPath,
+      $selectedAutomation.data
+    )
+  }
+
+  const onMouseUp = () => {
+    if ($view.droptarget) {
+      handleDragDrop()
+    }
+
+    view.update(state => ({
+      ...state,
+      dragging: false,
+      moveStep: null,
+      dragSpot: null,
+      dropzones: {},
+      droptarget: null,
+    }))
+
+    // Clear the scroll offset for dragging
+    contentPos.update(state => ({
+      ...state,
+      scrollY: 0,
+      scrollX: 0,
+    }))
+  }
+
+  const onMouseMove = async e => {
+    if (!viewPort) {
+      return
+    }
+    // Update viewDims to get the latest viewport dimensions
+    viewDims = viewPort.getBoundingClientRect()
+
+    if ($view.moveStep && $view.dragging === false) {
+      view.update(state => ({
+        ...state,
+        dragging: true,
+      }))
+    }
+
+    const checkIntersection = (pos, dzRect) => {
+      return (
+        pos.x < dzRect.right &&
+        pos.x > dzRect.left &&
+        pos.y < dzRect.bottom &&
+        pos.y > dzRect.top
+      )
+    }
+
+    if ($view.dragging) {
+      const adjustedX =
+        (e.clientX - viewDims.left - $view.moveStep.offsetX) / $view.scale
+      const adjustedY =
+        (e.clientY - viewDims.top - $view.moveStep.offsetY) / $view.scale
+
+      view.update(state => ({
+        ...state,
+        dragSpot: {
+          x: adjustedX,
+          y: adjustedY,
+        },
+      }))
+    }
+
+    if ($view.moveStep && $view.dragging) {
+      let hovering = false
+      Object.entries($view.dropzones).forEach(entry => {
+        const [dzKey, dz] = entry
+        if (checkIntersection({ x: e.clientX, y: e.clientY }, dz.dims)) {
+          hovering = true
+          view.update(state => ({
+            ...state,
+            droptarget: dzKey,
+          }))
+        }
+      })
+      // Ensure that when it stops hovering, it clears the drop target
+      if (!hovering) {
+        view.update(state => ({
+          ...state,
+          droptarget: null,
+        }))
+      }
+    }
+  }
+
+  const onMoveContent = e => {
+    if (down || !viewPort) {
+      return
+    }
+    const { x, y } = eleXY(e, viewPort)
+
+    dragOffset = [Math.abs(x - $contentPos.x), Math.abs(y - $contentPos.y)]
+  }
+
+  const focusOnLoad = () => {
+    if ($view.focusEle && !loaded) {
+      const focusEleDims = $view.focusEle
+      const viewWidth = viewDims.width
+
+      // The amount to shift the content in order to center the trigger on load.
+      // The content is also padded with `contentDragPadding`
+      // The sidebar offset factors into the left positioning of the content here.
+      const targetX =
+        contentWrap.getBoundingClientRect().x -
+        focusEleDims.x +
+        (viewWidth / 2 - focusEleDims.width / 2)
+
+      // Update the content position state
+      // Shift the content up slightly to accommodate the padding
+      contentPos.update(state => ({
+        ...state,
+        x: targetX,
+        y: -(contentDragPadding / 2),
+      }))
+
+      loaded = true
+    }
+  }
+
+  // Update dims after scaling
+  $: {
+    $view.scale
+    onScale()
+  }
+
+  // Focus on a registered element
+  $: {
+    $view.focusEle
+    focusOnLoad()
+  }
+
+  // Content mouse pos and scale to css variables.
+  // The wrap is set to match the content size
+  $: wrapStyles = buildWrapStyles($contentPos, $view.scale, contentDims)
+
+  onMount(() => {
+    observer = new ResizeObserver(getDims)
+    observer.observe(viewPort)
+  })
+
+  onDestroy(() => {
+    observer.disconnect()
+  })
+</script>
+
+<!-- svelte-ignore a11y-no-static-element-interactions -->
+<!-- svelte-ignore a11y-no-noninteractive-element-interactions -->
+<div
+  class="draggable-canvas"
+  role="region"
+  aria-label="Viewport for building automations"
+  on:mouseup={onMouseUp}
+  on:mousemove={Utils.domDebounce(onMouseMove)}
+  style={`--dragPadding: ${contentDragPadding}px;`}
+>
+  <div
+    class="draggable-view"
+    bind:this={viewPort}
+    on:wheel={Utils.domDebounce(onViewScroll)}
+    on:mousemove={Utils.domDebounce(onViewMouseMove)}
+    on:mouseup={onViewDragEnd}
+    on:mouseleave={onViewDragEnd}
+  >
+    <!-- <div class="debug">
+      <span>
+        View Pos [{$internalPos.x}, {$internalPos.y}]
+      </span>
+      <span>View Dims [{viewDims.width}, {viewDims.height}]</span>
+      <span>Mouse Down [{down}]</span>
+      <span>Drag [{$view.dragging}]</span>
+      <span>Dragging [{$view?.moveStep?.id || "no"}]</span>
+      <span>Scale [{$view.scale}]</span>
+      <span>Content [{JSON.stringify($contentPos)}]</span>
+    </div> -->
+    <div
+      class="content-wrap"
+      style={wrapStyles}
+      bind:this={contentWrap}
+      class:dragging={down}
+    >
+      <div
+        class="content"
+        bind:this={mainContent}
+        on:mousemove={Utils.domDebounce(onMoveContent)}
+        on:mousedown={e => {
+          if (e.which === 1 || e.button === 0) {
+            down = true
+          }
+        }}
+        on:mouseup={e => {
+          if (e.which === 1 || e.button === 0) {
+            down = false
+          }
+        }}
+        on:mouseleave={() => {
+          down = false
+          view.update(state => ({
+            ...state,
+            dragging: false,
+            moveStep: null,
+            dragSpot: null,
+            dropzones: {},
+          }))
+        }}
+      >
+        <slot name="content" />
+      </div>
+    </div>
+  </div>
+</div>
+
+<style>
+  .draggable-canvas {
+    width: 100%;
+    height: 100%;
+  }
+  .draggable-view {
+    width: 100%;
+    height: 100%;
+    overflow: hidden;
+    position: relative;
+  }
+  .content-wrap {
+    min-width: 100%;
+    position: absolute;
+    top: 0;
+    left: 0;
+    width: var(--wrapW);
+    height: var(--wrapH);
+    cursor: grab;
+    transform: translate(var(--posX), var(--posY));
+  }
+  .content {
+    transform: scale(var(--scale));
+    user-select: none;
+    padding: var(--dragPadding);
+  }
+
+  .content-wrap.dragging {
+    cursor: grabbing;
+  }
+
+  /* .debug {
+    display: flex;
+    align-items: center;
+    gap: 8px;
+    position: fixed;
+    padding: 8px;
+    z-index: 2;
+  } */
+</style>

packages/builder/src/components/automation/AutomationBuilder/FlowChart/ActionModal.svelte

@@ -9,27 +9,42 @@
     Tags,
     Tag,
   } from "@budibase/bbui"
+  import { AutomationActionStepId } from "@budibase/types"
   import { automationStore, selectedAutomation } from "stores/builder"
   import { admin, licensing } from "stores/portal"
   import { externalActions } from "./ExternalActions"
   import { TriggerStepID, ActionStepID } from "constants/backend/automations"
-  import { checkForCollectStep } from "helpers/utils"
 
-  export let blockIdx
-  export let lastStep
+  export let block
   export let modal
 
   let syncAutomationsEnabled = $licensing.syncAutomationsEnabled
   let triggerAutomationRunEnabled = $licensing.triggerAutomationRunEnabled
   let collectBlockAllowedSteps = [TriggerStepID.APP, TriggerStepID.WEBHOOK]
   let selectedAction
-  let actions = Object.entries($automationStore.blockDefinitions.ACTION)
+  let actions = Object.entries($automationStore.blockDefinitions.ACTION).filter(
+    entry => {
+      const [key] = entry
+      return key !== AutomationActionStepId.BRANCH
+    }
+  )
   let lockedFeatures = [
     ActionStepID.COLLECT,
     ActionStepID.TRIGGER_AUTOMATION_RUN,
   ]
 
-  $: collectBlockExists = checkForCollectStep($selectedAutomation)
+  $: blockRef = $selectedAutomation.blockRefs?.[block.id]
+  $: lastStep = blockRef?.terminating
+  $: pathSteps = block.id
+    ? automationStore.actions.getPathSteps(
+        blockRef.pathTo,
+        $selectedAutomation?.data
+      )
+    : []
+
+  $: collectBlockExists = pathSteps?.some(
+    step => step.stepId === ActionStepID.COLLECT
+  )
 
   const disabled = () => {
     return {
@@ -75,7 +90,7 @@
     // Filter out Collect block if not App Action or Webhook
     if (
       !collectBlockAllowedSteps.includes(
-        $selectedAutomation.definition.trigger.stepId
+        $selectedAutomation.data.definition.trigger.stepId
       )
     ) {
       delete acc.COLLECT
@@ -100,9 +115,14 @@
         action.stepId,
         action
       )
-      await automationStore.actions.addBlockToAutomation(newBlock, blockIdx + 1)
+
+      await automationStore.actions.addBlockToAutomation(
+        newBlock,
+        blockRef ? blockRef.pathTo : block.pathTo
+      )
       modal.hide()
     } catch (error) {
+      console.error(error)
       notifications.error("Error saving automation")
     }
   }

packages/builder/src/components/automation/AutomationBuilder/FlowChart/BranchNode.svelte

@@ -0,0 +1,283 @@
+<script>
+  import FilterBuilder from "components/design/settings/controls/FilterEditor/FilterBuilder.svelte"
+  import {
+    Drawer,
+    DrawerContent,
+    ActionButton,
+    Icon,
+    Layout,
+    Body,
+    Divider,
+    TooltipPosition,
+    TooltipType,
+    Button,
+    Modal,
+    ModalContent,
+  } from "@budibase/bbui"
+  import PropField from "components/automation/SetupPanel/PropField.svelte"
+  import AutomationBindingPanel from "components/common/bindings/ServerBindingPanel.svelte"
+  import FlowItemHeader from "./FlowItemHeader.svelte"
+  import FlowItemActions from "./FlowItemActions.svelte"
+  import { automationStore, selectedAutomation } from "stores/builder"
+  import { QueryUtils } from "@budibase/frontend-core"
+  import { cloneDeep } from "lodash/fp"
+  import { createEventDispatcher, getContext } from "svelte"
+  import DragZone from "./DragZone.svelte"
+
+  const dispatch = createEventDispatcher()
+
+  export let pathTo
+  export let branchIdx
+  export let step
+  export let isLast
+  export let bindings
+  export let automation
+
+  const view = getContext("draggableView")
+
+  let drawer
+  let condition
+  let open = true
+  let confirmDeleteModal
+
+  $: branch = step.inputs?.branches?.[branchIdx]
+  $: editableConditionUI = cloneDeep(branch.conditionUI || {})
+  $: condition = QueryUtils.buildQuery(editableConditionUI)
+
+  // Parse all the bindings into fields for the condition builder
+  $: schemaFields = bindings.map(binding => {
+    return {
+      name: `{{${binding.runtimeBinding}}}`,
+      displayName: `${binding.category} - ${binding.display.name}`,
+      type: "string",
+    }
+  })
+  $: branchBlockRef = {
+    branchNode: true,
+    pathTo: (pathTo || []).concat({ branchIdx, branchStepId: step.id }),
+  }
+</script>
+
+<Modal bind:this={confirmDeleteModal}>
+  <ModalContent
+    size="M"
+    title={"Are you sure you want to delete?"}
+    confirmText="Delete"
+    onConfirm={async () => {
+      await automationStore.actions.deleteBranch(
+        branchBlockRef.pathTo,
+        $selectedAutomation.data
+      )
+    }}
+  >
+    <Body>By deleting this branch, you will delete all of its contents.</Body>
+  </ModalContent>
+</Modal>
+
+<Drawer bind:this={drawer} title="Branch condition" forceModal>
+  <Button
+    cta
+    slot="buttons"
+    on:click={() => {
+      drawer.hide()
+      dispatch("change", {
+        conditionUI: editableConditionUI,
+        condition,
+      })
+    }}
+  >
+    Save
+  </Button>
+  <DrawerContent slot="body">
+    <FilterBuilder
+      filters={editableConditionUI}
+      {bindings}
+      {schemaFields}
+      datasource={{ type: "custom" }}
+      panel={AutomationBindingPanel}
+      on:change={e => {
+        editableConditionUI = e.detail
+      }}
+      allowOnEmpty={false}
+      builderType={"condition"}
+      docsURL={null}
+    />
+  </DrawerContent>
+</Drawer>
+
+<div class="flow-item">
+  <!-- svelte-ignore a11y-no-static-element-interactions -->
+  <div
+    class={`block branch-node hoverable`}
+    class:selected={false}
+    on:mousedown={e => {
+      e.stopPropagation()
+    }}
+  >
+    <FlowItemHeader
+      {automation}
+      {open}
+      itemName={branch.name}
+      block={step}
+      deleteStep={async () => {
+        const branchSteps = step.inputs?.children[branch.id]
+        if (branchSteps.length) {
+          confirmDeleteModal.show()
+        } else {
+          await automationStore.actions.deleteBranch(
+            branchBlockRef.pathTo,
+            $selectedAutomation.data
+          )
+        }
+      }}
+      on:update={async e => {
+        let stepUpdate = cloneDeep(step)
+        let branchUpdate = stepUpdate.inputs?.branches.find(
+          stepBranch => stepBranch.id == branch.id
+        )
+        branchUpdate.name = e.detail
+
+        const updatedAuto = automationStore.actions.updateStep(
+          pathTo,
+          $selectedAutomation.data,
+          stepUpdate
+        )
+        await automationStore.actions.save(updatedAuto)
+      }}
+      on:toggle={() => (open = !open)}
+    >
+      <div slot="custom-actions" class="branch-actions">
+        <Icon
+          on:click={() => {
+            automationStore.actions.branchLeft(
+              branchBlockRef.pathTo,
+              $selectedAutomation.data,
+              step
+            )
+          }}
+          tooltip={"Move left"}
+          tooltipType={TooltipType.Info}
+          tooltipPosition={TooltipPosition.Top}
+          hoverable
+          disabled={branchIdx == 0}
+          name="ArrowLeft"
+        />
+        <Icon
+          on:click={() => {
+            automationStore.actions.branchRight(
+              branchBlockRef.pathTo,
+              $selectedAutomation.data,
+              step
+            )
+          }}
+          tooltip={"Move right"}
+          tooltipType={TooltipType.Info}
+          tooltipPosition={TooltipPosition.Top}
+          hoverable
+          disabled={isLast}
+          name="ArrowRight"
+        />
+      </div>
+    </FlowItemHeader>
+    {#if open}
+      <Divider noMargin />
+      <div class="blockSection">
+        <!-- Content body for possible slot -->
+        <Layout noPadding>
+          <PropField label="Only run when">
+            <ActionButton fullWidth on:click={drawer.show}>
+              {editableConditionUI?.groups?.length
+                ? "Update condition"
+                : "Add condition"}
+            </ActionButton>
+          </PropField>
+          <div class="footer">
+            <Icon
+              name="InfoOutline"
+              size="S"
+              color="var(--spectrum-global-color-gray-700)"
+            />
+            <Body size="XS" color="var(--spectrum-global-color-gray-700)">
+              Only the first branch which matches its condition will run
+            </Body>
+          </div>
+        </Layout>
+      </div>
+    {/if}
+  </div>
+
+  <div class="separator" />
+
+  {#if $view.dragging}
+    <DragZone path={branchBlockRef.pathTo} />
+  {:else}
+    <FlowItemActions block={branchBlockRef} />
+  {/if}
+
+  {#if step.inputs.children[branch.id]?.length}
+    <div class="separator" />
+  {/if}
+</div>
+
+<style>
+  .branch-actions {
+    display: flex;
+    gap: var(--spacing-l);
+  }
+  .footer {
+    display: flex;
+    align-items: center;
+    gap: var(--spacing-m);
+  }
+
+  .flow-item {
+    display: flex;
+    flex-direction: column;
+    align-items: center;
+  }
+
+  .block-options {
+    justify-content: flex-end;
+    align-items: center;
+    display: flex;
+    gap: var(--spacing-m);
+  }
+  .center-items {
+    display: flex;
+    align-items: center;
+  }
+  .splitHeader {
+    display: flex;
+    justify-content: space-between;
+    align-items: center;
+  }
+  .iconAlign {
+    padding: 0 0 0 var(--spacing-m);
+    display: inline-block;
+  }
+  .block {
+    width: 480px;
+    font-size: 16px;
+    background-color: var(--background);
+    border: 1px solid var(--spectrum-global-color-gray-300);
+    border-radius: 4px 4px 4px 4px;
+  }
+
+  .blockSection {
+    padding: var(--spacing-xl);
+  }
+
+  .separator {
+    width: 1px;
+    height: 25px;
+    border-left: 1px dashed var(--grey-4);
+    color: var(--grey-4);
+    align-self: center;
+  }
+
+  .blockTitle {
+    display: flex;
+    align-items: center;
+    gap: var(--spacing-s);
+  }
+</style>

packages/builder/src/components/automation/AutomationBuilder/FlowChart/DragZone.svelte

@@ -0,0 +1,66 @@
+<script>
+  import { getContext, onMount } from "svelte"
+  import { generate } from "shortid"
+
+  export let path
+
+  let dropEle
+  let dzid = generate()
+
+  const view = getContext("draggableView")
+
+  onMount(() => {
+    // Always return up-to-date values
+    view.update(state => {
+      return {
+        ...state,
+        dropzones: {
+          ...(state.dropzones || {}),
+          [dzid]: {
+            get dims() {
+              return dropEle.getBoundingClientRect()
+            },
+            path,
+          },
+        },
+      }
+    })
+  })
+</script>
+
+<div
+  id={`dz-${dzid}`}
+  bind:this={dropEle}
+  class="drag-zone"
+  class:drag-over={$view?.droptarget === dzid}
+>
+  <span class="move-to">Move to</span>
+</div>
+
+<style>
+  .drag-zone.drag-over {
+    background-color: #1ca872b8;
+  }
+  .drag-zone {
+    min-height: calc(var(--spectrum-global-dimension-size-225) + 12px);
+    min-width: 100%;
+    background-color: rgba(28, 168, 114, 0.2);
+    border-radius: 4px;
+    border: 1px dashed #1ca872;
+    position: relative;
+    text-align: center;
+  }
+  .move-to {
+    position: absolute;
+    left: 0;
+    right: 0;
+    top: -50%;
+    margin: auto;
+    width: fit-content;
+    font-weight: 600;
+    border-radius: 8px;
+    padding: 4px 8px;
+    background-color: rgb(28, 168, 114);
+    color: var(--spectrum-global-color-gray-50);
+  }
+</style>

packages/builder/src/components/automation/AutomationBuilder/FlowChart/FlowChart.svelte

@@ -1,27 +1,63 @@
 <script>
   import {
     automationStore,
-    selectedAutomation,
     automationHistoryStore,
+    selectedAutomation,
   } from "stores/builder"
   import ConfirmDialog from "components/common/ConfirmDialog.svelte"
-  import FlowItem from "./FlowItem.svelte"
   import TestDataModal from "./TestDataModal.svelte"
-  import { flip } from "svelte/animate"
-  import { fly } from "svelte/transition"
-  import { Icon, notifications, Modal, Toggle } from "@budibase/bbui"
+  import {
+    Icon,
+    notifications,
+    Modal,
+    Toggle,
+    Button,
+    ActionButton,
+  } from "@budibase/bbui"
   import { ActionStepID } from "constants/backend/automations"
   import UndoRedoControl from "components/common/UndoRedoControl.svelte"
+  import StepNode from "./StepNode.svelte"
+  import { memo } from "@budibase/frontend-core"
   import { sdk } from "@budibase/shared-core"
+  import DraggableCanvas from "../DraggableCanvas.svelte"
 
   export let automation
 
+  const memoAutomation = memo(automation)
+
   let testDataModal
   let confirmDeleteDialog
   let scrolling = false
+  let blockRefs = {}
+  let treeEle
+  let draggable
 
-  $: blocks = getBlocks(automation).filter(x => x.stepId !== ActionStepID.LOOP)
-  $: isRowAction = sdk.automations.isRowAction(automation)
+  // Memo auto - selectedAutomation
+  $: memoAutomation.set(automation)
+
+  // Parse the automation tree state
+  $: refresh($memoAutomation)
+
+  $: blocks = getBlocks($memoAutomation).filter(
+    x => x.stepId !== ActionStepID.LOOP
+  )
+  $: isRowAction = sdk.automations.isRowAction($memoAutomation)
+
+  const refresh = () => {
+    // Build global automation bindings.
+    const environmentBindings =
+      automationStore.actions.buildEnvironmentBindings()
+
+    // Get all processed block references
+    blockRefs = $selectedAutomation.blockRefs
+
+    automationStore.update(state => {
+      return {
+        ...state,
+        bindings: [...environmentBindings],
+      }
+    })
+  }
 
   const getBlocks = automation => {
     let blocks = []
@@ -34,39 +70,46 @@
 
   const deleteAutomation = async () => {
     try {
-      await automationStore.actions.delete($selectedAutomation)
+      await automationStore.actions.delete(automation)
     } catch (error) {
       notifications.error("Error deleting automation")
     }
   }
-
-  const handleScroll = e => {
-    if (e.target.scrollTop >= 30) {
-      scrolling = true
-    } else if (e.target.scrollTop) {
-      // Set scrolling back to false if scrolled back to less than 100px
-      scrolling = false
-    }
-  }
 </script>
 
 <!-- svelte-ignore a11y-click-events-have-key-events -->
 <!-- svelte-ignore a11y-no-static-element-interactions -->
 <div class="header" class:scrolling>
   <div class="header-left">
-    <UndoRedoControl store={automationHistoryStore} />
+    <UndoRedoControl store={automationHistoryStore} showButtonGroup />
+
+    <div class="zoom">
+      <div class="group">
+        <ActionButton icon="Add" quiet on:click={draggable.zoomIn} />
+        <ActionButton icon="Remove" quiet on:click={draggable.zoomOut} />
+      </div>
+    </div>
+
+    <Button
+      secondary
+      on:click={() => {
+        draggable.zoomToFit()
+      }}
+    >
+      Zoom to fit
+    </Button>
   </div>
   <div class="controls">
-    <div
-      class:disabled={!$selectedAutomation?.definition?.trigger}
+    <Button
+      icon={"Play"}
+      cta
+      disabled={!automation?.definition?.trigger}
       on:click={() => {
         testDataModal.show()
       }}
-      class="buttons"
     >
-      <Icon size="M" name="Play" />
-      <div>Run test</div>
-    </div>
+      Run test
+    </Button>
     <div class="buttons">
       <Icon disabled={!$automationStore.testResults} size="M" name="Multiple" />
       <div
@@ -79,36 +122,39 @@
       </div>
     </div>
     {#if !isRowAction}
-      <div class="setting-spacing">
+      <div class="toggle-active setting-spacing">
         <Toggle
           text={automation.disabled ? "Paused" : "Activated"}
           on:change={automationStore.actions.toggleDisabled(
             automation._id,
             automation.disabled
           )}
-          disabled={!$selectedAutomation?.definition?.trigger}
+          disabled={!automation?.definition?.trigger}
           value={!automation.disabled}
         />
       </div>
     {/if}
   </div>
 </div>
-<div class="canvas" on:scroll={handleScroll}>
-  <div class="content">
-    {#each blocks as block, idx (block.id)}
-      <div
-        class="block"
-        animate:flip={{ duration: 500 }}
-        in:fly={{ x: 500, duration: 500 }}
-        out:fly|local={{ x: 500, duration: 500 }}
-      >
-        {#if block.stepId !== ActionStepID.LOOP}
-          <FlowItem {testDataModal} {block} {idx} />
-        {/if}
-      </div>
-    {/each}
-  </div>
+
+<div class="root" bind:this={treeEle}>
+  <DraggableCanvas bind:this={draggable}>
+    <span class="main-content" slot="content">
+      {#if Object.keys(blockRefs).length}
+        {#each blocks as block, idx (block.id)}
+          <StepNode
+            step={blocks[idx]}
+            stepIdx={idx}
+            isLast={blocks?.length - 1 === idx}
+            automation={$memoAutomation}
+            blocks={blockRefs}
+          />
+        {/each}
+      {/if}
+    </span>
+  </DraggableCanvas>
 </div>
+
 <ConfirmDialog
   bind:this={confirmDeleteDialog}
   okText="Delete Automation"
@@ -125,32 +171,43 @@
 </Modal>
 
 <style>
-  .canvas {
-    padding: var(--spacing-l) var(--spacing-xl);
-    overflow-y: auto;
+  .toggle-active :global(.spectrum-Switch) {
+    margin: 0px;
+  }
+
+  .root :global(.main-content) {
+    display: flex;
+    flex-direction: column;
+    align-items: center;
     max-height: 100%;
+    height: 100%;
+    width: 100%;
+  }
+
+  .header-left {
+    display: flex;
+    gap: var(--spacing-l);
   }
 
   .header-left :global(div) {
     border-right: none;
   }
-  /* Fix for firefox not respecting bottom padding in scrolling containers */
-  .canvas > *:last-child {
-    padding-bottom: 40px;
+
+  .root {
+    height: 100%;
+    width: 100%;
   }
 
-  .block {
+  .root :global(.block) {
     display: flex;
     flex-direction: column;
     justify-content: flex-start;
     align-items: center;
   }
 
-  .content {
-    flex-grow: 1;
-    padding: 23px 23px 80px;
+  .root :global(.blockSection) {
+    width: 100%;
     box-sizing: border-box;
-    overflow-x: hidden;
   }
 
   .header.scrolling {
@@ -166,13 +223,15 @@
     align-items: center;
     padding-left: var(--spacing-l);
     transition: background 130ms ease-out;
-    flex: 0 0 48px;
+    flex: 0 0 60px;
     padding-right: var(--spacing-xl);
   }
+
   .controls {
     display: flex;
     gap: var(--spacing-xl);
   }
+
   .buttons {
     display: flex;
     justify-content: flex-end;
@@ -188,4 +247,33 @@
     pointer-events: none;
     color: var(--spectrum-global-color-gray-500) !important;
   }
+
+  .group {
+    border-radius: 4px;
+    display: flex;
+    flex-direction: row;
+  }
+  .group :global(> *:not(:first-child)) {
+    border-top-left-radius: 0;
+    border-bottom-left-radius: 0;
+    border-left: 2px solid var(--spectrum-global-color-gray-300);
+  }
+  .group :global(> *:not(:last-child)) {
+    border-top-right-radius: 0;
+    border-bottom-right-radius: 0;
+  }
+
+  .header-left .group :global(.spectrum-Button),
+  .header-left .group :global(.spectrum-ActionButton),
+  .header-left .group :global(.spectrum-Icon) {
+    color: var(--spectrum-global-color-gray-900) !important;
+  }
+  .header-left .group :global(.spectrum-Button),
+  .header-left .group :global(.spectrum-ActionButton) {
+    background: var(--spectrum-global-color-gray-200) !important;
+  }
+  .header-left .group :global(.spectrum-Button:hover),
+  .header-left .group :global(.spectrum-ActionButton:hover) {
+    background: var(--spectrum-global-color-gray-300) !important;
+  }
 </style>

packages/builder/src/components/automation/AutomationBuilder/FlowChart/FlowItem.svelte

@@ -1,8 +1,8 @@
 <script>
   import {
     automationStore,
-    selectedAutomation,
     permissions,
+    selectedAutomation,
     tables,
   } from "stores/builder"
   import {
@@ -11,54 +11,115 @@
     Layout,
     Detail,
     Modal,
-    Button,
-    notifications,
     Label,
     AbsTooltip,
-    InlineAlert,
   } from "@budibase/bbui"
   import { sdk } from "@budibase/shared-core"
   import AutomationBlockSetup from "../../SetupPanel/AutomationBlockSetup.svelte"
   import CreateWebhookModal from "components/automation/Shared/CreateWebhookModal.svelte"
-  import ActionModal from "./ActionModal.svelte"
   import FlowItemHeader from "./FlowItemHeader.svelte"
   import RoleSelect from "components/design/settings/controls/RoleSelect.svelte"
   import { ActionStepID, TriggerStepID } from "constants/backend/automations"
+  import { AutomationStepType } from "@budibase/types"
+  import FlowItemActions from "./FlowItemActions.svelte"
+  import DragHandle from "components/design/settings/controls/DraggableList/drag-handle.svelte"
+  import { getContext } from "svelte"
+  import DragZone from "./DragZone.svelte"
+  import InfoDisplay from "pages/builder/app/[application]/design/[screenId]/[componentId]/_components/Component/InfoDisplay.svelte"
 
   export let block
+  export let blockRef
   export let testDataModal
   export let idx
+  export let automation
+  export let bindings
+  export let draggable = true
+
+  const view = getContext("draggableView")
+  const pos = getContext("viewPos")
+  const contentPos = getContext("contentPos")
 
-  let selected
   let webhookModal
-  let actionModal
   let open = true
   let showLooping = false
   let role
+  let blockEle
+  let positionStyles
+  let blockDims
 
-  $: collectBlockExists = $selectedAutomation.definition.steps.some(
+  const updateBlockDims = () => {
+    if (!blockEle) {
+      return
+    }
+    const { width, height } = blockEle.getBoundingClientRect()
+    blockDims = { width: width / $view.scale, height: height / $view.scale }
+  }
+
+  const loadSteps = blockRef => {
+    return blockRef
+      ? automationStore.actions.getPathSteps(blockRef.pathTo, automation)
+      : []
+  }
+
+  $: pathSteps = loadSteps(blockRef)
+
+  $: collectBlockExists = pathSteps.some(
     step => step.stepId === ActionStepID.COLLECT
   )
-  $: automationId = $selectedAutomation?._id
-  $: isTrigger = block.type === "TRIGGER"
-  $: steps = $selectedAutomation?.definition?.steps ?? []
-  $: blockIdx = steps.findIndex(step => step.id === block.id)
-  $: lastStep = !isTrigger && blockIdx + 1 === steps.length
-  $: totalBlocks = $selectedAutomation?.definition?.steps.length + 1
-  $: loopBlock = $selectedAutomation?.definition.steps.find(
-    x => x.blockToLoop === block.id
-  )
+  $: automationId = automation?._id
+  $: isTrigger = block.type === AutomationStepType.TRIGGER
+  $: lastStep = blockRef?.terminating
+
+  $: loopBlock = pathSteps.find(x => x.blockToLoop === block.id)
   $: isAppAction = block?.stepId === TriggerStepID.APP
   $: isAppAction && setPermissions(role)
   $: isAppAction && getPermissions(automationId)
 
-  $: triggerInfo = sdk.automations.isRowAction($selectedAutomation) && {
+  $: triggerInfo = sdk.automations.isRowAction($selectedAutomation?.data) && {
     title: "Automation trigger",
     tableName: $tables.list.find(
-      x => x._id === $selectedAutomation.definition.trigger.inputs?.tableId
+      x =>
+        x._id === $selectedAutomation.data?.definition?.trigger?.inputs?.tableId
     )?.name,
   }
 
+  $: selected = $view?.moveStep && $view?.moveStep?.id === block.id
+
+  $: if (selected && blockEle) {
+    updateBlockDims()
+  }
+
+  $: placeholderDims = buildPlaceholderStyles(blockDims)
+
+  // Move the selected item
+  // Listen for scrolling in the content. As its scrolled this will be updated
+  $: move(
+    blockEle,
+    $view?.dragSpot,
+    selected,
+    $contentPos?.scrollX,
+    $contentPos?.scrollY
+  )
+
+  const move = (block, dragPos, selected, scrollX, scrollY) => {
+    if ((!block && !selected) || !dragPos) {
+      return
+    }
+    positionStyles = `
+      --blockPosX: ${Math.round(dragPos.x - scrollX / $view.scale)}px;
+      --blockPosY: ${Math.round(dragPos.y - scrollY / $view.scale)}px;
+    `
+  }
+
+  const buildPlaceholderStyles = dims => {
+    if (!dims) {
+      return ""
+    }
+    const { width, height } = dims
+    return `--pswidth: ${Math.round(width)}px;
+            --psheight: ${Math.round(height)}px;`
+  }
+
   async function setPermissions(role) {
     if (!role || !automationId) {
       return
@@ -82,23 +143,13 @@
     }
   }
 
-  async function removeLooping() {
-    try {
-      await automationStore.actions.deleteAutomationBlock(loopBlock)
-    } catch (error) {
-      notifications.error("Error saving automation")
-    }
+  async function deleteStep() {
+    await automationStore.actions.deleteAutomationBlock(blockRef.pathTo)
   }
 
-  async function deleteStep() {
-    try {
-      if (loopBlock) {
-        await automationStore.actions.deleteAutomationBlock(loopBlock)
-      }
-      await automationStore.actions.deleteAutomationBlock(block, blockIdx)
-    } catch (error) {
-      notifications.error("Error saving automation")
-    }
+  async function removeLooping() {
+    let loopBlockRef = $selectedAutomation.blockRefs[blockRef.looped]
+    await automationStore.actions.deleteAutomationBlock(loopBlockRef.pathTo)
   }
 
   async function addLooping() {
@@ -109,122 +160,201 @@
       loopDefinition
     )
     loopBlock.blockToLoop = block.id
-    await automationStore.actions.addBlockToAutomation(loopBlock, blockIdx)
+    await automationStore.actions.addBlockToAutomation(
+      loopBlock,
+      blockRef.pathTo
+    )
+  }
+
+  const onHandleMouseDown = e => {
+    if (isTrigger) {
+      e.preventDefault()
+      return
+    }
+
+    e.stopPropagation()
+
+    view.update(state => ({
+      ...state,
+      moveStep: {
+        id: block.id,
+        offsetX: $pos.x,
+        offsetY: $pos.y,
+      },
+    }))
   }
 </script>
 
-<!-- svelte-ignore a11y-click-events-have-key-events -->
-<!-- svelte-ignore a11y-no-static-element-interactions -->
-<div class={`block ${block.type} hoverable`} class:selected on:click={() => {}}>
-  {#if loopBlock}
-    <div class="blockSection">
+{#if block.stepId !== "LOOP"}
+  <!-- svelte-ignore a11y-click-events-have-key-events -->
+  <!-- svelte-ignore a11y-no-static-element-interactions -->
+  <div
+    id={`block-${block.id}`}
+    class={`block ${block.type} hoverable`}
+    class:selected
+    class:draggable
+  >
+    <div class="wrap">
+      {#if $view.dragging && selected}
+        <div class="drag-placeholder" style={placeholderDims} />
+      {/if}
+
       <div
-        on:click={() => {
-          showLooping = !showLooping
+        bind:this={blockEle}
+        class="block-content"
+        class:dragging={$view.dragging && selected}
+        style={positionStyles}
+        on:mousedown={e => {
+          e.stopPropagation()
         }}
-        class="splitHeader"
       >
-        <div class="center-items">
-          <svg
-            width="28px"
-            height="28px"
-            class="spectrum-Icon"
-            style="color:var(--spectrum-global-color-gray-700);"
-            focusable="false"
+        {#if draggable}
+          <div
+            class="handle"
+            class:grabbing={selected}
+            on:mousedown={onHandleMouseDown}
           >
-            <use xlink:href="#spectrum-icon-18-Reuse" />
-          </svg>
-          <div class="iconAlign">
-            <Detail size="S">Looping</Detail>
+            <DragHandle />
           </div>
-        </div>
+        {/if}
+        <div class="block-core">
+          {#if loopBlock}
+            <div class="blockSection">
+              <div
+                on:click={() => {
+                  showLooping = !showLooping
+                }}
+                class="splitHeader"
+              >
+                <div class="center-items">
+                  <svg
+                    width="28px"
+                    height="28px"
+                    class="spectrum-Icon"
+                    style="color:var(--spectrum-global-color-gray-700);"
+                    focusable="false"
+                  >
+                    <use xlink:href="#spectrum-icon-18-Reuse" />
+                  </svg>
+                  <div class="iconAlign">
+                    <Detail size="S">Looping</Detail>
+                  </div>
+                </div>
 
-        <div class="blockTitle">
-          <AbsTooltip type="negative" text="Remove looping">
-            <Icon on:click={removeLooping} hoverable name="DeleteOutline" />
-          </AbsTooltip>
+                <div class="blockTitle">
+                  <AbsTooltip type="negative" text="Remove looping">
+                    <Icon
+                      on:click={removeLooping}
+                      hoverable
+                      name="DeleteOutline"
+                    />
+                  </AbsTooltip>
 
-          <div style="margin-left: 10px;" on:click={() => {}}>
-            <Icon hoverable name={showLooping ? "ChevronDown" : "ChevronUp"} />
-          </div>
+                  <div style="margin-left: 10px;" on:click={() => {}}>
+                    <Icon
+                      hoverable
+                      name={showLooping ? "ChevronDown" : "ChevronUp"}
+                    />
+                  </div>
+                </div>
+              </div>
+            </div>
+
+            <Divider noMargin />
+            {#if !showLooping}
+              <div class="blockSection">
+                <Layout noPadding gap="S">
+                  <AutomationBlockSetup
+                    schemaProperties={Object.entries(
+                      $automationStore.blockDefinitions.ACTION.LOOP.schema
+                        .inputs.properties
+                    )}
+                    {webhookModal}
+                    block={loopBlock}
+                    {automation}
+                    {bindings}
+                  />
+                </Layout>
+              </div>
+              <Divider noMargin />
+            {/if}
+          {/if}
+
+          <FlowItemHeader
+            {automation}
+            {open}
+            {block}
+            {testDataModal}
+            {idx}
+            {addLooping}
+            {deleteStep}
+            on:toggle={() => (open = !open)}
+            on:update={async e => {
+              const newName = e.detail
+              if (newName.length === 0) {
+                await automationStore.actions.deleteAutomationName(block.id)
+              } else {
+                await automationStore.actions.saveAutomationName(
+                  block.id,
+                  newName
+                )
+              }
+            }}
+          />
+          {#if open}
+            <Divider noMargin />
+            <div class="blockSection">
+              <Layout noPadding gap="S">
+                {#if isAppAction}
+                  <div>
+                    <Label>Role</Label>
+                    <RoleSelect bind:value={role} />
+                  </div>
+                {/if}
+                <AutomationBlockSetup
+                  schemaProperties={Object.entries(
+                    block?.schema?.inputs?.properties || {}
+                  )}
+                  {block}
+                  {webhookModal}
+                  {automation}
+                  {bindings}
+                />
+                {#if isTrigger && triggerInfo}
+                  <InfoDisplay
+                    title={triggerInfo.title}
+                    body="This trigger is tied to your '{triggerInfo.tableName}' table"
+                    icon="InfoOutline"
+                  />
+                {/if}
+              </Layout>
+            </div>
+          {/if}
         </div>
       </div>
     </div>
-
-    <Divider noMargin />
-    {#if !showLooping}
-      <div class="blockSection">
-        <Layout noPadding gap="S">
-          <AutomationBlockSetup
-            schemaProperties={Object.entries(
-              $automationStore.blockDefinitions.ACTION.LOOP.schema.inputs
-                .properties
-            )}
-            {webhookModal}
-            block={loopBlock}
-          />
-        </Layout>
-      </div>
-      <Divider noMargin />
+  </div>
+  {#if !collectBlockExists || !lastStep}
+    <div class="separator" />
+    {#if $view.dragging}
+      <DragZone path={blockRef?.pathTo} />
+    {:else}
+      <FlowItemActions
+        {block}
+        on:branch={() => {
+          automationStore.actions.branchAutomation(
+            $selectedAutomation.blockRefs[block.id].pathTo,
+            automation
+          )
+        }}
+      />
+    {/if}
+    {#if !lastStep}
+      <div class="separator" />
     {/if}
   {/if}
-
-  <FlowItemHeader
-    {open}
-    {block}
-    {testDataModal}
-    {idx}
-    {addLooping}
-    {deleteStep}
-    on:toggle={() => (open = !open)}
-  />
-  {#if open}
-    <Divider noMargin />
-    <div class="blockSection">
-      <Layout noPadding gap="S">
-        {#if isAppAction}
-          <div>
-            <Label>Role</Label>
-            <RoleSelect bind:value={role} />
-          </div>
-        {/if}
-        <AutomationBlockSetup
-          schemaProperties={Object.entries(block.schema.inputs.properties)}
-          {block}
-          {webhookModal}
-        />
-        {#if triggerInfo}
-          <InlineAlert
-            header={triggerInfo.title}
-            message={`This trigger is tied to your "${triggerInfo.tableName}" table`}
-          />
-        {/if}
-        {#if lastStep}
-          <Button on:click={() => testDataModal.show()} cta>
-            Finish and test automation
-          </Button>
-        {/if}
-      </Layout>
-    </div>
-  {/if}
-</div>
-{#if !collectBlockExists || !lastStep}
-  <div class="separator" />
-  <Icon
-    on:click={() => actionModal.show()}
-    hoverable
-    name="AddCircle"
-    size="S"
-  />
-  {#if isTrigger ? totalBlocks > 1 : blockIdx !== totalBlocks - 2}
-    <div class="separator" />
-  {/if}
 {/if}
 
-<Modal bind:this={actionModal} width="30%">
-  <ActionModal modal={actionModal} {lastStep} {blockIdx} />
-</Modal>
-
 <Modal bind:this={webhookModal} width="30%">
   <CreateWebhookModal />
 </Modal>
@@ -255,27 +385,73 @@
   .block {
     width: 480px;
     font-size: 16px;
-    background-color: var(--background);
-    border: 1px solid var(--spectrum-global-color-gray-300);
-    border-radius: 4px 4px 4px 4px;
+    border-radius: 4px;
+  }
+  .block .wrap {
+    width: 100%;
+    position: relative;
+  }
+  .block.draggable .wrap {
+    display: flex;
+    flex-direction: row;
+  }
+  .block.draggable .wrap .handle {
+    height: auto;
+    display: flex;
+    justify-content: center;
+    align-items: center;
+    background-color: var(--grey-3);
+    padding: 6px;
+    color: var(--grey-6);
+    cursor: grab;
+  }
+  .block.draggable .wrap .handle.grabbing {
+    cursor: grabbing;
+  }
+  .block.draggable .wrap .handle :global(.drag-handle) {
+    width: 6px;
+  }
+  .block .wrap .block-content {
+    width: 100%;
+    display: flex;
+    flex-direction: row;
+    background-color: var(--background);
+    border: 1px solid var(--grey-3);
+    border-radius: 4px;
   }
-
   .blockSection {
     padding: var(--spacing-xl);
   }
-
   .separator {
     width: 1px;
     height: 25px;
     border-left: 1px dashed var(--grey-4);
     color: var(--grey-4);
-    /* center horizontally */
     align-self: center;
   }
-
   .blockTitle {
     display: flex;
     align-items: center;
     gap: var(--spacing-s);
   }
+  .drag-placeholder {
+    height: calc(var(--psheight) - 2px);
+    width: var(--pswidth);
+    background-color: rgba(92, 92, 92, 0.1);
+    border: 1px dashed #5c5c5c;
+    border-radius: 4px;
+    display: block;
+  }
+  .block-core {
+    flex: 1;
+  }
+  .block-core.dragging {
+    pointer-events: none;
+  }
+  .block-content.dragging {
+    position: absolute;
+    z-index: 3;
+    top: var(--blockPosY);
+    left: var(--blockPosX);
+  }
 </style>

packages/builder/src/components/automation/AutomationBuilder/FlowChart/FlowItemActions.svelte

@@ -0,0 +1,51 @@
+<script>
+  import { Icon, TooltipPosition, TooltipType, Modal } from "@budibase/bbui"
+  import { createEventDispatcher } from "svelte"
+  import ActionModal from "./ActionModal.svelte"
+
+  export let block
+
+  const dispatch = createEventDispatcher()
+  let actionModal
+</script>
+
+<Modal bind:this={actionModal} width="30%">
+  <ActionModal modal={actionModal} {block} />
+</Modal>
+
+<div class="action-bar">
+  {#if !block.branchNode}
+    <Icon
+      hoverable
+      name="Branch3"
+      on:click={() => {
+        dispatch("branch")
+      }}
+      tooltipType={TooltipType.Info}
+      tooltipPosition={TooltipPosition.Left}
+      tooltip={"Create branch"}
+      size={"S"}
+    />
+  {/if}
+  <Icon
+    hoverable
+    name="AddCircle"
+    on:click={() => {
+      actionModal.show()
+    }}
+    tooltipType={TooltipType.Info}
+    tooltipPosition={TooltipPosition.Right}
+    tooltip={"Add a step"}
+    size={"S"}
+  />
+</div>
+
+<style>
+  .action-bar {
+    background-color: var(--background);
+    border-radius: 4px 4px 4px 4px;
+    display: flex;
+    gap: var(--spacing-m);
+    padding: 8px 12px;
+  }
+</style>

packages/builder/src/components/automation/AutomationBuilder/FlowChart/FlowItemHeader.svelte

@@ -10,21 +10,25 @@
   export let showTestStatus = false
   export let testResult
   export let isTrigger
-  export let idx
   export let addLooping
   export let deleteStep
   export let enableNaming = true
+  export let itemName
+  export let automation
+
   let validRegex = /^[A-Za-z0-9_\s]+$/
   let typing = false
   let editing = false
   const dispatch = createEventDispatcher()
 
-  $: stepNames = $selectedAutomation?.definition.stepNames
-  $: allSteps = $selectedAutomation?.definition.steps || []
-  $: automationName = stepNames?.[block.id] || block?.name || ""
+  $: blockRefs = $selectedAutomation?.blockRefs || {}
+  $: stepNames = automation?.definition.stepNames
+  $: allSteps = automation?.definition.steps || []
+  $: automationName = itemName || stepNames?.[block.id] || block?.name || ""
   $: automationNameError = getAutomationNameError(automationName)
   $: status = updateStatus(testResult)
   $: isHeaderTrigger = isTrigger || block.type === "TRIGGER"
+  $: isBranch = block.stepId === "BRANCH"
 
   $: {
     if (!testResult) {
@@ -33,12 +37,12 @@
       )?.[0]
     }
   }
-  $: loopBlock = $selectedAutomation?.definition.steps.find(
-    x => x.blockToLoop === block?.id
-  )
+
+  $: blockRef = blockRefs[block.id]
+  $: isLooped = blockRef?.looped
 
   async function onSelect(block) {
-    await automationStore.update(state => {
+    automationStore.update(state => {
       state.selectedBlock = block
       return state
     })
@@ -84,30 +88,18 @@
     return null
   }
 
-  const saveName = async () => {
-    if (automationNameError || block.name === automationName) {
-      return
-    }
-
-    if (automationName.length === 0) {
-      await automationStore.actions.deleteAutomationName(block.id)
-    } else {
-      await automationStore.actions.saveAutomationName(block.id, automationName)
-    }
-  }
-
   const startEditing = () => {
     editing = true
     typing = true
   }
 
-  const stopEditing = async () => {
+  const stopEditing = () => {
     editing = false
     typing = false
     if (automationNameError) {
       automationName = stepNames[block.id] || block?.name
     } else {
-      await saveName()
+      dispatch("update", automationName)
     }
   }
 </script>
@@ -118,7 +110,6 @@
   class:typing={typing && !automationNameError && editing}
   class:typing-error={automationNameError && editing}
   class="blockSection"
-  on:click={() => dispatch("toggle")}
 >
   <div class="splitHeader">
     <div class="center-items">
@@ -144,16 +135,14 @@
         {#if isHeaderTrigger}
           <Body size="XS"><b>Trigger</b></Body>
         {:else}
-          <div style="margin-left: 2px;">
-            <Body size="XS"><b>Step {idx}</b></Body>
-          </div>
+          <Body size="XS"><b>{isBranch ? "Branch" : "Step"}</b></Body>
         {/if}
 
         {#if enableNaming}
           <input
             class="input-text"
             disabled={!enableNaming}
-            placeholder="Enter step name"
+            placeholder={`Enter ${isBranch ? "branch" : "step"} name`}
             name="name"
             autocomplete="off"
             value={automationName}
@@ -208,8 +197,9 @@
           onSelect(block)
         }}
       >
+        <slot name="custom-actions" />
         {#if !showTestStatus}
-          {#if !isHeaderTrigger && !loopBlock && (block?.features?.[Features.LOOPING] || !block.features)}
+          {#if !isHeaderTrigger && !isLooped && !isBranch && (block?.features?.[Features.LOOPING] || !block.features)}
             <AbsTooltip type="info" text="Add looping">
               <Icon on:click={addLooping} hoverable name="RotateCW" />
             </AbsTooltip>
@@ -220,6 +210,9 @@
             </AbsTooltip>
           {/if}
         {/if}
+        {#if !showTestStatus && !isHeaderTrigger}
+          <span class="action-spacer" />
+        {/if}
         {#if !showTestStatus}
           <Icon
             on:click={e => {
@@ -245,6 +238,9 @@
 </div>
 
 <style>
+  .action-spacer {
+    border-left: 1px solid var(--spectrum-global-color-gray-300);
+  }
   .status-container {
     display: flex;
     align-items: center;
@@ -298,6 +294,8 @@
     font-size: var(--spectrum-alias-font-size-default);
     font-family: var(--font-sans);
     text-overflow: ellipsis;
+    padding-left: 0px;
+    border: 0px;
   }
 
   input:focus {

packages/builder/src/components/automation/AutomationBuilder/FlowChart/StepNode.svelte

@@ -0,0 +1,227 @@
+<script>
+  import FlowItem from "./FlowItem.svelte"
+  import BranchNode from "./BranchNode.svelte"
+  import { AutomationActionStepId } from "@budibase/types"
+  import { ActionButton, notifications } from "@budibase/bbui"
+  import { automationStore } from "stores/builder"
+  import { environment } from "stores/portal"
+  import { cloneDeep } from "lodash"
+  import { memo } from "@budibase/frontend-core"
+  import { getContext, onMount } from "svelte"
+
+  export let step = {}
+  export let stepIdx
+  export let automation
+  export let blocks
+  export let isLast = false
+
+  const memoEnvVariables = memo($environment.variables)
+  const view = getContext("draggableView")
+
+  let stepEle
+
+  $: memoEnvVariables.set($environment.variables)
+  $: blockRef = blocks?.[step.id]
+  $: pathToCurrentNode = blockRef?.pathTo
+  $: isBranch = step.stepId === AutomationActionStepId.BRANCH
+  $: branches = step.inputs?.branches
+
+  // All bindings available to this point
+  $: availableBindings = automationStore.actions.getPathBindings(
+    step.id,
+    automation
+  )
+
+  // Fetch the env bindings
+  $: environmentBindings =
+    automationStore.actions.buildEnvironmentBindings($memoEnvVariables)
+
+  $: userBindings = automationStore.actions.buildUserBindings()
+  $: settingBindings = automationStore.actions.buildSettingBindings()
+
+  // Combine all bindings for the step
+  $: bindings = [
+    ...availableBindings,
+    ...environmentBindings,
+    ...userBindings,
+    ...settingBindings,
+  ]
+  onMount(() => {
+    // Register the trigger as the focus element for the automation
+    // Onload, the canvas will use the dimensions to center the step
+    if (stepEle && step.type === "TRIGGER" && !$view.focusEle) {
+      view.update(state => ({
+        ...state,
+        focusEle: stepEle.getBoundingClientRect(),
+      }))
+    }
+  })
+</script>
+
+{#if isBranch}
+  <div class="split-branch-btn">
+    <ActionButton
+      icon="AddCircle"
+      on:click={() => {
+        automationStore.actions.branchAutomation(pathToCurrentNode, automation)
+      }}
+    >
+      Add branch
+    </ActionButton>
+  </div>
+  <div class="branched">
+    {#each branches as branch, bIdx}
+      {@const leftMost = bIdx === 0}
+      {@const rightMost = branches?.length - 1 === bIdx}
+      <div class="branch-wrap">
+        <div
+          class="branch"
+          class:left={leftMost}
+          class:right={rightMost}
+          class:middle={!leftMost && !rightMost}
+        >
+          <div class="branch-node">
+            <BranchNode
+              {automation}
+              {step}
+              {bindings}
+              pathTo={pathToCurrentNode}
+              branchIdx={bIdx}
+              isLast={rightMost}
+              on:change={async e => {
+                const updatedBranch = { ...branch, ...e.detail }
+
+                if (!step?.inputs?.branches?.[bIdx]) {
+                  console.error(`Cannot load target branch: ${bIdx}`)
+                  return
+                }
+
+                let branchStepUpdate = cloneDeep(step)
+                branchStepUpdate.inputs.branches[bIdx] = updatedBranch
+
+                // Ensure valid base configuration for all branches
+                // Reinitialise empty branch conditions on update
+                branchStepUpdate.inputs.branches.forEach(
+                  (branch, i, branchArray) => {
+                    if (!Object.keys(branch.condition).length) {
+                      branchArray[i] = {
+                        ...branch,
+                        ...automationStore.actions.generateDefaultConditions(),
+                      }
+                    }
+                  }
+                )
+
+                const updated = automationStore.actions.updateStep(
+                  blockRef?.pathTo,
+                  automation,
+                  branchStepUpdate
+                )
+
+                try {
+                  await automationStore.actions.save(updated)
+                } catch (e) {
+                  notifications.error("Error saving branch update")
+                  console.error("Error saving automation branch", e)
+                }
+              }}
+            />
+          </div>
+
+          <!-- Branch steps -->
+          {#each step.inputs?.children[branch.id] || [] as bStep, sIdx}
+            <!-- Recursive StepNode -->
+            <svelte:self
+              step={bStep}
+              stepIdx={sIdx}
+              branchIdx={bIdx}
+              isLast={blockRef.terminating}
+              pathTo={pathToCurrentNode}
+              {automation}
+              {blocks}
+            />
+          {/each}
+        </div>
+      </div>
+    {/each}
+  </div>
+{:else}
+  <div class="block" bind:this={stepEle}>
+    <FlowItem
+      block={step}
+      idx={stepIdx}
+      {blockRef}
+      {isLast}
+      {automation}
+      {bindings}
+      draggable={step.type !== "TRIGGER"}
+    />
+  </div>
+{/if}
+
+<style>
+  .branch-wrap {
+    width: inherit;
+  }
+
+  .branch {
+    display: flex;
+    align-items: center;
+    flex-direction: column;
+    position: relative;
+    width: inherit;
+  }
+
+  .branched {
+    display: flex;
+    gap: 64px;
+  }
+
+  .branch::before {
+    height: 64px;
+    border-left: 1px dashed var(--grey-4);
+    border-top: 1px dashed var(--grey-4);
+    content: "";
+    color: var(--grey-4);
+    width: 50%;
+    position: absolute;
+    left: 50%;
+    top: -16px;
+  }
+
+  .branch.left::before {
+    color: var(--grey-4);
+    width: calc(50% + 62px);
+  }
+
+  .branch.middle::after {
+    height: 64px;
+    border-top: 1px dashed var(--grey-4);
+    content: "";
+    color: var(--grey-4);
+    width: calc(50% + 62px);
+    position: absolute;
+    left: 50%;
+    top: -16px;
+  }
+
+  .branch.right::before {
+    left: 0px;
+    border-right: 1px dashed var(--grey-4);
+    border-left: none;
+  }
+
+  .branch.middle::before {
+    left: 0px;
+    border-right: 1px dashed var(--grey-4);
+    border-left: none;
+  }
+
+  .branch-node {
+    margin-top: 48px;
+  }
+
+  .split-branch-btn {
+    z-index: 2;
+  }
+</style>

packages/builder/src/components/automation/AutomationBuilder/FlowChart/TestDataModal.svelte

@@ -1,4 +1,5 @@
 <script>
+  import { tick } from "svelte"
   import {
     ModalContent,
     TextArea,
@@ -8,7 +9,6 @@
   import { automationStore, selectedAutomation } from "stores/builder"
   import AutomationBlockSetup from "../../SetupPanel/AutomationBlockSetup.svelte"
   import { cloneDeep } from "lodash/fp"
-  import { memo } from "@budibase/frontend-core"
   import { AutomationEventType } from "@budibase/types"
 
   let failedParse = null
@@ -29,7 +29,7 @@
    * @todo Parse *all* data for each trigger type and relay adequate feedback
    */
   const parseTestData = testData => {
-    const autoTrigger = $selectedAutomation?.definition?.trigger
+    const autoTrigger = $selectedAutomation.data?.definition?.trigger
     const { tableId } = autoTrigger?.inputs || {}
 
     // Ensure the tableId matches the trigger table for row trigger automations
@@ -63,12 +63,11 @@
     return true
   }
 
-  const memoTestData = memo(parseTestData($selectedAutomation.testData))
-  $: memoTestData.set(parseTestData($selectedAutomation.testData))
+  $: testData = testData || parseTestData($selectedAutomation.data.testData)
 
   $: {
     // clone the trigger so we're not mutating the reference
-    trigger = cloneDeep($selectedAutomation.definition.trigger)
+    trigger = cloneDeep($selectedAutomation.data.definition.trigger)
 
     // get the outputs so we can define the fields
     let schema = Object.entries(trigger.schema?.outputs?.properties || {})
@@ -83,7 +82,7 @@
   $: isError =
     !isTriggerValid(trigger) ||
     !(trigger.schema.outputs.required || []).every(
-      required => $memoTestData?.[required] || required !== "row"
+      required => testData?.[required] || required !== "row"
     )
 
   function parseTestJSON(e) {
@@ -110,8 +109,10 @@
   }
 
   const testAutomation = async () => {
+    // Ensure testData reactiveness is processed
+    await tick()
     try {
-      await automationStore.actions.test($selectedAutomation, $memoTestData)
+      await automationStore.actions.test($selectedAutomation.data, testData)
       $automationStore.showTestPanel = true
     } catch (error) {
       notifications.error(error)
@@ -149,7 +150,7 @@
   {#if selectedValues}
     <div class="tab-content-padding">
       <AutomationBlockSetup
-        testData={$memoTestData}
+        bind:testData
         {schemaProperties}
         isTestModal
         block={trigger}
@@ -159,7 +160,7 @@
   {#if selectedJSON}
     <div class="text-area-container">
       <TextArea
-        value={JSON.stringify($selectedAutomation.testData, null, 2)}
+        value={JSON.stringify($selectedAutomation.data.testData, null, 2)}
         error={failedParse}
         on:change={e => parseTestJSON(e)}
       />

packages/builder/src/components/automation/AutomationBuilder/TestDisplay.svelte

@@ -3,10 +3,12 @@
   import FlowItemHeader from "./FlowChart/FlowItemHeader.svelte"
   import { ActionStepID } from "constants/backend/automations"
   import { JsonView } from "@zerodevx/svelte-json-view"
+  import { automationStore } from "stores/builder"
+  import { AutomationActionStepId } from "@budibase/types"
 
   export let automation
+  export let automationBlockRefs = {}
   export let testResults
-  export let width = "400px"
 
   let openBlocks = {}
   let blocks
@@ -28,21 +30,28 @@
     }
   }
 
-  $: filteredResults = prepTestResults(testResults)
+  const getBranchName = (step, id) => {
+    if (!step || !id) {
+      return
+    }
+    return step.inputs.branches.find(branch => branch.id === id)?.name
+  }
 
+  $: filteredResults = prepTestResults(testResults)
   $: {
     if (testResults.message) {
-      blocks = automation?.definition?.trigger
-        ? [automation.definition.trigger]
-        : []
+      const trigger = automation?.definition?.trigger
+      blocks = trigger ? [trigger] : []
     } else if (automation) {
-      blocks = []
-      if (automation.definition.trigger) {
-        blocks.push(automation.definition.trigger)
+      const terminatingStep = filteredResults.at(-1)
+      const terminatingBlockRef = automationBlockRefs[terminatingStep.id]
+      if (terminatingBlockRef) {
+        const pathSteps = automationStore.actions.getPathSteps(
+          terminatingBlockRef.pathTo,
+          automation
+        )
+        blocks = [...pathSteps].filter(x => x.stepId !== ActionStepID.LOOP)
       }
-      blocks = blocks
-        .concat(automation.definition.steps || [])
-        .filter(x => x.stepId !== ActionStepID.LOOP)
     } else if (filteredResults) {
       blocks = filteredResults || []
       // make sure there is an ID for each block being displayed
@@ -56,10 +65,14 @@
 
 <div class="container">
   {#each blocks as block, idx}
-    <div class="block" style={width ? `width: ${width}` : ""}>
+    <div class="block">
       {#if block.stepId !== ActionStepID.LOOP}
         <FlowItemHeader
+          {automation}
           enableNaming={false}
+          itemName={block.stepId === AutomationActionStepId.BRANCH
+            ? getBranchName(block, filteredResults?.[idx].outputs?.branchId)
+            : null}
           open={!!openBlocks[block.id]}
           on:toggle={() => (openBlocks[block.id] = !openBlocks[block.id])}
           isTrigger={idx === 0}
@@ -133,7 +146,10 @@
   .container {
     padding: 0 30px 30px 30px;
     height: 100%;
-    overflow: auto;
+    overflow-y: auto;
+    overflow-x: hidden;
+    display: flex;
+    flex-direction: column;
   }
 
   .wrap {
@@ -175,17 +191,17 @@
 
   .block {
     display: inline-block;
-    width: 400px;
-    height: auto;
+    height: fit-content;
     font-size: 16px;
     background-color: var(--background);
     border: 1px solid var(--spectrum-global-color-gray-300);
     border-radius: 4px 4px 4px 4px;
+    min-width: 425px;
   }
 
   .separator {
     width: 1px;
-    height: 40px;
+    flex: 0 0 40px;
     border-left: 1px dashed var(--grey-4);
     color: var(--grey-4);
     /* center horizontally */

packages/builder/src/components/automation/AutomationBuilder/TestPanel.svelte

@@ -1,7 +1,7 @@
 <script>
   import { Icon, Divider } from "@budibase/bbui"
   import TestDisplay from "./TestDisplay.svelte"
-  import { automationStore } from "stores/builder"
+  import { automationStore, selectedAutomation } from "stores/builder"
 
   export let automation
 </script>
@@ -9,9 +9,9 @@
 <div class="title">
   <div class="title-text">
     <Icon name="MultipleCheck" />
-    <div style="padding-left: var(--spacing-l); ">Test Details</div>
+    <div>Test Details</div>
   </div>
-  <div style="padding-right: var(--spacing-xl)">
+  <div>
     <Icon
       on:click={async () => {
         $automationStore.showTestPanel = false
@@ -24,7 +24,11 @@
 
 <Divider />
 
-<TestDisplay {automation} testResults={$automationStore.testResults} />
+<TestDisplay
+  {automation}
+  testResults={$automationStore.testResults}
+  automationBlockRefs={$selectedAutomation.blockRefs}
+/>
 
 <style>
   .title {
@@ -32,7 +36,8 @@
     flex-direction: row;
     align-items: center;
     gap: var(--spacing-xs);
-    padding-left: var(--spacing-xl);
+    padding: 0px 30px;
+    padding-top: var(--spacing-l);
     justify-content: space-between;
   }
 
@@ -40,7 +45,7 @@
     display: flex;
     flex-direction: row;
     align-items: center;
-    padding-top: var(--spacing-s);
+    gap: var(--spacing-l);
   }
 
   .title :global(h1) {

packages/builder/src/components/automation/AutomationPanel/AutomationNavItem.svelte

@@ -113,7 +113,7 @@
     ? "var(--spectrum-global-color-gray-600)"
     : "var(--spectrum-global-color-gray-900)"}
   text={automation.name}
-  selected={automation._id === $selectedAutomation?._id}
+  selected={automation._id === $selectedAutomation?.data?._id}
   hovering={automation._id === $contextMenuStore.id}
   on:click={() => automationStore.actions.select(automation._id)}
   selectedBy={$userSelectedResourceMap[automation._id]}

packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte

@@ -21,8 +21,8 @@
   } from "@budibase/bbui"
 
   import CreateWebhookModal from "components/automation/Shared/CreateWebhookModal.svelte"
-  import { automationStore, selectedAutomation, tables } from "stores/builder"
-  import { environment, licensing } from "stores/portal"
+  import { automationStore, tables } from "stores/builder"
+  import { environment } from "stores/portal"
   import WebhookDisplay from "../Shared/WebhookDisplay.svelte"
   import {
     BindingSidePanel,
@@ -46,10 +46,7 @@
   } from "components/common/CodeEditor"
   import FilterBuilder from "components/design/settings/controls/FilterEditor/FilterBuilder.svelte"
   import { QueryUtils, Utils, search, memo } from "@budibase/frontend-core"
-  import {
-    getSchemaForDatasourcePlus,
-    getEnvironmentBindings,
-  } from "dataBinding"
+  import { getSchemaForDatasourcePlus } from "dataBinding"
   import { TriggerStepID, ActionStepID } from "constants/backend/automations"
   import { onMount } from "svelte"
   import { writable } from "svelte/store"
@@ -60,18 +57,18 @@
     AutomationActionStepId,
     AutomationCustomIOType,
   } from "@budibase/types"
-  import { FIELDS } from "constants/backend"
   import PropField from "./PropField.svelte"
   import { utils } from "@budibase/shared-core"
 
+  export let automation
   export let block
   export let testData
   export let schemaProperties
   export let isTestModal = false
+  export let bindings = []
 
   // Stop unnecessary rendering
   const memoBlock = memo(block)
-  const memoEnvVariables = memo($environment.variables)
 
   const rowTriggers = [
     TriggerStepID.ROW_UPDATED,
@@ -94,7 +91,6 @@
   let insertAtPos, getCaretPosition
   let stepLayouts = {}
 
-  $: memoEnvVariables.set($environment.variables)
   $: memoBlock.set(block)
 
   $: filters = lookForFilters(schemaProperties)
@@ -107,13 +103,6 @@
   $: tempFilters = cloneDeep(filters)
   $: stepId = $memoBlock.stepId
 
-  $: automationBindings = getAvailableBindings(
-    $memoBlock,
-    $selectedAutomation?.definition
-  )
-  $: environmentBindings = buildEnvironmentBindings($memoEnvVariables)
-  $: bindings = [...automationBindings, ...environmentBindings]
-
   $: getInputData(testData, $memoBlock.inputs)
   $: tableId = inputData ? inputData.tableId : null
   $: table = tableId
@@ -146,21 +135,6 @@
       ? [hbAutocomplete([...bindingsToCompletions(bindings, codeMode)])]
       : []
 
-  const buildEnvironmentBindings = () => {
-    if ($licensing.environmentVariablesEnabled) {
-      return getEnvironmentBindings().map(binding => {
-        return {
-          ...binding,
-          display: {
-            ...binding.display,
-            rank: 98,
-          },
-        }
-      })
-    }
-    return []
-  }
-
   const getInputData = (testData, blockInputs) => {
     // Test data is not cloned for reactivity
     let newInputData = testData || cloneDeep(blockInputs)
@@ -437,7 +411,7 @@
 
         if (
           Object.hasOwn(update, "tableId") &&
-          $selectedAutomation.testData?.row?.tableId !== update.tableId
+          automation.testData?.row?.tableId !== update.tableId
         ) {
           const reqSchema = getSchemaForDatasourcePlus(update.tableId, {
             searchableSchema: true,
@@ -529,7 +503,15 @@
       row: { "Active": true, "Order Id" : 14, ... }
     })
    */
-  const onChange = Utils.sequential(async update => {
+  const onChange = async update => {
+    if (isTestModal) {
+      testData = update
+    }
+
+    updateAutomation(update)
+  }
+
+  const updateAutomation = Utils.sequential(async update => {
     const request = cloneDeep(update)
     // Process app trigger updates
     if (isTrigger && !isTestModal) {
@@ -566,7 +548,7 @@
             ...newTestData,
             body: {
               ...update,
-              ...$selectedAutomation.testData?.body,
+              ...automation.testData?.body,
             },
           }
         }
@@ -574,6 +556,7 @@
           ...newTestData,
           ...request,
         }
+
         await automationStore.actions.addTestDataToAutomation(newTestData)
       } else {
         const data = { schema, ...request }
@@ -585,201 +568,6 @@
     }
   })
 
-  function getAvailableBindings(block, automation) {
-    if (!block || !automation) {
-      return []
-    }
-
-    // Find previous steps to the selected one
-    let allSteps = [...automation.steps]
-
-    if (automation.trigger) {
-      allSteps = [automation.trigger, ...allSteps]
-    }
-    let blockIdx = allSteps.findIndex(step => step.id === block.id)
-
-    // Extract all outputs from all previous steps as available bindingsx§x
-    let bindings = []
-    let loopBlockCount = 0
-    const addBinding = (name, value, icon, idx, isLoopBlock, bindingName) => {
-      if (!name) return
-      const runtimeBinding = determineRuntimeBinding(
-        name,
-        idx,
-        isLoopBlock,
-        bindingName
-      )
-      const categoryName = determineCategoryName(idx, isLoopBlock, bindingName)
-      bindings.push(
-        createBindingObject(
-          name,
-          value,
-          icon,
-          idx,
-          loopBlockCount,
-          isLoopBlock,
-          runtimeBinding,
-          categoryName,
-          bindingName
-        )
-      )
-    }
-
-    const determineRuntimeBinding = (name, idx, isLoopBlock, bindingName) => {
-      let runtimeName
-
-      /* Begin special cases for generating custom schemas based on triggers */
-      if (
-        idx === 0 &&
-        automation.trigger?.event === AutomationEventType.APP_TRIGGER
-      ) {
-        return `trigger.fields.${name}`
-      }
-
-      if (
-        idx === 0 &&
-        (automation.trigger?.event === AutomationEventType.ROW_UPDATE ||
-          automation.trigger?.event === AutomationEventType.ROW_SAVE)
-      ) {
-        let noRowKeywordBindings = ["id", "revision", "oldRow"]
-        if (!noRowKeywordBindings.includes(name)) return `trigger.row.${name}`
-      }
-      /* End special cases for generating custom schemas based on triggers */
-
-      let hasUserDefinedName = automation.stepNames?.[allSteps[idx]?.id]
-      if (isLoopBlock) {
-        runtimeName = `loop.${name}`
-      } else if (idx === 0) {
-        runtimeName = `trigger.${name}`
-      } else if (block.name.startsWith("JS")) {
-        runtimeName = hasUserDefinedName
-          ? `stepsByName["${bindingName}"].${name}`
-          : `steps["${idx - loopBlockCount}"].${name}`
-      } else {
-        runtimeName = hasUserDefinedName
-          ? `stepsByName.${bindingName}.${name}`
-          : `steps.${idx - loopBlockCount}.${name}`
-      }
-      return runtimeName
-    }
-
-    const determineCategoryName = (idx, isLoopBlock, bindingName) => {
-      if (idx === 0) return "Trigger outputs"
-      if (isLoopBlock) return "Loop Outputs"
-      return bindingName
-        ? `${bindingName} outputs`
-        : `Step ${idx - loopBlockCount} outputs`
-    }
-
-    const createBindingObject = (
-      name,
-      value,
-      icon,
-      idx,
-      loopBlockCount,
-      isLoopBlock,
-      runtimeBinding,
-      categoryName,
-      bindingName
-    ) => {
-      const field = Object.values(FIELDS).find(
-        field => field.type === value.type && field.subtype === value.subtype
-      )
-      return {
-        readableBinding:
-          bindingName && !isLoopBlock && idx !== 0
-            ? `steps.${bindingName}.${name}`
-            : runtimeBinding,
-        runtimeBinding,
-        type: value.type,
-        description: value.description,
-        icon,
-        category: categoryName,
-        display: {
-          type: field?.name || value.type,
-          name,
-          rank: isLoopBlock ? idx + 1 : idx - loopBlockCount,
-        },
-      }
-    }
-
-    for (let idx = 0; idx < blockIdx; idx++) {
-      let wasLoopBlock = allSteps[idx - 1]?.stepId === ActionStepID.LOOP
-      let isLoopBlock =
-        allSteps[idx]?.stepId === ActionStepID.LOOP &&
-        allSteps.some(x => x.blockToLoop === block.id)
-      let schema = cloneDeep(allSteps[idx]?.schema?.outputs?.properties) ?? {}
-      if (allSteps[idx]?.name.includes("Looping")) {
-        isLoopBlock = true
-        loopBlockCount++
-      }
-      let bindingName =
-        automation.stepNames?.[allSteps[idx].id] || allSteps[idx].name
-
-      if (isLoopBlock) {
-        schema = {
-          currentItem: {
-            type: "string",
-            description: "the item currently being executed",
-          },
-        }
-      }
-
-      if (
-        idx === 0 &&
-        automation.trigger?.event === AutomationEventType.APP_TRIGGER
-      ) {
-        schema = Object.fromEntries(
-          Object.keys(automation.trigger.inputs.fields || []).map(key => [
-            key,
-            { type: automation.trigger.inputs.fields[key] },
-          ])
-        )
-      }
-      if (
-        (idx === 0 &&
-          automation.trigger.event === AutomationEventType.ROW_UPDATE) ||
-        (idx === 0 && automation.trigger.event === AutomationEventType.ROW_SAVE)
-      ) {
-        let table = $tables.list.find(
-          table => table._id === automation.trigger.inputs.tableId
-        )
-        // We want to generate our own schema for the bindings from the table schema itself
-        for (const key in table?.schema) {
-          schema[key] = {
-            type: table.schema[key].type,
-            subtype: table.schema[key].subtype,
-          }
-        }
-        // remove the original binding
-        delete schema.row
-      }
-      let icon =
-        idx === 0
-          ? automation.trigger.icon
-          : isLoopBlock
-          ? "Reuse"
-          : allSteps[idx].icon
-
-      if (wasLoopBlock) {
-        schema = cloneDeep(allSteps[idx - 1]?.schema?.outputs?.properties)
-      }
-      Object.entries(schema).forEach(([name, value]) => {
-        addBinding(name, value, icon, idx, isLoopBlock, bindingName)
-      })
-    }
-
-    if (
-      allSteps[blockIdx - 1]?.stepId !== ActionStepID.LOOP &&
-      allSteps
-        .slice(0, blockIdx)
-        .some(step => step.stepId === ActionStepID.LOOP)
-    ) {
-      bindings = bindings.filter(x => !x.readableBinding.includes("loop"))
-    }
-    return bindings
-  }
-
   function lookForFilters(properties) {
     if (!properties) {
       return []
@@ -865,7 +653,7 @@
   <!-- Custom Layouts -->
   {#if stepLayouts[block.stepId]}
     {#each Object.keys(stepLayouts[block.stepId] || {}) as key}
-      {#if canShowField(key, stepLayouts[block.stepId].schema)}
+      {#if canShowField(stepLayouts[block.stepId].schema)}
         {#each stepLayouts[block.stepId][key].content as config}
           {#if config.title}
             <PropField label={config.title} labelTooltip={config.tooltip}>
@@ -890,7 +678,7 @@
   {:else}
     <!-- Default Schema Property Layout -->
     {#each schemaProperties as [key, value]}
-      {#if canShowField(key, value)}
+      {#if canShowField(value)}
         {@const label = getFieldLabel(key, value)}
         <div class:block-field={shouldRenderField(value)}>
           {#if key !== "fields" && value.type !== "boolean" && shouldRenderField(value)}
@@ -912,8 +700,8 @@
               {/if}
             </div>
           {/if}
-          <div class:field-width={shouldRenderField(value)}>
-            {#if value.type === "string" && value.enum && canShowField(key, value)}
+          <div>
+            {#if value.type === "string" && value.enum && canShowField(value)}
               <Select
                 on:change={e => onChange({ [key]: e.detail })}
                 value={inputData[key]}
@@ -1208,8 +996,9 @@
     align-items: center;
     gap: var(--spacing-s);
   }
-  .field-width {
-    width: 320px;
+
+  .label-container :global(label) {
+    white-space: unset;
   }
 
   .step-fields {
@@ -1221,12 +1010,9 @@
   }
 
   .block-field {
-    display: flex;
     justify-content: space-between;
-    flex-direction: row;
-    align-items: center;
-    gap: 10px;
-    flex: 1;
+    display: grid;
+    grid-template-columns: 1fr 320px;
   }
 
   .attachment-field-width {

packages/builder/src/components/automation/SetupPanel/AutomationSelector.svelte

@@ -29,7 +29,7 @@
   $: filteredAutomations = $automationStore.automations.filter(
     automation =>
       automation.definition.trigger.stepId === TriggerStepID.APP &&
-      automation._id !== $selectedAutomation._id
+      automation._id !== $selectedAutomation.data._id
   )
 </script>
 

packages/builder/src/components/automation/Shared/CreateWebhookModal.svelte

@@ -11,7 +11,7 @@
   let schemaURL
   let propCount = 0
 
-  $: automation = $selectedAutomation
+  $: automation = $selectedAutomation?.data
 
   onMount(async () => {
     if (!automation?.definition?.trigger?.inputs.schemaUrl) {

packages/builder/src/components/backend/DataTable/buttons/grid/ColumnsSettingContent.svelte

@@ -11,7 +11,6 @@
   export let disabledPermissions = []
   export let columns
   export let fromRelationshipField
-  export let canSetRelationshipSchemas
 
   const { datasource, dispatch } = getContext("grid")
 
@@ -129,6 +128,8 @@
     }
   })
 
+  $: hasLinkColumns = columns.some(c => c.schema.type === FieldType.LINK)
+
   async function toggleColumn(column, permission) {
     const visible = permission !== FieldPermissions.HIDDEN
     const readonly = permission === FieldPermissions.READONLY
@@ -184,7 +185,7 @@
           value={columnToPermissionOptions(column)}
           options={column.options}
         />
-        {#if canSetRelationshipSchemas && column.schema.type === FieldType.LINK && columnToPermissionOptions(column) !== FieldPermissions.HIDDEN}
+        {#if column.schema.type === FieldType.LINK && columnToPermissionOptions(column) !== FieldPermissions.HIDDEN}
           <div class="relationship-columns">
             <ActionButton
               on:click={e => {
@@ -203,7 +204,7 @@
   </div>
 </div>
 
-{#if canSetRelationshipSchemas}
+{#if hasLinkColumns}
   <Popover
     on:close={() => (relationshipFieldName = null)}
     open={relationshipFieldName}

packages/builder/src/components/backend/DataTable/buttons/grid/GridColumnsSettingButton.svelte

@@ -10,8 +10,6 @@
   import { getContext } from "svelte"
   import { ActionButton } from "@budibase/bbui"
   import ColumnsSettingContent from "./ColumnsSettingContent.svelte"
-  import { isEnabled } from "helpers/featureFlags"
-  import { FeatureFlag } from "@budibase/types"
   import DetailPopover from "components/common/DetailPopover.svelte"
 
   const { tableColumns, datasource } = getContext("grid")
@@ -46,9 +44,5 @@
       {text}
     </ActionButton>
   </svelte:fragment>
-  <ColumnsSettingContent
-    columns={$tableColumns}
-    canSetRelationshipSchemas={isEnabled(FeatureFlag.ENRICHED_RELATIONSHIPS)}
-    {permissions}
-  />
+  <ColumnsSettingContent columns={$tableColumns} {permissions} />
 </DetailPopover>

packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte

@@ -371,6 +371,7 @@
     delete editableColumn.relationshipType
     delete editableColumn.formulaType
     delete editableColumn.constraints
+    delete editableColumn.responseType
 
     // Add in defaults and initial definition
     const definition = fieldDefinitions[type?.toUpperCase()]
@@ -386,6 +387,7 @@
       editableColumn.relationshipType = RelationshipType.MANY_TO_MANY
     } else if (editableColumn.type === FieldType.FORMULA) {
       editableColumn.formulaType = "dynamic"
+      editableColumn.responseType = field.responseType || FIELDS.STRING.type
     }
   }
 
@@ -767,6 +769,25 @@
         </div>
       </div>
     {/if}
+    <div class="split-label">
+      <div class="label-length">
+        <Label size="M">Response Type</Label>
+      </div>
+      <div class="input-length">
+        <Select
+          bind:value={editableColumn.responseType}
+          options={[
+            FIELDS.STRING,
+            FIELDS.NUMBER,
+            FIELDS.BOOLEAN,
+            FIELDS.DATETIME,
+          ]}
+          getOptionLabel={option => option.name}
+          getOptionValue={option => option.type}
+          tooltip="Formulas by default will return a string - however if you need another type the response can be coerced."
+        />
+      </div>
+    </div>
     <div class="split-label">
       <div class="label-length">
         <Label size="M">Formula</Label>

packages/builder/src/components/common/NavItem.svelte

@@ -84,8 +84,8 @@
   on:mouseleave
   on:click={onClick}
   on:contextmenu
-  ondragover="return false"
-  ondragenter="return false"
+  on:dragover={e => e.preventDefault()}
+  on:dragenter={e => e.preventDefault()}
   {id}
   {style}
   {draggable}

packages/builder/src/components/common/UndoRedoControl.svelte

@@ -1,9 +1,10 @@
 <script>
-  import { Icon } from "@budibase/bbui"
+  import { Icon, ActionButton } from "@budibase/bbui"
   import { onMount } from "svelte"
   import { isBuilderInputFocused } from "helpers"
 
   export let store
+  export let showButtonGroup = false
 
   const handleKeyPress = e => {
     if (!(e.ctrlKey || e.metaKey)) {
@@ -32,19 +33,36 @@
   })
 </script>
 
-<div class="undo-redo">
-  <Icon
-    name="Undo"
-    hoverable
-    on:click={store.undo}
-    disabled={!$store.canUndo}
-  />
-  <Icon
-    name="Redo"
-    hoverable
-    on:click={store.redo}
-    disabled={!$store.canRedo}
-  />
+<div class="undo-redo" class:buttons={showButtonGroup}>
+  {#if showButtonGroup}
+    <div class="group">
+      <ActionButton
+        icon="Undo"
+        quiet
+        on:click={store.undo}
+        disabled={!$store.canUndo}
+      />
+      <ActionButton
+        icon="Redo"
+        quiet
+        on:click={store.redo}
+        disabled={!$store.canRedo}
+      />
+    </div>
+  {:else}
+    <Icon
+      name="Undo"
+      hoverable
+      on:click={store.undo}
+      disabled={!$store.canUndo}
+    />
+    <Icon
+      name="Redo"
+      hoverable
+      on:click={store.redo}
+      disabled={!$store.canRedo}
+    />
+  {/if}
 </div>
 
 <style>
@@ -60,4 +78,36 @@
   .undo-redo :global(svg) {
     padding: 6px;
   }
+
+  .undo-redo.buttons :global(svg) {
+    padding: 0px;
+  }
+
+  .undo-redo.buttons {
+    padding-right: 0px;
+  }
+
+  .group {
+    border-radius: 4px;
+    display: flex;
+    flex-direction: row;
+  }
+  .group :global(> *:not(:first-child)) {
+    border-top-left-radius: 0;
+    border-bottom-left-radius: 0;
+    border-left: 2px solid var(--spectrum-global-color-gray-300);
+  }
+  .group :global(> *:not(:last-child)) {
+    border-top-right-radius: 0;
+    border-bottom-right-radius: 0;
+  }
+
+  .group :global(.spectrum-Button),
+  .group :global(.spectrum-ActionButton) {
+    background: var(--spectrum-global-color-gray-200) !important;
+  }
+  .group :global(.spectrum-Button:hover),
+  .group :global(.spectrum-ActionButton:hover) {
+    background: var(--spectrum-global-color-gray-300) !important;
+  }
 </style>

packages/builder/src/components/design/settings/controls/FilterEditor/FilterBuilder.svelte

@@ -12,8 +12,10 @@
   export let bindings = []
   export let panel = ClientBindingPanel
   export let allowBindings = true
+  export let allowOnEmpty
   export let datasource
-  export let showFilterEmptyDropdown
+  export let builderType
+  export let docsURL
 </script>
 
 <CoreFilterBuilder
@@ -26,7 +28,9 @@
   {schemaFields}
   {datasource}
   {allowBindings}
-  {showFilterEmptyDropdown}
   {bindings}
+  {allowOnEmpty}
+  {builderType}
+  {docsURL}
   on:change
 />

packages/builder/src/dataBinding.js

@@ -614,6 +614,40 @@ const getDeviceBindings = () => {
   return bindings
 }
 
+export const getSettingBindings = () => {
+  let bindings = []
+  const safeSetting = makePropSafe("settings")
+
+  bindings = [
+    {
+      type: "context",
+      runtimeBinding: `${safeSetting}.${makePropSafe("url")}`,
+      readableBinding: `Settings.url`,
+      category: "Settings",
+      icon: "Settings",
+      display: { type: "string", name: "url" },
+    },
+    {
+      type: "context",
+      runtimeBinding: `${safeSetting}.${makePropSafe("logo")}`,
+      readableBinding: `Settings.logo`,
+      category: "Settings",
+      icon: "Settings",
+      display: { type: "string", name: "logo" },
+    },
+    {
+      type: "context",
+      runtimeBinding: `${safeSetting}.${makePropSafe("company")}`,
+      readableBinding: `Settings.company`,
+      category: "Settings",
+      icon: "Settings",
+      display: { type: "string", name: "company" },
+    },
+  ]
+
+  return bindings
+}
+
 /**
  * Gets all selected rows bindings for tables in the current asset.
  * TODO: remove in future because we don't need a separate store for this
@@ -1469,3 +1503,31 @@ export const updateReferencesInObject = ({
     }
   }
 }
+
+// Migrate references
+// Switch all bindings to reference their ids
+export const migrateReferencesInObject = ({ obj, label = "steps", steps }) => {
+  const stepIndexRegex = new RegExp(`{{\\s*${label}\\.(\\d+)\\.`, "g")
+  const updateActionStep = (str, index, replaceWith) =>
+    str.replace(`{{ ${label}.${index}.`, `{{ ${label}.${replaceWith}.`)
+
+  for (const key in obj) {
+    if (typeof obj[key] === "string") {
+      let matches
+      while ((matches = stepIndexRegex.exec(obj[key])) !== null) {
+        const referencedStep = parseInt(matches[1])
+
+        obj[key] = updateActionStep(
+          obj[key],
+          referencedStep,
+          steps[referencedStep]?.id
+        )
+      }
+    } else if (typeof obj[key] === "object" && obj[key] !== null) {
+      migrateReferencesInObject({
+        obj: obj[key],
+        steps,
+      })
+    }
+  }
+}

packages/builder/src/pages/builder/app/[application]/automation/_layout.svelte

@@ -13,7 +13,7 @@
     selectedAutomation,
   } from "stores/builder"
 
-  $: automationId = $selectedAutomation?._id
+  $: automationId = $selectedAutomation?.data?._id
   $: builderStore.selectResource(automationId)
 
   // Keep URL and state in sync for selected screen ID
@@ -68,7 +68,7 @@
 
   {#if $automationStore.showTestPanel}
     <div class="setup">
-      <TestPanel automation={$selectedAutomation} />
+      <TestPanel automation={$selectedAutomation.data} />
     </div>
   {/if}
   <Modal bind:this={modal}>

packages/builder/src/pages/builder/app/[application]/design/[screenId]/_components/ComponentList/ComponentScrollWrapper.svelte

@@ -68,8 +68,8 @@
   on:scroll
   bind:this={scrollRef}
   on:drop={onDrop}
-  ondragover="return false"
-  ondragenter="return false"
+  on:dragover={e => e.preventDefault()}
+  on:dragenter={e => e.preventDefault()}
 >
   <slot />
 </div>

packages/builder/src/pages/builder/app/[application]/settings/automations/_components/HistoryDetailsPanel.svelte

@@ -56,7 +56,7 @@
     {/if}
     {#key history}
       <div class="history">
-        <TestDisplay testResults={history} width="320px" />
+        <TestDisplay testResults={history} />
       </div>
     {/key}
   </Layout>
@@ -65,6 +65,14 @@
 {/if}
 
 <style>
+  .history :global(.block) {
+    min-width: unset;
+  }
+  .history :global(> .container) {
+    max-width: 320px;
+    width: 320px;
+    padding: 0px;
+  }
   .controls {
     display: flex;
     gap: var(--spacing-s);
@@ -76,7 +84,4 @@
     width: 100%;
     justify-content: center;
   }
-  .history {
-    margin: 0 -30px;
-  }
 </style>

packages/builder/src/pages/builder/portal/settings/index.svelte

@@ -1,8 +1,8 @@
 <script>
   import { redirect } from "@roxi/routify"
-  import { licensing, featureFlags } from "stores/portal"
+  import { featureFlags } from "stores/portal"
 
-  if ($featureFlags.AI_CUSTOM_CONFIGS && $licensing.customAIConfigsEnabled) {
+  if ($featureFlags.AI_CUSTOM_CONFIGS) {
     $redirect("./ai")
   } else {
     $redirect("./auth")

packages/builder/src/pages/builder/portal/settings/version.svelte

@@ -4,12 +4,10 @@
     Layout,
     Heading,
     Body,
-    Button,
     Divider,
     notifications,
     Label,
-    Modal,
-    ModalContent,
+    Link,
   } from "@budibase/bbui"
   import { API } from "api"
   import { auth, admin } from "stores/portal"
@@ -21,8 +19,6 @@
   let githubVersion
   let githubPublishedDate
   let githubPublishedTime
-  let needsUpdate = true
-  let updateModal
 
   // Only admins allowed here
   $: {
@@ -31,21 +27,6 @@
     }
   }
 
-  async function updateBudibase() {
-    try {
-      notifications.info("Updating budibase..")
-      await fetch("/v1/update", {
-        headers: {
-          Authorization: "Bearer budibase",
-        },
-      })
-      notifications.success("Your budibase installation is up to date.")
-      getVersion()
-    } catch (err) {
-      notifications.error(`Error installing budibase update ${err}`)
-    }
-  }
-
   async function getVersion() {
     try {
       version = await API.getBudibaseVersion()
@@ -69,13 +50,6 @@
       githubPublishedDate = new Date(githubResponse.published_at)
       githubPublishedTime = githubPublishedDate.toLocaleTimeString()
       githubPublishedDate = githubPublishedDate.toLocaleDateString()
-
-      //Does Budibase need to be updated?
-      if (githubVersion === version) {
-        needsUpdate = false
-      } else {
-        needsUpdate = true
-      }
     } catch (error) {
       notifications.error("Error getting the latest Budibase version")
       githubVersion = null
@@ -115,23 +89,15 @@
         >
       </Layout>
       <Divider />
-      <div>
-        <Button cta on:click={updateModal.show} disabled={!needsUpdate}
-          >Update Budibase</Button
+      <Layout noPadding gap="XS">
+        <Heading>Updating Budibase</Heading>
+        <Body
+          >To update your self-host installation, follow the docs found <Link
+            size="L"
+            href="https://docs.budibase.com/docs/updating-budibase">here.</Link
+          ></Body
         >
-        <Modal bind:this={updateModal}>
-          <ModalContent
-            title="Update Budibase"
-            confirmText="Update"
-            onConfirm={updateBudibase}
-          >
-            <span
-              >Are you sure you want to update your budibase installation to the
-              latest version?</span
-            >
-          </ModalContent>
-        </Modal>
-      </div>
+      </Layout>
     {/if}
   </Layout>
 {/if}

packages/client/package.json

@@ -33,8 +33,7 @@
     "sanitize-html": "^2.13.0",
     "screenfull": "^6.0.1",
     "shortid": "^2.2.15",
-    "svelte-spa-router": "^4.0.1",
-    "atrament": "^4.3.0"
+    "svelte-spa-router": "^4.0.1"
   },
   "devDependencies": {
     "@rollup/plugin-alias": "^5.1.0",

packages/client/src/utils/blocks.js

@@ -1,5 +1,7 @@
 import { makePropSafe as safe } from "@budibase/string-templates"
 import { API } from "../api/index.js"
+import { UILogicalOperator } from "@budibase/types"
+import { OnEmptyFilter } from "@budibase/frontend-core/src/constants.js"
 
 // Map of data types to component types for search fields inside blocks
 const schemaComponentMap = {
@@ -60,7 +62,11 @@ export const enrichSearchColumns = async (searchColumns, schema) => {
  * @param formId the ID of the form containing the search fields
  */
 export const enrichFilter = (filter, columns, formId) => {
-  let enrichedFilter = [...(filter || [])]
+  if (!columns?.length) {
+    return filter
+  }
+
+  let newFilters = []
   columns?.forEach(column => {
     const safePath = column.name.split(".").map(safe).join(".")
     const stringType = column.type === "string" || column.type === "formula"
@@ -69,7 +75,7 @@ export const enrichFilter = (filter, columns, formId) => {
 
     // For dates, use a range of the entire day selected
     if (dateType) {
-      enrichedFilter.push({
+      newFilters.push({
         field: column.name,
         type: column.type,
         operator: "rangeLow",
@@ -79,7 +85,7 @@ export const enrichFilter = (filter, columns, formId) => {
       const format = "YYYY-MM-DDTHH:mm:ss.SSSZ"
       let hbs = `{{ date (add (date ${binding} "x") 86399999) "${format}" }}`
       hbs = `{{#if ${binding} }}${hbs}{{/if}}`
-      enrichedFilter.push({
+      newFilters.push({
         field: column.name,
         type: column.type,
         operator: "rangeHigh",
@@ -90,7 +96,7 @@ export const enrichFilter = (filter, columns, formId) => {
 
     // For other fields, do an exact match
     else {
-      enrichedFilter.push({
+      newFilters.push({
         field: column.name,
         type: column.type,
         operator: stringType ? "string" : "equal",
@@ -99,5 +105,16 @@ export const enrichFilter = (filter, columns, formId) => {
       })
     }
   })
-  return enrichedFilter
+
+  return {
+    logicalOperator: UILogicalOperator.ALL,
+    onEmptyFilter: OnEmptyFilter.RETURN_ALL,
+    groups: [
+      ...(filter?.groups || []),
+      {
+        logicalOperator: UILogicalOperator.ALL,
+        filters: newFilters,
+      },
+    ],
+  }
 }

packages/frontend-core/src/components/ConditionField.svelte

@@ -0,0 +1,177 @@
+<script>
+  import { Input, Icon, Drawer, Button } from "@budibase/bbui"
+  import { isJSBinding } from "@budibase/string-templates"
+  import { createEventDispatcher } from "svelte"
+
+  export let filter
+  export let disabled = false
+  export let bindings = []
+  export let panel
+  export let drawerTitle
+  export let toReadable
+  export let toRuntime
+
+  const dispatch = createEventDispatcher()
+
+  let bindingDrawer
+  let fieldValue
+
+  $: fieldValue = filter?.field
+  $: readableValue = toReadable ? toReadable(bindings, fieldValue) : fieldValue
+  $: drawerValue = toDrawerValue(fieldValue)
+  $: isJS = isJSBinding(fieldValue)
+
+  const drawerOnChange = e => {
+    drawerValue = e.detail
+  }
+
+  const onChange = e => {
+    fieldValue = e.detail
+    dispatch("change", {
+      field: toRuntime ? toRuntime(bindings, fieldValue) : fieldValue,
+    })
+  }
+
+  const onConfirmBinding = () => {
+    dispatch("change", {
+      field: toRuntime ? toRuntime(bindings, drawerValue) : drawerValue,
+    })
+  }
+
+  /**
+   * Converts arrays into strings. The CodeEditor expects a string or encoded JS
+   *
+   * @param{string} fieldValue
+   */
+  const toDrawerValue = fieldValue => {
+    return Array.isArray(fieldValue) ? fieldValue.join(",") : readableValue
+  }
+</script>
+
+<div>
+  <Drawer
+    on:drawerHide
+    on:drawerShow
+    bind:this={bindingDrawer}
+    title={drawerTitle || ""}
+    forceModal
+  >
+    <Button
+      cta
+      slot="buttons"
+      on:click={() => {
+        onConfirmBinding()
+        bindingDrawer.hide()
+      }}
+    >
+      Confirm
+    </Button>
+
+    <svelte:component
+      this={panel}
+      slot="body"
+      value={drawerValue}
+      allowJS
+      allowHelpers
+      allowHBS
+      on:change={drawerOnChange}
+      {bindings}
+    />
+  </Drawer>
+
+  <div class="field-wrap" class:bindings={true}>
+    <div class="field">
+      <Input
+        disabled={filter.noValue}
+        readonly={isJS}
+        value={isJS ? "(JavaScript function)" : readableValue}
+        on:change={onChange}
+      />
+    </div>
+
+    <div class="binding-control">
+      {#if !disabled}
+        <!-- svelte-ignore a11y-click-events-have-key-events -->
+        <!-- svelte-ignore a11y-no-static-element-interactions -->
+        <div
+          class="icon binding"
+          on:click={() => {
+            bindingDrawer.show()
+          }}
+        >
+          <Icon size="S" name="FlashOn" />
+        </div>
+      {/if}
+    </div>
+  </div>
+</div>
+
+<style>
+  .field-wrap {
+    display: flex;
+  }
+  .field {
+    flex: 1;
+  }
+
+  .field-wrap.bindings .field :global(.spectrum-Form-itemField),
+  .field-wrap.bindings .field :global(input),
+  .field-wrap.bindings .field :global(.spectrum-Picker) {
+    border-top-right-radius: 0px;
+    border-bottom-right-radius: 0px;
+  }
+
+  .field-wrap.bindings
+    .field
+    :global(.spectrum-InputGroup.spectrum-Datepicker) {
+    min-width: unset;
+    border-radius: 0px;
+  }
+
+  .field-wrap.bindings
+    .field
+    :global(
+      .spectrum-InputGroup.spectrum-Datepicker
+        .spectrum-Textfield-input.spectrum-InputGroup-input
+    ) {
+    width: 100%;
+  }
+
+  .binding-control .icon {
+    border: 1px solid
+      var(
+        --spectrum-textfield-m-border-color,
+        var(--spectrum-alias-border-color)
+      );
+    border-left: 0px;
+    border-top-right-radius: 4px;
+    border-bottom-right-radius: 4px;
+    justify-content: center;
+    align-items: center;
+    display: flex;
+    flex-direction: row;
+    box-sizing: border-box;
+    width: 31px;
+    color: var(--spectrum-alias-text-color);
+    background-color: var(--spectrum-global-color-gray-75);
+    transition: background-color
+        var(--spectrum-global-animation-duration-100, 130ms),
+      box-shadow var(--spectrum-global-animation-duration-100, 130ms),
+      border-color var(--spectrum-global-animation-duration-100, 130ms);
+    height: calc(var(--spectrum-alias-item-height-m));
+  }
+  .binding-control .icon.binding {
+    color: var(--yellow);
+  }
+
+  .binding-control .icon:hover {
+    cursor: pointer;
+    background-color: var(--spectrum-global-color-gray-50);
+    border-color: var(--spectrum-alias-border-color-hover);
+    color: var(--spectrum-alias-text-color-hover);
+  }
+
+  .binding-control .icon.binding:hover {
+    color: var(--yellow);
+  }
+</style>

packages/frontend-core/src/components/CoreFilterBuilder.svelte

@@ -16,6 +16,7 @@
   import { QueryUtils, Constants } from "@budibase/frontend-core"
   import { getContext, createEventDispatcher } from "svelte"
   import FilterField from "./FilterField.svelte"
+  import ConditionField from "./ConditionField.svelte"
   import { utils } from "@budibase/shared-core"
 
   const dispatch = createEventDispatcher()
@@ -33,8 +34,10 @@
   export let datasource
   export let behaviourFilters = false
   export let allowBindings = false
+  export let allowOnEmpty = true
+  export let builderType = "filter"
+  export let docsURL = "https://docs.budibase.com/docs/searchfilter-data"
 
-  // Review
   export let bindings
   export let panel
   export let toReadable
@@ -53,6 +56,10 @@
       schemaFields = [...schemaFields, { name: "_id", type: "string" }]
     }
   }
+  $: prefix =
+    builderType === "filter"
+      ? "Show data which matches"
+      : "Run branch when matching"
 
   // We still may need to migrate this even though the backend does it automatically now
   // for query definitions. This is because we might be editing saved filter definitions
@@ -103,6 +110,10 @@
   }
 
   const getValidOperatorsForType = filter => {
+    if (builderType === "condition") {
+      return [OperatorOptions.Equals, OperatorOptions.NotEquals]
+    }
+
     if (!filter?.field && !filter?.name) {
       return []
     }
@@ -222,6 +233,9 @@
       } else if (addFilter) {
         targetGroup.filters.push({
           valueType: FilterValueType.VALUE,
+          ...(builderType === "condition"
+            ? { operator: OperatorOptions.Equals.value, type: FieldType.STRING }
+            : {}),
         })
       } else if (group) {
         editable.groups[groupIdx] = {
@@ -242,6 +256,11 @@
         filters: [
           {
             valueType: FilterValueType.VALUE,
+            ...(builderType === "condition"
+              ? {
+                  operator: OperatorOptions.Equals.value,
+                }
+              : {}),
           },
         ],
       })
@@ -271,7 +290,7 @@
 
       {#if editableFilters?.groups?.length}
         <div class="global-filter-header">
-          <span>Show data which matches</span>
+          <span>{prefix}</span>
           <span class="operator-picker">
             <Select
               value={editableFilters?.logicalOperator}
@@ -286,7 +305,7 @@
               placeholder={false}
             />
           </span>
-          <span>of the following filter groups:</span>
+          <span>of the following {builderType} groups:</span>
         </div>
       {/if}
       {#if editableFilters?.groups?.length}
@@ -315,7 +334,7 @@
                       placeholder={false}
                     />
                   </span>
-                  <span>of the following filters are matched:</span>
+                  <span>of the following {builderType}s are matched:</span>
                 </div>
                 <div class="group-actions">
                   <Icon
@@ -346,20 +365,38 @@
               <div class="filters">
                 {#each group.filters as filter, filterIdx}
                   <div class="filter">
-                    <Select
-                      value={filter.field}
-                      options={fieldOptions}
-                      on:change={e => {
-                        const updated = { ...filter, field: e.detail }
-                        onFieldChange(updated)
-                        onFilterFieldUpdate(updated, groupIdx, filterIdx)
-                      }}
-                      placeholder="Column"
-                    />
-
+                    {#if builderType === "filter"}
+                      <Select
+                        value={filter.field}
+                        options={fieldOptions}
+                        on:change={e => {
+                          const updated = { ...filter, field: e.detail }
+                          onFieldChange(updated)
+                          onFilterFieldUpdate(updated, groupIdx, filterIdx)
+                        }}
+                        placeholder="Column"
+                      />
+                    {:else}
+                      <ConditionField
+                        placeholder="Value"
+                        {filter}
+                        drawerTitle={"Edit Binding"}
+                        {bindings}
+                        {panel}
+                        {toReadable}
+                        {toRuntime}
+                        on:change={e => {
+                          const updated = {
+                            ...filter,
+                            field: e.detail.field,
+                          }
+                          onFilterFieldUpdate(updated, groupIdx, filterIdx)
+                        }}
+                      />
+                    {/if}
                     <Select
                       value={filter.operator}
-                      disabled={!filter.field}
+                      disabled={!filter.field && builderType === "filter"}
                       options={getValidOperatorsForType(filter)}
                       on:change={e => {
                         const updated = { ...filter, operator: e.detail }
@@ -368,11 +405,18 @@
                       }}
                       placeholder={false}
                     />
-
                     <FilterField
                       placeholder="Value"
+                      drawerTitle={builderType === "condition"
+                        ? "Edit binding"
+                        : null}
                       {allowBindings}
-                      {filter}
+                      filter={{
+                        ...filter,
+                        ...(builderType === "condition"
+                          ? { type: FieldType.STRING }
+                          : {}),
+                      }}
                       {schemaFields}
                       {bindings}
                       {panel}
@@ -408,7 +452,7 @@
 
       <div class="filters-footer">
         <Layout noPadding>
-          {#if behaviourFilters && editableFilters?.groups?.length}
+          {#if behaviourFilters && allowOnEmpty && editableFilters?.groups?.length}
             <div class="empty-filter">
               <span>Return</span>
               <span class="empty-filter-picker">
@@ -425,7 +469,7 @@
                   placeholder={false}
                 />
               </span>
-              <span>when all filters are empty</span>
+              <span>when all {builderType}s are empty</span>
             </div>
           {/if}
           <div class="add-group">
@@ -439,17 +483,16 @@
                 })
               }}
             >
-              Add filter group
+              Add {builderType} group
             </Button>
-            <a
-              href="https://docs.budibase.com/docs/searchfilter-data"
-              target="_blank"
-            >
-              <Icon
-                name="HelpOutline"
-                color="var(--spectrum-global-color-gray-600)"
-              />
-            </a>
+            {#if docsURL}
+              <a href={docsURL} target="_blank">
+                <Icon
+                  name="HelpOutline"
+                  color="var(--spectrum-global-color-gray-600)"
+                />
+              </a>
+            {/if}
           </div>
         </Layout>
       </div>

packages/frontend-core/src/components/FilterField.svelte

@@ -21,6 +21,7 @@
   export let allowBindings = false
   export let schemaFields
   export let panel
+  export let drawerTitle
   export let toReadable
   export let toRuntime
 
@@ -28,7 +29,6 @@
   const { OperatorOptions, FilterValueType } = Constants
 
   let bindingDrawer
-  let fieldValue
 
   $: fieldValue = filter?.value
   $: readableValue = toReadable ? toReadable(bindings, fieldValue) : fieldValue
@@ -133,7 +133,7 @@
     on:drawerHide
     on:drawerShow
     bind:this={bindingDrawer}
-    title={filter.field}
+    title={drawerTitle || filter.field}
     forceModal
   >
     <Button
@@ -168,7 +168,7 @@
       {#if filter.valueType === FilterValueType.BINDING}
         <Input
           disabled={filter.noValue}
-          readonly={true}
+          readonly={isJS}
           value={isJS ? "(JavaScript function)" : readableValue}
           on:change={onChange}
         />
@@ -231,7 +231,7 @@
 
     <div class="binding-control">
       <!-- needs field, operator -->
-      {#if !disabled && allowBindings && filter.field && !filter.noValue}
+      {#if !disabled && allowBindings && !filter.noValue}
         <!-- svelte-ignore a11y-click-events-have-key-events -->
         <!-- svelte-ignore a11y-no-static-element-interactions -->
         <div

packages/frontend-core/src/components/grid/cells/FormulaCell.svelte

@@ -1,5 +1,21 @@
 <script>
   import TextCell from "./TextCell.svelte"
+  import DateCell from "./DateCell.svelte"
+  import NumberCell from "./NumberCell.svelte"
+  import BooleanCell from "./BooleanCell.svelte"
+  import { FieldType } from "@budibase/types"
+
+  export let schema
+
+  $: responseType = schema.responseType
 </script>
 
-<TextCell {...$$props} readonly />
+{#if responseType === FieldType.NUMBER}
+  <NumberCell {...$$props} readonly />
+{:else if responseType === FieldType.BOOLEAN}
+  <BooleanCell {...$$props} readonly />
+{:else if responseType === FieldType.DATETIME}
+  <DateCell {...$$props} readonly />
+{:else}
+  <TextCell {...$$props} readonly />
+{/if}

packages/frontend-core/src/components/grid/layout/NewColumnButton.svelte

@@ -53,6 +53,7 @@
     on:close={close}
     maxHeight={null}
     resizable
+    minWidth={360}
   >
     <div class="content">
       <slot />
@@ -80,7 +81,6 @@
   }
 
   .content {
-    width: 300px;
     padding: 20px;
     display: flex;
     flex-direction: column;

packages/frontend-core/src/fetch/NestedProviderFetch.js

@@ -5,6 +5,7 @@ export default class NestedProviderFetch extends DataFetch {
     // Nested providers should already have exposed their own schema
     return {
       schema: datasource?.value?.schema,
+      primaryDisplay: datasource?.value?.primaryDisplay,
     }
   }
 

packages/pro

@@ -1 +1 @@
-Subproject commit 2ab8536b6005576684810d774f1ac22239218546
+Subproject commit 25dd40ee12b048307b558ebcedb36548d6e042cd

packages/server/package.json

@@ -13,6 +13,7 @@
     "build": "node ./scripts/build.js",
     "postbuild": "copyfiles -f ../client/dist/budibase-client.js ../client/manifest.json client && copyfiles -f ../../yarn.lock ./dist/",
     "check:types": "tsc -p tsconfig.json --noEmit --paths null --target es2020",
+    "check:dependencies": "node ../../scripts/depcheck.js",
     "build:isolated-vm-lib:snippets": "esbuild --minify --bundle src/jsRunner/bundles/snippets.ts --outfile=src/jsRunner/bundles/snippets.ivm.bundle.js --platform=node --format=iife --global-name=snippets",
     "build:isolated-vm-lib:string-templates": "esbuild --minify --bundle src/jsRunner/bundles/index-helpers.ts --outfile=src/jsRunner/bundles/index-helpers.ivm.bundle.js --platform=node --format=iife --external:handlebars --global-name=helpers",
     "build:isolated-vm-lib:bson": "esbuild --minify --bundle src/jsRunner/bundles/bsonPackage.ts --outfile=src/jsRunner/bundles/bson.ivm.bundle.js --platform=node --format=iife --global-name=bson",
@@ -49,9 +50,11 @@
   "license": "GPL-3.0",
   "dependencies": {
     "@apidevtools/swagger-parser": "10.0.3",
+    "@azure/msal-node": "^2.5.1",
     "@budibase/backend-core": "0.0.0",
     "@budibase/client": "0.0.0",
     "@budibase/frontend-core": "0.0.0",
+    "@budibase/nano": "10.1.5",
     "@budibase/pro": "0.0.0",
     "@budibase/shared-core": "0.0.0",
     "@budibase/string-templates": "0.0.0",
@@ -60,15 +63,17 @@
     "@bull-board/koa": "5.10.2",
     "@elastic/elasticsearch": "7.10.0",
     "@google-cloud/firestore": "7.8.0",
-    "@koa/router": "8.0.8",
+    "@koa/cors": "5.0.0",
+    "@koa/router": "13.1.0",
     "@socket.io/redis-adapter": "^8.2.1",
     "@types/xml2js": "^0.4.14",
     "airtable": "0.12.2",
     "arangojs": "7.2.0",
     "archiver": "7.0.1",
-    "aws-sdk": "2.1030.0",
+    "aws-sdk": "2.1692.0",
     "bcrypt": "5.1.0",
     "bcryptjs": "2.4.3",
+    "bson": "^6.9.0",
     "buffer": "6.0.3",
     "bull": "4.10.1",
     "chokidar": "3.5.3",
@@ -76,17 +81,20 @@
     "cookies": "0.8.0",
     "csvtojson": "2.0.10",
     "curlconverter": "3.21.0",
-    "dd-trace": "5.2.0",
+    "dayjs": "^1.10.8",
+    "dd-trace": "5.26.0",
     "dotenv": "8.2.0",
     "form-data": "4.0.0",
     "global-agent": "3.0.0",
+    "google-auth-library": "^8.0.1",
     "google-spreadsheet": "npm:@budibase/google-spreadsheet@4.1.5",
     "ioredis": "5.3.2",
     "isolated-vm": "^4.7.2",
-    "jimp": "0.22.12",
+    "jimp": "1.1.4",
     "joi": "17.6.0",
     "js-yaml": "4.1.0",
     "jsonschema": "1.4.0",
+    "jsonwebtoken": "9.0.2",
     "knex": "2.4.2",
     "koa": "2.13.4",
     "koa-body": "4.2.0",
@@ -97,7 +105,7 @@
     "lodash": "4.17.21",
     "memorystream": "0.3.1",
     "mongodb": "6.7.0",
-    "mssql": "10.0.1",
+    "mssql": "11.0.1",
     "mysql2": "3.9.8",
     "node-fetch": "2.6.7",
     "object-sizeof": "2.6.1",
@@ -105,24 +113,28 @@
     "openapi-types": "9.3.1",
     "oracledb": "6.5.1",
     "pg": "8.10.0",
-    "pouchdb": "7.3.0",
+    "pouchdb": "9.0.0",
     "pouchdb-all-dbs": "1.1.1",
-    "pouchdb-find": "7.2.2",
+    "pouchdb-find": "9.0.0",
     "redis": "4",
+    "semver": "^7.5.4",
     "serialize-error": "^7.0.1",
     "server-destroy": "1.0.1",
-    "snowflake-promise": "^4.5.0",
-    "socket.io": "4.7.5",
+    "snowflake-sdk": "^1.15.0",
+    "socket.io": "4.8.1",
+    "svelte": "^4.2.10",
     "tar": "6.2.1",
     "tmp": "0.2.3",
     "to-json-schema": "0.2.5",
     "uuid": "^8.3.2",
     "validate.js": "0.13.1",
     "worker-farm": "1.7.0",
-    "xml2js": "0.5.0"
+    "xml2js": "0.6.2"
   },
   "devDependencies": {
+    "@babel/core": "^7.22.5",
     "@babel/preset-env": "7.16.11",
+    "@jest/types": "^29.6.3",
     "@swc/core": "1.3.71",
     "@swc/jest": "0.2.27",
     "@types/archiver": "6.0.2",
@@ -130,19 +142,24 @@
     "@types/jest": "29.5.5",
     "@types/koa": "2.13.4",
     "@types/koa-send": "^4.1.6",
-    "@types/koa__router": "8.0.8",
+    "@types/koa__cors": "5.0.0",
+    "@types/koa__router": "12.0.4",
     "@types/lodash": "4.14.200",
-    "@types/mssql": "9.1.4",
+    "@types/mssql": "9.1.5",
+    "@types/node": "^22.9.0",
     "@types/node-fetch": "2.6.4",
     "@types/oracledb": "6.5.1",
     "@types/pg": "8.6.6",
+    "@types/pouchdb": "6.4.2",
     "@types/server-destroy": "1.0.1",
     "@types/supertest": "2.0.14",
     "@types/tar": "6.1.5",
     "@types/tmp": "0.2.6",
     "@types/uuid": "8.3.4",
+    "chance": "^1.1.12",
     "copyfiles": "2.4.1",
     "docker-compose": "0.23.17",
+    "ioredis-mock": "8.9.0",
     "jest": "29.7.0",
     "jest-extended": "^4.0.2",
     "jest-openapi": "0.14.2",

packages/server/scripts/test.sh

@@ -1,12 +1,12 @@
 #!/bin/bash
-set -e
+set -ex
 
 if [[ -n $CI ]]
 then
   export NODE_OPTIONS="--max-old-space-size=4096 --no-node-snapshot $NODE_OPTIONS"
-  jest --coverage --maxWorkers=4 --forceExit --workerIdleMemoryLimit=2000MB --bail $@
+  jest --coverage --maxWorkers=4 --forceExit --workerIdleMemoryLimit=2000MB --bail "$@"
 else
   # --maxWorkers performs better in development
   export NODE_OPTIONS="--no-node-snapshot $NODE_OPTIONS"
-  jest --coverage --maxWorkers=2 --forceExit $@
+  jest --coverage --maxWorkers=2 --forceExit "$@"
 fi

packages/server/specs/openapi.yaml

@@ -23,6 +23,13 @@ components:
       description: The ID of the table which this request is targeting.
       schema:
         type: string
+    viewId:
+      in: path
+      name: viewId
+      required: true
+      description: The ID of the view which this request is targeting.
+      schema:
+        type: string
     rowId:
       in: path
       name: rowId
@@ -36,7 +43,7 @@ components:
       required: true
       description: The ID of the app which this request is targeting.
       schema:
-        default: "{{ appId }}"
+        default: "{{appId}}"
         type: string
     appIdUrl:
       in: path
@@ -44,7 +51,7 @@ components:
       required: true
       description: The ID of the app which this request is targeting.
       schema:
-        default: "{{ appId }}"
+        default: "{{appId}}"
         type: string
     queryId:
       in: path
@@ -442,6 +449,74 @@ components:
         # TYPE budibase_quota_limit_automations gauge
 
         budibase_quota_limit_automations 9007199254740991
+    view:
+      value:
+        data:
+          name: peopleView
+          tableId: ta_896a325f7e8147d2a2cda93c5d236511
+          schema:
+            name:
+              visible: true
+              readonly: false
+              order: 1
+              width: 300
+            age:
+              visible: true
+              readonly: true
+              order: 2
+              width: 200
+            salary:
+              visible: false
+              readonly: false
+          query:
+            logicalOperator: all
+            onEmptyFilter: none
+            groups:
+              - logicalOperator: any
+                filters:
+                  - operator: string
+                    field: name
+                    value: John
+                  - operator: range
+                    field: age
+                    value:
+                      low: 18
+                      high: 100
+          primaryDisplay: name
+    views:
+      value:
+        data:
+          - name: peopleView
+            tableId: ta_896a325f7e8147d2a2cda93c5d236511
+            schema:
+              name:
+                visible: true
+                readonly: false
+                order: 1
+                width: 300
+              age:
+                visible: true
+                readonly: true
+                order: 2
+                width: 200
+              salary:
+                visible: false
+                readonly: false
+            query:
+              logicalOperator: all
+              onEmptyFilter: none
+              groups:
+                - logicalOperator: any
+                  filters:
+                    - operator: string
+                      field: name
+                      value: John
+                    - operator: range
+                      field: age
+                      value:
+                        low: 18
+                        high: 100
+            primaryDisplay: name
   securitySchemes:
     ApiKeyAuth:
       type: apiKey
@@ -761,7 +836,6 @@ components:
                     enum:
                       - static
                       - dynamic
-                      - ai
                     description: Defines whether this is a static or dynamic formula.
               - type: object
                 properties:
@@ -931,7 +1005,6 @@ components:
                         enum:
                           - static
                           - dynamic
-                          - ai
                         description: Defines whether this is a static or dynamic formula.
                   - type: object
                     properties:
@@ -1108,7 +1181,6 @@ components:
                           enum:
                             - static
                             - dynamic
-                            - ai
                           description: Defines whether this is a static or dynamic formula.
                     - type: object
                       properties:
@@ -1704,6 +1776,644 @@ components:
             - userIds
       required:
         - data
+    view:
+      description: The view to be created/updated.
+      type: object
+      required:
+        - name
+        - schema
+        - tableId
+      properties:
+        name:
+          description: The name of the view.
+          type: string
+        tableId:
+          description: The ID of the table this view is based on.
+          type: string
+        type:
+          description: The type of view - standard (empty value) or calculation.
+          type: string
+          enum:
+            - calculation
+        primaryDisplay:
+          type: string
+          description: A column used to display rows from this view - usually used when
+            rendered in tables.
+        query:
+          description: Search parameters for view
+          type: object
+          required: []
+          properties:
+            logicalOperator:
+              description: When using groups this defines whether all of the filters must
+                match, or only one of them.
+              type: string
+              enum:
+                - all
+                - any
+            onEmptyFilter:
+              description: If no filters match, should the view return all rows, or no rows.
+              type: string
+              enum:
+                - all
+                - none
+            groups:
+              description: A grouping of filters to be applied.
+              type: array
+              items:
+                type: object
+                properties:
+                  logicalOperator:
+                    description: When using groups this defines whether all of the filters must
+                      match, or only one of them.
+                    type: string
+                    enum:
+                      - all
+                      - any
+                  filters:
+                    description: A list of filters to apply
+                    type: array
+                    items:
+                      type: object
+                      properties:
+                        operator:
+                          type: string
+                          description: The type of search operation which is being performed.
+                          enum:
+                            - equal
+                            - notEqual
+                            - empty
+                            - notEmpty
+                            - fuzzy
+                            - string
+                            - contains
+                            - notContains
+                            - containsAny
+                            - oneOf
+                            - range
+                        field:
+                          type: string
+                          description: The field in the view to perform the search on.
+                        value:
+                          description: The value to search for - the type will depend on the operator in
+                            use.
+                          oneOf:
+                            - type: string
+                            - type: number
+                            - type: boolean
+                            - type: object
+                            - type: array
+                  groups:
+                    description: A grouping of filters to be applied.
+                    type: array
+                    items:
+                      type: object
+                      properties:
+                        logicalOperator:
+                          description: When using groups this defines whether all of the filters must
+                            match, or only one of them.
+                          type: string
+                          enum:
+                            - all
+                            - any
+                        filters:
+                          description: A list of filters to apply
+                          type: array
+                          items:
+                            type: object
+                            properties:
+                              operator:
+                                type: string
+                                description: The type of search operation which is being performed.
+                                enum:
+                                  - equal
+                                  - notEqual
+                                  - empty
+                                  - notEmpty
+                                  - fuzzy
+                                  - string
+                                  - contains
+                                  - notContains
+                                  - containsAny
+                                  - oneOf
+                                  - range
+                              field:
+                                type: string
+                                description: The field in the view to perform the search on.
+                              value:
+                                description: The value to search for - the type will depend on the operator in
+                                  use.
+                                oneOf:
+                                  - type: string
+                                  - type: number
+                                  - type: boolean
+                                  - type: object
+                                  - type: array
+        sort:
+          type: object
+          required:
+            - field
+          properties:
+            field:
+              type: string
+              description: The field from the table/view schema to sort on.
+            order:
+              type: string
+              description: The order in which to sort.
+              enum:
+                - ascending
+                - descending
+            type:
+              type: string
+              description: The type of sort to perform (by number, or by alphabetically).
+              enum:
+                - string
+                - number
+        schema:
+          type: object
+          additionalProperties:
+            oneOf:
+              - type: object
+                properties:
+                  visible:
+                    type: boolean
+                    description: Defines whether the column is visible or not - rows
+                      retrieved/updated through this view will not be able to
+                      access it.
+                  readonly:
+                    type: boolean
+                    description: "When used in combination with 'visible: true' the column will be
+                      visible in row responses but cannot be updated."
+                  order:
+                    type: integer
+                    description: A number defining where the column shows up in tables, lowest being
+                      first.
+                  width:
+                    type: integer
+                    description: A width for the column, defined in pixels - this affects rendering
+                      in tables.
+                  column:
+                    type: array
+                    description: If this is a relationship column, we can set the columns we wish to
+                      include
+                    items:
+                      type: object
+                      properties:
+                        readonly:
+                          type: boolean
+              - type: object
+                properties:
+                  calculationType:
+                    type: string
+                    description: This column should be built from a calculation, specifying a type
+                      and field. It is important to note when a calculation is
+                      configured all non-calculation columns will be used for
+                      grouping.
+                    enum:
+                      - sum
+                      - avg
+                      - count
+                      - min
+                      - max
+                  field:
+                    type: string
+                    description: The field from the table to perform the calculation on.
+                  distinct:
+                    type: boolean
+                    description: Can be used in tandem with the count calculation type, to count
+                      unique entries.
+    viewOutput:
+      type: object
+      properties:
+        data:
+          description: The view to be created/updated.
+          type: object
+          required:
+            - name
+            - schema
+            - tableId
+            - id
+          properties:
+            name:
+              description: The name of the view.
+              type: string
+            tableId:
+              description: The ID of the table this view is based on.
+              type: string
+            type:
+              description: The type of view - standard (empty value) or calculation.
+              type: string
+              enum:
+                - calculation
+            primaryDisplay:
+              type: string
+              description: A column used to display rows from this view - usually used when
+                rendered in tables.
+            query:
+              description: Search parameters for view
+              type: object
+              required: []
+              properties:
+                logicalOperator:
+                  description: When using groups this defines whether all of the filters must
+                    match, or only one of them.
+                  type: string
+                  enum:
+                    - all
+                    - any
+                onEmptyFilter:
+                  description: If no filters match, should the view return all rows, or no rows.
+                  type: string
+                  enum:
+                    - all
+                    - none
+                groups:
+                  description: A grouping of filters to be applied.
+                  type: array
+                  items:
+                    type: object
+                    properties:
+                      logicalOperator:
+                        description: When using groups this defines whether all of the filters must
+                          match, or only one of them.
+                        type: string
+                        enum:
+                          - all
+                          - any
+                      filters:
+                        description: A list of filters to apply
+                        type: array
+                        items:
+                          type: object
+                          properties:
+                            operator:
+                              type: string
+                              description: The type of search operation which is being performed.
+                              enum:
+                                - equal
+                                - notEqual
+                                - empty
+                                - notEmpty
+                                - fuzzy
+                                - string
+                                - contains
+                                - notContains
+                                - containsAny
+                                - oneOf
+                                - range
+                            field:
+                              type: string
+                              description: The field in the view to perform the search on.
+                            value:
+                              description: The value to search for - the type will depend on the operator in
+                                use.
+                              oneOf:
+                                - type: string
+                                - type: number
+                                - type: boolean
+                                - type: object
+                                - type: array
+                      groups:
+                        description: A grouping of filters to be applied.
+                        type: array
+                        items:
+                          type: object
+                          properties:
+                            logicalOperator:
+                              description: When using groups this defines whether all of the filters must
+                                match, or only one of them.
+                              type: string
+                              enum:
+                                - all
+                                - any
+                            filters:
+                              description: A list of filters to apply
+                              type: array
+                              items:
+                                type: object
+                                properties:
+                                  operator:
+                                    type: string
+                                    description: The type of search operation which is being performed.
+                                    enum:
+                                      - equal
+                                      - notEqual
+                                      - empty
+                                      - notEmpty
+                                      - fuzzy
+                                      - string
+                                      - contains
+                                      - notContains
+                                      - containsAny
+                                      - oneOf
+                                      - range
+                                  field:
+                                    type: string
+                                    description: The field in the view to perform the search on.
+                                  value:
+                                    description: The value to search for - the type will depend on the operator in
+                                      use.
+                                    oneOf:
+                                      - type: string
+                                      - type: number
+                                      - type: boolean
+                                      - type: object
+                                      - type: array
+            sort:
+              type: object
+              required:
+                - field
+              properties:
+                field:
+                  type: string
+                  description: The field from the table/view schema to sort on.
+                order:
+                  type: string
+                  description: The order in which to sort.
+                  enum:
+                    - ascending
+                    - descending
+                type:
+                  type: string
+                  description: The type of sort to perform (by number, or by alphabetically).
+                  enum:
+                    - string
+                    - number
+            schema:
+              type: object
+              additionalProperties:
+                oneOf:
+                  - type: object
+                    properties:
+                      visible:
+                        type: boolean
+                        description: Defines whether the column is visible or not - rows
+                          retrieved/updated through this view will not be able
+                          to access it.
+                      readonly:
+                        type: boolean
+                        description: "When used in combination with 'visible: true' the column will be
+                          visible in row responses but cannot be updated."
+                      order:
+                        type: integer
+                        description: A number defining where the column shows up in tables, lowest being
+                          first.
+                      width:
+                        type: integer
+                        description: A width for the column, defined in pixels - this affects rendering
+                          in tables.
+                      column:
+                        type: array
+                        description: If this is a relationship column, we can set the columns we wish to
+                          include
+                        items:
+                          type: object
+                          properties:
+                            readonly:
+                              type: boolean
+                  - type: object
+                    properties:
+                      calculationType:
+                        type: string
+                        description: This column should be built from a calculation, specifying a type
+                          and field. It is important to note when a calculation
+                          is configured all non-calculation columns will be used
+                          for grouping.
+                        enum:
+                          - sum
+                          - avg
+                          - count
+                          - min
+                          - max
+                      field:
+                        type: string
+                        description: The field from the table to perform the calculation on.
+                      distinct:
+                        type: boolean
+                        description: Can be used in tandem with the count calculation type, to count
+                          unique entries.
+            id:
+              description: The ID of the view.
+              type: string
+      required:
+        - data
+    viewSearch:
+      type: object
+      properties:
+        data:
+          type: array
+          items:
+            description: The view to be created/updated.
+            type: object
+            required:
+              - name
+              - schema
+              - tableId
+              - id
+            properties:
+              name:
+                description: The name of the view.
+                type: string
+              tableId:
+                description: The ID of the table this view is based on.
+                type: string
+              type:
+                description: The type of view - standard (empty value) or calculation.
+                type: string
+                enum:
+                  - calculation
+              primaryDisplay:
+                type: string
+                description: A column used to display rows from this view - usually used when
+                  rendered in tables.
+              query:
+                description: Search parameters for view
+                type: object
+                required: []
+                properties:
+                  logicalOperator:
+                    description: When using groups this defines whether all of the filters must
+                      match, or only one of them.
+                    type: string
+                    enum:
+                      - all
+                      - any
+                  onEmptyFilter:
+                    description: If no filters match, should the view return all rows, or no rows.
+                    type: string
+                    enum:
+                      - all
+                      - none
+                  groups:
+                    description: A grouping of filters to be applied.
+                    type: array
+                    items:
+                      type: object
+                      properties:
+                        logicalOperator:
+                          description: When using groups this defines whether all of the filters must
+                            match, or only one of them.
+                          type: string
+                          enum:
+                            - all
+                            - any
+                        filters:
+                          description: A list of filters to apply
+                          type: array
+                          items:
+                            type: object
+                            properties:
+                              operator:
+                                type: string
+                                description: The type of search operation which is being performed.
+                                enum:
+                                  - equal
+                                  - notEqual
+                                  - empty
+                                  - notEmpty
+                                  - fuzzy
+                                  - string
+                                  - contains
+                                  - notContains
+                                  - containsAny
+                                  - oneOf
+                                  - range
+                              field:
+                                type: string
+                                description: The field in the view to perform the search on.
+                              value:
+                                description: The value to search for - the type will depend on the operator in
+                                  use.
+                                oneOf:
+                                  - type: string
+                                  - type: number
+                                  - type: boolean
+                                  - type: object
+                                  - type: array
+                        groups:
+                          description: A grouping of filters to be applied.
+                          type: array
+                          items:
+                            type: object
+                            properties:
+                              logicalOperator:
+                                description: When using groups this defines whether all of the filters must
+                                  match, or only one of them.
+                                type: string
+                                enum:
+                                  - all
+                                  - any
+                              filters:
+                                description: A list of filters to apply
+                                type: array
+                                items:
+                                  type: object
+                                  properties:
+                                    operator:
+                                      type: string
+                                      description: The type of search operation which is being performed.
+                                      enum:
+                                        - equal
+                                        - notEqual
+                                        - empty
+                                        - notEmpty
+                                        - fuzzy
+                                        - string
+                                        - contains
+                                        - notContains
+                                        - containsAny
+                                        - oneOf
+                                        - range
+                                    field:
+                                      type: string
+                                      description: The field in the view to perform the search on.
+                                    value:
+                                      description: The value to search for - the type will depend on the operator in
+                                        use.
+                                      oneOf:
+                                        - type: string
+                                        - type: number
+                                        - type: boolean
+                                        - type: object
+                                        - type: array
+              sort:
+                type: object
+                required:
+                  - field
+                properties:
+                  field:
+                    type: string
+                    description: The field from the table/view schema to sort on.
+                  order:
+                    type: string
+                    description: The order in which to sort.
+                    enum:
+                      - ascending
+                      - descending
+                  type:
+                    type: string
+                    description: The type of sort to perform (by number, or by alphabetically).
+                    enum:
+                      - string
+                      - number
+              schema:
+                type: object
+                additionalProperties:
+                  oneOf:
+                    - type: object
+                      properties:
+                        visible:
+                          type: boolean
+                          description: Defines whether the column is visible or not - rows
+                            retrieved/updated through this view will not be able
+                            to access it.
+                        readonly:
+                          type: boolean
+                          description: "When used in combination with 'visible: true' the column will be
+                            visible in row responses but cannot be updated."
+                        order:
+                          type: integer
+                          description: A number defining where the column shows up in tables, lowest being
+                            first.
+                        width:
+                          type: integer
+                          description: A width for the column, defined in pixels - this affects rendering
+                            in tables.
+                        column:
+                          type: array
+                          description: If this is a relationship column, we can set the columns we wish to
+                            include
+                          items:
+                            type: object
+                            properties:
+                              readonly:
+                                type: boolean
+                    - type: object
+                      properties:
+                        calculationType:
+                          type: string
+                          description: This column should be built from a calculation, specifying a type
+                            and field. It is important to note when a
+                            calculation is configured all non-calculation
+                            columns will be used for grouping.
+                          enum:
+                            - sum
+                            - avg
+                            - count
+                            - min
+                            - max
+                        field:
+                          type: string
+                          description: The field from the table to perform the calculation on.
+                        distinct:
+                          type: boolean
+                          description: Can be used in tandem with the count calculation type, to count
+                            unique entries.
+              id:
+                description: The ID of the view.
+                type: string
+      required:
+        - data
 security:
   - ApiKeyAuth: []
 paths:
@@ -2136,6 +2846,32 @@ paths:
               examples:
                 search:
                   $ref: "#/components/examples/rows"
+  "/views/{viewId}/rows/search":
+    post:
+      operationId: rowViewSearch
+      summary: Search for rows in a view
+      tags:
+        - rows
+      parameters:
+        - $ref: "#/components/parameters/viewId"
+        - $ref: "#/components/parameters/appId"
+      requestBody:
+        required: true
+        content:
+          application/json:
+            schema:
+              $ref: "#/components/schemas/rowSearch"
+      responses:
+        "200":
+          description: The response will contain an array of rows that match the search
+            parameters.
+          content:
+            application/json:
+              schema:
+                $ref: "#/components/schemas/searchOutput"
+              examples:
+                search:
+                  $ref: "#/components/examples/rows"
   /tables:
     post:
       operationId: tableCreate
@@ -2359,4 +3095,123 @@ paths:
               examples:
                 users:
                   $ref: "#/components/examples/users"
+  /views:
+    post:
+      operationId: viewCreate
+      summary: Create a view
+      description: Create a view, this can be against an internal or external table.
+      tags:
+        - views
+      parameters:
+        - $ref: "#/components/parameters/appId"
+      requestBody:
+        content:
+          application/json:
+            schema:
+              $ref: "#/components/schemas/view"
+            examples:
+              view:
+                $ref: "#/components/examples/view"
+      responses:
+        "200":
+          description: Returns the created view, including the ID which has been generated
+            for it.
+          content:
+            application/json:
+              schema:
+                $ref: "#/components/schemas/viewOutput"
+              examples:
+                view:
+                  $ref: "#/components/examples/view"
+  "/views/{viewId}":
+    put:
+      operationId: viewUpdate
+      summary: Update a view
+      description: Update a view, this can be against an internal or external table.
+      tags:
+        - views
+      parameters:
+        - $ref: "#/components/parameters/viewId"
+        - $ref: "#/components/parameters/appId"
+      requestBody:
+        content:
+          application/json:
+            schema:
+              $ref: "#/components/schemas/view"
+            examples:
+              view:
+                $ref: "#/components/examples/view"
+      responses:
+        "200":
+          description: Returns the updated view.
+          content:
+            application/json:
+              schema:
+                $ref: "#/components/schemas/viewOutput"
+              examples:
+                view:
+                  $ref: "#/components/examples/view"
+    delete:
+      operationId: viewDestroy
+      summary: Delete a view
+      description: Delete a view, this can be against an internal or external table.
+      tags:
+        - views
+      parameters:
+        - $ref: "#/components/parameters/viewId"
+        - $ref: "#/components/parameters/appId"
+      responses:
+        "200":
+          description: Returns the deleted view.
+          content:
+            application/json:
+              schema:
+                $ref: "#/components/schemas/viewOutput"
+              examples:
+                view:
+                  $ref: "#/components/examples/view"
+    get:
+      operationId: viewGetById
+      summary: Retrieve a view
+      description: Lookup a view, this could be internal or external.
+      tags:
+        - views
+      parameters:
+        - $ref: "#/components/parameters/viewId"
+        - $ref: "#/components/parameters/appId"
+      responses:
+        "200":
+          description: Returns the retrieved view.
+          content:
+            application/json:
+              schema:
+                $ref: "#/components/schemas/viewOutput"
+              examples:
+                view:
+                  $ref: "#/components/examples/view"
+  /views/search:
+    post:
+      operationId: viewSearch
+      summary: Search for views
+      description: Based on view properties (currently only name) search for views.
+      tags:
+        - views
+      parameters:
+        - $ref: "#/components/parameters/appId"
+      requestBody:
+        required: true
+        content:
+          application/json:
+            schema:
+              $ref: "#/components/schemas/nameSearch"
+      responses:
+        "200":
+          description: Returns the found views, based on the search parameters.
+          content:
+            application/json:
+              schema:
+                $ref: "#/components/schemas/viewSearch"
+              examples:
+                views:
+                  $ref: "#/components/examples/views"
 tags: []

packages/server/specs/parameters.ts

@@ -8,6 +8,16 @@ export const tableId = {
   },
 }
 
+export const viewId = {
+  in: "path",
+  name: "viewId",
+  required: true,
+  description: "The ID of the view which this request is targeting.",
+  schema: {
+    type: "string",
+  },
+}
+
 export const rowId = {
   in: "path",
   name: "rowId",

packages/server/specs/resources/index.ts

@@ -6,6 +6,7 @@ import user from "./user"
 import metrics from "./metrics"
 import misc from "./misc"
 import roles from "./roles"
+import view from "./view"
 
 export const examples = {
   ...application.getExamples(),
@@ -16,6 +17,7 @@ export const examples = {
   ...misc.getExamples(),
   ...metrics.getExamples(),
   ...roles.getExamples(),
+  ...view.getExamples(),
 }
 
 export const schemas = {
@@ -26,4 +28,5 @@ export const schemas = {
   ...user.getSchemas(),
   ...misc.getSchemas(),
   ...roles.getSchemas(),
+  ...view.getSchemas(),
 }

packages/server/specs/resources/misc.ts

@@ -1,99 +1,101 @@
 import { object } from "./utils"
 import Resource from "./utils/Resource"
 
+export const searchSchema = {
+  type: "object",
+  properties: {
+    allOr: {
+      type: "boolean",
+      description:
+        "Specifies that a row should be returned if it satisfies any of the specified options, rather than requiring it to fulfill all the search parameters. This defaults to false, meaning AND logic will be used.",
+    },
+    string: {
+      type: "object",
+      example: {
+        columnName1: "value",
+        columnName2: "value",
+      },
+      description:
+        "A map of field name to the string to search for, this will look for rows that have a value starting with the string value.",
+      additionalProperties: {
+        type: "string",
+        description: "The value to search for in the column.",
+      },
+    },
+    fuzzy: {
+      type: "object",
+      description:
+        "Searches for a sub-string within a string column, e.g. searching for 'dib' will match 'Budibase'.",
+    },
+    range: {
+      type: "object",
+      description:
+        'Searches within a range, the format of this must be in the format of an object with a "low" and "high" property.',
+      example: {
+        columnName1: {
+          low: 10,
+          high: 20,
+        },
+      },
+    },
+    equal: {
+      type: "object",
+      description:
+        "Searches for rows that have a column value that is exactly the value set.",
+    },
+    notEqual: {
+      type: "object",
+      description:
+        "Searches for any row which does not contain the specified column value.",
+    },
+    empty: {
+      type: "object",
+      description:
+        "Searches for rows which do not contain the specified column. The object should simply contain keys of the column names, these can map to any value.",
+      example: {
+        columnName1: "",
+      },
+    },
+    notEmpty: {
+      type: "object",
+      description: "Searches for rows which have the specified column.",
+    },
+    oneOf: {
+      type: "object",
+      description:
+        "Searches for rows which have a column value that is any of the specified values. The format of this must be columnName -> [value1, value2].",
+    },
+    contains: {
+      type: "object",
+      description:
+        "Searches for a value, or set of values in array column types (such as a multi-select). If an array of search options is provided then it must match all.",
+      example: {
+        arrayColumn: ["a", "b"],
+      },
+    },
+    notContains: {
+      type: "object",
+      description:
+        "The logical inverse of contains. Only works on array column types. If an array of values is passed, the row must not match any of them to be returned in the response.",
+      example: {
+        arrayColumn: ["a", "b"],
+      },
+    },
+    containsAny: {
+      type: "object",
+      description:
+        "As with the contains search, only works on array column types and searches for any of the provided values when given an array.",
+      example: {
+        arrayColumn: ["a", "b"],
+      },
+    },
+  },
+}
+
 export default new Resource().setSchemas({
   rowSearch: object(
     {
-      query: {
-        type: "object",
-        properties: {
-          allOr: {
-            type: "boolean",
-            description:
-              "Specifies that a row should be returned if it satisfies any of the specified options, rather than requiring it to fulfill all the search parameters. This defaults to false, meaning AND logic will be used.",
-          },
-          string: {
-            type: "object",
-            example: {
-              columnName1: "value",
-              columnName2: "value",
-            },
-            description:
-              "A map of field name to the string to search for, this will look for rows that have a value starting with the string value.",
-            additionalProperties: {
-              type: "string",
-              description: "The value to search for in the column.",
-            },
-          },
-          fuzzy: {
-            type: "object",
-            description:
-              "Searches for a sub-string within a string column, e.g. searching for 'dib' will match 'Budibase'.",
-          },
-          range: {
-            type: "object",
-            description:
-              'Searches within a range, the format of this must be in the format of an object with a "low" and "high" property.',
-            example: {
-              columnName1: {
-                low: 10,
-                high: 20,
-              },
-            },
-          },
-          equal: {
-            type: "object",
-            description:
-              "Searches for rows that have a column value that is exactly the value set.",
-          },
-          notEqual: {
-            type: "object",
-            description:
-              "Searches for any row which does not contain the specified column value.",
-          },
-          empty: {
-            type: "object",
-            description:
-              "Searches for rows which do not contain the specified column. The object should simply contain keys of the column names, these can map to any value.",
-            example: {
-              columnName1: "",
-            },
-          },
-          notEmpty: {
-            type: "object",
-            description: "Searches for rows which have the specified column.",
-          },
-          oneOf: {
-            type: "object",
-            description:
-              "Searches for rows which have a column value that is any of the specified values. The format of this must be columnName -> [value1, value2].",
-          },
-          contains: {
-            type: "object",
-            description:
-              "Searches for a value, or set of values in array column types (such as a multi-select). If an array of search options is provided then it must match all.",
-            example: {
-              arrayColumn: ["a", "b"],
-            },
-          },
-          notContains: {
-            type: "object",
-            description:
-              "The logical inverse of contains. Only works on array column types. If an array of values is passed, the row must not match any of them to be returned in the response.",
-            example: {
-              arrayColumn: ["a", "b"],
-            },
-          },
-          containsAny: {
-            type: "object",
-            description:
-              "As with the contains search, only works on array column types and searches for any of the provided values when given an array.",
-            example: {
-              arrayColumn: ["a", "b"],
-            },
-          },
-        },
-      },
+      query: searchSchema,
       paginate: {
         type: "boolean",
         description: "Enables pagination, by default this is disabled.",

packages/server/specs/resources/view.ts

@@ -0,0 +1,274 @@
+import { object } from "./utils"
+import Resource from "./utils/Resource"
+import {
+  ArrayOperator,
+  BasicOperator,
+  CalculationType,
+  RangeOperator,
+  SortOrder,
+  SortType,
+} from "@budibase/types"
+import { cloneDeep } from "lodash"
+
+const view = {
+  name: "peopleView",
+  tableId: "ta_896a325f7e8147d2a2cda93c5d236511",
+  schema: {
+    name: {
+      visible: true,
+      readonly: false,
+      order: 1,
+      width: 300,
+    },
+    age: {
+      visible: true,
+      readonly: true,
+      order: 2,
+      width: 200,
+    },
+    salary: {
+      visible: false,
+      readonly: false,
+    },
+  },
+  query: {
+    logicalOperator: "all",
+    onEmptyFilter: "none",
+    groups: [
+      {
+        logicalOperator: "any",
+        filters: [
+          { operator: "string", field: "name", value: "John" },
+          { operator: "range", field: "age", value: { low: 18, high: 100 } },
+        ],
+      },
+    ],
+  },
+  primaryDisplay: "name",
+}
+
+const baseColumnDef = {
+  visible: {
+    type: "boolean",
+    description:
+      "Defines whether the column is visible or not - rows retrieved/updated through this view will not be able to access it.",
+  },
+  readonly: {
+    type: "boolean",
+    description:
+      "When used in combination with 'visible: true' the column will be visible in row responses but cannot be updated.",
+  },
+  order: {
+    type: "integer",
+    description:
+      "A number defining where the column shows up in tables, lowest being first.",
+  },
+  width: {
+    type: "integer",
+    description:
+      "A width for the column, defined in pixels - this affects rendering in tables.",
+  },
+  column: {
+    type: "array",
+    description:
+      "If this is a relationship column, we can set the columns we wish to include",
+    items: {
+      type: "object",
+      properties: {
+        readonly: {
+          type: "boolean",
+        },
+      },
+    },
+  },
+}
+
+const logicalOperator = {
+  description:
+    "When using groups this defines whether all of the filters must match, or only one of them.",
+  type: "string",
+  enum: ["all", "any"],
+}
+
+const filterGroup = {
+  description: "A grouping of filters to be applied.",
+  type: "array",
+  items: {
+    type: "object",
+    properties: {
+      logicalOperator,
+      filters: {
+        description: "A list of filters to apply",
+        type: "array",
+        items: {
+          type: "object",
+          properties: {
+            operator: {
+              type: "string",
+              description:
+                "The type of search operation which is being performed.",
+              enum: [
+                ...Object.values(BasicOperator),
+                ...Object.values(ArrayOperator),
+                ...Object.values(RangeOperator),
+              ],
+            },
+            field: {
+              type: "string",
+              description: "The field in the view to perform the search on.",
+            },
+            value: {
+              description:
+                "The value to search for - the type will depend on the operator in use.",
+              oneOf: [
+                { type: "string" },
+                { type: "number" },
+                { type: "boolean" },
+                { type: "object" },
+                { type: "array" },
+              ],
+            },
+          },
+        },
+      },
+    },
+  },
+}
+
+// have to clone to avoid constantly recursive structure - we can't represent this easily
+const layeredFilterGroup: any = cloneDeep(filterGroup)
+layeredFilterGroup.items.properties.groups = filterGroup
+
+const viewQuerySchema = {
+  description: "Search parameters for view",
+  type: "object",
+  required: [],
+  properties: {
+    logicalOperator,
+    onEmptyFilter: {
+      description:
+        "If no filters match, should the view return all rows, or no rows.",
+      type: "string",
+      enum: ["all", "none"],
+    },
+    groups: layeredFilterGroup,
+  },
+}
+
+const viewSchema = {
+  description: "The view to be created/updated.",
+  type: "object",
+  required: ["name", "schema", "tableId"],
+  properties: {
+    name: {
+      description: "The name of the view.",
+      type: "string",
+    },
+    tableId: {
+      description: "The ID of the table this view is based on.",
+      type: "string",
+    },
+    type: {
+      description: "The type of view - standard (empty value) or calculation.",
+      type: "string",
+      enum: ["calculation"],
+    },
+    primaryDisplay: {
+      type: "string",
+      description:
+        "A column used to display rows from this view - usually used when rendered in tables.",
+    },
+    query: viewQuerySchema,
+    sort: {
+      type: "object",
+      required: ["field"],
+      properties: {
+        field: {
+          type: "string",
+          description: "The field from the table/view schema to sort on.",
+        },
+        order: {
+          type: "string",
+          description: "The order in which to sort.",
+          enum: Object.values(SortOrder),
+        },
+        type: {
+          type: "string",
+          description:
+            "The type of sort to perform (by number, or by alphabetically).",
+          enum: Object.values(SortType),
+        },
+      },
+    },
+    schema: {
+      type: "object",
+      additionalProperties: {
+        oneOf: [
+          {
+            type: "object",
+            properties: baseColumnDef,
+          },
+          {
+            type: "object",
+            properties: {
+              calculationType: {
+                type: "string",
+                description:
+                  "This column should be built from a calculation, specifying a type and field. It is important to note when a calculation is configured all non-calculation columns will be used for grouping.",
+                enum: Object.values(CalculationType),
+              },
+              field: {
+                type: "string",
+                description:
+                  "The field from the table to perform the calculation on.",
+              },
+              distinct: {
+                type: "boolean",
+                description:
+                  "Can be used in tandem with the count calculation type, to count unique entries.",
+              },
+            },
+          },
+        ],
+      },
+    },
+  },
+}
+
+const viewOutputSchema = {
+  ...viewSchema,
+  properties: {
+    ...viewSchema.properties,
+    id: {
+      description: "The ID of the view.",
+      type: "string",
+    },
+  },
+  required: [...viewSchema.required, "id"],
+}
+
+export default new Resource()
+  .setExamples({
+    view: {
+      value: {
+        data: view,
+      },
+    },
+    views: {
+      value: {
+        data: [view],
+      },
+    },
+  })
+  .setSchemas({
+    view: viewSchema,
+    viewOutput: object({
+      data: viewOutputSchema,
+    }),
+    viewSearch: object({
+      data: {
+        type: "array",
+        items: viewOutputSchema,
+      },
+    }),
+  })

packages/server/src/api/controllers/application.ts

@@ -153,7 +153,11 @@ async function createInstance(appId: string, template: AppTemplate) {
   await createAllSearchIndex()
 
   if (template && template.useTemplate) {
-    await sdk.backups.importApp(appId, db, template)
+    const opts = {
+      importObjStoreContents: true,
+      updateAttachmentColumns: !template.key, // preserve attachments when using Budibase templates
+    }
+    await sdk.backups.importApp(appId, db, template, opts)
   } else {
     // create the users table
     await db.put(USERS_TABLE_SCHEMA)

packages/server/src/api/controllers/public/mapping/applications.ts

@@ -1,6 +1,7 @@
 import { Application } from "./types"
+import { RequiredKeys } from "@budibase/types"
 
-function application(body: any): Application {
+function application(body: any): RequiredKeys<Application> {
   let app = body?.application ? body.application : body
   return {
     _id: app.appId,

packages/server/src/api/controllers/public/mapping/index.ts

@@ -3,6 +3,7 @@ import applications from "./applications"
 import users from "./users"
 import rows from "./rows"
 import queries from "./queries"
+import views from "./views"
 
 export default {
   ...tables,
@@ -10,4 +11,5 @@ export default {
   ...users,
   ...rows,
   ...queries,
+  ...views,
 }

packages/server/src/api/controllers/public/mapping/queries.ts

@@ -1,6 +1,7 @@
 import { Query, ExecuteQuery } from "./types"
+import { RequiredKeys } from "@budibase/types"
 
-function query(body: any): Query {
+function query(body: any): RequiredKeys<Query> {
   return {
     _id: body._id,
     datasourceId: body.datasourceId,

packages/server/src/api/controllers/public/mapping/rows.ts

@@ -1,6 +1,7 @@
 import { Row, RowSearch } from "./types"
+import { RequiredKeys } from "@budibase/types"
 
-function row(body: any): Row {
+function row(body: any): RequiredKeys<Row> {
   delete body._rev
   // have to input everything, since structure unknown
   return {

packages/server/src/api/controllers/public/mapping/tables.ts

@@ -1,6 +1,7 @@
 import { Table } from "./types"
+import { RequiredKeys } from "@budibase/types"
 
-function table(body: any): Table {
+function table(body: any): RequiredKeys<Table> {
   return {
     _id: body._id,
     name: body.name,

packages/server/src/api/controllers/public/mapping/types.ts

@@ -9,6 +9,9 @@ export type CreateApplicationParams = components["schemas"]["application"]
 export type Table = components["schemas"]["tableOutput"]["data"]
 export type CreateTableParams = components["schemas"]["table"]
 
+export type View = components["schemas"]["viewOutput"]["data"]
+export type CreateViewParams = components["schemas"]["view"]
+
 export type Row = components["schemas"]["rowOutput"]["data"]
 export type RowSearch = components["schemas"]["searchOutput"]
 export type CreateRowParams = components["schemas"]["row"]

packages/server/src/api/controllers/public/mapping/users.ts

@@ -1,6 +1,7 @@
 import { User } from "./types"
+import { RequiredKeys } from "@budibase/types"
 
-function user(body: any): User {
+function user(body: any): RequiredKeys<User> {
   return {
     _id: body._id,
     email: body.email,

packages/server/src/api/controllers/public/mapping/views.ts

@@ -0,0 +1,32 @@
+import { View } from "./types"
+import { ViewV2, Ctx, RequiredKeys } from "@budibase/types"
+import { dataFilters } from "@budibase/shared-core"
+
+function view(body: ViewV2): RequiredKeys<View> {
+  return {
+    id: body.id,
+    tableId: body.tableId,
+    type: body.type,
+    name: body.name,
+    schema: body.schema!,
+    primaryDisplay: body.primaryDisplay,
+    query: dataFilters.buildQuery(body.query),
+    sort: body.sort,
+  }
+}
+
+function mapView(ctx: Ctx<{ data: ViewV2 }>): { data: View } {
+  return {
+    data: view(ctx.body.data),
+  }
+}
+
+function mapViews(ctx: Ctx<{ data: ViewV2[] }>): { data: View[] } {
+  const views = ctx.body.data.map((body: ViewV2) => view(body))
+  return { data: views }
+}
+
+export default {
+  mapView,
+  mapViews,
+}

packages/server/src/api/controllers/public/rows.ts

@@ -22,13 +22,13 @@ export function fixRow(row: Row, params: any) {
   return row
 }
 
-export async function search(ctx: UserCtx, next: Next) {
+function buildSearchRequestBody(ctx: UserCtx) {
   let { sort, paginate, bookmark, limit, query } = ctx.request.body
   // update the body to the correct format of the internal search
   if (!sort) {
     sort = {}
   }
-  ctx.request.body = {
+  return {
     sort: sort.column,
     sortType: sort.type,
     sortOrder: sort.order,
@@ -37,10 +37,23 @@ export async function search(ctx: UserCtx, next: Next) {
     limit,
     query,
   }
+}
+
+export async function search(ctx: UserCtx, next: Next) {
+  ctx.request.body = buildSearchRequestBody(ctx)
   await rowController.search(ctx)
   await next()
 }
 
+export async function viewSearch(ctx: UserCtx, next: Next) {
+  ctx.request.body = buildSearchRequestBody(ctx)
+  ctx.params = {
+    viewId: ctx.params.viewId,
+  }
+  await rowController.views.searchView(ctx)
+  await next()
+}
+
 export async function create(ctx: UserCtx, next: Next) {
   ctx.request.body = fixRow(ctx.request.body, ctx.params)
   await rowController.save(ctx)
@@ -79,4 +92,5 @@ export default {
   update,
   destroy,
   search,
+  viewSearch,
 }

packages/server/src/api/controllers/public/views.ts

@@ -0,0 +1,95 @@
+import { search as stringSearch } from "./utils"
+import * as controller from "../view"
+import { ViewV2, UserCtx, UISearchFilter, PublicAPIView } from "@budibase/types"
+import { Next } from "koa"
+import { merge } from "lodash"
+
+function viewRequest(view: PublicAPIView, params?: { viewId: string }) {
+  const viewV2: ViewV2 = view
+  if (!viewV2) {
+    return viewV2
+  }
+  if (params?.viewId) {
+    viewV2.id = params.viewId
+  }
+  if (!view.query) {
+    viewV2.query = {}
+  } else {
+    // public API only has one form of query
+    viewV2.queryUI = viewV2.query as UISearchFilter
+  }
+  viewV2.version = 2
+  return viewV2
+}
+
+function viewResponse(view: ViewV2): PublicAPIView {
+  // remove our internal structure - always un-necessary
+  delete view.query
+  return {
+    ...view,
+    query: view.queryUI,
+  }
+}
+
+function viewsResponse(views: ViewV2[]): PublicAPIView[] {
+  return views.map(viewResponse)
+}
+
+export async function search(ctx: UserCtx, next: Next) {
+  const { name } = ctx.request.body
+  await controller.v2.fetch(ctx)
+  ctx.body.data = viewsResponse(stringSearch(ctx.body.data, name))
+  await next()
+}
+
+export async function create(ctx: UserCtx, next: Next) {
+  ctx = merge(ctx, {
+    request: {
+      body: viewRequest(ctx.request.body),
+    },
+  })
+  await controller.v2.create(ctx)
+  ctx.body.data = viewResponse(ctx.body.data)
+  await next()
+}
+
+export async function read(ctx: UserCtx, next: Next) {
+  ctx = merge(ctx, {
+    params: {
+      viewId: ctx.params.viewId,
+    },
+  })
+  await controller.v2.get(ctx)
+  ctx.body.data = viewResponse(ctx.body.data)
+  await next()
+}
+
+export async function update(ctx: UserCtx, next: Next) {
+  const viewId = ctx.params.viewId
+  ctx = merge(ctx, {
+    request: {
+      body: {
+        data: viewRequest(ctx.request.body, { viewId }),
+      },
+    },
+    params: {
+      viewId,
+    },
+  })
+  await controller.v2.update(ctx)
+  ctx.body.data = viewResponse(ctx.body.data)
+  await next()
+}
+
+export async function destroy(ctx: UserCtx, next: Next) {
+  await controller.v2.remove(ctx)
+  await next()
+}
+
+export default {
+  create,
+  read,
+  update,
+  destroy,
+  search,
+}