Merge branch 'master' of github.com:Budibase/budibase into chore/refactor-finalizerow

.eslintignore

@@ -9,8 +9,5 @@ packages/server/client
 packages/server/coverage
 packages/builder/.routify
 packages/sdk/sdk
-packages/account-portal/packages/server/build
-packages/account-portal/packages/ui/.routify
-packages/account-portal/packages/ui/build
 **/*.ivm.bundle.js
 packages/server/build/oldClientVersions/**/**

.github/workflows/budibase_ci.yml

@@ -64,18 +64,15 @@ jobs:
       - run: yarn --frozen-lockfile
 
       # Run build all the projects
-      - name: Build OSS
-        run: yarn build:oss
-      - name: Build account portal
-        run: yarn build:account-portal
-        if: ${{ env.IS_OSS_CONTRIBUTOR == 'false' }}
+      - name: Build
+        run: yarn build
       # Check the types of the projects built via esbuild
       - name: Check types
         run: |
           if ${{ env.ONLY_AFFECTED_TASKS }}; then
-            yarn check:types --since=${{ env.NX_BASE_BRANCH }} --ignore @budibase/account-portal-server
+            yarn check:types --since=${{ env.NX_BASE_BRANCH }}
           else
-            yarn check:types --ignore @budibase/account-portal-server
+            yarn check:types
           fi
 
   helm-lint:
@@ -117,9 +114,11 @@ jobs:
       - name: Test
         run: |
           if ${{ env.ONLY_AFFECTED_TASKS }}; then
-            yarn test --ignore=@budibase/worker --ignore=@budibase/server --since=${{ env.NX_BASE_BRANCH }}
+            yarn test -- --ignore=@budibase/worker --ignore=@budibase/server --ignore=@budibase/builder --no-prefix --since=${{ env.NX_BASE_BRANCH }} -- --verbose --reporters=default --reporters=github-actions
+            yarn test -- --scope=@budibase/builder --since=${{ env.NX_BASE_BRANCH }}
           else
-            yarn test --ignore=@budibase/worker --ignore=@budibase/server
+            yarn test -- --ignore=@budibase/worker --ignore=@budibase/server --ignore=@budibase/builder --no-prefix -- --verbose --reporters=default --reporters=github-actions
+            yarn test -- --scope=@budibase/builder --no-prefix
           fi
 
   test-worker:
@@ -141,13 +140,22 @@ jobs:
       - name: Test worker
         run: |
           if ${{ env.ONLY_AFFECTED_TASKS }}; then
-            node scripts/run-affected.js --task=test --scope=@budibase/worker --since=${{ env.NX_BASE_BRANCH }}
-          else
-            yarn test --scope=@budibase/worker
+            AFFECTED=$(yarn --silent nx show projects --affected -t test --base=${{ env.NX_BASE_BRANCH }} -p @budibase/worker)
+            if [ -z "$AFFECTED" ]; then
+              echo "No affected tests to run"
+              exit 0
+            fi
           fi
 
+          cd packages/worker
+          yarn test --verbose --reporters=default --reporters=github-actions
+
   test-server:
-    runs-on: budi-tubby-tornado-quad-core-300gb
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        datasource:
+          [mssql, mysql, postgres, mongodb, mariadb, oracle, sqs, none]
     steps:
       - name: Checkout repo
         uses: actions/checkout@v4
@@ -170,12 +178,19 @@ jobs:
 
       - name: Pull testcontainers images
         run: |
-          docker pull mcr.microsoft.com/mssql/server@${{ steps.dotenv.outputs.MSSQL_SHA }} &
-          docker pull mysql@${{ steps.dotenv.outputs.MYSQL_SHA }} &
-          docker pull postgres@${{ steps.dotenv.outputs.POSTGRES_SHA }} &
-          docker pull mongo@${{ steps.dotenv.outputs.MONGODB_SHA }} &
-          docker pull mariadb@${{ steps.dotenv.outputs.MARIADB_SHA }} &
-          docker pull budibase/oracle-database:23.2-slim-faststart &
+          if [ "${{ matrix.datasource }}" == "mssql" ]; then
+            docker pull mcr.microsoft.com/mssql/server@${{ steps.dotenv.outputs.MSSQL_SHA }}
+          elif [ "${{ matrix.datasource }}" == "mysql" ]; then
+            docker pull mysql@${{ steps.dotenv.outputs.MYSQL_SHA }}
+          elif [ "${{ matrix.datasource }}" == "postgres" ]; then
+            docker pull postgres@${{ steps.dotenv.outputs.POSTGRES_SHA }}
+          elif [ "${{ matrix.datasource }}" == "mongodb" ]; then
+            docker pull mongo@${{ steps.dotenv.outputs.MONGODB_SHA }}
+          elif [ "${{ matrix.datasource }}" == "mariadb" ]; then
+            docker pull mariadb@${{ steps.dotenv.outputs.MARIADB_SHA }}
+          elif [ "${{ matrix.datasource }}" == "oracle" ]; then
+            docker pull budibase/oracle-database:23.2-slim-faststart
+          fi
           docker pull minio/minio &
           docker pull redis &
           docker pull testcontainers/ryuk:0.5.1 &
@@ -186,12 +201,24 @@ jobs:
       - run: yarn --frozen-lockfile
 
       - name: Test server
+        env:
+          DATASOURCE: ${{ matrix.datasource }}
         run: |
           if ${{ env.ONLY_AFFECTED_TASKS }}; then
-            node scripts/run-affected.js --task=test --scope=@budibase/server --since=${{ env.NX_BASE_BRANCH }}
-          else
-            yarn test --scope=@budibase/server
+            AFFECTED=$(yarn --silent nx show projects --affected -t test --base=${{ env.NX_BASE_BRANCH }} -p @budibase/server)
+            if [ -z "$AFFECTED" ]; then
+              echo "No affected tests to run"
+              exit 0
             fi
+          fi
+
+          FILTER="./src/tests/filters/datasource-tests.js"
+          if [ "${{ matrix.datasource }}" == "none" ]; then
+            FILTER="./src/tests/filters/non-datasource-tests.js"
+          fi
+
+          cd packages/server
+          yarn test --filter $FILTER --verbose --reporters=default --reporters=github-actions
 
   check-pro-submodule:
     runs-on: ubuntu-latest
@@ -252,64 +279,6 @@ jobs:
             echo 'All good, the submodule had been merged and setup correctly!'
           fi
 
-  check-accountportal-submodule:
-    runs-on: ubuntu-latest
-    if: inputs.run_as_oss != true  && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == 'Budibase/budibase')
-    steps:
-      - name: Checkout repo and submodules
-        uses: actions/checkout@v4
-        with:
-          submodules: true
-          token: ${{ secrets.PERSONAL_ACCESS_TOKEN || github.token }}
-          fetch-depth: 0
-
-      - uses: dorny/paths-filter@v3
-        id: changes
-        with:
-          filters: |
-            src:
-              - packages/account-portal/**
-
-      - if: steps.changes.outputs.src == 'true'
-        name: Check account portal commit
-        id: get_accountportal_commits
-        run: |
-          cd packages/account-portal
-          accountportal_commit=$(git rev-parse HEAD)
-
-          branch="${{ github.base_ref || github.ref_name  }}"
-          echo "Running on branch '$branch' (base_ref=${{ github.base_ref }}, ref_name=${{ github.head_ref }})"
-
-          base_commit=$(git rev-parse origin/master)
-
-          if [[ ! -z $base_commit ]]; then
-            echo "target_branch=$branch"
-            echo "target_branch=$branch" >> "$GITHUB_OUTPUT"
-            echo "accountportal_commit=$accountportal_commit"
-            echo "accountportal_commit=$accountportal_commit" >> "$GITHUB_OUTPUT"
-            echo "base_commit=$base_commit"
-            echo "base_commit=$base_commit" >> "$GITHUB_OUTPUT"
-          else
-            echo "Nothing to do - branch to branch merge."
-          fi
-
-      - name: Check submodule merged to base branch
-        if: ${{ steps.get_accountportal_commits.outputs.base_commit != '' }}
-        uses: actions/github-script@v7
-        with:
-          github-token: ${{ secrets.GITHUB_TOKEN }}
-          script: |
-            const submoduleCommit = '${{ steps.get_accountportal_commits.outputs.accountportal_commit }}';
-            const baseCommit = '${{ steps.get_accountportal_commits.outputs.base_commit }}';
-
-            if (submoduleCommit !== baseCommit) {
-              console.error('Submodule commit does not match the latest commit on the "${{ steps.get_accountportal_commits.outputs.target_branch }}" branch.');
-              console.error('Refer to the account portal repo to merge your changes: https://github.com/Budibase/account-portal/blob/master/docs/index.md')
-              process.exit(1);
-            } else {
-              console.log('All good, the submodule had been merged and setup correctly!')
-            }
-
   check-lockfile:
     runs-on: ubuntu-latest
     steps:

.gitignore

@@ -8,6 +8,7 @@ packages/server/build/oldClientVersions/**/*
 packages/builder/src/components/deploy/clientVersions.json
 packages/server/src/integrations/tests/utils/*.lock
 packages/builder/vite.config.mjs.timestamp*
+packages/account-portal
 
 # Logs
 logs

.gitmodules

@@ -1,6 +1,3 @@
 [submodule "packages/pro"]
 	path = packages/pro
 	url = git@github.com:Budibase/budibase-pro.git
-[submodule "packages/account-portal"]
-	path = packages/account-portal
-	url = git@github.com:Budibase/account-portal.git

.prettierignore

@@ -9,8 +9,4 @@ packages/backend-core/coverage
 packages/builder/.routify
 packages/sdk/sdk
 packages/pro/coverage
-packages/account-portal/packages/ui/build
-packages/account-portal/packages/ui/.routify
-packages/account-portal/packages/server/build
-packages/account-portal/packages/server/coverage
 **/*.ivm.bundle.js

.vscode/launch.json

@@ -20,16 +20,6 @@
       "args": ["${workspaceFolder}/packages/worker/src/index.ts"],
       "cwd": "${workspaceFolder}/packages/worker"
     },
-    {
-      "name": "Camunda Worker",
-      "type": "node",
-      "request": "launch",
-      "runtimeArgs": ["--nolazy", "-r", "ts-node/register/transpile-only"],
-      "args": [
-        "${workspaceFolder}/packages/account-portal/packages/server/src/v2/run.ts"
-      ],
-      "cwd": "${workspaceFolder}/packages/account-portal/packages/server"
-    },
     {
       "type": "chrome",
       "request": "launch",

charts/budibase/templates/secrets.yaml

@@ -12,12 +12,12 @@ metadata:
 type: Opaque
 data:
   {{- if $existingSecret }}
-  internalApiKey: {{ index $existingSecret.data "internalApiKey" }}
-  jwtSecret: {{ index $existingSecret.data "jwtSecret" }}
-  objectStoreAccess: {{ index $existingSecret.data "objectStoreAccess" }}
-  objectStoreSecret: {{ index $existingSecret.data "objectStoreSecret" }}
-  bbEncryptionKey: {{ index $existingSecret.data "bbEncryptionKey" }}
-  apiEncryptionKey: {{ index $existingSecret.data "apiEncryptionKey" }}
+  internalApiKey: {{ index $existingSecret.data "internalApiKey" | quote }}
+  jwtSecret: {{ index $existingSecret.data "jwtSecret" | quote }}
+  objectStoreAccess: {{ index $existingSecret.data "objectStoreAccess" | quote }}
+  objectStoreSecret: {{ index $existingSecret.data "objectStoreSecret" | quote }}
+  bbEncryptionKey: {{ index $existingSecret.data "bbEncryptionKey" | quote }}
+  apiEncryptionKey: {{ index $existingSecret.data "apiEncryptionKey" | quote }}
   {{- else }}
   internalApiKey: {{ template "budibase.defaultsecret" .Values.globals.internalApiKey }}
   jwtSecret: {{ template "budibase.defaultsecret" .Values.globals.jwtSecret }}

examples/nextjs-api-sales/yarn.lock

@@ -423,9 +423,9 @@ core-js-pure@^3.20.2:
   integrity sha512-12VZfFIu+wyVbBebyHmRTuEE/tZrB4tJToWcwAMcsp3h4+sHR+fMJWbKpYiCRWlhFBq+KNyO8rIV9rTkeVmznQ==
 
 cross-spawn@^7.0.2:
-  version "7.0.3"
-  resolved "https://registry.yarnpkg.com/cross-spawn/-/cross-spawn-7.0.3.tgz#f73a85b9d5d41d045551c177e2882d4ac85728a6"
-  integrity sha512-iRDPJKUPVEND7dHPO8rkbOnPpyDygcDFtWjpeWNCgy8WP2rXcxXL8TskReQl6OrB2G7+UJrags1q15Fudc7G6w==
+  version "7.0.6"
+  resolved "https://registry.yarnpkg.com/cross-spawn/-/cross-spawn-7.0.6.tgz#8a58fe78f00dcd70c370451759dfbfaf03e8ee9f"
+  integrity sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA==
   dependencies:
     path-key "^3.1.0"
     shebang-command "^2.0.0"

globalSetup.ts

@@ -62,6 +62,7 @@ export default async function setup() {
         },
       ])
       .withLabels({ "com.budibase": "true" })
+      .withTmpFs({ "/data": "rw" })
       .withReuse()
       .withWaitStrategy(
         Wait.forSuccessfulCommand(
@@ -72,6 +73,7 @@ export default async function setup() {
     const minio = new GenericContainer("minio/minio")
       .withExposedPorts(9000)
       .withCommand(["server", "/data"])
+      .withTmpFs({ "/data": "rw" })
       .withEnvironment({
         MINIO_ACCESS_KEY: "budibase",
         MINIO_SECRET_KEY: "budibase",

hosting/.env

@@ -19,7 +19,6 @@ MINIO_PORT=4004
 COUCH_DB_PORT=4005
 COUCH_DB_SQS_PORT=4006
 REDIS_PORT=6379
-WATCHTOWER_PORT=6161
 BUDIBASE_ENVIRONMENT=PRODUCTION
 SQL_MAX_ROWS=
 

hosting/docker-compose.build.yaml

@@ -74,7 +74,6 @@ services:
       - WORKER_UPSTREAM_URL=http://worker-service:4003
       - MINIO_UPSTREAM_URL=http://minio-service:9000
       - COUCHDB_UPSTREAM_URL=http://couchdb-service:5984
-      - WATCHTOWER_UPSTREAM_URL=http://watchtower-service:8080
       - RESOLVER=127.0.0.11
     depends_on:
       - minio-service

hosting/docker-compose.yaml

@@ -87,7 +87,6 @@ services:
       - WORKER_UPSTREAM_URL=http://worker-service:4003
       - MINIO_UPSTREAM_URL=http://minio-service:9000
       - COUCHDB_UPSTREAM_URL=http://couchdb-service:5984
-      - WATCHTOWER_UPSTREAM_URL=http://watchtower-service:8080
       - RESOLVER=127.0.0.11
     depends_on:
       - minio-service
@@ -112,19 +111,6 @@ services:
     volumes:
       - redis_data:/data
 
-  watchtower-service:
-    restart: always
-    image: containrrr/watchtower
-    volumes:
-      - /var/run/docker.sock:/var/run/docker.sock
-    command: --debug --http-api-update bbapps bbworker bbproxy
-    environment:
-      - WATCHTOWER_HTTP_API=true
-      - WATCHTOWER_HTTP_API_TOKEN=budibase
-      - WATCHTOWER_CLEANUP=true
-    labels:
-      - "com.centurylinklabs.watchtower.enable=false"
-
 volumes:
   couchdb3_data:
     driver: local

hosting/envoy.yaml

@@ -1,152 +0,0 @@
-static_resources:
-  listeners:
-  - name: main_listener
-    address:
-      socket_address: { address: 0.0.0.0, port_value: 10000 }
-    filter_chains:
-    - filters:
-      - name: envoy.filters.network.http_connection_manager
-        typed_config:
-          "@type": type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager
-          stat_prefix: ingress
-          codec_type: auto
-          route_config:
-            name: local_route
-            virtual_hosts:
-            - name: local_services
-              domains: ["*"]
-              routes:
-              - match: { prefix: "/app/" }
-                route:
-                  cluster: app-service
-                  prefix_rewrite: "/"
-
-              - match: { path: "/v1/update" }
-                route:
-                  cluster: watchtower-service
-
-              - match: { prefix: "/builder/" }
-                route:
-                  cluster: app-service
-
-              - match: { prefix: "/builder" }
-                route:
-                  cluster: app-service
-              
-              - match: { prefix: "/app_" }
-                route:
-                  cluster: app-service
-
-              # special cases for worker admin (deprecated), global and system API
-              - match: { prefix: "/api/global/" }
-                route:
-                  cluster: worker-service
-
-              - match: { prefix: "/api/admin/" }
-                route:
-                  cluster: worker-service
-
-              - match: { prefix: "/api/system/" }
-                route:
-                  cluster: worker-service
-
-              - match: { path: "/" }
-                route:
-                  cluster: app-service
-
-              # special case for when API requests are made, can just forward, not to minio    
-              - match: { prefix: "/api/" }
-                route:
-                  cluster: app-service
-                  timeout: 120s
-
-              - match: { prefix: "/worker/" }
-                route: 
-                  cluster: worker-service
-                  prefix_rewrite: "/"
-
-              - match: { prefix: "/db/" }
-                route:
-                  cluster: couchdb-service
-                  prefix_rewrite: "/"
-
-              # minio is on the default route because this works
-              # best, minio + AWS SDK doesn't handle path proxy
-              - match: { prefix: "/" }
-                route:
-                  cluster: minio-service
-
-          http_filters:
-          - name: envoy.filters.http.router
-
-  clusters:
-  - name: app-service
-    connect_timeout: 0.25s
-    type: strict_dns
-    lb_policy: round_robin
-    load_assignment:
-      cluster_name: app-service
-      endpoints:
-      - lb_endpoints:
-        - endpoint:
-            address:
-              socket_address: 
-                address: app-service
-                port_value: 4002
-
-  - name: minio-service
-    connect_timeout: 0.25s
-    type: strict_dns
-    lb_policy: round_robin
-    load_assignment:
-      cluster_name: minio-service
-      endpoints:
-      - lb_endpoints:
-        - endpoint:
-            address:
-              socket_address:
-                address: minio-service
-                port_value: 9000
-
-  - name: worker-service
-    connect_timeout: 0.25s
-    type: strict_dns
-    lb_policy: round_robin
-    load_assignment:
-      cluster_name: worker-service
-      endpoints:
-      - lb_endpoints:
-        - endpoint:
-            address:
-              socket_address: 
-                address: worker-service
-                port_value: 4003
-
-  - name: couchdb-service
-    connect_timeout: 0.25s
-    type: strict_dns
-    lb_policy: round_robin
-    load_assignment:
-      cluster_name: couchdb-service
-      endpoints:
-      - lb_endpoints:
-        - endpoint:
-            address:
-              socket_address:
-                address: couchdb-service
-                port_value: 5984
-
-  - name: watchtower-service
-    connect_timeout: 0.25s
-    type: strict_dns
-    lb_policy: round_robin
-    load_assignment:
-      cluster_name: watchtower-service
-      endpoints:
-      - lb_endpoints:
-        - endpoint:
-            address:
-              socket_address: 
-                address: watchtower-service
-                port_value: 8080
-

hosting/hosting.properties

@@ -18,7 +18,6 @@ WORKER_PORT=4003
 MINIO_PORT=4004
 COUCH_DB_PORT=4005
 REDIS_PORT=6379
-WATCHTOWER_PORT=6161
 BUDIBASE_ENVIRONMENT=PRODUCTION
 
 # An admin user can be automatically created initially if these are set

hosting/portainer/template.json

@@ -78,11 +78,6 @@
           "default": "6379",
           "preset": true
         },
-        {
-          "name": "WATCHTOWER_PORT",
-          "default": "6161",
-          "preset": true
-        },
         {
           "name": "BUDIBASE_ENVIRONMENT",
           "default": "PRODUCTION",

hosting/proxy/Dockerfile

@@ -22,5 +22,4 @@ ENV APPS_UPSTREAM_URL=http://app-service:4002
 ENV WORKER_UPSTREAM_URL=http://worker-service:4003
 ENV MINIO_UPSTREAM_URL=http://minio-service:9000
 ENV COUCHDB_UPSTREAM_URL=http://couchdb-service:5984
-ENV WATCHTOWER_UPSTREAM_URL=http://watchtower-service:8080
 ENV RESOLVER=127.0.0.11

hosting/proxy/nginx.prod.conf

@@ -50,19 +50,6 @@ http {
     ignore_invalid_headers off;
     proxy_buffering off;
 
-    set $csp_default "default-src 'self'";
-    set $csp_script "script-src 'self' 'unsafe-inline' 'unsafe-eval' https://*.budibase.net https://cdn.budi.live https://js.intercomcdn.com https://widget.intercom.io https://d2l5prqdbvm3op.cloudfront.net https://us-assets.i.posthog.com";
-    set $csp_style "style-src 'self' 'unsafe-inline' https://cdn.jsdelivr.net https://fonts.googleapis.com https://rsms.me https://maxcdn.bootstrapcdn.com";
-    set $csp_object "object-src 'none'";
-    set $csp_base_uri "base-uri 'self'";
-    set $csp_connect "connect-src 'self' https://*.budibase.app https://*.budibaseqa.app https://*.budibase.net https://api-iam.intercom.io https://api-iam.intercom.io  https://api-ping.intercom.io https://app.posthog.com https://us.i.posthog.com wss://nexus-websocket-a.intercom.io  wss://nexus-websocket-b.intercom.io https://nexus-websocket-a.intercom.io https://nexus-websocket-b.intercom.io  https://uploads.intercomcdn.com  https://uploads.intercomusercontent.com https://*.amazonaws.com https://*.s3.amazonaws.com https://*.s3.us-east-2.amazonaws.com https://*.s3.us-east-1.amazonaws.com https://*.s3.us-west-1.amazonaws.com https://*.s3.us-west-2.amazonaws.com https://*.s3.af-south-1.amazonaws.com https://*.s3.ap-east-1.amazonaws.com https://*.s3.ap-southeast-3.amazonaws.com https://*.s3.ap-south-1.amazonaws.com https://*.s3.ap-northeast-3.amazonaws.com https://*.s3.ap-northeast-2.amazonaws.com https://*.s3.ap-southeast-1.amazonaws.com https://*.s3.ap-southeast-2.amazonaws.com https://*.s3.ap-northeast-1.amazonaws.com https://*.s3.ca-central-1.amazonaws.com https://*.s3.cn-north-1.amazonaws.com https://*.s3.cn-northwest-1.amazonaws.com https://*.s3.eu-central-1.amazonaws.com https://*.s3.eu-west-1.amazonaws.com https://*.s3.eu-west-2.amazonaws.com https://*.s3.eu-south-1.amazonaws.com https://*.s3.eu-west-3.amazonaws.com https://*.s3.eu-north-1.amazonaws.com https://*.s3.sa-east-1.amazonaws.com https://*.s3.me-south-1.amazonaws.com https://*.s3.us-gov-east-1.amazonaws.com https://*.s3.us-gov-west-1.amazonaws.com https://api.github.com";    
-    set $csp_font "font-src 'self' data: https://cdn.jsdelivr.net https://fonts.gstatic.com https://rsms.me https://maxcdn.bootstrapcdn.com  https://js.intercomcdn.com  https://fonts.intercomcdn.com";
-    set $csp_frame "frame-src 'self' https:";
-    set $csp_img "img-src http: https: data: blob:";
-    set $csp_manifest "manifest-src 'self'";
-    set $csp_media "media-src 'self' https://js.intercomcdn.com https://cdn.budi.live"; 
-    set $csp_worker "worker-src blob:";
-
     error_page 502 503 504 /error.html;
     location = /error.html {
       root /usr/share/nginx/html;
@@ -73,7 +60,6 @@ http {
     add_header X-Frame-Options SAMEORIGIN always;
     add_header X-Content-Type-Options nosniff always;
     add_header X-XSS-Protection "1; mode=block" always;
-    add_header Content-Security-Policy "${csp_default}; ${csp_script}; ${csp_style}; ${csp_object}; ${csp_base_uri}; ${csp_connect}; ${csp_font}; ${csp_frame}; ${csp_img}; ${csp_manifest}; ${csp_media}; ${csp_worker};" always;
     add_header Strict-Transport-Security "max-age=31536000; includeSubDomains; preload" always;
 
     # upstreams
@@ -81,7 +67,6 @@ http {
     set $worker ${WORKER_UPSTREAM_URL};
     set $minio ${MINIO_UPSTREAM_URL};
     set $couchdb ${COUCHDB_UPSTREAM_URL};
-    set $watchtower ${WATCHTOWER_UPSTREAM_URL};
 
     location /health {
       access_log off;
@@ -107,10 +92,6 @@ http {
       proxy_pass      $apps;
     }
 
-    location = /v1/update {
-      proxy_pass      $watchtower;
-    }
-
     location ~ ^/(builder|app_) {
       proxy_http_version  1.1;
 
@@ -125,6 +106,12 @@ http {
 
     location ~ ^/api/(system|admin|global)/ {
       proxy_set_header    Host                $host;
+
+      # Enable buffering for potentially large OIDC configs
+      proxy_buffering on;
+      proxy_buffer_size 16k;
+      proxy_buffers 4 32k;
+
       proxy_pass      $worker;
     }
 

hosting/scripts/airgapped/airgappedDockerBuild.js

@@ -12,7 +12,6 @@ let IMAGES = {
 	couch: "ibmcom/couchdb3",
 	curl: "curlimages/curl",
 	redis: "redis",
-	watchtower: "containrrr/watchtower",
 }
 
 if (IS_SINGLE_IMAGE) {

lerna.json

@@ -1,12 +1,7 @@
 {
   "$schema": "node_modules/lerna/schemas/lerna-schema.json",
-  "version": "3.0.3",
+  "version": "3.2.12",
   "npmClient": "yarn",
-  "packages": [
-    "packages/*",
-    "!packages/account-portal",
-    "packages/account-portal/packages/*"
-  ],
   "concurrency": 20,
   "command": {
     "publish": {

nx.json

@@ -2,7 +2,6 @@
   "$schema": "./node_modules/nx/schemas/nx-schema.json",
   "tasksRunnerOptions": {
     "default": {
-      "runner": "nx-cloud",
       "options": {
         "cacheableOperations": ["build", "test", "check:types"]
       }

package.json

@@ -9,6 +9,7 @@
     "@types/node": "20.10.0",
     "@types/proper-lockfile": "^4.1.4",
     "@typescript-eslint/parser": "6.9.0",
+    "depcheck": "^1.4.7",
     "esbuild": "^0.18.17",
     "esbuild-node-externals": "^1.14.0",
     "eslint": "^8.52.0",
@@ -24,22 +25,22 @@
     "prettier": "2.8.8",
     "prettier-plugin-svelte": "^2.3.0",
     "proper-lockfile": "^4.1.2",
-    "svelte": "^4.2.10",
+    "svelte": "4.2.19",
     "svelte-eslint-parser": "^0.33.1",
     "typescript": "5.5.2",
     "typescript-eslint": "^7.3.1",
-    "yargs": "^17.7.2"
+    "yargs": "^17.7.2",
+    "cross-spawn": "7.0.6"
   },
   "scripts": {
     "get-past-client-version": "node scripts/getPastClientVersion.js",
     "setup": "git config submodule.recurse true && git submodule update && node ./hosting/scripts/setup.js && yarn && yarn build && yarn dev",
     "build": "DISABLE_V8_COMPILE_CACHE=1 NODE_OPTIONS=--max-old-space-size=1500 lerna run build --stream",
     "build:apps": "DISABLE_V8_COMPILE_CACHE=1 yarn build --scope @budibase/server --scope @budibase/worker",
-    "build:oss": "DISABLE_V8_COMPILE_CACHE=1 NODE_OPTIONS=--max-old-space-size=1500 lerna run build --stream --ignore @budibase/account-portal-server --ignore @budibase/account-portal-ui",
     "build:cli": "yarn build --scope @budibase/cli",
-    "build:account-portal": "NODE_OPTIONS=--max-old-space-size=1500 lerna run build --stream --scope @budibase/account-portal-server --scope @budibase/account-portal-ui",
     "build:dev": "lerna run --stream prebuild && yarn nx run-many --target=build --output-style=dynamic --watch --preserveWatchOutput",
-    "check:types": "lerna run --concurrency 2 check:types --ignore @budibase/account-portal-server",
+    "check:types": "yarn check:dependencies && lerna run --concurrency 2 check:types",
+    "check:dependencies": "lerna run --concurrency 2 check:dependencies",
     "build:sdk": "lerna run --stream build:sdk",
     "deps:circular": "madge packages/server/dist/index.js packages/worker/src/index.ts packages/backend-core/dist/src/index.js packages/cli/src/index.js --circular",
     "release": "lerna publish from-package --yes --force-publish --no-git-tag-version --no-push --no-git-reset",
@@ -52,15 +53,12 @@
     "kill-server": "kill-port 4001 4002",
     "kill-accountportal": "kill-port 3001 4003",
     "kill-all": "yarn run kill-builder && yarn run kill-server && yarn kill-accountportal",
-    "dev": "yarn run kill-all && lerna run --parallel prebuild && lerna run --stream dev --ignore=@budibase/account-portal-ui --ignore @budibase/account-portal-server",
-    "dev:noserver": "yarn run kill-builder && lerna run --stream dev:stack:up --ignore @budibase/account-portal-server && lerna run --stream dev --ignore @budibase/backend-core --ignore @budibase/server --ignore @budibase/worker --ignore=@budibase/account-portal-ui --ignore @budibase/account-portal-server",
+    "dev": "yarn run kill-all && lerna run --parallel prebuild && lerna run --stream dev",
+    "dev:noserver": "yarn run kill-builder && lerna run --stream dev:stack:up && lerna run --stream dev --ignore @budibase/backend-core --ignore @budibase/server --ignore @budibase/worker",
     "dev:server": "yarn run kill-server && lerna run --stream dev --scope @budibase/worker --scope @budibase/server",
-    "dev:accountportal": "yarn kill-accountportal && lerna run dev --stream --scope @budibase/account-portal-ui --scope @budibase/account-portal-server",
-    "dev:camunda": "./scripts/deploy-camunda.sh",
-    "dev:all": "yarn run kill-all && lerna run --stream dev",
     "dev:built": "yarn run kill-all && cd packages/server && yarn dev:stack:up && cd ../../ && lerna run --stream dev:built",
     "dev:docker": "./scripts/devDocker.sh",
-    "test": "lerna run --concurrency 1 --stream test --stream",
+    "test": "lerna run --concurrency 1 --stream test",
     "test:containers:kill": "./scripts/killTestcontainers.sh",
     "lint:eslint": "eslint packages --max-warnings=0",
     "lint:prettier": "prettier --check \"packages/**/*.{js,ts,svelte}\" && prettier --write \"examples/**/*.{js,ts,svelte}\"",
@@ -98,9 +96,7 @@
   },
   "workspaces": {
     "packages": [
-      "packages/*",
-      "!packages/account-portal",
-      "packages/account-portal/packages/*"
+      "packages/*"
     ]
   },
   "resolutions": {
@@ -114,7 +110,7 @@
     "semver": "7.5.3",
     "http-cache-semantics": "4.1.1",
     "msgpackr": "1.10.1",
-    "axios": "1.6.3",
+    "axios": "1.7.7",
     "xml2js": "0.6.2",
     "unset-value": "2.0.1",
     "passport": "0.6.0",
@@ -124,6 +120,5 @@
   },
   "engines": {
     "node": ">=20.0.0 <21.0.0"
-  },
-  "dependencies": {}
+  }
 }

packages/account-portal

@@ -1 +0,0 @@
-Subproject commit 9bef5d1656b4f3c991447ded6d65b0eba393a140

packages/backend-core/.npmignore

@@ -2,5 +2,3 @@
 !dist/**/*
 dist/tsconfig.build.tsbuildinfo
 !package.json
-!src/**
-!tests/**

packages/backend-core/package.json

@@ -9,6 +9,13 @@
     "./tests": "./dist/tests/index.js",
     "./*": "./dist/*.js"
   },
+  "typesVersions": {
+    "*": {
+      "tests": [
+        "dist/tests/index.d.ts"
+      ]
+    }
+  },
   "author": "Budibase",
   "license": "GPL-3.0",
   "scripts": {
@@ -17,6 +24,7 @@
     "build": "tsc -p tsconfig.build.json --paths null && node ./scripts/build.js",
     "build:dev": "yarn prebuild && tsc --build --watch --preserveWatchOutput",
     "check:types": "tsc -p tsconfig.json --noEmit --paths null --target es2020",
+    "check:dependencies": "node ../../scripts/depcheck.js",
     "test": "bash scripts/test.sh",
     "test:watch": "jest --watchAll"
   },
@@ -25,17 +33,21 @@
     "@budibase/pouchdb-replication-stream": "1.2.11",
     "@budibase/shared-core": "0.0.0",
     "@budibase/types": "0.0.0",
+    "@techpass/passport-openidconnect": "0.3.3",
     "aws-cloudfront-sign": "3.0.2",
-    "aws-sdk": "2.1030.0",
+    "aws-sdk": "2.1692.0",
     "bcrypt": "5.1.0",
     "bcryptjs": "2.4.3",
     "bull": "4.10.1",
     "correlation-id": "4.0.0",
-    "dd-trace": "5.2.0",
+    "dd-trace": "5.26.0",
     "dotenv": "16.0.1",
+    "google-auth-library": "^8.0.1",
+    "google-spreadsheet": "npm:@budibase/google-spreadsheet@4.1.5",
     "ioredis": "5.3.2",
     "joi": "17.6.0",
     "jsonwebtoken": "9.0.2",
+    "knex": "2.4.2",
     "koa-passport": "^6.0.0",
     "koa-pino-logger": "4.0.0",
     "lodash": "4.17.21",
@@ -46,17 +58,17 @@
     "pino": "8.11.0",
     "pino-http": "8.3.3",
     "posthog-node": "4.0.1",
-    "pouchdb": "7.3.0",
-    "pouchdb-find": "7.2.2",
+    "pouchdb": "9.0.0",
+    "pouchdb-find": "9.0.0",
     "redlock": "4.2.0",
     "rotating-file-stream": "3.1.0",
     "sanitize-s3-objectkey": "0.0.1",
     "semver": "^7.5.4",
     "tar-fs": "2.1.1",
-    "uuid": "^8.3.2",
-    "knex": "2.4.2"
+    "uuid": "^8.3.2"
   },
   "devDependencies": {
+    "@jest/types": "^29.6.3",
     "@shopify/jest-koa-mocks": "5.1.1",
     "@swc/core": "1.3.71",
     "@swc/jest": "0.2.27",
@@ -64,8 +76,9 @@
     "@types/cookies": "0.7.8",
     "@types/jest": "29.5.5",
     "@types/lodash": "4.14.200",
+    "@types/node": "^22.9.0",
     "@types/node-fetch": "2.6.4",
-    "@types/pouchdb": "6.4.0",
+    "@types/pouchdb": "6.4.2",
     "@types/redlock": "4.0.7",
     "@types/semver": "7.3.7",
     "@types/tar-fs": "2.0.1",
@@ -74,6 +87,7 @@
     "ioredis-mock": "8.9.0",
     "jest": "29.7.0",
     "jest-serial-runner": "1.2.1",
+    "nock": "^13.5.6",
     "pino-pretty": "10.0.0",
     "pouchdb-adapter-memory": "7.2.2",
     "testcontainers": "^10.7.2",

packages/backend-core/src/cache/tests/docWritethrough.spec.ts

@@ -1,7 +1,12 @@
 import tk from "timekeeper"
 
 import _ from "lodash"
-import { DBTestConfiguration, generator, structures } from "../../../tests"
+import {
+  DBTestConfiguration,
+  generator,
+  structures,
+  utils,
+} from "../../../tests"
 import { getDB } from "../../db"
 
 import {
@@ -10,15 +15,14 @@ import {
   init,
 } from "../docWritethrough"
 
-import InMemoryQueue from "../../queue/inMemoryQueue"
-
 const initialTime = Date.now()
 
 async function waitForQueueCompletion() {
-  const queue: InMemoryQueue = DocWritethroughProcessor.queue as never
-  await queue.waitForCompletion()
+  await utils.queue.processMessages(DocWritethroughProcessor.queue)
 }
 
+beforeAll(() => utils.queue.useRealQueues())
+
 describe("docWritethrough", () => {
   beforeAll(() => {
     init()
@@ -67,7 +71,7 @@ describe("docWritethrough", () => {
         const patch3 = generatePatchObject(3)
         await docWritethrough.patch(patch3)
 
-        expect(await db.get(documentId)).toEqual({
+        expect(await db.tryGet(documentId)).toEqual({
           _id: documentId,
           ...patch1,
           ...patch2,
@@ -92,7 +96,7 @@ describe("docWritethrough", () => {
 
         await waitForQueueCompletion()
 
-        expect(await db.get(documentId)).toEqual(
+        expect(await db.tryGet(documentId)).toEqual(
           expect.objectContaining({
             _id: documentId,
             ...patch1,
@@ -117,7 +121,7 @@ describe("docWritethrough", () => {
         await waitForQueueCompletion()
 
         expect(date1).not.toEqual(date2)
-        expect(await db.get(documentId)).toEqual(
+        expect(await db.tryGet(documentId)).toEqual(
           expect.objectContaining({
             createdAt: date1.toISOString(),
             updatedAt: date2.toISOString(),
@@ -135,7 +139,7 @@ describe("docWritethrough", () => {
         await docWritethrough.patch(patch2)
 
         const keyToOverride = _.sample(Object.keys(patch1))!
-        expect(await db.get(documentId)).toEqual(
+        expect(await db.tryGet(documentId)).toEqual(
           expect.objectContaining({
             [keyToOverride]: patch1[keyToOverride],
           })
@@ -150,7 +154,7 @@ describe("docWritethrough", () => {
         await docWritethrough.patch(patch3)
         await waitForQueueCompletion()
 
-        expect(await db.get(documentId)).toEqual(
+        expect(await db.tryGet(documentId)).toEqual(
           expect.objectContaining({
             ...patch1,
             ...patch2,
@@ -180,14 +184,14 @@ describe("docWritethrough", () => {
         await secondDocWritethrough.patch(doc2Patch2)
         await waitForQueueCompletion()
 
-        expect(await db.get(docWritethrough.docId)).toEqual(
+        expect(await db.tryGet(docWritethrough.docId)).toEqual(
           expect.objectContaining({
             ...doc1Patch,
             ...doc1Patch2,
           })
         )
 
-        expect(await db.get(secondDocWritethrough.docId)).toEqual(
+        expect(await db.tryGet(secondDocWritethrough.docId)).toEqual(
           expect.objectContaining({
             ...doc2Patch,
             ...doc2Patch2,
@@ -203,7 +207,7 @@ describe("docWritethrough", () => {
         await docWritethrough.patch(initialPatch)
         await waitForQueueCompletion()
 
-        expect(await db.get(documentId)).toEqual(
+        expect(await db.tryGet(documentId)).toEqual(
           expect.objectContaining(initialPatch)
         )
 
@@ -214,10 +218,10 @@ describe("docWritethrough", () => {
         await docWritethrough.patch(extraPatch)
         await waitForQueueCompletion()
 
-        expect(await db.get(documentId)).toEqual(
+        expect(await db.tryGet(documentId)).toEqual(
           expect.objectContaining(extraPatch)
         )
-        expect(await db.get(documentId)).not.toEqual(
+        expect(await db.tryGet(documentId)).not.toEqual(
           expect.objectContaining(initialPatch)
         )
       })
@@ -242,7 +246,7 @@ describe("docWritethrough", () => {
         expect(queueMessageSpy).toHaveBeenCalledTimes(5)
 
         await waitForQueueCompletion()
-        expect(await db.get(documentId)).toEqual(
+        expect(await db.tryGet(documentId)).toEqual(
           expect.objectContaining(patches)
         )
 
@@ -250,7 +254,7 @@ describe("docWritethrough", () => {
         expect(queueMessageSpy).toHaveBeenCalledTimes(45)
 
         await waitForQueueCompletion()
-        expect(await db.get(documentId)).toEqual(
+        expect(await db.tryGet(documentId)).toEqual(
           expect.objectContaining(patches)
         )
 
@@ -258,20 +262,18 @@ describe("docWritethrough", () => {
         expect(queueMessageSpy).toHaveBeenCalledTimes(55)
 
         await waitForQueueCompletion()
-        expect(await db.get(documentId)).toEqual(
+        expect(await db.tryGet(documentId)).toEqual(
           expect.objectContaining(patches)
         )
       })
     })
 
-    // This is not yet supported
-    // eslint-disable-next-line jest/no-disabled-tests
-    it.skip("patches will execute in order", async () => {
+    it("patches will execute in order", async () => {
       let incrementalValue = 0
       const keyToOverride = generator.word()
       async function incrementalPatches(count: number) {
         for (let i = 0; i < count; i++) {
-          await docWritethrough.patch({ [keyToOverride]: incrementalValue++ })
+          await docWritethrough.patch({ [keyToOverride]: ++incrementalValue })
         }
       }
 
@@ -279,13 +281,13 @@ describe("docWritethrough", () => {
         await incrementalPatches(5)
 
         await waitForQueueCompletion()
-        expect(await db.get(documentId)).toEqual(
+        expect(await db.tryGet(documentId)).toEqual(
           expect.objectContaining({ [keyToOverride]: 5 })
         )
 
         await incrementalPatches(40)
         await waitForQueueCompletion()
-        expect(await db.get(documentId)).toEqual(
+        expect(await db.tryGet(documentId)).toEqual(
           expect.objectContaining({ [keyToOverride]: 45 })
         )
       })

packages/backend-core/src/db/couch/DatabaseImpl.ts

@@ -10,7 +10,6 @@ import {
   DatabaseQueryOpts,
   DBError,
   Document,
-  FeatureFlag,
   isDocument,
   RowResponse,
   RowValue,
@@ -27,7 +26,6 @@ import { SQLITE_DESIGN_DOC_ID } from "../../constants"
 import { DDInstrumentedDatabase } from "../instrumentation"
 import { checkSlashesInUrl } from "../../helpers"
 import { sqlLog } from "../../sql/utils"
-import { flags } from "../../features"
 
 const DATABASE_NOT_FOUND = "Database does not exist."
 
@@ -192,7 +190,7 @@ export class DatabaseImpl implements Database {
     }
   }
 
-  private async performCall<T>(call: DBCallback<T>): Promise<any> {
+  private async performCall<T>(call: DBCallback<T>): Promise<T> {
     const db = this.getDb()
     const fnc = await call(db)
     try {
@@ -456,10 +454,7 @@ export class DatabaseImpl implements Database {
   }
 
   async destroy() {
-    if (
-      (await flags.isEnabled(FeatureFlag.SQS)) &&
-      (await this.exists(SQLITE_DESIGN_DOC_ID))
-    ) {
+    if (await this.exists(SQLITE_DESIGN_DOC_ID)) {
       // delete the design document, then run the cleanup operation
       const definition = await this.get<SQLiteDefinition>(SQLITE_DESIGN_DOC_ID)
       // remove all tables - save the definition then trigger a cleanup
@@ -472,7 +467,7 @@ export class DatabaseImpl implements Database {
     } catch (err: any) {
       // didn't exist, don't worry
       if (err.statusCode === 404) {
-        return
+        return { ok: true }
       } else {
         throw new CouchDBError(err.message, err)
       }

packages/backend-core/src/db/instrumentation.ts

@@ -27,7 +27,7 @@ export class DDInstrumentedDatabase implements Database {
 
   exists(docId?: string): Promise<boolean> {
     return tracer.trace("db.exists", span => {
-      span?.addTags({ db_name: this.name, doc_id: docId })
+      span.addTags({ db_name: this.name, doc_id: docId })
       if (docId) {
         return this.db.exists(docId)
       }
@@ -37,15 +37,17 @@ export class DDInstrumentedDatabase implements Database {
 
   get<T extends Document>(id?: string | undefined): Promise<T> {
     return tracer.trace("db.get", span => {
-      span?.addTags({ db_name: this.name, doc_id: id })
+      span.addTags({ db_name: this.name, doc_id: id })
       return this.db.get(id)
     })
   }
 
   tryGet<T extends Document>(id?: string | undefined): Promise<T | undefined> {
-    return tracer.trace("db.tryGet", span => {
-      span?.addTags({ db_name: this.name, doc_id: id })
-      return this.db.tryGet(id)
+    return tracer.trace("db.tryGet", async span => {
+      span.addTags({ db_name: this.name, doc_id: id })
+      const doc = await this.db.tryGet<T>(id)
+      span.addTags({ doc_found: doc !== undefined })
+      return doc
     })
   }
 
@@ -53,13 +55,15 @@ export class DDInstrumentedDatabase implements Database {
     ids: string[],
     opts?: { allowMissing?: boolean | undefined } | undefined
   ): Promise<T[]> {
-    return tracer.trace("db.getMultiple", span => {
-      span?.addTags({
+    return tracer.trace("db.getMultiple", async span => {
+      span.addTags({
         db_name: this.name,
         num_docs: ids.length,
         allow_missing: opts?.allowMissing,
       })
-      return this.db.getMultiple(ids, opts)
+      const docs = await this.db.getMultiple<T>(ids, opts)
+      span.addTags({ num_docs_found: docs.length })
+      return docs
     })
   }
 
@@ -69,12 +73,14 @@ export class DDInstrumentedDatabase implements Database {
     idOrDoc: string | Document,
     rev?: string
   ): Promise<DocumentDestroyResponse> {
-    return tracer.trace("db.remove", span => {
-      span?.addTags({ db_name: this.name, doc_id: idOrDoc })
+    return tracer.trace("db.remove", async span => {
+      span.addTags({ db_name: this.name, doc_id: idOrDoc, rev })
       const isDocument = typeof idOrDoc === "object"
       const id = isDocument ? idOrDoc._id! : idOrDoc
       rev = isDocument ? idOrDoc._rev : rev
-      return this.db.remove(id, rev)
+      const resp = await this.db.remove(id, rev)
+      span.addTags({ ok: resp.ok })
+      return resp
     })
   }
 
@@ -83,7 +89,11 @@ export class DDInstrumentedDatabase implements Database {
     opts?: { silenceErrors?: boolean }
   ): Promise<void> {
     return tracer.trace("db.bulkRemove", span => {
-      span?.addTags({ db_name: this.name, num_docs: documents.length })
+      span.addTags({
+        db_name: this.name,
+        num_docs: documents.length,
+        silence_errors: opts?.silenceErrors,
+      })
       return this.db.bulkRemove(documents, opts)
     })
   }
@@ -92,15 +102,21 @@ export class DDInstrumentedDatabase implements Database {
     document: AnyDocument,
     opts?: DatabasePutOpts | undefined
   ): Promise<DocumentInsertResponse> {
-    return tracer.trace("db.put", span => {
-      span?.addTags({ db_name: this.name, doc_id: document._id })
-      return this.db.put(document, opts)
+    return tracer.trace("db.put", async span => {
+      span.addTags({
+        db_name: this.name,
+        doc_id: document._id,
+        force: opts?.force,
+      })
+      const resp = await this.db.put(document, opts)
+      span.addTags({ ok: resp.ok })
+      return resp
     })
   }
 
   bulkDocs(documents: AnyDocument[]): Promise<DocumentBulkResponse[]> {
     return tracer.trace("db.bulkDocs", span => {
-      span?.addTags({ db_name: this.name, num_docs: documents.length })
+      span.addTags({ db_name: this.name, num_docs: documents.length })
       return this.db.bulkDocs(documents)
     })
   }
@@ -108,9 +124,15 @@ export class DDInstrumentedDatabase implements Database {
   allDocs<T extends Document | RowValue>(
     params: DatabaseQueryOpts
   ): Promise<AllDocsResponse<T>> {
-    return tracer.trace("db.allDocs", span => {
-      span?.addTags({ db_name: this.name })
-      return this.db.allDocs(params)
+    return tracer.trace("db.allDocs", async span => {
+      span.addTags({ db_name: this.name, ...params })
+      const resp = await this.db.allDocs<T>(params)
+      span.addTags({
+        total_rows: resp.total_rows,
+        rows_length: resp.rows.length,
+        offset: resp.offset,
+      })
+      return resp
     })
   }
 
@@ -118,57 +140,75 @@ export class DDInstrumentedDatabase implements Database {
     viewName: string,
     params: DatabaseQueryOpts
   ): Promise<AllDocsResponse<T>> {
-    return tracer.trace("db.query", span => {
-      span?.addTags({ db_name: this.name, view_name: viewName })
-      return this.db.query(viewName, params)
+    return tracer.trace("db.query", async span => {
+      span.addTags({ db_name: this.name, view_name: viewName, ...params })
+      const resp = await this.db.query<T>(viewName, params)
+      span.addTags({
+        total_rows: resp.total_rows,
+        rows_length: resp.rows.length,
+        offset: resp.offset,
+      })
+      return resp
     })
   }
 
-  destroy(): Promise<void | OkResponse> {
-    return tracer.trace("db.destroy", span => {
-      span?.addTags({ db_name: this.name })
-      return this.db.destroy()
+  destroy(): Promise<OkResponse> {
+    return tracer.trace("db.destroy", async span => {
+      span.addTags({ db_name: this.name })
+      const resp = await this.db.destroy()
+      span.addTags({ ok: resp.ok })
+      return resp
     })
   }
 
-  compact(): Promise<void | OkResponse> {
-    return tracer.trace("db.compact", span => {
-      span?.addTags({ db_name: this.name })
-      return this.db.compact()
+  compact(): Promise<OkResponse> {
+    return tracer.trace("db.compact", async span => {
+      span.addTags({ db_name: this.name })
+      const resp = await this.db.compact()
+      span.addTags({ ok: resp.ok })
+      return resp
     })
   }
 
   dump(stream: Writable, opts?: DatabaseDumpOpts | undefined): Promise<any> {
     return tracer.trace("db.dump", span => {
-      span?.addTags({ db_name: this.name })
+      span.addTags({
+        db_name: this.name,
+        batch_limit: opts?.batch_limit,
+        batch_size: opts?.batch_size,
+        style: opts?.style,
+        timeout: opts?.timeout,
+        num_doc_ids: opts?.doc_ids?.length,
+        view: opts?.view,
+      })
       return this.db.dump(stream, opts)
     })
   }
 
   load(...args: any[]): Promise<any> {
     return tracer.trace("db.load", span => {
-      span?.addTags({ db_name: this.name })
+      span.addTags({ db_name: this.name, num_args: args.length })
       return this.db.load(...args)
     })
   }
 
   createIndex(...args: any[]): Promise<any> {
     return tracer.trace("db.createIndex", span => {
-      span?.addTags({ db_name: this.name })
+      span.addTags({ db_name: this.name, num_args: args.length })
       return this.db.createIndex(...args)
     })
   }
 
   deleteIndex(...args: any[]): Promise<any> {
     return tracer.trace("db.deleteIndex", span => {
-      span?.addTags({ db_name: this.name })
+      span.addTags({ db_name: this.name, num_args: args.length })
       return this.db.deleteIndex(...args)
     })
   }
 
   getIndexes(...args: any[]): Promise<any> {
     return tracer.trace("db.getIndexes", span => {
-      span?.addTags({ db_name: this.name })
+      span.addTags({ db_name: this.name, num_args: args.length })
       return this.db.getIndexes(...args)
     })
   }
@@ -177,22 +217,27 @@ export class DDInstrumentedDatabase implements Database {
     sql: string,
     parameters?: SqlQueryBinding
   ): Promise<T[]> {
-    return tracer.trace("db.sql", span => {
-      span?.addTags({ db_name: this.name })
-      return this.db.sql(sql, parameters)
+    return tracer.trace("db.sql", async span => {
+      span.addTags({ db_name: this.name, num_bindings: parameters?.length })
+      const resp = await this.db.sql<T>(sql, parameters)
+      span.addTags({ num_rows: resp.length })
+      return resp
     })
   }
 
   sqlPurgeDocument(docIds: string[] | string): Promise<void> {
     return tracer.trace("db.sqlPurgeDocument", span => {
-      span?.addTags({ db_name: this.name })
+      span.addTags({
+        db_name: this.name,
+        num_docs: Array.isArray(docIds) ? docIds.length : 1,
+      })
       return this.db.sqlPurgeDocument(docIds)
     })
   }
 
   sqlDiskCleanup(): Promise<void> {
     return tracer.trace("db.sqlDiskCleanup", span => {
-      span?.addTags({ db_name: this.name })
+      span.addTags({ db_name: this.name })
       return this.db.sqlDiskCleanup()
     })
   }

packages/backend-core/src/environment.ts

@@ -1,6 +1,7 @@
 import { existsSync, readFileSync } from "fs"
 import { ServiceType } from "@budibase/types"
 import { cloneDeep } from "lodash"
+import { createSecretKey } from "crypto"
 
 function isTest() {
   return isJest()
@@ -18,6 +19,12 @@ function isDev() {
   return process.env.NODE_ENV !== "production"
 }
 
+function parseIntSafe(number?: string) {
+  if (number) {
+    return parseInt(number)
+  }
+}
+
 let LOADED = false
 if (!LOADED && isDev() && !isTest()) {
   require("dotenv").config()
@@ -126,8 +133,12 @@ const environment = {
   },
   BUDIBASE_ENVIRONMENT: process.env.BUDIBASE_ENVIRONMENT,
   JS_BCRYPT: process.env.JS_BCRYPT,
-  JWT_SECRET: process.env.JWT_SECRET,
-  JWT_SECRET_FALLBACK: process.env.JWT_SECRET_FALLBACK,
+  JWT_SECRET: process.env.JWT_SECRET
+    ? createSecretKey(Buffer.from(process.env.JWT_SECRET))
+    : undefined,
+  JWT_SECRET_FALLBACK: process.env.JWT_SECRET_FALLBACK
+    ? createSecretKey(Buffer.from(process.env.JWT_SECRET_FALLBACK))
+    : undefined,
   ENCRYPTION_KEY: process.env.ENCRYPTION_KEY,
   API_ENCRYPTION_KEY: getAPIEncryptionKey(),
   COUCH_DB_URL: process.env.COUCH_DB_URL || "http://localhost:4005",
@@ -225,6 +236,8 @@ const environment = {
   OPENAI_API_KEY: process.env.OPENAI_API_KEY,
   MIN_VERSION_WITHOUT_POWER_ROLE:
     process.env.MIN_VERSION_WITHOUT_POWER_ROLE || "3.0.0",
+  DISABLE_CONTENT_SECURITY_POLICY: process.env.DISABLE_CONTENT_SECURITY_POLICY,
+  BSON_BUFFER_SIZE: parseIntSafe(process.env.BSON_BUFFER_SIZE),
 }
 
 export function setEnv(newEnvVars: Partial<typeof environment>): () => void {

packages/backend-core/src/features/features.ts

@@ -267,12 +267,10 @@ export class FlagSet<V extends Flag<any>, T extends { [key: string]: V }> {
 // All of the machinery in this file is to make sure that flags have their
 // default values set correctly and their types flow through the system.
 export const flags = new FlagSet({
-  [FeatureFlag.DEFAULT_VALUES]: Flag.boolean(env.isDev()),
-  [FeatureFlag.AUTOMATION_BRANCHING]: Flag.boolean(env.isDev()),
-  [FeatureFlag.SQS]: Flag.boolean(true),
-  [FeatureFlag.AI_CUSTOM_CONFIGS]: Flag.boolean(env.isDev()),
-  [FeatureFlag.ENRICHED_RELATIONSHIPS]: Flag.boolean(env.isDev()),
-  [FeatureFlag.BUDIBASE_AI]: Flag.boolean(env.isDev()),
+  [FeatureFlag.DEFAULT_VALUES]: Flag.boolean(true),
+  [FeatureFlag.AUTOMATION_BRANCHING]: Flag.boolean(true),
+  [FeatureFlag.AI_CUSTOM_CONFIGS]: Flag.boolean(true),
+  [FeatureFlag.BUDIBASE_AI]: Flag.boolean(true),
 })
 
 type UnwrapPromise<T> = T extends Promise<infer U> ? U : T

packages/backend-core/src/middleware/contentSecurityPolicy.ts

@@ -0,0 +1,113 @@
+import crypto from "crypto"
+
+const CSP_DIRECTIVES = {
+  "default-src": ["'self'"],
+  "script-src": [
+    "'self'",
+    "'unsafe-eval'",
+    "https://*.budibase.net",
+    "https://cdn.budi.live",
+    "https://js.intercomcdn.com",
+    "https://widget.intercom.io",
+    "https://d2l5prqdbvm3op.cloudfront.net",
+    "https://us-assets.i.posthog.com",
+  ],
+  "style-src": [
+    "'self'",
+    "'unsafe-inline'",
+    "https://cdn.jsdelivr.net",
+    "https://fonts.googleapis.com",
+    "https://rsms.me",
+    "https://maxcdn.bootstrapcdn.com",
+  ],
+  "object-src": ["'none'"],
+  "base-uri": ["'self'"],
+  "connect-src": [
+    "'self'",
+    "https://*.budibase.app",
+    "https://*.budibaseqa.app",
+    "https://*.budibase.net",
+    "https://api-iam.intercom.io",
+    "https://api-ping.intercom.io",
+    "https://app.posthog.com",
+    "https://us.i.posthog.com",
+    "wss://nexus-websocket-a.intercom.io",
+    "wss://nexus-websocket-b.intercom.io",
+    "https://nexus-websocket-a.intercom.io",
+    "https://nexus-websocket-b.intercom.io",
+    "https://uploads.intercomcdn.com",
+    "https://uploads.intercomusercontent.com",
+    "https://*.amazonaws.com",
+    "https://*.s3.amazonaws.com",
+    "https://*.s3.us-east-2.amazonaws.com",
+    "https://*.s3.us-east-1.amazonaws.com",
+    "https://*.s3.us-west-1.amazonaws.com",
+    "https://*.s3.us-west-2.amazonaws.com",
+    "https://*.s3.af-south-1.amazonaws.com",
+    "https://*.s3.ap-east-1.amazonaws.com",
+    "https://*.s3.ap-south-1.amazonaws.com",
+    "https://*.s3.ap-northeast-2.amazonaws.com",
+    "https://*.s3.ap-southeast-1.amazonaws.com",
+    "https://*.s3.ap-southeast-2.amazonaws.com",
+    "https://*.s3.ap-northeast-1.amazonaws.com",
+    "https://*.s3.ca-central-1.amazonaws.com",
+    "https://*.s3.cn-north-1.amazonaws.com",
+    "https://*.s3.cn-northwest-1.amazonaws.com",
+    "https://*.s3.eu-central-1.amazonaws.com",
+    "https://*.s3.eu-west-1.amazonaws.com",
+    "https://*.s3.eu-west-2.amazonaws.com",
+    "https://*.s3.eu-south-1.amazonaws.com",
+    "https://*.s3.eu-west-3.amazonaws.com",
+    "https://*.s3.eu-north-1.amazonaws.com",
+    "https://*.s3.sa-east-1.amazonaws.com",
+    "https://*.s3.me-south-1.amazonaws.com",
+    "https://*.s3.us-gov-east-1.amazonaws.com",
+    "https://*.s3.us-gov-west-1.amazonaws.com",
+    "https://api.github.com",
+  ],
+  "font-src": [
+    "'self'",
+    "data:",
+    "https://cdn.jsdelivr.net",
+    "https://fonts.gstatic.com",
+    "https://rsms.me",
+    "https://maxcdn.bootstrapcdn.com",
+    "https://js.intercomcdn.com",
+    "https://fonts.intercomcdn.com",
+  ],
+  "frame-src": ["'self'", "https:"],
+  "img-src": ["http:", "https:", "data:", "blob:"],
+  "manifest-src": ["'self'"],
+  "media-src": [
+    "'self'",
+    "https://js.intercomcdn.com",
+    "https://cdn.budi.live",
+  ],
+  "worker-src": ["blob:"],
+}
+
+export async function contentSecurityPolicy(ctx: any, next: any) {
+  try {
+    const nonce = crypto.randomBytes(16).toString("base64")
+
+    const directives = { ...CSP_DIRECTIVES }
+    directives["script-src"] = [
+      ...CSP_DIRECTIVES["script-src"],
+      `'nonce-${nonce}'`,
+    ]
+
+    ctx.state.nonce = nonce
+
+    const cspHeader = Object.entries(directives)
+      .map(([key, sources]) => `${key} ${sources.join(" ")}`)
+      .join("; ")
+    ctx.set("Content-Security-Policy", cspHeader)
+    await next()
+  } catch (err: any) {
+    console.error(
+      `Error occurred in Content-Security-Policy middleware: ${err}`
+    )
+  }
+}
+
+export default contentSecurityPolicy

packages/backend-core/src/middleware/index.ts

@@ -19,5 +19,6 @@ export { default as pino } from "../logging/pino/middleware"
 export { default as correlation } from "../logging/correlation/middleware"
 export { default as errorHandling } from "./errorHandling"
 export { default as querystringToBody } from "./querystringToBody"
+export { default as csp } from "./contentSecurityPolicy"
 export * as joiValidator from "./joi-validator"
 export { default as ip } from "./ip"

packages/backend-core/src/middleware/tests/contentSecurityPolicy.spec.ts

@@ -0,0 +1,75 @@
+import crypto from "crypto"
+import contentSecurityPolicy from "../contentSecurityPolicy"
+
+jest.mock("crypto", () => ({
+  randomBytes: jest.fn(),
+  randomUUID: jest.fn(),
+}))
+
+describe("contentSecurityPolicy middleware", () => {
+  let ctx: any
+  let next: any
+  const mockNonce = "mocked/nonce"
+
+  beforeEach(() => {
+    ctx = {
+      state: {},
+      set: jest.fn(),
+    }
+    next = jest.fn()
+    // @ts-ignore
+    crypto.randomBytes.mockReturnValue(Buffer.from(mockNonce, "base64"))
+  })
+
+  afterEach(() => {
+    jest.clearAllMocks()
+  })
+
+  it("should generate a nonce and set it in the script-src directive", async () => {
+    await contentSecurityPolicy(ctx, next)
+
+    expect(ctx.state.nonce).toBe(mockNonce)
+    expect(ctx.set).toHaveBeenCalledWith(
+      "Content-Security-Policy",
+      expect.stringContaining(
+        `script-src 'self' 'unsafe-eval' https://*.budibase.net https://cdn.budi.live https://js.intercomcdn.com https://widget.intercom.io https://d2l5prqdbvm3op.cloudfront.net https://us-assets.i.posthog.com 'nonce-${mockNonce}'`
+      )
+    )
+    expect(next).toHaveBeenCalled()
+  })
+
+  it("should include all CSP directives in the header", async () => {
+    await contentSecurityPolicy(ctx, next)
+
+    const cspHeader = ctx.set.mock.calls[0][1]
+    expect(cspHeader).toContain("default-src 'self'")
+    expect(cspHeader).toContain("script-src 'self' 'unsafe-eval'")
+    expect(cspHeader).toContain("style-src 'self' 'unsafe-inline'")
+    expect(cspHeader).toContain("object-src 'none'")
+    expect(cspHeader).toContain("base-uri 'self'")
+    expect(cspHeader).toContain("connect-src 'self'")
+    expect(cspHeader).toContain("font-src 'self'")
+    expect(cspHeader).toContain("frame-src 'self'")
+    expect(cspHeader).toContain("img-src http: https: data: blob:")
+    expect(cspHeader).toContain("manifest-src 'self'")
+    expect(cspHeader).toContain("media-src 'self'")
+    expect(cspHeader).toContain("worker-src blob:")
+  })
+
+  it("should handle errors and log an error message", async () => {
+    const consoleSpy = jest.spyOn(console, "error").mockImplementation()
+    const error = new Error("Test error")
+    // @ts-ignore
+    crypto.randomBytes.mockImplementation(() => {
+      throw error
+    })
+
+    await contentSecurityPolicy(ctx, next)
+
+    expect(consoleSpy).toHaveBeenCalledWith(
+      `Error occurred in Content-Security-Policy middleware: ${error}`
+    )
+    expect(next).not.toHaveBeenCalled()
+    consoleSpy.mockRestore()
+  })
+})

packages/backend-core/src/queue/inMemoryQueue.ts

@@ -1,5 +1,5 @@
 import events from "events"
-import { newid, timeout } from "../utils"
+import { newid } from "../utils"
 import { Queue, QueueOptions, JobOptions } from "./queue"
 
 interface JobMessage {
@@ -141,7 +141,7 @@ class InMemoryQueue implements Partial<Queue> {
     } else {
       pushMessage()
     }
-    return {} as any
+    return { id: jobId } as any
   }
 
   /**
@@ -184,16 +184,6 @@ class InMemoryQueue implements Partial<Queue> {
     // do nothing
     return this as any
   }
-
-  async waitForCompletion() {
-    do {
-      await timeout(50)
-    } while (this.hasRunningJobs())
-  }
-
-  hasRunningJobs() {
-    return this._addCount > this._runCount
-  }
 }
 
 export default InMemoryQueue

packages/backend-core/src/queue/queue.ts

@@ -15,7 +15,7 @@ const QUEUE_LOCK_MS = Duration.fromMinutes(5).toMs()
 const QUEUE_LOCK_RENEW_INTERNAL_MS = Duration.fromSeconds(30).toMs()
 // cleanup the queue every 60 seconds
 const CLEANUP_PERIOD_MS = Duration.fromSeconds(60).toMs()
-let QUEUES: BullQueue.Queue[] | InMemoryQueue[] = []
+let QUEUES: BullQueue.Queue[] = []
 let cleanupInterval: NodeJS.Timeout
 
 async function cleanup() {
@@ -45,11 +45,18 @@ export function createQueue<T>(
   if (opts.jobOptions) {
     queueConfig.defaultJobOptions = opts.jobOptions
   }
-  let queue: any
+  let queue: BullQueue.Queue<T>
   if (!env.isTest()) {
     queue = new BullQueue(jobQueue, queueConfig)
+  } else if (
+    process.env.BULL_TEST_REDIS_PORT &&
+    !isNaN(+process.env.BULL_TEST_REDIS_PORT)
+  ) {
+    queue = new BullQueue(jobQueue, {
+      redis: { host: "localhost", port: +process.env.BULL_TEST_REDIS_PORT },
+    })
   } else {
-    queue = new InMemoryQueue(jobQueue, queueConfig)
+    queue = new InMemoryQueue(jobQueue, queueConfig) as any
   }
   addListeners(queue, jobQueue, opts?.removeStalledCb)
   QUEUES.push(queue)

packages/backend-core/src/security/tests/encryption.spec.ts

@@ -4,7 +4,7 @@ import env from "../../environment"
 describe("encryption", () => {
   it("should throw an error if API encryption key is not set", () => {
     const jwt = getSecret(SecretOption.API)
-    expect(jwt).toBe(env.JWT_SECRET)
+    expect(jwt).toBe(env.JWT_SECRET?.export().toString())
   })
 
   it("should throw an error if encryption key is not set", () => {

packages/backend-core/tests/core/utilities/index.ts

@@ -4,3 +4,4 @@ export { generator } from "./structures"
 export * as testContainerUtils from "./testContainerUtils"
 export * as utils from "./utils"
 export * from "./jestUtils"
+export * as queue from "./queue"

packages/backend-core/tests/core/utilities/queue.ts

@@ -0,0 +1,9 @@
+import { Queue } from "bull"
+
+export async function processMessages(queue: Queue) {
+  do {
+    await queue.whenCurrentJobsFinished()
+  } while (await queue.count())
+
+  await queue.whenCurrentJobsFinished()
+}

packages/backend-core/tests/core/utilities/testContainerUtils.ts

@@ -1,4 +1,6 @@
 import { execSync } from "child_process"
+import { cloneDeep } from "lodash"
+import { GenericContainer, StartedTestContainer } from "testcontainers"
 
 const IPV4_PORT_REGEX = new RegExp(`0\\.0\\.0\\.0:(\\d+)->(\\d+)/tcp`, "g")
 
@@ -106,3 +108,58 @@ export function setupEnv(...envs: any[]) {
     }
   }
 }
+
+export async function startContainer(container: GenericContainer) {
+  const imageName = (container as any).imageName.string as string
+  let key: string = imageName
+  if (imageName.includes("@sha256")) {
+    key = imageName.split("@")[0]
+  }
+  key = key.replace(/\//g, "-").replace(/:/g, "-")
+
+  container = container
+    .withReuse()
+    .withLabels({ "com.budibase": "true" })
+    .withName(`${key}_testcontainer`)
+
+  let startedContainer: StartedTestContainer | undefined = undefined
+  let lastError = undefined
+  for (let i = 0; i < 10; i++) {
+    try {
+      // container.start() is not an idempotent operation, calling `start`
+      // modifies the internal state of a GenericContainer instance such that
+      // the hash it uses to determine reuse changes. We need to clone the
+      // container before calling start to ensure that we're using the same
+      // reuse hash every time.
+      const containerCopy = cloneDeep(container)
+      startedContainer = await containerCopy.start()
+      lastError = undefined
+      break
+    } catch (e: any) {
+      lastError = e
+      await new Promise(resolve => setTimeout(resolve, 1000))
+    }
+  }
+
+  if (!startedContainer) {
+    if (lastError) {
+      throw lastError
+    }
+    throw new Error(`failed to start container: ${imageName}`)
+  }
+
+  const info = getContainerById(startedContainer.getId())
+  if (!info) {
+    throw new Error("Container not found")
+  }
+
+  // Some Docker runtimes, when you expose a port, will bind it to both
+  // 127.0.0.1 and ::1, so ipv4 and ipv6. The port spaces of ipv4 and ipv6
+  // addresses are not shared, and testcontainers will sometimes give you back
+  // the ipv6 port. There's no way to know that this has happened, and if you
+  // try to then connect to `localhost:port` you may attempt to bind to the v4
+  // address which could be unbound or even an entirely different container. For
+  // that reason, we don't use testcontainers' `getExposedPort` function,
+  // preferring instead our own method that guaranteed v4 ports.
+  return getExposedV4Ports(info)
+}

packages/backend-core/tests/core/utilities/utils/index.ts

@@ -1 +1,2 @@
 export * as time from "./time"
+export * as queue from "./queue"

packages/backend-core/tests/core/utilities/utils/queue.ts

@@ -0,0 +1,27 @@
+import { Queue } from "bull"
+import { GenericContainer, Wait } from "testcontainers"
+import { startContainer } from "../testContainerUtils"
+
+export async function useRealQueues() {
+  const ports = await startContainer(
+    new GenericContainer("redis")
+      .withExposedPorts(6379)
+      .withWaitStrategy(
+        Wait.forSuccessfulCommand(`redis-cli`).withStartupTimeout(10000)
+      )
+  )
+
+  const port = ports.find(x => x.container === 6379)?.host
+  if (!port) {
+    throw new Error("Redis port not found")
+  }
+  process.env.BULL_TEST_REDIS_PORT = port.toString()
+}
+
+export async function processMessages(queue: Queue) {
+  do {
+    await queue.whenCurrentJobsFinished()
+  } while (await queue.count())
+
+  await queue.whenCurrentJobsFinished()
+}

packages/bbui/package.json

@@ -81,6 +81,7 @@
     "@spectrum-css/typography": "3.0.1",
     "@spectrum-css/underlay": "2.0.9",
     "@spectrum-css/vars": "3.0.1",
+    "atrament": "^4.3.0",
     "dayjs": "^1.10.8",
     "easymde": "^2.16.1",
     "svelte-dnd-action": "^0.9.8",

packages/bbui/src/Form/Core/Dropzone.svelte

@@ -8,6 +8,7 @@
   import Link from "../../Link/Link.svelte"
   import Tag from "../../Tags/Tag.svelte"
   import Tags from "../../Tags/Tags.svelte"
+  import ProgressCircle from "../../ProgressCircle/ProgressCircle.svelte"
 
   const BYTES_IN_KB = 1000
   const BYTES_IN_MB = 1000000
@@ -39,12 +40,14 @@
     "jfif",
     "webp",
   ]
-
   const fieldId = id || uuid()
+
   let selectedImageIdx = 0
   let fileDragged = false
   let selectedUrl
   let fileInput
+  let loading = false
+
   $: selectedImage = value?.[selectedImageIdx] ?? null
   $: fileCount = value?.length ?? 0
   $: isImage =
@@ -86,10 +89,15 @@
     }
 
     if (processFiles) {
+      loading = true
+      try {
         const processedFiles = await processFiles(fileList)
         const newValue = [...value, ...processedFiles]
         dispatch("change", newValue)
         selectedImageIdx = newValue.length - 1
+      } finally {
+        loading = false
+      }
     } else {
       dispatch("change", fileList)
     }
@@ -227,7 +235,7 @@
   {#if showDropzone}
     <div
       class="spectrum-Dropzone"
-      class:disabled
+      class:disabled={disabled || loading}
       role="region"
       tabindex="0"
       on:dragover={handleDragOver}
@@ -241,7 +249,7 @@
           id={fieldId}
           {disabled}
           type="file"
-          multiple
+          multiple={maximum !== 1}
           accept={extensions}
           bind:this={fileInput}
           on:change={handleFile}
@@ -339,6 +347,12 @@
           {/if}
         {/if}
       </div>
+
+      {#if loading}
+        <div class="loading">
+          <ProgressCircle size="M" />
+        </div>
+      {/if}
     </div>
   {/if}
 </div>
@@ -464,6 +478,7 @@
 
   .spectrum-Dropzone {
     height: 220px;
+    position: relative;
   }
   .compact .spectrum-Dropzone {
     height: 40px;
@@ -488,4 +503,14 @@
   .tag {
     margin-top: 8px;
   }
+
+  .loading {
+    position: absolute;
+    display: grid;
+    place-items: center;
+    height: 100%;
+    width: 100%;
+    top: 0;
+    left: 0;
+  }
 </style>

packages/builder/src/components/automation/AutomationBuilder/AutomationBuilder.svelte

@@ -3,8 +3,8 @@
   import Flowchart from "./FlowChart/FlowChart.svelte"
 </script>
 
-{#if $selectedAutomation}
-  {#key $selectedAutomation._id}
-    <Flowchart automation={$selectedAutomation} />
+{#if $selectedAutomation?.data}
+  {#key $selectedAutomation.data._id}
+    <Flowchart automation={$selectedAutomation.data} />
   {/key}
 {/if}

packages/builder/src/components/automation/AutomationBuilder/DraggableCanvas.svelte

@@ -0,0 +1,497 @@
+<script>
+  import { writable } from "svelte/store"
+  import {
+    setContext,
+    onMount,
+    createEventDispatcher,
+    onDestroy,
+    tick,
+  } from "svelte"
+  import { Utils } from "@budibase/frontend-core"
+  import { selectedAutomation, automationStore } from "stores/builder"
+
+  export function zoomIn() {
+    const scale = Number(Math.min($view.scale + 0.1, 1.5).toFixed(2))
+    view.update(state => ({
+      ...state,
+      scale,
+    }))
+  }
+
+  export function zoomOut() {
+    const scale = Number(Math.max($view.scale - 0.1, 0).toFixed(2))
+    view.update(state => ({
+      ...state,
+      scale,
+    }))
+  }
+
+  export async function reset() {
+    contentDims = {
+      ...contentDims,
+      w: contentDims.original.w,
+      h: contentDims.original.h,
+    }
+    dragOffset = []
+    contentPos.update(state => ({
+      ...state,
+      x: 0,
+      y: 0,
+    }))
+    view.update(state => ({
+      ...state,
+      scale: 1,
+    }))
+  }
+
+  export async function zoomToFit() {
+    const { width: wViewPort, height: hViewPort } =
+      viewPort.getBoundingClientRect()
+
+    const scaleTarget = Math.min(
+      wViewPort / contentDims.original.w,
+      hViewPort / contentDims.original.h
+    )
+
+    // Smallest ratio determines which dimension needs squeezed
+    view.update(state => ({
+      ...state,
+      scale: scaleTarget,
+    }))
+
+    await tick()
+
+    const adjustedY = (hViewPort - contentDims.original.h) / 2
+
+    contentPos.update(state => ({
+      ...state,
+      x: 0,
+      y: parseInt(0 + adjustedY),
+    }))
+  }
+
+  const dispatch = createEventDispatcher()
+
+  // View State
+  const view = writable({
+    dragging: false,
+    moveStep: null,
+    dragSpot: null,
+    scale: 1,
+    dropzones: {},
+    //focus - node to center on?
+  })
+
+  setContext("draggableView", view)
+
+  // View internal pos tracking
+  const internalPos = writable({ x: 0, y: 0 })
+  setContext("viewPos", internalPos)
+
+  // Content pos tracking
+  const contentPos = writable({ x: 0, y: 0, scrollX: 0, scrollY: 0 })
+  setContext("contentPos", contentPos)
+
+  // Elements
+  let mainContent
+  let viewPort
+  let contentWrap
+
+  // Mouse down
+  let down = false
+
+  // Monitor the size of the viewPort
+  let observer
+
+  // Size of the core display content
+  let contentDims = {}
+
+  // Size of the view port
+  let viewDims = {}
+
+  // When dragging the content, maintain the drag start offset
+  let dragOffset
+
+  // Used when focusing the UI on trigger
+  let loaded = false
+
+  // Edge around the draggable content
+  let contentDragPadding = 200
+
+  const onScale = async () => {
+    dispatch("zoom", $view.scale)
+    await getDims()
+  }
+
+  const getDims = async () => {
+    if (!mainContent) return
+
+    if (!contentDims.original) {
+      contentDims.original = {
+        w: parseInt(mainContent.getBoundingClientRect().width),
+        h: parseInt(mainContent.getBoundingClientRect().height),
+      }
+    }
+
+    viewDims = viewPort.getBoundingClientRect()
+
+    contentDims = {
+      ...contentDims,
+      w: contentDims.original.w * $view.scale,
+      h: contentDims.original.h * $view.scale,
+    }
+  }
+
+  const eleXY = (coords, ele) => {
+    const { clientX, clientY } = coords
+    const rect = ele.getBoundingClientRect()
+    const x = Math.round(clientX - rect.left)
+    const y = Math.round(clientY - rect.top)
+    return { x: Math.max(x, 0), y: Math.max(y, 0) }
+  }
+
+  const buildWrapStyles = (pos, scale, dims) => {
+    const { x, y } = pos
+    const { w, h } = dims
+    return `--posX: ${x}px; --posY: ${y}px; --scale: ${scale}; --wrapH: ${h}px; --wrapW: ${w}px`
+  }
+
+  const onViewScroll = e => {
+    e.preventDefault()
+    let currentScale = $view.scale
+    let scrollIncrement = 35
+    let xBump = 0
+    let yBump = 0
+
+    if (e.shiftKey) {
+      // Scroll horizontal - Needs Limits
+      xBump = scrollIncrement * (e.deltaX < 0 ? -1 : 1)
+      contentPos.update(state => ({
+        ...state,
+        x: state.x - xBump,
+        y: state.y,
+        // If scrolling *and* dragging, maintain a record of the scroll offset
+        ...($view.dragging
+          ? {
+              scrollX: state.scrollX - xBump,
+            }
+          : {}),
+      }))
+    } else if (e.ctrlKey || e.metaKey) {
+      // Scale the content on scrolling
+      let updatedScale
+      if (e.deltaY < 0) {
+        updatedScale = Math.min(1, currentScale + 0.05)
+      } else if (e.deltaY > 0) {
+        updatedScale = Math.max(0, currentScale - 0.05)
+      }
+
+      view.update(state => ({
+        ...state,
+        scale: Number(updatedScale.toFixed(2)),
+      }))
+    } else {
+      yBump = scrollIncrement * (e.deltaY < 0 ? -1 : 1)
+      contentPos.update(state => ({
+        ...state,
+        x: state.x,
+        y: state.y - yBump,
+        // If scrolling *and* dragging, maintain a record of the scroll offset
+        ...($view.dragging
+          ? {
+              scrollY: state.scrollY - yBump,
+            }
+          : {}),
+      }))
+    }
+  }
+
+  // Optimization options
+  const onViewMouseMove = async e => {
+    if (!viewPort) {
+      return
+    }
+
+    const { x, y } = eleXY(e, viewPort)
+
+    internalPos.update(() => ({
+      x,
+      y,
+    }))
+
+    if (down && !$view.dragging && dragOffset) {
+      contentPos.update(state => ({
+        ...state,
+        x: x - dragOffset[0],
+        y: y - dragOffset[1],
+      }))
+    }
+  }
+
+  const onViewDragEnd = () => {
+    down = false
+    dragOffset = [0, 0]
+  }
+
+  const handleDragDrop = () => {
+    const sourceBlock = $selectedAutomation.blockRefs[$view.moveStep.id]
+    const sourcePath = sourceBlock.pathTo
+
+    const dropZone = $view.dropzones[$view.droptarget]
+    const destPath = dropZone?.path
+
+    automationStore.actions.moveBlock(
+      sourcePath,
+      destPath,
+      $selectedAutomation.data
+    )
+  }
+
+  const onMouseUp = () => {
+    if ($view.droptarget) {
+      handleDragDrop()
+    }
+
+    view.update(state => ({
+      ...state,
+      dragging: false,
+      moveStep: null,
+      dragSpot: null,
+      dropzones: {},
+      droptarget: null,
+    }))
+
+    // Clear the scroll offset for dragging
+    contentPos.update(state => ({
+      ...state,
+      scrollY: 0,
+      scrollX: 0,
+    }))
+  }
+
+  const onMouseMove = async e => {
+    if (!viewPort) {
+      return
+    }
+    // Update viewDims to get the latest viewport dimensions
+    viewDims = viewPort.getBoundingClientRect()
+
+    if ($view.moveStep && $view.dragging === false) {
+      view.update(state => ({
+        ...state,
+        dragging: true,
+      }))
+    }
+
+    const checkIntersection = (pos, dzRect) => {
+      return (
+        pos.x < dzRect.right &&
+        pos.x > dzRect.left &&
+        pos.y < dzRect.bottom &&
+        pos.y > dzRect.top
+      )
+    }
+
+    if ($view.dragging) {
+      const adjustedX =
+        (e.clientX - viewDims.left - $view.moveStep.offsetX) / $view.scale
+      const adjustedY =
+        (e.clientY - viewDims.top - $view.moveStep.offsetY) / $view.scale
+
+      view.update(state => ({
+        ...state,
+        dragSpot: {
+          x: adjustedX,
+          y: adjustedY,
+        },
+      }))
+    }
+
+    if ($view.moveStep && $view.dragging) {
+      let hovering = false
+      Object.entries($view.dropzones).forEach(entry => {
+        const [dzKey, dz] = entry
+        if (checkIntersection({ x: e.clientX, y: e.clientY }, dz.dims)) {
+          hovering = true
+          view.update(state => ({
+            ...state,
+            droptarget: dzKey,
+          }))
+        }
+      })
+      // Ensure that when it stops hovering, it clears the drop target
+      if (!hovering) {
+        view.update(state => ({
+          ...state,
+          droptarget: null,
+        }))
+      }
+    }
+  }
+
+  const onMoveContent = e => {
+    if (down || !viewPort) {
+      return
+    }
+    const { x, y } = eleXY(e, viewPort)
+
+    dragOffset = [Math.abs(x - $contentPos.x), Math.abs(y - $contentPos.y)]
+  }
+
+  const focusOnLoad = () => {
+    if ($view.focusEle && !loaded) {
+      const focusEleDims = $view.focusEle
+      const viewWidth = viewDims.width
+
+      // The amount to shift the content in order to center the trigger on load.
+      // The content is also padded with `contentDragPadding`
+      // The sidebar offset factors into the left positioning of the content here.
+      const targetX =
+        contentWrap.getBoundingClientRect().x -
+        focusEleDims.x +
+        (viewWidth / 2 - focusEleDims.width / 2)
+
+      // Update the content position state
+      // Shift the content up slightly to accommodate the padding
+      contentPos.update(state => ({
+        ...state,
+        x: targetX,
+        y: -(contentDragPadding / 2),
+      }))
+
+      loaded = true
+    }
+  }
+
+  // Update dims after scaling
+  $: {
+    $view.scale
+    onScale()
+  }
+
+  // Focus on a registered element
+  $: {
+    $view.focusEle
+    focusOnLoad()
+  }
+
+  // Content mouse pos and scale to css variables.
+  // The wrap is set to match the content size
+  $: wrapStyles = buildWrapStyles($contentPos, $view.scale, contentDims)
+
+  onMount(() => {
+    observer = new ResizeObserver(getDims)
+    observer.observe(viewPort)
+  })
+
+  onDestroy(() => {
+    observer.disconnect()
+  })
+</script>
+
+<!-- svelte-ignore a11y-no-static-element-interactions -->
+<!-- svelte-ignore a11y-no-noninteractive-element-interactions -->
+<div
+  class="draggable-canvas"
+  role="region"
+  aria-label="Viewport for building automations"
+  on:mouseup={onMouseUp}
+  on:mousemove={Utils.domDebounce(onMouseMove)}
+  style={`--dragPadding: ${contentDragPadding}px;`}
+>
+  <div
+    class="draggable-view"
+    bind:this={viewPort}
+    on:wheel={Utils.domDebounce(onViewScroll)}
+    on:mousemove={Utils.domDebounce(onViewMouseMove)}
+    on:mouseup={onViewDragEnd}
+    on:mouseleave={onViewDragEnd}
+  >
+    <!-- <div class="debug">
+      <span>
+        View Pos [{$internalPos.x}, {$internalPos.y}]
+      </span>
+      <span>View Dims [{viewDims.width}, {viewDims.height}]</span>
+      <span>Mouse Down [{down}]</span>
+      <span>Drag [{$view.dragging}]</span>
+      <span>Dragging [{$view?.moveStep?.id || "no"}]</span>
+      <span>Scale [{$view.scale}]</span>
+      <span>Content [{JSON.stringify($contentPos)}]</span>
+    </div> -->
+    <div
+      class="content-wrap"
+      style={wrapStyles}
+      bind:this={contentWrap}
+      class:dragging={down}
+    >
+      <div
+        class="content"
+        bind:this={mainContent}
+        on:mousemove={Utils.domDebounce(onMoveContent)}
+        on:mousedown={e => {
+          if (e.which === 1 || e.button === 0) {
+            down = true
+          }
+        }}
+        on:mouseup={e => {
+          if (e.which === 1 || e.button === 0) {
+            down = false
+          }
+        }}
+        on:mouseleave={() => {
+          down = false
+          view.update(state => ({
+            ...state,
+            dragging: false,
+            moveStep: null,
+            dragSpot: null,
+            dropzones: {},
+          }))
+        }}
+      >
+        <slot name="content" />
+      </div>
+    </div>
+  </div>
+</div>
+
+<style>
+  .draggable-canvas {
+    width: 100%;
+    height: 100%;
+  }
+  .draggable-view {
+    width: 100%;
+    height: 100%;
+    overflow: hidden;
+    position: relative;
+  }
+  .content-wrap {
+    min-width: 100%;
+    position: absolute;
+    top: 0;
+    left: 0;
+    width: var(--wrapW);
+    height: var(--wrapH);
+    cursor: grab;
+    transform: translate(var(--posX), var(--posY));
+  }
+  .content {
+    transform: scale(var(--scale));
+    user-select: none;
+    padding: var(--dragPadding);
+  }
+
+  .content-wrap.dragging {
+    cursor: grabbing;
+  }
+
+  /* .debug {
+    display: flex;
+    align-items: center;
+    gap: 8px;
+    position: fixed;
+    padding: 8px;
+    z-index: 2;
+  } */
+</style>

packages/builder/src/components/automation/AutomationBuilder/FlowChart/ActionModal.svelte

@@ -9,27 +9,42 @@
     Tags,
     Tag,
   } from "@budibase/bbui"
+  import { AutomationActionStepId } from "@budibase/types"
   import { automationStore, selectedAutomation } from "stores/builder"
   import { admin, licensing } from "stores/portal"
   import { externalActions } from "./ExternalActions"
   import { TriggerStepID, ActionStepID } from "constants/backend/automations"
-  import { checkForCollectStep } from "helpers/utils"
 
-  export let blockIdx
-  export let lastStep
+  export let block
   export let modal
 
   let syncAutomationsEnabled = $licensing.syncAutomationsEnabled
   let triggerAutomationRunEnabled = $licensing.triggerAutomationRunEnabled
   let collectBlockAllowedSteps = [TriggerStepID.APP, TriggerStepID.WEBHOOK]
   let selectedAction
-  let actions = Object.entries($automationStore.blockDefinitions.ACTION)
+  let actions = Object.entries($automationStore.blockDefinitions.ACTION).filter(
+    entry => {
+      const [key] = entry
+      return key !== AutomationActionStepId.BRANCH
+    }
+  )
   let lockedFeatures = [
     ActionStepID.COLLECT,
     ActionStepID.TRIGGER_AUTOMATION_RUN,
   ]
 
-  $: collectBlockExists = checkForCollectStep($selectedAutomation)
+  $: blockRef = $selectedAutomation.blockRefs?.[block.id]
+  $: lastStep = blockRef?.terminating
+  $: pathSteps = block.id
+    ? automationStore.actions.getPathSteps(
+        blockRef.pathTo,
+        $selectedAutomation?.data
+      )
+    : []
+
+  $: collectBlockExists = pathSteps?.some(
+    step => step.stepId === ActionStepID.COLLECT
+  )
 
   const disabled = () => {
     return {
@@ -75,7 +90,7 @@
     // Filter out Collect block if not App Action or Webhook
     if (
       !collectBlockAllowedSteps.includes(
-        $selectedAutomation.definition.trigger.stepId
+        $selectedAutomation.data.definition.trigger.stepId
       )
     ) {
       delete acc.COLLECT
@@ -100,9 +115,14 @@
         action.stepId,
         action
       )
-      await automationStore.actions.addBlockToAutomation(newBlock, blockIdx + 1)
+
+      await automationStore.actions.addBlockToAutomation(
+        newBlock,
+        blockRef ? blockRef.pathTo : block.pathTo
+      )
       modal.hide()
     } catch (error) {
+      console.error(error)
       notifications.error("Error saving automation")
     }
   }

packages/builder/src/components/automation/AutomationBuilder/FlowChart/BranchNode.svelte

@@ -0,0 +1,283 @@
+<script>
+  import FilterBuilder from "components/design/settings/controls/FilterEditor/FilterBuilder.svelte"
+  import {
+    Drawer,
+    DrawerContent,
+    ActionButton,
+    Icon,
+    Layout,
+    Body,
+    Divider,
+    TooltipPosition,
+    TooltipType,
+    Button,
+    Modal,
+    ModalContent,
+  } from "@budibase/bbui"
+  import PropField from "components/automation/SetupPanel/PropField.svelte"
+  import AutomationBindingPanel from "components/common/bindings/ServerBindingPanel.svelte"
+  import FlowItemHeader from "./FlowItemHeader.svelte"
+  import FlowItemActions from "./FlowItemActions.svelte"
+  import { automationStore, selectedAutomation } from "stores/builder"
+  import { QueryUtils } from "@budibase/frontend-core"
+  import { cloneDeep } from "lodash/fp"
+  import { createEventDispatcher, getContext } from "svelte"
+  import DragZone from "./DragZone.svelte"
+
+  const dispatch = createEventDispatcher()
+
+  export let pathTo
+  export let branchIdx
+  export let step
+  export let isLast
+  export let bindings
+  export let automation
+
+  const view = getContext("draggableView")
+
+  let drawer
+  let condition
+  let open = true
+  let confirmDeleteModal
+
+  $: branch = step.inputs?.branches?.[branchIdx]
+  $: editableConditionUI = cloneDeep(branch.conditionUI || {})
+  $: condition = QueryUtils.buildQuery(editableConditionUI)
+
+  // Parse all the bindings into fields for the condition builder
+  $: schemaFields = bindings.map(binding => {
+    return {
+      name: `{{${binding.runtimeBinding}}}`,
+      displayName: `${binding.category} - ${binding.display.name}`,
+      type: "string",
+    }
+  })
+  $: branchBlockRef = {
+    branchNode: true,
+    pathTo: (pathTo || []).concat({ branchIdx, branchStepId: step.id }),
+  }
+</script>
+
+<Modal bind:this={confirmDeleteModal}>
+  <ModalContent
+    size="M"
+    title={"Are you sure you want to delete?"}
+    confirmText="Delete"
+    onConfirm={async () => {
+      await automationStore.actions.deleteBranch(
+        branchBlockRef.pathTo,
+        $selectedAutomation.data
+      )
+    }}
+  >
+    <Body>By deleting this branch, you will delete all of its contents.</Body>
+  </ModalContent>
+</Modal>
+
+<Drawer bind:this={drawer} title="Branch condition" forceModal>
+  <Button
+    cta
+    slot="buttons"
+    on:click={() => {
+      drawer.hide()
+      dispatch("change", {
+        conditionUI: editableConditionUI,
+        condition,
+      })
+    }}
+  >
+    Save
+  </Button>
+  <DrawerContent slot="body">
+    <FilterBuilder
+      filters={editableConditionUI}
+      {bindings}
+      {schemaFields}
+      datasource={{ type: "custom" }}
+      panel={AutomationBindingPanel}
+      on:change={e => {
+        editableConditionUI = e.detail
+      }}
+      allowOnEmpty={false}
+      builderType={"condition"}
+      docsURL={null}
+    />
+  </DrawerContent>
+</Drawer>
+
+<div class="flow-item">
+  <!-- svelte-ignore a11y-no-static-element-interactions -->
+  <div
+    class={`block branch-node hoverable`}
+    class:selected={false}
+    on:mousedown={e => {
+      e.stopPropagation()
+    }}
+  >
+    <FlowItemHeader
+      {automation}
+      {open}
+      itemName={branch.name}
+      block={step}
+      deleteStep={async () => {
+        const branchSteps = step.inputs?.children[branch.id]
+        if (branchSteps.length) {
+          confirmDeleteModal.show()
+        } else {
+          await automationStore.actions.deleteBranch(
+            branchBlockRef.pathTo,
+            $selectedAutomation.data
+          )
+        }
+      }}
+      on:update={async e => {
+        let stepUpdate = cloneDeep(step)
+        let branchUpdate = stepUpdate.inputs?.branches.find(
+          stepBranch => stepBranch.id == branch.id
+        )
+        branchUpdate.name = e.detail
+
+        const updatedAuto = automationStore.actions.updateStep(
+          pathTo,
+          $selectedAutomation.data,
+          stepUpdate
+        )
+        await automationStore.actions.save(updatedAuto)
+      }}
+      on:toggle={() => (open = !open)}
+    >
+      <div slot="custom-actions" class="branch-actions">
+        <Icon
+          on:click={() => {
+            automationStore.actions.branchLeft(
+              branchBlockRef.pathTo,
+              $selectedAutomation.data,
+              step
+            )
+          }}
+          tooltip={"Move left"}
+          tooltipType={TooltipType.Info}
+          tooltipPosition={TooltipPosition.Top}
+          hoverable
+          disabled={branchIdx == 0}
+          name="ArrowLeft"
+        />
+        <Icon
+          on:click={() => {
+            automationStore.actions.branchRight(
+              branchBlockRef.pathTo,
+              $selectedAutomation.data,
+              step
+            )
+          }}
+          tooltip={"Move right"}
+          tooltipType={TooltipType.Info}
+          tooltipPosition={TooltipPosition.Top}
+          hoverable
+          disabled={isLast}
+          name="ArrowRight"
+        />
+      </div>
+    </FlowItemHeader>
+    {#if open}
+      <Divider noMargin />
+      <div class="blockSection">
+        <!-- Content body for possible slot -->
+        <Layout noPadding>
+          <PropField label="Only run when">
+            <ActionButton fullWidth on:click={drawer.show}>
+              {editableConditionUI?.groups?.length
+                ? "Update condition"
+                : "Add condition"}
+            </ActionButton>
+          </PropField>
+          <div class="footer">
+            <Icon
+              name="InfoOutline"
+              size="S"
+              color="var(--spectrum-global-color-gray-700)"
+            />
+            <Body size="XS" color="var(--spectrum-global-color-gray-700)">
+              Only the first branch which matches its condition will run
+            </Body>
+          </div>
+        </Layout>
+      </div>
+    {/if}
+  </div>
+
+  <div class="separator" />
+
+  {#if $view.dragging}
+    <DragZone path={branchBlockRef.pathTo} />
+  {:else}
+    <FlowItemActions block={branchBlockRef} />
+  {/if}
+
+  {#if step.inputs.children[branch.id]?.length}
+    <div class="separator" />
+  {/if}
+</div>
+
+<style>
+  .branch-actions {
+    display: flex;
+    gap: var(--spacing-l);
+  }
+  .footer {
+    display: flex;
+    align-items: center;
+    gap: var(--spacing-m);
+  }
+
+  .flow-item {
+    display: flex;
+    flex-direction: column;
+    align-items: center;
+  }
+
+  .block-options {
+    justify-content: flex-end;
+    align-items: center;
+    display: flex;
+    gap: var(--spacing-m);
+  }
+  .center-items {
+    display: flex;
+    align-items: center;
+  }
+  .splitHeader {
+    display: flex;
+    justify-content: space-between;
+    align-items: center;
+  }
+  .iconAlign {
+    padding: 0 0 0 var(--spacing-m);
+    display: inline-block;
+  }
+  .block {
+    width: 480px;
+    font-size: 16px;
+    background-color: var(--background);
+    border: 1px solid var(--spectrum-global-color-gray-300);
+    border-radius: 4px 4px 4px 4px;
+  }
+
+  .blockSection {
+    padding: var(--spacing-xl);
+  }
+
+  .separator {
+    width: 1px;
+    height: 25px;
+    border-left: 1px dashed var(--grey-4);
+    color: var(--grey-4);
+    align-self: center;
+  }
+
+  .blockTitle {
+    display: flex;
+    align-items: center;
+    gap: var(--spacing-s);
+  }
+</style>

packages/builder/src/components/automation/AutomationBuilder/FlowChart/DragZone.svelte

@@ -0,0 +1,66 @@
+<script>
+  import { getContext, onMount } from "svelte"
+  import { generate } from "shortid"
+
+  export let path
+
+  let dropEle
+  let dzid = generate()
+
+  const view = getContext("draggableView")
+
+  onMount(() => {
+    // Always return up-to-date values
+    view.update(state => {
+      return {
+        ...state,
+        dropzones: {
+          ...(state.dropzones || {}),
+          [dzid]: {
+            get dims() {
+              return dropEle.getBoundingClientRect()
+            },
+            path,
+          },
+        },
+      }
+    })
+  })
+</script>
+
+<div
+  id={`dz-${dzid}`}
+  bind:this={dropEle}
+  class="drag-zone"
+  class:drag-over={$view?.droptarget === dzid}
+>
+  <span class="move-to">Move to</span>
+</div>
+
+<style>
+  .drag-zone.drag-over {
+    background-color: #1ca872b8;
+  }
+  .drag-zone {
+    min-height: calc(var(--spectrum-global-dimension-size-225) + 12px);
+    min-width: 100%;
+    background-color: rgba(28, 168, 114, 0.2);
+    border-radius: 4px;
+    border: 1px dashed #1ca872;
+    position: relative;
+    text-align: center;
+  }
+  .move-to {
+    position: absolute;
+    left: 0;
+    right: 0;
+    top: -50%;
+    margin: auto;
+    width: fit-content;
+    font-weight: 600;
+    border-radius: 8px;
+    padding: 4px 8px;
+    background-color: rgb(28, 168, 114);
+    color: var(--spectrum-global-color-gray-50);
+  }
+</style>

packages/builder/src/components/automation/AutomationBuilder/FlowChart/FlowChart.svelte

@@ -1,27 +1,63 @@
 <script>
   import {
     automationStore,
-    selectedAutomation,
     automationHistoryStore,
+    selectedAutomation,
   } from "stores/builder"
   import ConfirmDialog from "components/common/ConfirmDialog.svelte"
-  import FlowItem from "./FlowItem.svelte"
   import TestDataModal from "./TestDataModal.svelte"
-  import { flip } from "svelte/animate"
-  import { fly } from "svelte/transition"
-  import { Icon, notifications, Modal, Toggle } from "@budibase/bbui"
+  import {
+    Icon,
+    notifications,
+    Modal,
+    Toggle,
+    Button,
+    ActionButton,
+  } from "@budibase/bbui"
   import { ActionStepID } from "constants/backend/automations"
   import UndoRedoControl from "components/common/UndoRedoControl.svelte"
+  import StepNode from "./StepNode.svelte"
+  import { memo } from "@budibase/frontend-core"
   import { sdk } from "@budibase/shared-core"
+  import DraggableCanvas from "../DraggableCanvas.svelte"
 
   export let automation
 
+  const memoAutomation = memo(automation)
+
   let testDataModal
   let confirmDeleteDialog
   let scrolling = false
+  let blockRefs = {}
+  let treeEle
+  let draggable
 
-  $: blocks = getBlocks(automation).filter(x => x.stepId !== ActionStepID.LOOP)
-  $: isRowAction = sdk.automations.isRowAction(automation)
+  // Memo auto - selectedAutomation
+  $: memoAutomation.set(automation)
+
+  // Parse the automation tree state
+  $: refresh($memoAutomation)
+
+  $: blocks = getBlocks($memoAutomation).filter(
+    x => x.stepId !== ActionStepID.LOOP
+  )
+  $: isRowAction = sdk.automations.isRowAction($memoAutomation)
+
+  const refresh = () => {
+    // Build global automation bindings.
+    const environmentBindings =
+      automationStore.actions.buildEnvironmentBindings()
+
+    // Get all processed block references
+    blockRefs = $selectedAutomation.blockRefs
+
+    automationStore.update(state => {
+      return {
+        ...state,
+        bindings: [...environmentBindings],
+      }
+    })
+  }
 
   const getBlocks = automation => {
     let blocks = []
@@ -34,39 +70,46 @@
 
   const deleteAutomation = async () => {
     try {
-      await automationStore.actions.delete($selectedAutomation)
+      await automationStore.actions.delete(automation)
     } catch (error) {
       notifications.error("Error deleting automation")
     }
   }
-
-  const handleScroll = e => {
-    if (e.target.scrollTop >= 30) {
-      scrolling = true
-    } else if (e.target.scrollTop) {
-      // Set scrolling back to false if scrolled back to less than 100px
-      scrolling = false
-    }
-  }
 </script>
 
 <!-- svelte-ignore a11y-click-events-have-key-events -->
 <!-- svelte-ignore a11y-no-static-element-interactions -->
 <div class="header" class:scrolling>
   <div class="header-left">
-    <UndoRedoControl store={automationHistoryStore} />
+    <UndoRedoControl store={automationHistoryStore} showButtonGroup />
+
+    <div class="zoom">
+      <div class="group">
+        <ActionButton icon="Add" quiet on:click={draggable.zoomIn} />
+        <ActionButton icon="Remove" quiet on:click={draggable.zoomOut} />
+      </div>
+    </div>
+
+    <Button
+      secondary
+      on:click={() => {
+        draggable.zoomToFit()
+      }}
+    >
+      Zoom to fit
+    </Button>
   </div>
   <div class="controls">
-    <div
-      class:disabled={!$selectedAutomation?.definition?.trigger}
+    <Button
+      icon={"Play"}
+      cta
+      disabled={!automation?.definition?.trigger}
       on:click={() => {
         testDataModal.show()
       }}
-      class="buttons"
     >
-      <Icon size="M" name="Play" />
-      <div>Run test</div>
-    </div>
+      Run test
+    </Button>
     <div class="buttons">
       <Icon disabled={!$automationStore.testResults} size="M" name="Multiple" />
       <div
@@ -79,36 +122,39 @@
       </div>
     </div>
     {#if !isRowAction}
-      <div class="setting-spacing">
+      <div class="toggle-active setting-spacing">
         <Toggle
           text={automation.disabled ? "Paused" : "Activated"}
           on:change={automationStore.actions.toggleDisabled(
             automation._id,
             automation.disabled
           )}
-          disabled={!$selectedAutomation?.definition?.trigger}
+          disabled={!automation?.definition?.trigger}
           value={!automation.disabled}
         />
       </div>
     {/if}
   </div>
 </div>
-<div class="canvas" on:scroll={handleScroll}>
-  <div class="content">
+
+<div class="root" bind:this={treeEle}>
+  <DraggableCanvas bind:this={draggable}>
+    <span class="main-content" slot="content">
+      {#if Object.keys(blockRefs).length}
         {#each blocks as block, idx (block.id)}
-      <div
-        class="block"
-        animate:flip={{ duration: 500 }}
-        in:fly={{ x: 500, duration: 500 }}
-        out:fly|local={{ x: 500, duration: 500 }}
-      >
-        {#if block.stepId !== ActionStepID.LOOP}
-          <FlowItem {testDataModal} {block} {idx} />
-        {/if}
-      </div>
+          <StepNode
+            step={blocks[idx]}
+            stepIdx={idx}
+            isLast={blocks?.length - 1 === idx}
+            automation={$memoAutomation}
+            blocks={blockRefs}
+          />
         {/each}
+      {/if}
+    </span>
+  </DraggableCanvas>
 </div>
-</div>
+
 <ConfirmDialog
   bind:this={confirmDeleteDialog}
   okText="Delete Automation"
@@ -125,32 +171,43 @@
 </Modal>
 
 <style>
-  .canvas {
-    padding: var(--spacing-l) var(--spacing-xl);
-    overflow-y: auto;
+  .toggle-active :global(.spectrum-Switch) {
+    margin: 0px;
+  }
+
+  .root :global(.main-content) {
+    display: flex;
+    flex-direction: column;
+    align-items: center;
     max-height: 100%;
+    height: 100%;
+    width: 100%;
+  }
+
+  .header-left {
+    display: flex;
+    gap: var(--spacing-l);
   }
 
   .header-left :global(div) {
     border-right: none;
   }
-  /* Fix for firefox not respecting bottom padding in scrolling containers */
-  .canvas > *:last-child {
-    padding-bottom: 40px;
+
+  .root {
+    height: 100%;
+    width: 100%;
   }
 
-  .block {
+  .root :global(.block) {
     display: flex;
     flex-direction: column;
     justify-content: flex-start;
     align-items: center;
   }
 
-  .content {
-    flex-grow: 1;
-    padding: 23px 23px 80px;
+  .root :global(.blockSection) {
+    width: 100%;
     box-sizing: border-box;
-    overflow-x: hidden;
   }
 
   .header.scrolling {
@@ -166,13 +223,15 @@
     align-items: center;
     padding-left: var(--spacing-l);
     transition: background 130ms ease-out;
-    flex: 0 0 48px;
+    flex: 0 0 60px;
     padding-right: var(--spacing-xl);
   }
+
   .controls {
     display: flex;
     gap: var(--spacing-xl);
   }
+
   .buttons {
     display: flex;
     justify-content: flex-end;
@@ -188,4 +247,33 @@
     pointer-events: none;
     color: var(--spectrum-global-color-gray-500) !important;
   }
+
+  .group {
+    border-radius: 4px;
+    display: flex;
+    flex-direction: row;
+  }
+  .group :global(> *:not(:first-child)) {
+    border-top-left-radius: 0;
+    border-bottom-left-radius: 0;
+    border-left: 2px solid var(--spectrum-global-color-gray-300);
+  }
+  .group :global(> *:not(:last-child)) {
+    border-top-right-radius: 0;
+    border-bottom-right-radius: 0;
+  }
+
+  .header-left .group :global(.spectrum-Button),
+  .header-left .group :global(.spectrum-ActionButton),
+  .header-left .group :global(.spectrum-Icon) {
+    color: var(--spectrum-global-color-gray-900) !important;
+  }
+  .header-left .group :global(.spectrum-Button),
+  .header-left .group :global(.spectrum-ActionButton) {
+    background: var(--spectrum-global-color-gray-200) !important;
+  }
+  .header-left .group :global(.spectrum-Button:hover),
+  .header-left .group :global(.spectrum-ActionButton:hover) {
+    background: var(--spectrum-global-color-gray-300) !important;
+  }
 </style>

packages/builder/src/components/automation/AutomationBuilder/FlowChart/FlowItem.svelte

@@ -1,8 +1,8 @@
 <script>
   import {
     automationStore,
-    selectedAutomation,
     permissions,
+    selectedAutomation,
     tables,
   } from "stores/builder"
   import {
@@ -11,54 +11,115 @@
     Layout,
     Detail,
     Modal,
-    Button,
-    notifications,
     Label,
     AbsTooltip,
-    InlineAlert,
   } from "@budibase/bbui"
   import { sdk } from "@budibase/shared-core"
   import AutomationBlockSetup from "../../SetupPanel/AutomationBlockSetup.svelte"
   import CreateWebhookModal from "components/automation/Shared/CreateWebhookModal.svelte"
-  import ActionModal from "./ActionModal.svelte"
   import FlowItemHeader from "./FlowItemHeader.svelte"
   import RoleSelect from "components/design/settings/controls/RoleSelect.svelte"
   import { ActionStepID, TriggerStepID } from "constants/backend/automations"
+  import { AutomationStepType } from "@budibase/types"
+  import FlowItemActions from "./FlowItemActions.svelte"
+  import DragHandle from "components/design/settings/controls/DraggableList/drag-handle.svelte"
+  import { getContext } from "svelte"
+  import DragZone from "./DragZone.svelte"
+  import InfoDisplay from "pages/builder/app/[application]/design/[screenId]/[componentId]/_components/Component/InfoDisplay.svelte"
 
   export let block
+  export let blockRef
   export let testDataModal
   export let idx
+  export let automation
+  export let bindings
+  export let draggable = true
+
+  const view = getContext("draggableView")
+  const pos = getContext("viewPos")
+  const contentPos = getContext("contentPos")
 
-  let selected
   let webhookModal
-  let actionModal
   let open = true
   let showLooping = false
   let role
+  let blockEle
+  let positionStyles
+  let blockDims
 
-  $: collectBlockExists = $selectedAutomation.definition.steps.some(
+  const updateBlockDims = () => {
+    if (!blockEle) {
+      return
+    }
+    const { width, height } = blockEle.getBoundingClientRect()
+    blockDims = { width: width / $view.scale, height: height / $view.scale }
+  }
+
+  const loadSteps = blockRef => {
+    return blockRef
+      ? automationStore.actions.getPathSteps(blockRef.pathTo, automation)
+      : []
+  }
+
+  $: pathSteps = loadSteps(blockRef)
+
+  $: collectBlockExists = pathSteps.some(
     step => step.stepId === ActionStepID.COLLECT
   )
-  $: automationId = $selectedAutomation?._id
-  $: isTrigger = block.type === "TRIGGER"
-  $: steps = $selectedAutomation?.definition?.steps ?? []
-  $: blockIdx = steps.findIndex(step => step.id === block.id)
-  $: lastStep = !isTrigger && blockIdx + 1 === steps.length
-  $: totalBlocks = $selectedAutomation?.definition?.steps.length + 1
-  $: loopBlock = $selectedAutomation?.definition.steps.find(
-    x => x.blockToLoop === block.id
-  )
+  $: automationId = automation?._id
+  $: isTrigger = block.type === AutomationStepType.TRIGGER
+  $: lastStep = blockRef?.terminating
+
+  $: loopBlock = pathSteps.find(x => x.blockToLoop === block.id)
   $: isAppAction = block?.stepId === TriggerStepID.APP
   $: isAppAction && setPermissions(role)
   $: isAppAction && getPermissions(automationId)
 
-  $: triggerInfo = sdk.automations.isRowAction($selectedAutomation) && {
+  $: triggerInfo = sdk.automations.isRowAction($selectedAutomation?.data) && {
     title: "Automation trigger",
     tableName: $tables.list.find(
-      x => x._id === $selectedAutomation.definition.trigger.inputs?.tableId
+      x =>
+        x._id === $selectedAutomation.data?.definition?.trigger?.inputs?.tableId
     )?.name,
   }
 
+  $: selected = $view?.moveStep && $view?.moveStep?.id === block.id
+
+  $: if (selected && blockEle) {
+    updateBlockDims()
+  }
+
+  $: placeholderDims = buildPlaceholderStyles(blockDims)
+
+  // Move the selected item
+  // Listen for scrolling in the content. As its scrolled this will be updated
+  $: move(
+    blockEle,
+    $view?.dragSpot,
+    selected,
+    $contentPos?.scrollX,
+    $contentPos?.scrollY
+  )
+
+  const move = (block, dragPos, selected, scrollX, scrollY) => {
+    if ((!block && !selected) || !dragPos) {
+      return
+    }
+    positionStyles = `
+      --blockPosX: ${Math.round(dragPos.x - scrollX / $view.scale)}px;
+      --blockPosY: ${Math.round(dragPos.y - scrollY / $view.scale)}px;
+    `
+  }
+
+  const buildPlaceholderStyles = dims => {
+    if (!dims) {
+      return ""
+    }
+    const { width, height } = dims
+    return `--pswidth: ${Math.round(width)}px;
+            --psheight: ${Math.round(height)}px;`
+  }
+
   async function setPermissions(role) {
     if (!role || !automationId) {
       return
@@ -82,23 +143,13 @@
     }
   }
 
-  async function removeLooping() {
-    try {
-      await automationStore.actions.deleteAutomationBlock(loopBlock)
-    } catch (error) {
-      notifications.error("Error saving automation")
-    }
+  async function deleteStep() {
+    await automationStore.actions.deleteAutomationBlock(blockRef.pathTo)
   }
 
-  async function deleteStep() {
-    try {
-      if (loopBlock) {
-        await automationStore.actions.deleteAutomationBlock(loopBlock)
-      }
-      await automationStore.actions.deleteAutomationBlock(block, blockIdx)
-    } catch (error) {
-      notifications.error("Error saving automation")
-    }
+  async function removeLooping() {
+    let loopBlockRef = $selectedAutomation.blockRefs[blockRef.looped]
+    await automationStore.actions.deleteAutomationBlock(loopBlockRef.pathTo)
   }
 
   async function addLooping() {
@@ -109,13 +160,64 @@
       loopDefinition
     )
     loopBlock.blockToLoop = block.id
-    await automationStore.actions.addBlockToAutomation(loopBlock, blockIdx)
+    await automationStore.actions.addBlockToAutomation(
+      loopBlock,
+      blockRef.pathTo
+    )
+  }
+
+  const onHandleMouseDown = e => {
+    if (isTrigger) {
+      e.preventDefault()
+      return
+    }
+
+    e.stopPropagation()
+
+    view.update(state => ({
+      ...state,
+      moveStep: {
+        id: block.id,
+        offsetX: $pos.x,
+        offsetY: $pos.y,
+      },
+    }))
   }
 </script>
 
+{#if block.stepId !== "LOOP"}
   <!-- svelte-ignore a11y-click-events-have-key-events -->
   <!-- svelte-ignore a11y-no-static-element-interactions -->
-<div class={`block ${block.type} hoverable`} class:selected on:click={() => {}}>
+  <div
+    id={`block-${block.id}`}
+    class={`block ${block.type} hoverable`}
+    class:selected
+    class:draggable
+  >
+    <div class="wrap">
+      {#if $view.dragging && selected}
+        <div class="drag-placeholder" style={placeholderDims} />
+      {/if}
+
+      <div
+        bind:this={blockEle}
+        class="block-content"
+        class:dragging={$view.dragging && selected}
+        style={positionStyles}
+        on:mousedown={e => {
+          e.stopPropagation()
+        }}
+      >
+        {#if draggable}
+          <div
+            class="handle"
+            class:grabbing={selected}
+            on:mousedown={onHandleMouseDown}
+          >
+            <DragHandle />
+          </div>
+        {/if}
+        <div class="block-core">
           {#if loopBlock}
             <div class="blockSection">
               <div
@@ -141,11 +243,18 @@
 
                 <div class="blockTitle">
                   <AbsTooltip type="negative" text="Remove looping">
-            <Icon on:click={removeLooping} hoverable name="DeleteOutline" />
+                    <Icon
+                      on:click={removeLooping}
+                      hoverable
+                      name="DeleteOutline"
+                    />
                   </AbsTooltip>
 
                   <div style="margin-left: 10px;" on:click={() => {}}>
-            <Icon hoverable name={showLooping ? "ChevronDown" : "ChevronUp"} />
+                    <Icon
+                      hoverable
+                      name={showLooping ? "ChevronDown" : "ChevronUp"}
+                    />
                   </div>
                 </div>
               </div>
@@ -157,11 +266,13 @@
                 <Layout noPadding gap="S">
                   <AutomationBlockSetup
                     schemaProperties={Object.entries(
-              $automationStore.blockDefinitions.ACTION.LOOP.schema.inputs
-                .properties
+                      $automationStore.blockDefinitions.ACTION.LOOP.schema
+                        .inputs.properties
                     )}
                     {webhookModal}
                     block={loopBlock}
+                    {automation}
+                    {bindings}
                   />
                 </Layout>
               </div>
@@ -170,6 +281,7 @@
           {/if}
 
           <FlowItemHeader
+            {automation}
             {open}
             {block}
             {testDataModal}
@@ -177,6 +289,17 @@
             {addLooping}
             {deleteStep}
             on:toggle={() => (open = !open)}
+            on:update={async e => {
+              const newName = e.detail
+              if (newName.length === 0) {
+                await automationStore.actions.deleteAutomationName(block.id)
+              } else {
+                await automationStore.actions.saveAutomationName(
+                  block.id,
+                  newName
+                )
+              }
+            }}
           />
           {#if open}
             <Divider noMargin />
@@ -189,41 +312,48 @@
                   </div>
                 {/if}
                 <AutomationBlockSetup
-          schemaProperties={Object.entries(block.schema.inputs.properties)}
+                  schemaProperties={Object.entries(
+                    block?.schema?.inputs?.properties || {}
+                  )}
                   {block}
                   {webhookModal}
+                  {automation}
+                  {bindings}
                 />
-        {#if triggerInfo}
-          <InlineAlert
-            header={triggerInfo.title}
-            message={`This trigger is tied to your "${triggerInfo.tableName}" table`}
+                {#if isTrigger && triggerInfo}
+                  <InfoDisplay
+                    title={triggerInfo.title}
+                    body="This trigger is tied to your '{triggerInfo.tableName}' table"
+                    icon="InfoOutline"
                   />
                 {/if}
-        {#if lastStep}
-          <Button on:click={() => testDataModal.show()} cta>
-            Finish and test automation
-          </Button>
-        {/if}
               </Layout>
             </div>
           {/if}
         </div>
+      </div>
+    </div>
+  </div>
   {#if !collectBlockExists || !lastStep}
     <div class="separator" />
-  <Icon
-    on:click={() => actionModal.show()}
-    hoverable
-    name="AddCircle"
-    size="S"
+    {#if $view.dragging}
+      <DragZone path={blockRef?.pathTo} />
+    {:else}
+      <FlowItemActions
+        {block}
+        on:branch={() => {
+          automationStore.actions.branchAutomation(
+            $selectedAutomation.blockRefs[block.id].pathTo,
+            automation
+          )
+        }}
       />
-  {#if isTrigger ? totalBlocks > 1 : blockIdx !== totalBlocks - 2}
+    {/if}
+    {#if !lastStep}
       <div class="separator" />
     {/if}
   {/if}
-
-<Modal bind:this={actionModal} width="30%">
-  <ActionModal modal={actionModal} {lastStep} {blockIdx} />
-</Modal>
+{/if}
 
 <Modal bind:this={webhookModal} width="30%">
   <CreateWebhookModal />
@@ -255,27 +385,73 @@
   .block {
     width: 480px;
     font-size: 16px;
-    background-color: var(--background);
-    border: 1px solid var(--spectrum-global-color-gray-300);
-    border-radius: 4px 4px 4px 4px;
+    border-radius: 4px;
+  }
+  .block .wrap {
+    width: 100%;
+    position: relative;
+  }
+  .block.draggable .wrap {
+    display: flex;
+    flex-direction: row;
+  }
+  .block.draggable .wrap .handle {
+    height: auto;
+    display: flex;
+    justify-content: center;
+    align-items: center;
+    background-color: var(--grey-3);
+    padding: 6px;
+    color: var(--grey-6);
+    cursor: grab;
+  }
+  .block.draggable .wrap .handle.grabbing {
+    cursor: grabbing;
+  }
+  .block.draggable .wrap .handle :global(.drag-handle) {
+    width: 6px;
+  }
+  .block .wrap .block-content {
+    width: 100%;
+    display: flex;
+    flex-direction: row;
+    background-color: var(--background);
+    border: 1px solid var(--grey-3);
+    border-radius: 4px;
   }
-
   .blockSection {
     padding: var(--spacing-xl);
   }
-
   .separator {
     width: 1px;
     height: 25px;
     border-left: 1px dashed var(--grey-4);
     color: var(--grey-4);
-    /* center horizontally */
     align-self: center;
   }
-
   .blockTitle {
     display: flex;
     align-items: center;
     gap: var(--spacing-s);
   }
+  .drag-placeholder {
+    height: calc(var(--psheight) - 2px);
+    width: var(--pswidth);
+    background-color: rgba(92, 92, 92, 0.1);
+    border: 1px dashed #5c5c5c;
+    border-radius: 4px;
+    display: block;
+  }
+  .block-core {
+    flex: 1;
+  }
+  .block-core.dragging {
+    pointer-events: none;
+  }
+  .block-content.dragging {
+    position: absolute;
+    z-index: 3;
+    top: var(--blockPosY);
+    left: var(--blockPosX);
+  }
 </style>

packages/builder/src/components/automation/AutomationBuilder/FlowChart/FlowItemActions.svelte

@@ -0,0 +1,51 @@
+<script>
+  import { Icon, TooltipPosition, TooltipType, Modal } from "@budibase/bbui"
+  import { createEventDispatcher } from "svelte"
+  import ActionModal from "./ActionModal.svelte"
+
+  export let block
+
+  const dispatch = createEventDispatcher()
+  let actionModal
+</script>
+
+<Modal bind:this={actionModal} width="30%">
+  <ActionModal modal={actionModal} {block} />
+</Modal>
+
+<div class="action-bar">
+  {#if !block.branchNode}
+    <Icon
+      hoverable
+      name="Branch3"
+      on:click={() => {
+        dispatch("branch")
+      }}
+      tooltipType={TooltipType.Info}
+      tooltipPosition={TooltipPosition.Left}
+      tooltip={"Create branch"}
+      size={"S"}
+    />
+  {/if}
+  <Icon
+    hoverable
+    name="AddCircle"
+    on:click={() => {
+      actionModal.show()
+    }}
+    tooltipType={TooltipType.Info}
+    tooltipPosition={TooltipPosition.Right}
+    tooltip={"Add a step"}
+    size={"S"}
+  />
+</div>
+
+<style>
+  .action-bar {
+    background-color: var(--background);
+    border-radius: 4px 4px 4px 4px;
+    display: flex;
+    gap: var(--spacing-m);
+    padding: 8px 12px;
+  }
+</style>

packages/builder/src/components/automation/AutomationBuilder/FlowChart/FlowItemHeader.svelte

@@ -10,21 +10,25 @@
   export let showTestStatus = false
   export let testResult
   export let isTrigger
-  export let idx
   export let addLooping
   export let deleteStep
   export let enableNaming = true
+  export let itemName
+  export let automation
+
   let validRegex = /^[A-Za-z0-9_\s]+$/
   let typing = false
   let editing = false
   const dispatch = createEventDispatcher()
 
-  $: stepNames = $selectedAutomation?.definition.stepNames
-  $: allSteps = $selectedAutomation?.definition.steps || []
-  $: automationName = stepNames?.[block.id] || block?.name || ""
+  $: blockRefs = $selectedAutomation?.blockRefs || {}
+  $: stepNames = automation?.definition.stepNames
+  $: allSteps = automation?.definition.steps || []
+  $: automationName = itemName || stepNames?.[block.id] || block?.name || ""
   $: automationNameError = getAutomationNameError(automationName)
   $: status = updateStatus(testResult)
   $: isHeaderTrigger = isTrigger || block.type === "TRIGGER"
+  $: isBranch = block.stepId === "BRANCH"
 
   $: {
     if (!testResult) {
@@ -33,12 +37,12 @@
       )?.[0]
     }
   }
-  $: loopBlock = $selectedAutomation?.definition.steps.find(
-    x => x.blockToLoop === block?.id
-  )
+
+  $: blockRef = blockRefs[block.id]
+  $: isLooped = blockRef?.looped
 
   async function onSelect(block) {
-    await automationStore.update(state => {
+    automationStore.update(state => {
       state.selectedBlock = block
       return state
     })
@@ -84,30 +88,18 @@
     return null
   }
 
-  const saveName = async () => {
-    if (automationNameError || block.name === automationName) {
-      return
-    }
-
-    if (automationName.length === 0) {
-      await automationStore.actions.deleteAutomationName(block.id)
-    } else {
-      await automationStore.actions.saveAutomationName(block.id, automationName)
-    }
-  }
-
   const startEditing = () => {
     editing = true
     typing = true
   }
 
-  const stopEditing = async () => {
+  const stopEditing = () => {
     editing = false
     typing = false
     if (automationNameError) {
       automationName = stepNames[block.id] || block?.name
     } else {
-      await saveName()
+      dispatch("update", automationName)
     }
   }
 </script>
@@ -118,7 +110,6 @@
   class:typing={typing && !automationNameError && editing}
   class:typing-error={automationNameError && editing}
   class="blockSection"
-  on:click={() => dispatch("toggle")}
 >
   <div class="splitHeader">
     <div class="center-items">
@@ -144,16 +135,14 @@
         {#if isHeaderTrigger}
           <Body size="XS"><b>Trigger</b></Body>
         {:else}
-          <div style="margin-left: 2px;">
-            <Body size="XS"><b>Step {idx}</b></Body>
-          </div>
+          <Body size="XS"><b>{isBranch ? "Branch" : "Step"}</b></Body>
         {/if}
 
         {#if enableNaming}
           <input
             class="input-text"
             disabled={!enableNaming}
-            placeholder="Enter step name"
+            placeholder={`Enter ${isBranch ? "branch" : "step"} name`}
             name="name"
             autocomplete="off"
             value={automationName}
@@ -208,8 +197,9 @@
           onSelect(block)
         }}
       >
+        <slot name="custom-actions" />
         {#if !showTestStatus}
-          {#if !isHeaderTrigger && !loopBlock && (block?.features?.[Features.LOOPING] || !block.features)}
+          {#if !isHeaderTrigger && !isLooped && !isBranch && (block?.features?.[Features.LOOPING] || !block.features)}
             <AbsTooltip type="info" text="Add looping">
               <Icon on:click={addLooping} hoverable name="RotateCW" />
             </AbsTooltip>
@@ -220,6 +210,9 @@
             </AbsTooltip>
           {/if}
         {/if}
+        {#if !showTestStatus && !isHeaderTrigger}
+          <span class="action-spacer" />
+        {/if}
         {#if !showTestStatus}
           <Icon
             on:click={e => {
@@ -245,6 +238,9 @@
 </div>
 
 <style>
+  .action-spacer {
+    border-left: 1px solid var(--spectrum-global-color-gray-300);
+  }
   .status-container {
     display: flex;
     align-items: center;
@@ -298,6 +294,8 @@
     font-size: var(--spectrum-alias-font-size-default);
     font-family: var(--font-sans);
     text-overflow: ellipsis;
+    padding-left: 0px;
+    border: 0px;
   }
 
   input:focus {

packages/builder/src/components/automation/AutomationBuilder/FlowChart/StepNode.svelte

@@ -0,0 +1,227 @@
+<script>
+  import FlowItem from "./FlowItem.svelte"
+  import BranchNode from "./BranchNode.svelte"
+  import { AutomationActionStepId } from "@budibase/types"
+  import { ActionButton, notifications } from "@budibase/bbui"
+  import { automationStore } from "stores/builder"
+  import { environment } from "stores/portal"
+  import { cloneDeep } from "lodash"
+  import { memo } from "@budibase/frontend-core"
+  import { getContext, onMount } from "svelte"
+
+  export let step = {}
+  export let stepIdx
+  export let automation
+  export let blocks
+  export let isLast = false
+
+  const memoEnvVariables = memo($environment.variables)
+  const view = getContext("draggableView")
+
+  let stepEle
+
+  $: memoEnvVariables.set($environment.variables)
+  $: blockRef = blocks?.[step.id]
+  $: pathToCurrentNode = blockRef?.pathTo
+  $: isBranch = step.stepId === AutomationActionStepId.BRANCH
+  $: branches = step.inputs?.branches
+
+  // All bindings available to this point
+  $: availableBindings = automationStore.actions.getPathBindings(
+    step.id,
+    automation
+  )
+
+  // Fetch the env bindings
+  $: environmentBindings =
+    automationStore.actions.buildEnvironmentBindings($memoEnvVariables)
+
+  $: userBindings = automationStore.actions.buildUserBindings()
+  $: settingBindings = automationStore.actions.buildSettingBindings()
+
+  // Combine all bindings for the step
+  $: bindings = [
+    ...availableBindings,
+    ...environmentBindings,
+    ...userBindings,
+    ...settingBindings,
+  ]
+  onMount(() => {
+    // Register the trigger as the focus element for the automation
+    // Onload, the canvas will use the dimensions to center the step
+    if (stepEle && step.type === "TRIGGER" && !$view.focusEle) {
+      view.update(state => ({
+        ...state,
+        focusEle: stepEle.getBoundingClientRect(),
+      }))
+    }
+  })
+</script>
+
+{#if isBranch}
+  <div class="split-branch-btn">
+    <ActionButton
+      icon="AddCircle"
+      on:click={() => {
+        automationStore.actions.branchAutomation(pathToCurrentNode, automation)
+      }}
+    >
+      Add branch
+    </ActionButton>
+  </div>
+  <div class="branched">
+    {#each branches as branch, bIdx}
+      {@const leftMost = bIdx === 0}
+      {@const rightMost = branches?.length - 1 === bIdx}
+      <div class="branch-wrap">
+        <div
+          class="branch"
+          class:left={leftMost}
+          class:right={rightMost}
+          class:middle={!leftMost && !rightMost}
+        >
+          <div class="branch-node">
+            <BranchNode
+              {automation}
+              {step}
+              {bindings}
+              pathTo={pathToCurrentNode}
+              branchIdx={bIdx}
+              isLast={rightMost}
+              on:change={async e => {
+                const updatedBranch = { ...branch, ...e.detail }
+
+                if (!step?.inputs?.branches?.[bIdx]) {
+                  console.error(`Cannot load target branch: ${bIdx}`)
+                  return
+                }
+
+                let branchStepUpdate = cloneDeep(step)
+                branchStepUpdate.inputs.branches[bIdx] = updatedBranch
+
+                // Ensure valid base configuration for all branches
+                // Reinitialise empty branch conditions on update
+                branchStepUpdate.inputs.branches.forEach(
+                  (branch, i, branchArray) => {
+                    if (!Object.keys(branch.condition).length) {
+                      branchArray[i] = {
+                        ...branch,
+                        ...automationStore.actions.generateDefaultConditions(),
+                      }
+                    }
+                  }
+                )
+
+                const updated = automationStore.actions.updateStep(
+                  blockRef?.pathTo,
+                  automation,
+                  branchStepUpdate
+                )
+
+                try {
+                  await automationStore.actions.save(updated)
+                } catch (e) {
+                  notifications.error("Error saving branch update")
+                  console.error("Error saving automation branch", e)
+                }
+              }}
+            />
+          </div>
+
+          <!-- Branch steps -->
+          {#each step.inputs?.children[branch.id] || [] as bStep, sIdx}
+            <!-- Recursive StepNode -->
+            <svelte:self
+              step={bStep}
+              stepIdx={sIdx}
+              branchIdx={bIdx}
+              isLast={blockRef.terminating}
+              pathTo={pathToCurrentNode}
+              {automation}
+              {blocks}
+            />
+          {/each}
+        </div>
+      </div>
+    {/each}
+  </div>
+{:else}
+  <div class="block" bind:this={stepEle}>
+    <FlowItem
+      block={step}
+      idx={stepIdx}
+      {blockRef}
+      {isLast}
+      {automation}
+      {bindings}
+      draggable={step.type !== "TRIGGER"}
+    />
+  </div>
+{/if}
+
+<style>
+  .branch-wrap {
+    width: inherit;
+  }
+
+  .branch {
+    display: flex;
+    align-items: center;
+    flex-direction: column;
+    position: relative;
+    width: inherit;
+  }
+
+  .branched {
+    display: flex;
+    gap: 64px;
+  }
+
+  .branch::before {
+    height: 64px;
+    border-left: 1px dashed var(--grey-4);
+    border-top: 1px dashed var(--grey-4);
+    content: "";
+    color: var(--grey-4);
+    width: 50%;
+    position: absolute;
+    left: 50%;
+    top: -16px;
+  }
+
+  .branch.left::before {
+    color: var(--grey-4);
+    width: calc(50% + 62px);
+  }
+
+  .branch.middle::after {
+    height: 64px;
+    border-top: 1px dashed var(--grey-4);
+    content: "";
+    color: var(--grey-4);
+    width: calc(50% + 62px);
+    position: absolute;
+    left: 50%;
+    top: -16px;
+  }
+
+  .branch.right::before {
+    left: 0px;
+    border-right: 1px dashed var(--grey-4);
+    border-left: none;
+  }
+
+  .branch.middle::before {
+    left: 0px;
+    border-right: 1px dashed var(--grey-4);
+    border-left: none;
+  }
+
+  .branch-node {
+    margin-top: 48px;
+  }
+
+  .split-branch-btn {
+    z-index: 2;
+  }
+</style>

packages/builder/src/components/automation/AutomationBuilder/FlowChart/TestDataModal.svelte

@@ -1,4 +1,5 @@
 <script>
+  import { tick } from "svelte"
   import {
     ModalContent,
     TextArea,
@@ -8,7 +9,6 @@
   import { automationStore, selectedAutomation } from "stores/builder"
   import AutomationBlockSetup from "../../SetupPanel/AutomationBlockSetup.svelte"
   import { cloneDeep } from "lodash/fp"
-  import { memo } from "@budibase/frontend-core"
   import { AutomationEventType } from "@budibase/types"
 
   let failedParse = null
@@ -29,7 +29,7 @@
    * @todo Parse *all* data for each trigger type and relay adequate feedback
    */
   const parseTestData = testData => {
-    const autoTrigger = $selectedAutomation?.definition?.trigger
+    const autoTrigger = $selectedAutomation.data?.definition?.trigger
     const { tableId } = autoTrigger?.inputs || {}
 
     // Ensure the tableId matches the trigger table for row trigger automations
@@ -63,12 +63,11 @@
     return true
   }
 
-  const memoTestData = memo(parseTestData($selectedAutomation.testData))
-  $: memoTestData.set(parseTestData($selectedAutomation.testData))
+  $: testData = testData || parseTestData($selectedAutomation.data.testData)
 
   $: {
     // clone the trigger so we're not mutating the reference
-    trigger = cloneDeep($selectedAutomation.definition.trigger)
+    trigger = cloneDeep($selectedAutomation.data.definition.trigger)
 
     // get the outputs so we can define the fields
     let schema = Object.entries(trigger.schema?.outputs?.properties || {})
@@ -83,7 +82,7 @@
   $: isError =
     !isTriggerValid(trigger) ||
     !(trigger.schema.outputs.required || []).every(
-      required => $memoTestData?.[required] || required !== "row"
+      required => testData?.[required] || required !== "row"
     )
 
   function parseTestJSON(e) {
@@ -110,8 +109,10 @@
   }
 
   const testAutomation = async () => {
+    // Ensure testData reactiveness is processed
+    await tick()
     try {
-      await automationStore.actions.test($selectedAutomation, $memoTestData)
+      await automationStore.actions.test($selectedAutomation.data, testData)
       $automationStore.showTestPanel = true
     } catch (error) {
       notifications.error(error)
@@ -149,7 +150,7 @@
   {#if selectedValues}
     <div class="tab-content-padding">
       <AutomationBlockSetup
-        testData={$memoTestData}
+        bind:testData
         {schemaProperties}
         isTestModal
         block={trigger}
@@ -159,7 +160,7 @@
   {#if selectedJSON}
     <div class="text-area-container">
       <TextArea
-        value={JSON.stringify($selectedAutomation.testData, null, 2)}
+        value={JSON.stringify($selectedAutomation.data.testData, null, 2)}
         error={failedParse}
         on:change={e => parseTestJSON(e)}
       />

packages/builder/src/components/automation/AutomationBuilder/TestDisplay.svelte

@@ -3,10 +3,12 @@
   import FlowItemHeader from "./FlowChart/FlowItemHeader.svelte"
   import { ActionStepID } from "constants/backend/automations"
   import { JsonView } from "@zerodevx/svelte-json-view"
+  import { automationStore } from "stores/builder"
+  import { AutomationActionStepId } from "@budibase/types"
 
   export let automation
+  export let automationBlockRefs = {}
   export let testResults
-  export let width = "400px"
 
   let openBlocks = {}
   let blocks
@@ -28,21 +30,28 @@
     }
   }
 
-  $: filteredResults = prepTestResults(testResults)
+  const getBranchName = (step, id) => {
+    if (!step || !id) {
+      return
+    }
+    return step.inputs.branches.find(branch => branch.id === id)?.name
+  }
 
+  $: filteredResults = prepTestResults(testResults)
   $: {
     if (testResults.message) {
-      blocks = automation?.definition?.trigger
-        ? [automation.definition.trigger]
-        : []
+      const trigger = automation?.definition?.trigger
+      blocks = trigger ? [trigger] : []
     } else if (automation) {
-      blocks = []
-      if (automation.definition.trigger) {
-        blocks.push(automation.definition.trigger)
+      const terminatingStep = filteredResults.at(-1)
+      const terminatingBlockRef = automationBlockRefs[terminatingStep.id]
+      if (terminatingBlockRef) {
+        const pathSteps = automationStore.actions.getPathSteps(
+          terminatingBlockRef.pathTo,
+          automation
+        )
+        blocks = [...pathSteps].filter(x => x.stepId !== ActionStepID.LOOP)
       }
-      blocks = blocks
-        .concat(automation.definition.steps || [])
-        .filter(x => x.stepId !== ActionStepID.LOOP)
     } else if (filteredResults) {
       blocks = filteredResults || []
       // make sure there is an ID for each block being displayed
@@ -56,10 +65,14 @@
 
 <div class="container">
   {#each blocks as block, idx}
-    <div class="block" style={width ? `width: ${width}` : ""}>
+    <div class="block">
       {#if block.stepId !== ActionStepID.LOOP}
         <FlowItemHeader
+          {automation}
           enableNaming={false}
+          itemName={block.stepId === AutomationActionStepId.BRANCH
+            ? getBranchName(block, filteredResults?.[idx].outputs?.branchId)
+            : null}
           open={!!openBlocks[block.id]}
           on:toggle={() => (openBlocks[block.id] = !openBlocks[block.id])}
           isTrigger={idx === 0}
@@ -133,7 +146,10 @@
   .container {
     padding: 0 30px 30px 30px;
     height: 100%;
-    overflow: auto;
+    overflow-y: auto;
+    overflow-x: hidden;
+    display: flex;
+    flex-direction: column;
   }
 
   .wrap {
@@ -175,17 +191,17 @@
 
   .block {
     display: inline-block;
-    width: 400px;
-    height: auto;
+    height: fit-content;
     font-size: 16px;
     background-color: var(--background);
     border: 1px solid var(--spectrum-global-color-gray-300);
     border-radius: 4px 4px 4px 4px;
+    min-width: 425px;
   }
 
   .separator {
     width: 1px;
-    height: 40px;
+    flex: 0 0 40px;
     border-left: 1px dashed var(--grey-4);
     color: var(--grey-4);
     /* center horizontally */

packages/builder/src/components/automation/AutomationBuilder/TestPanel.svelte

@@ -1,7 +1,7 @@
 <script>
   import { Icon, Divider } from "@budibase/bbui"
   import TestDisplay from "./TestDisplay.svelte"
-  import { automationStore } from "stores/builder"
+  import { automationStore, selectedAutomation } from "stores/builder"
 
   export let automation
 </script>
@@ -9,9 +9,9 @@
 <div class="title">
   <div class="title-text">
     <Icon name="MultipleCheck" />
-    <div style="padding-left: var(--spacing-l); ">Test Details</div>
+    <div>Test Details</div>
   </div>
-  <div style="padding-right: var(--spacing-xl)">
+  <div>
     <Icon
       on:click={async () => {
         $automationStore.showTestPanel = false
@@ -24,7 +24,11 @@
 
 <Divider />
 
-<TestDisplay {automation} testResults={$automationStore.testResults} />
+<TestDisplay
+  {automation}
+  testResults={$automationStore.testResults}
+  automationBlockRefs={$selectedAutomation.blockRefs}
+/>
 
 <style>
   .title {
@@ -32,7 +36,8 @@
     flex-direction: row;
     align-items: center;
     gap: var(--spacing-xs);
-    padding-left: var(--spacing-xl);
+    padding: 0px 30px;
+    padding-top: var(--spacing-l);
     justify-content: space-between;
   }
 
@@ -40,7 +45,7 @@
     display: flex;
     flex-direction: row;
     align-items: center;
-    padding-top: var(--spacing-s);
+    gap: var(--spacing-l);
   }
 
   .title :global(h1) {

packages/builder/src/components/automation/AutomationPanel/AutomationNavItem.svelte

@@ -113,7 +113,7 @@
     ? "var(--spectrum-global-color-gray-600)"
     : "var(--spectrum-global-color-gray-900)"}
   text={automation.name}
-  selected={automation._id === $selectedAutomation?._id}
+  selected={automation._id === $selectedAutomation?.data?._id}
   hovering={automation._id === $contextMenuStore.id}
   on:click={() => automationStore.actions.select(automation._id)}
   selectedBy={$userSelectedResourceMap[automation._id]}

packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte

@@ -21,8 +21,8 @@
   } from "@budibase/bbui"
 
   import CreateWebhookModal from "components/automation/Shared/CreateWebhookModal.svelte"
-  import { automationStore, selectedAutomation, tables } from "stores/builder"
-  import { environment, licensing } from "stores/portal"
+  import { automationStore, tables } from "stores/builder"
+  import { environment } from "stores/portal"
   import WebhookDisplay from "../Shared/WebhookDisplay.svelte"
   import {
     BindingSidePanel,
@@ -46,10 +46,7 @@
   } from "components/common/CodeEditor"
   import FilterBuilder from "components/design/settings/controls/FilterEditor/FilterBuilder.svelte"
   import { QueryUtils, Utils, search, memo } from "@budibase/frontend-core"
-  import {
-    getSchemaForDatasourcePlus,
-    getEnvironmentBindings,
-  } from "dataBinding"
+  import { getSchemaForDatasourcePlus } from "dataBinding"
   import { TriggerStepID, ActionStepID } from "constants/backend/automations"
   import { onMount } from "svelte"
   import { writable } from "svelte/store"
@@ -60,18 +57,18 @@
     AutomationActionStepId,
     AutomationCustomIOType,
   } from "@budibase/types"
-  import { FIELDS } from "constants/backend"
   import PropField from "./PropField.svelte"
   import { utils } from "@budibase/shared-core"
 
+  export let automation
   export let block
   export let testData
   export let schemaProperties
   export let isTestModal = false
+  export let bindings = []
 
   // Stop unnecessary rendering
   const memoBlock = memo(block)
-  const memoEnvVariables = memo($environment.variables)
 
   const rowTriggers = [
     TriggerStepID.ROW_UPDATED,
@@ -94,7 +91,6 @@
   let insertAtPos, getCaretPosition
   let stepLayouts = {}
 
-  $: memoEnvVariables.set($environment.variables)
   $: memoBlock.set(block)
 
   $: filters = lookForFilters(schemaProperties)
@@ -107,13 +103,6 @@
   $: tempFilters = cloneDeep(filters)
   $: stepId = $memoBlock.stepId
 
-  $: automationBindings = getAvailableBindings(
-    $memoBlock,
-    $selectedAutomation?.definition
-  )
-  $: environmentBindings = buildEnvironmentBindings($memoEnvVariables)
-  $: bindings = [...automationBindings, ...environmentBindings]
-
   $: getInputData(testData, $memoBlock.inputs)
   $: tableId = inputData ? inputData.tableId : null
   $: table = tableId
@@ -146,21 +135,6 @@
       ? [hbAutocomplete([...bindingsToCompletions(bindings, codeMode)])]
       : []
 
-  const buildEnvironmentBindings = () => {
-    if ($licensing.environmentVariablesEnabled) {
-      return getEnvironmentBindings().map(binding => {
-        return {
-          ...binding,
-          display: {
-            ...binding.display,
-            rank: 98,
-          },
-        }
-      })
-    }
-    return []
-  }
-
   const getInputData = (testData, blockInputs) => {
     // Test data is not cloned for reactivity
     let newInputData = testData || cloneDeep(blockInputs)
@@ -437,7 +411,7 @@
 
         if (
           Object.hasOwn(update, "tableId") &&
-          $selectedAutomation.testData?.row?.tableId !== update.tableId
+          automation.testData?.row?.tableId !== update.tableId
         ) {
           const reqSchema = getSchemaForDatasourcePlus(update.tableId, {
             searchableSchema: true,
@@ -529,7 +503,15 @@
       row: { "Active": true, "Order Id" : 14, ... }
     })
    */
-  const onChange = Utils.sequential(async update => {
+  const onChange = async update => {
+    if (isTestModal) {
+      testData = update
+    }
+
+    updateAutomation(update)
+  }
+
+  const updateAutomation = Utils.sequential(async update => {
     const request = cloneDeep(update)
     // Process app trigger updates
     if (isTrigger && !isTestModal) {
@@ -566,7 +548,7 @@
             ...newTestData,
             body: {
               ...update,
-              ...$selectedAutomation.testData?.body,
+              ...automation.testData?.body,
             },
           }
         }
@@ -574,6 +556,7 @@
           ...newTestData,
           ...request,
         }
+
         await automationStore.actions.addTestDataToAutomation(newTestData)
       } else {
         const data = { schema, ...request }
@@ -585,201 +568,6 @@
     }
   })
 
-  function getAvailableBindings(block, automation) {
-    if (!block || !automation) {
-      return []
-    }
-
-    // Find previous steps to the selected one
-    let allSteps = [...automation.steps]
-
-    if (automation.trigger) {
-      allSteps = [automation.trigger, ...allSteps]
-    }
-    let blockIdx = allSteps.findIndex(step => step.id === block.id)
-
-    // Extract all outputs from all previous steps as available bindingsx§x
-    let bindings = []
-    let loopBlockCount = 0
-    const addBinding = (name, value, icon, idx, isLoopBlock, bindingName) => {
-      if (!name) return
-      const runtimeBinding = determineRuntimeBinding(
-        name,
-        idx,
-        isLoopBlock,
-        bindingName
-      )
-      const categoryName = determineCategoryName(idx, isLoopBlock, bindingName)
-      bindings.push(
-        createBindingObject(
-          name,
-          value,
-          icon,
-          idx,
-          loopBlockCount,
-          isLoopBlock,
-          runtimeBinding,
-          categoryName,
-          bindingName
-        )
-      )
-    }
-
-    const determineRuntimeBinding = (name, idx, isLoopBlock, bindingName) => {
-      let runtimeName
-
-      /* Begin special cases for generating custom schemas based on triggers */
-      if (
-        idx === 0 &&
-        automation.trigger?.event === AutomationEventType.APP_TRIGGER
-      ) {
-        return `trigger.fields.${name}`
-      }
-
-      if (
-        idx === 0 &&
-        (automation.trigger?.event === AutomationEventType.ROW_UPDATE ||
-          automation.trigger?.event === AutomationEventType.ROW_SAVE)
-      ) {
-        let noRowKeywordBindings = ["id", "revision", "oldRow"]
-        if (!noRowKeywordBindings.includes(name)) return `trigger.row.${name}`
-      }
-      /* End special cases for generating custom schemas based on triggers */
-
-      let hasUserDefinedName = automation.stepNames?.[allSteps[idx]?.id]
-      if (isLoopBlock) {
-        runtimeName = `loop.${name}`
-      } else if (idx === 0) {
-        runtimeName = `trigger.${name}`
-      } else if (block.name.startsWith("JS")) {
-        runtimeName = hasUserDefinedName
-          ? `stepsByName["${bindingName}"].${name}`
-          : `steps["${idx - loopBlockCount}"].${name}`
-      } else {
-        runtimeName = hasUserDefinedName
-          ? `stepsByName.${bindingName}.${name}`
-          : `steps.${idx - loopBlockCount}.${name}`
-      }
-      return runtimeName
-    }
-
-    const determineCategoryName = (idx, isLoopBlock, bindingName) => {
-      if (idx === 0) return "Trigger outputs"
-      if (isLoopBlock) return "Loop Outputs"
-      return bindingName
-        ? `${bindingName} outputs`
-        : `Step ${idx - loopBlockCount} outputs`
-    }
-
-    const createBindingObject = (
-      name,
-      value,
-      icon,
-      idx,
-      loopBlockCount,
-      isLoopBlock,
-      runtimeBinding,
-      categoryName,
-      bindingName
-    ) => {
-      const field = Object.values(FIELDS).find(
-        field => field.type === value.type && field.subtype === value.subtype
-      )
-      return {
-        readableBinding:
-          bindingName && !isLoopBlock && idx !== 0
-            ? `steps.${bindingName}.${name}`
-            : runtimeBinding,
-        runtimeBinding,
-        type: value.type,
-        description: value.description,
-        icon,
-        category: categoryName,
-        display: {
-          type: field?.name || value.type,
-          name,
-          rank: isLoopBlock ? idx + 1 : idx - loopBlockCount,
-        },
-      }
-    }
-
-    for (let idx = 0; idx < blockIdx; idx++) {
-      let wasLoopBlock = allSteps[idx - 1]?.stepId === ActionStepID.LOOP
-      let isLoopBlock =
-        allSteps[idx]?.stepId === ActionStepID.LOOP &&
-        allSteps.some(x => x.blockToLoop === block.id)
-      let schema = cloneDeep(allSteps[idx]?.schema?.outputs?.properties) ?? {}
-      if (allSteps[idx]?.name.includes("Looping")) {
-        isLoopBlock = true
-        loopBlockCount++
-      }
-      let bindingName =
-        automation.stepNames?.[allSteps[idx].id] || allSteps[idx].name
-
-      if (isLoopBlock) {
-        schema = {
-          currentItem: {
-            type: "string",
-            description: "the item currently being executed",
-          },
-        }
-      }
-
-      if (
-        idx === 0 &&
-        automation.trigger?.event === AutomationEventType.APP_TRIGGER
-      ) {
-        schema = Object.fromEntries(
-          Object.keys(automation.trigger.inputs.fields || []).map(key => [
-            key,
-            { type: automation.trigger.inputs.fields[key] },
-          ])
-        )
-      }
-      if (
-        (idx === 0 &&
-          automation.trigger.event === AutomationEventType.ROW_UPDATE) ||
-        (idx === 0 && automation.trigger.event === AutomationEventType.ROW_SAVE)
-      ) {
-        let table = $tables.list.find(
-          table => table._id === automation.trigger.inputs.tableId
-        )
-        // We want to generate our own schema for the bindings from the table schema itself
-        for (const key in table?.schema) {
-          schema[key] = {
-            type: table.schema[key].type,
-            subtype: table.schema[key].subtype,
-          }
-        }
-        // remove the original binding
-        delete schema.row
-      }
-      let icon =
-        idx === 0
-          ? automation.trigger.icon
-          : isLoopBlock
-          ? "Reuse"
-          : allSteps[idx].icon
-
-      if (wasLoopBlock) {
-        schema = cloneDeep(allSteps[idx - 1]?.schema?.outputs?.properties)
-      }
-      Object.entries(schema).forEach(([name, value]) => {
-        addBinding(name, value, icon, idx, isLoopBlock, bindingName)
-      })
-    }
-
-    if (
-      allSteps[blockIdx - 1]?.stepId !== ActionStepID.LOOP &&
-      allSteps
-        .slice(0, blockIdx)
-        .some(step => step.stepId === ActionStepID.LOOP)
-    ) {
-      bindings = bindings.filter(x => !x.readableBinding.includes("loop"))
-    }
-    return bindings
-  }
-
   function lookForFilters(properties) {
     if (!properties) {
       return []
@@ -865,7 +653,7 @@
   <!-- Custom Layouts -->
   {#if stepLayouts[block.stepId]}
     {#each Object.keys(stepLayouts[block.stepId] || {}) as key}
-      {#if canShowField(key, stepLayouts[block.stepId].schema)}
+      {#if canShowField(stepLayouts[block.stepId].schema)}
         {#each stepLayouts[block.stepId][key].content as config}
           {#if config.title}
             <PropField label={config.title} labelTooltip={config.tooltip}>
@@ -890,7 +678,7 @@
   {:else}
     <!-- Default Schema Property Layout -->
     {#each schemaProperties as [key, value]}
-      {#if canShowField(key, value)}
+      {#if canShowField(value)}
         {@const label = getFieldLabel(key, value)}
         <div class:block-field={shouldRenderField(value)}>
           {#if key !== "fields" && value.type !== "boolean" && shouldRenderField(value)}
@@ -912,8 +700,8 @@
               {/if}
             </div>
           {/if}
-          <div class:field-width={shouldRenderField(value)}>
-            {#if value.type === "string" && value.enum && canShowField(key, value)}
+          <div>
+            {#if value.type === "string" && value.enum && canShowField(value)}
               <Select
                 on:change={e => onChange({ [key]: e.detail })}
                 value={inputData[key]}
@@ -1208,8 +996,9 @@
     align-items: center;
     gap: var(--spacing-s);
   }
-  .field-width {
-    width: 320px;
+
+  .label-container :global(label) {
+    white-space: unset;
   }
 
   .step-fields {
@@ -1221,12 +1010,9 @@
   }
 
   .block-field {
-    display: flex;
     justify-content: space-between;
-    flex-direction: row;
-    align-items: center;
-    gap: 10px;
-    flex: 1;
+    display: grid;
+    grid-template-columns: 1fr 320px;
   }
 
   .attachment-field-width {

packages/builder/src/components/automation/SetupPanel/AutomationSelector.svelte

@@ -29,7 +29,7 @@
   $: filteredAutomations = $automationStore.automations.filter(
     automation =>
       automation.definition.trigger.stepId === TriggerStepID.APP &&
-      automation._id !== $selectedAutomation._id
+      automation._id !== $selectedAutomation.data._id
   )
 </script>
 

packages/builder/src/components/automation/Shared/CreateWebhookModal.svelte

@@ -11,7 +11,7 @@
   let schemaURL
   let propCount = 0
 
-  $: automation = $selectedAutomation
+  $: automation = $selectedAutomation?.data
 
   onMount(async () => {
     if (!automation?.definition?.trigger?.inputs.schemaUrl) {

packages/builder/src/components/backend/DataTable/buttons/grid/ColumnsSettingContent.svelte

@@ -11,7 +11,6 @@
   export let disabledPermissions = []
   export let columns
   export let fromRelationshipField
-  export let canSetRelationshipSchemas
 
   const { datasource, dispatch } = getContext("grid")
 
@@ -129,6 +128,8 @@
     }
   })
 
+  $: hasLinkColumns = columns.some(c => c.schema.type === FieldType.LINK)
+
   async function toggleColumn(column, permission) {
     const visible = permission !== FieldPermissions.HIDDEN
     const readonly = permission === FieldPermissions.READONLY
@@ -184,7 +185,7 @@
           value={columnToPermissionOptions(column)}
           options={column.options}
         />
-        {#if canSetRelationshipSchemas && column.schema.type === FieldType.LINK && columnToPermissionOptions(column) !== FieldPermissions.HIDDEN}
+        {#if column.schema.type === FieldType.LINK && columnToPermissionOptions(column) !== FieldPermissions.HIDDEN}
           <div class="relationship-columns">
             <ActionButton
               on:click={e => {
@@ -203,7 +204,7 @@
   </div>
 </div>
 
-{#if canSetRelationshipSchemas}
+{#if hasLinkColumns}
   <Popover
     on:close={() => (relationshipFieldName = null)}
     open={relationshipFieldName}

packages/builder/src/components/backend/DataTable/buttons/grid/GridColumnsSettingButton.svelte

@@ -10,8 +10,6 @@
   import { getContext } from "svelte"
   import { ActionButton } from "@budibase/bbui"
   import ColumnsSettingContent from "./ColumnsSettingContent.svelte"
-  import { isEnabled } from "helpers/featureFlags"
-  import { FeatureFlag } from "@budibase/types"
   import DetailPopover from "components/common/DetailPopover.svelte"
 
   const { tableColumns, datasource } = getContext("grid")
@@ -46,9 +44,5 @@
       {text}
     </ActionButton>
   </svelte:fragment>
-  <ColumnsSettingContent
-    columns={$tableColumns}
-    canSetRelationshipSchemas={isEnabled(FeatureFlag.ENRICHED_RELATIONSHIPS)}
-    {permissions}
-  />
+  <ColumnsSettingContent columns={$tableColumns} {permissions} />
 </DetailPopover>

packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte

@@ -371,6 +371,7 @@
     delete editableColumn.relationshipType
     delete editableColumn.formulaType
     delete editableColumn.constraints
+    delete editableColumn.responseType
 
     // Add in defaults and initial definition
     const definition = fieldDefinitions[type?.toUpperCase()]
@@ -386,6 +387,7 @@
       editableColumn.relationshipType = RelationshipType.MANY_TO_MANY
     } else if (editableColumn.type === FieldType.FORMULA) {
       editableColumn.formulaType = "dynamic"
+      editableColumn.responseType = field.responseType || FIELDS.STRING.type
     }
   }
 
@@ -767,6 +769,25 @@
         </div>
       </div>
     {/if}
+    <div class="split-label">
+      <div class="label-length">
+        <Label size="M">Response Type</Label>
+      </div>
+      <div class="input-length">
+        <Select
+          bind:value={editableColumn.responseType}
+          options={[
+            FIELDS.STRING,
+            FIELDS.NUMBER,
+            FIELDS.BOOLEAN,
+            FIELDS.DATETIME,
+          ]}
+          getOptionLabel={option => option.name}
+          getOptionValue={option => option.type}
+          tooltip="Formulas by default will return a string - however if you need another type the response can be coerced."
+        />
+      </div>
+    </div>
     <div class="split-label">
       <div class="label-length">
         <Label size="M">Formula</Label>

packages/builder/src/components/common/NavItem.svelte

@@ -84,8 +84,8 @@
   on:mouseleave
   on:click={onClick}
   on:contextmenu
-  ondragover="return false"
-  ondragenter="return false"
+  on:dragover={e => e.preventDefault()}
+  on:dragenter={e => e.preventDefault()}
   {id}
   {style}
   {draggable}

packages/builder/src/components/common/UndoRedoControl.svelte

@@ -1,9 +1,10 @@
 <script>
-  import { Icon } from "@budibase/bbui"
+  import { Icon, ActionButton } from "@budibase/bbui"
   import { onMount } from "svelte"
   import { isBuilderInputFocused } from "helpers"
 
   export let store
+  export let showButtonGroup = false
 
   const handleKeyPress = e => {
     if (!(e.ctrlKey || e.metaKey)) {
@@ -32,7 +33,23 @@
   })
 </script>
 
-<div class="undo-redo">
+<div class="undo-redo" class:buttons={showButtonGroup}>
+  {#if showButtonGroup}
+    <div class="group">
+      <ActionButton
+        icon="Undo"
+        quiet
+        on:click={store.undo}
+        disabled={!$store.canUndo}
+      />
+      <ActionButton
+        icon="Redo"
+        quiet
+        on:click={store.redo}
+        disabled={!$store.canRedo}
+      />
+    </div>
+  {:else}
     <Icon
       name="Undo"
       hoverable
@@ -45,6 +62,7 @@
       on:click={store.redo}
       disabled={!$store.canRedo}
     />
+  {/if}
 </div>
 
 <style>
@@ -60,4 +78,36 @@
   .undo-redo :global(svg) {
     padding: 6px;
   }
+
+  .undo-redo.buttons :global(svg) {
+    padding: 0px;
+  }
+
+  .undo-redo.buttons {
+    padding-right: 0px;
+  }
+
+  .group {
+    border-radius: 4px;
+    display: flex;
+    flex-direction: row;
+  }
+  .group :global(> *:not(:first-child)) {
+    border-top-left-radius: 0;
+    border-bottom-left-radius: 0;
+    border-left: 2px solid var(--spectrum-global-color-gray-300);
+  }
+  .group :global(> *:not(:last-child)) {
+    border-top-right-radius: 0;
+    border-bottom-right-radius: 0;
+  }
+
+  .group :global(.spectrum-Button),
+  .group :global(.spectrum-ActionButton) {
+    background: var(--spectrum-global-color-gray-200) !important;
+  }
+  .group :global(.spectrum-Button:hover),
+  .group :global(.spectrum-ActionButton:hover) {
+    background: var(--spectrum-global-color-gray-300) !important;
+  }
 </style>

packages/builder/src/components/design/settings/controls/FilterEditor/FilterBuilder.svelte

@@ -12,8 +12,10 @@
   export let bindings = []
   export let panel = ClientBindingPanel
   export let allowBindings = true
+  export let allowOnEmpty
   export let datasource
-  export let showFilterEmptyDropdown
+  export let builderType
+  export let docsURL
 </script>
 
 <CoreFilterBuilder
@@ -26,7 +28,9 @@
   {schemaFields}
   {datasource}
   {allowBindings}
-  {showFilterEmptyDropdown}
   {bindings}
+  {allowOnEmpty}
+  {builderType}
+  {docsURL}
   on:change
 />

packages/builder/src/dataBinding.js

@@ -614,6 +614,40 @@ const getDeviceBindings = () => {
   return bindings
 }
 
+export const getSettingBindings = () => {
+  let bindings = []
+  const safeSetting = makePropSafe("settings")
+
+  bindings = [
+    {
+      type: "context",
+      runtimeBinding: `${safeSetting}.${makePropSafe("url")}`,
+      readableBinding: `Settings.url`,
+      category: "Settings",
+      icon: "Settings",
+      display: { type: "string", name: "url" },
+    },
+    {
+      type: "context",
+      runtimeBinding: `${safeSetting}.${makePropSafe("logo")}`,
+      readableBinding: `Settings.logo`,
+      category: "Settings",
+      icon: "Settings",
+      display: { type: "string", name: "logo" },
+    },
+    {
+      type: "context",
+      runtimeBinding: `${safeSetting}.${makePropSafe("company")}`,
+      readableBinding: `Settings.company`,
+      category: "Settings",
+      icon: "Settings",
+      display: { type: "string", name: "company" },
+    },
+  ]
+
+  return bindings
+}
+
 /**
  * Gets all selected rows bindings for tables in the current asset.
  * TODO: remove in future because we don't need a separate store for this
@@ -1469,3 +1503,31 @@ export const updateReferencesInObject = ({
     }
   }
 }
+
+// Migrate references
+// Switch all bindings to reference their ids
+export const migrateReferencesInObject = ({ obj, label = "steps", steps }) => {
+  const stepIndexRegex = new RegExp(`{{\\s*${label}\\.(\\d+)\\.`, "g")
+  const updateActionStep = (str, index, replaceWith) =>
+    str.replace(`{{ ${label}.${index}.`, `{{ ${label}.${replaceWith}.`)
+
+  for (const key in obj) {
+    if (typeof obj[key] === "string") {
+      let matches
+      while ((matches = stepIndexRegex.exec(obj[key])) !== null) {
+        const referencedStep = parseInt(matches[1])
+
+        obj[key] = updateActionStep(
+          obj[key],
+          referencedStep,
+          steps[referencedStep]?.id
+        )
+      }
+    } else if (typeof obj[key] === "object" && obj[key] !== null) {
+      migrateReferencesInObject({
+        obj: obj[key],
+        steps,
+      })
+    }
+  }
+}

packages/builder/src/pages/builder/app/[application]/automation/_layout.svelte

@@ -13,7 +13,7 @@
     selectedAutomation,
   } from "stores/builder"
 
-  $: automationId = $selectedAutomation?._id
+  $: automationId = $selectedAutomation?.data?._id
   $: builderStore.selectResource(automationId)
 
   // Keep URL and state in sync for selected screen ID
@@ -68,7 +68,7 @@
 
   {#if $automationStore.showTestPanel}
     <div class="setup">
-      <TestPanel automation={$selectedAutomation} />
+      <TestPanel automation={$selectedAutomation.data} />
     </div>
   {/if}
   <Modal bind:this={modal}>

packages/builder/src/pages/builder/app/[application]/design/[screenId]/_components/ComponentList/ComponentScrollWrapper.svelte

@@ -68,8 +68,8 @@
   on:scroll
   bind:this={scrollRef}
   on:drop={onDrop}
-  ondragover="return false"
-  ondragenter="return false"
+  on:dragover={e => e.preventDefault()}
+  on:dragenter={e => e.preventDefault()}
 >
   <slot />
 </div>

packages/builder/src/pages/builder/app/[application]/settings/automations/_components/HistoryDetailsPanel.svelte

@@ -56,7 +56,7 @@
     {/if}
     {#key history}
       <div class="history">
-        <TestDisplay testResults={history} width="320px" />
+        <TestDisplay testResults={history} />
       </div>
     {/key}
   </Layout>
@@ -65,6 +65,14 @@
 {/if}
 
 <style>
+  .history :global(.block) {
+    min-width: unset;
+  }
+  .history :global(> .container) {
+    max-width: 320px;
+    width: 320px;
+    padding: 0px;
+  }
   .controls {
     display: flex;
     gap: var(--spacing-s);
@@ -76,7 +84,4 @@
     width: 100%;
     justify-content: center;
   }
-  .history {
-    margin: 0 -30px;
-  }
 </style>

packages/builder/src/pages/builder/portal/settings/index.svelte

@@ -1,8 +1,8 @@
 <script>
   import { redirect } from "@roxi/routify"
-  import { licensing, featureFlags } from "stores/portal"
+  import { featureFlags } from "stores/portal"
 
-  if ($featureFlags.AI_CUSTOM_CONFIGS && $licensing.customAIConfigsEnabled) {
+  if ($featureFlags.AI_CUSTOM_CONFIGS) {
     $redirect("./ai")
   } else {
     $redirect("./auth")

packages/builder/src/pages/builder/portal/settings/version.svelte

@@ -4,12 +4,10 @@
     Layout,
     Heading,
     Body,
-    Button,
     Divider,
     notifications,
     Label,
-    Modal,
-    ModalContent,
+    Link,
   } from "@budibase/bbui"
   import { API } from "api"
   import { auth, admin } from "stores/portal"
@@ -21,8 +19,6 @@
   let githubVersion
   let githubPublishedDate
   let githubPublishedTime
-  let needsUpdate = true
-  let updateModal
 
   // Only admins allowed here
   $: {
@@ -31,21 +27,6 @@
     }
   }
 
-  async function updateBudibase() {
-    try {
-      notifications.info("Updating budibase..")
-      await fetch("/v1/update", {
-        headers: {
-          Authorization: "Bearer budibase",
-        },
-      })
-      notifications.success("Your budibase installation is up to date.")
-      getVersion()
-    } catch (err) {
-      notifications.error(`Error installing budibase update ${err}`)
-    }
-  }
-
   async function getVersion() {
     try {
       version = await API.getBudibaseVersion()
@@ -69,13 +50,6 @@
       githubPublishedDate = new Date(githubResponse.published_at)
       githubPublishedTime = githubPublishedDate.toLocaleTimeString()
       githubPublishedDate = githubPublishedDate.toLocaleDateString()
-
-      //Does Budibase need to be updated?
-      if (githubVersion === version) {
-        needsUpdate = false
-      } else {
-        needsUpdate = true
-      }
     } catch (error) {
       notifications.error("Error getting the latest Budibase version")
       githubVersion = null
@@ -115,23 +89,15 @@
         >
       </Layout>
       <Divider />
-      <div>
-        <Button cta on:click={updateModal.show} disabled={!needsUpdate}
-          >Update Budibase</Button
+      <Layout noPadding gap="XS">
+        <Heading>Updating Budibase</Heading>
+        <Body
+          >To update your self-host installation, follow the docs found <Link
+            size="L"
+            href="https://docs.budibase.com/docs/updating-budibase">here.</Link
+          ></Body
         >
-        <Modal bind:this={updateModal}>
-          <ModalContent
-            title="Update Budibase"
-            confirmText="Update"
-            onConfirm={updateBudibase}
-          >
-            <span
-              >Are you sure you want to update your budibase installation to the
-              latest version?</span
-            >
-          </ModalContent>
-        </Modal>
-      </div>
+      </Layout>
     {/if}
   </Layout>
 {/if}

packages/client/package.json

@@ -33,8 +33,7 @@
     "sanitize-html": "^2.13.0",
     "screenfull": "^6.0.1",
     "shortid": "^2.2.15",
-    "svelte-spa-router": "^4.0.1",
-    "atrament": "^4.3.0"
+    "svelte-spa-router": "^4.0.1"
   },
   "devDependencies": {
     "@rollup/plugin-alias": "^5.1.0",

packages/client/src/utils/blocks.js

@@ -1,5 +1,7 @@
 import { makePropSafe as safe } from "@budibase/string-templates"
 import { API } from "../api/index.js"
+import { UILogicalOperator } from "@budibase/types"
+import { OnEmptyFilter } from "@budibase/frontend-core/src/constants.js"
 
 // Map of data types to component types for search fields inside blocks
 const schemaComponentMap = {
@@ -60,7 +62,11 @@ export const enrichSearchColumns = async (searchColumns, schema) => {
  * @param formId the ID of the form containing the search fields
  */
 export const enrichFilter = (filter, columns, formId) => {
-  let enrichedFilter = [...(filter || [])]
+  if (!columns?.length) {
+    return filter
+  }
+
+  let newFilters = []
   columns?.forEach(column => {
     const safePath = column.name.split(".").map(safe).join(".")
     const stringType = column.type === "string" || column.type === "formula"
@@ -69,7 +75,7 @@ export const enrichFilter = (filter, columns, formId) => {
 
     // For dates, use a range of the entire day selected
     if (dateType) {
-      enrichedFilter.push({
+      newFilters.push({
         field: column.name,
         type: column.type,
         operator: "rangeLow",
@@ -79,7 +85,7 @@ export const enrichFilter = (filter, columns, formId) => {
       const format = "YYYY-MM-DDTHH:mm:ss.SSSZ"
       let hbs = `{{ date (add (date ${binding} "x") 86399999) "${format}" }}`
       hbs = `{{#if ${binding} }}${hbs}{{/if}}`
-      enrichedFilter.push({
+      newFilters.push({
         field: column.name,
         type: column.type,
         operator: "rangeHigh",
@@ -90,7 +96,7 @@ export const enrichFilter = (filter, columns, formId) => {
 
     // For other fields, do an exact match
     else {
-      enrichedFilter.push({
+      newFilters.push({
         field: column.name,
         type: column.type,
         operator: stringType ? "string" : "equal",
@@ -99,5 +105,16 @@ export const enrichFilter = (filter, columns, formId) => {
       })
     }
   })
-  return enrichedFilter
+
+  return {
+    logicalOperator: UILogicalOperator.ALL,
+    onEmptyFilter: OnEmptyFilter.RETURN_ALL,
+    groups: [
+      ...(filter?.groups || []),
+      {
+        logicalOperator: UILogicalOperator.ALL,
+        filters: newFilters,
+      },
+    ],
+  }
 }

packages/frontend-core/src/components/ConditionField.svelte

@@ -0,0 +1,177 @@
+<script>
+  import { Input, Icon, Drawer, Button } from "@budibase/bbui"
+  import { isJSBinding } from "@budibase/string-templates"
+  import { createEventDispatcher } from "svelte"
+
+  export let filter
+  export let disabled = false
+  export let bindings = []
+  export let panel
+  export let drawerTitle
+  export let toReadable
+  export let toRuntime
+
+  const dispatch = createEventDispatcher()
+
+  let bindingDrawer
+  let fieldValue
+
+  $: fieldValue = filter?.field
+  $: readableValue = toReadable ? toReadable(bindings, fieldValue) : fieldValue
+  $: drawerValue = toDrawerValue(fieldValue)
+  $: isJS = isJSBinding(fieldValue)
+
+  const drawerOnChange = e => {
+    drawerValue = e.detail
+  }
+
+  const onChange = e => {
+    fieldValue = e.detail
+    dispatch("change", {
+      field: toRuntime ? toRuntime(bindings, fieldValue) : fieldValue,
+    })
+  }
+
+  const onConfirmBinding = () => {
+    dispatch("change", {
+      field: toRuntime ? toRuntime(bindings, drawerValue) : drawerValue,
+    })
+  }
+
+  /**
+   * Converts arrays into strings. The CodeEditor expects a string or encoded JS
+   *
+   * @param{string} fieldValue
+   */
+  const toDrawerValue = fieldValue => {
+    return Array.isArray(fieldValue) ? fieldValue.join(",") : readableValue
+  }
+</script>
+
+<div>
+  <Drawer
+    on:drawerHide
+    on:drawerShow
+    bind:this={bindingDrawer}
+    title={drawerTitle || ""}
+    forceModal
+  >
+    <Button
+      cta
+      slot="buttons"
+      on:click={() => {
+        onConfirmBinding()
+        bindingDrawer.hide()
+      }}
+    >
+      Confirm
+    </Button>
+
+    <svelte:component
+      this={panel}
+      slot="body"
+      value={drawerValue}
+      allowJS
+      allowHelpers
+      allowHBS
+      on:change={drawerOnChange}
+      {bindings}
+    />
+  </Drawer>
+
+  <div class="field-wrap" class:bindings={true}>
+    <div class="field">
+      <Input
+        disabled={filter.noValue}
+        readonly={isJS}
+        value={isJS ? "(JavaScript function)" : readableValue}
+        on:change={onChange}
+      />
+    </div>
+
+    <div class="binding-control">
+      {#if !disabled}
+        <!-- svelte-ignore a11y-click-events-have-key-events -->
+        <!-- svelte-ignore a11y-no-static-element-interactions -->
+        <div
+          class="icon binding"
+          on:click={() => {
+            bindingDrawer.show()
+          }}
+        >
+          <Icon size="S" name="FlashOn" />
+        </div>
+      {/if}
+    </div>
+  </div>
+</div>
+
+<style>
+  .field-wrap {
+    display: flex;
+  }
+  .field {
+    flex: 1;
+  }
+
+  .field-wrap.bindings .field :global(.spectrum-Form-itemField),
+  .field-wrap.bindings .field :global(input),
+  .field-wrap.bindings .field :global(.spectrum-Picker) {
+    border-top-right-radius: 0px;
+    border-bottom-right-radius: 0px;
+  }
+
+  .field-wrap.bindings
+    .field
+    :global(.spectrum-InputGroup.spectrum-Datepicker) {
+    min-width: unset;
+    border-radius: 0px;
+  }
+
+  .field-wrap.bindings
+    .field
+    :global(
+      .spectrum-InputGroup.spectrum-Datepicker
+        .spectrum-Textfield-input.spectrum-InputGroup-input
+    ) {
+    width: 100%;
+  }
+
+  .binding-control .icon {
+    border: 1px solid
+      var(
+        --spectrum-textfield-m-border-color,
+        var(--spectrum-alias-border-color)
+      );
+    border-left: 0px;
+    border-top-right-radius: 4px;
+    border-bottom-right-radius: 4px;
+    justify-content: center;
+    align-items: center;
+    display: flex;
+    flex-direction: row;
+    box-sizing: border-box;
+    width: 31px;
+    color: var(--spectrum-alias-text-color);
+    background-color: var(--spectrum-global-color-gray-75);
+    transition: background-color
+        var(--spectrum-global-animation-duration-100, 130ms),
+      box-shadow var(--spectrum-global-animation-duration-100, 130ms),
+      border-color var(--spectrum-global-animation-duration-100, 130ms);
+    height: calc(var(--spectrum-alias-item-height-m));
+  }
+  .binding-control .icon.binding {
+    color: var(--yellow);
+  }
+
+  .binding-control .icon:hover {
+    cursor: pointer;
+    background-color: var(--spectrum-global-color-gray-50);
+    border-color: var(--spectrum-alias-border-color-hover);
+    color: var(--spectrum-alias-text-color-hover);
+  }
+
+  .binding-control .icon.binding:hover {
+    color: var(--yellow);
+  }
+</style>

packages/frontend-core/src/components/CoreFilterBuilder.svelte

@@ -16,6 +16,7 @@
   import { QueryUtils, Constants } from "@budibase/frontend-core"
   import { getContext, createEventDispatcher } from "svelte"
   import FilterField from "./FilterField.svelte"
+  import ConditionField from "./ConditionField.svelte"
   import { utils } from "@budibase/shared-core"
 
   const dispatch = createEventDispatcher()
@@ -33,8 +34,10 @@
   export let datasource
   export let behaviourFilters = false
   export let allowBindings = false
+  export let allowOnEmpty = true
+  export let builderType = "filter"
+  export let docsURL = "https://docs.budibase.com/docs/searchfilter-data"
 
-  // Review
   export let bindings
   export let panel
   export let toReadable
@@ -53,6 +56,10 @@
       schemaFields = [...schemaFields, { name: "_id", type: "string" }]
     }
   }
+  $: prefix =
+    builderType === "filter"
+      ? "Show data which matches"
+      : "Run branch when matching"
 
   // We still may need to migrate this even though the backend does it automatically now
   // for query definitions. This is because we might be editing saved filter definitions
@@ -103,6 +110,10 @@
   }
 
   const getValidOperatorsForType = filter => {
+    if (builderType === "condition") {
+      return [OperatorOptions.Equals, OperatorOptions.NotEquals]
+    }
+
     if (!filter?.field && !filter?.name) {
       return []
     }
@@ -222,6 +233,9 @@
       } else if (addFilter) {
         targetGroup.filters.push({
           valueType: FilterValueType.VALUE,
+          ...(builderType === "condition"
+            ? { operator: OperatorOptions.Equals.value, type: FieldType.STRING }
+            : {}),
         })
       } else if (group) {
         editable.groups[groupIdx] = {
@@ -242,6 +256,11 @@
         filters: [
           {
             valueType: FilterValueType.VALUE,
+            ...(builderType === "condition"
+              ? {
+                  operator: OperatorOptions.Equals.value,
+                }
+              : {}),
           },
         ],
       })
@@ -271,7 +290,7 @@
 
       {#if editableFilters?.groups?.length}
         <div class="global-filter-header">
-          <span>Show data which matches</span>
+          <span>{prefix}</span>
           <span class="operator-picker">
             <Select
               value={editableFilters?.logicalOperator}
@@ -286,7 +305,7 @@
               placeholder={false}
             />
           </span>
-          <span>of the following filter groups:</span>
+          <span>of the following {builderType} groups:</span>
         </div>
       {/if}
       {#if editableFilters?.groups?.length}
@@ -315,7 +334,7 @@
                       placeholder={false}
                     />
                   </span>
-                  <span>of the following filters are matched:</span>
+                  <span>of the following {builderType}s are matched:</span>
                 </div>
                 <div class="group-actions">
                   <Icon
@@ -346,6 +365,7 @@
               <div class="filters">
                 {#each group.filters as filter, filterIdx}
                   <div class="filter">
+                    {#if builderType === "filter"}
                       <Select
                         value={filter.field}
                         options={fieldOptions}
@@ -356,10 +376,27 @@
                         }}
                         placeholder="Column"
                       />
-
+                    {:else}
+                      <ConditionField
+                        placeholder="Value"
+                        {filter}
+                        drawerTitle={"Edit Binding"}
+                        {bindings}
+                        {panel}
+                        {toReadable}
+                        {toRuntime}
+                        on:change={e => {
+                          const updated = {
+                            ...filter,
+                            field: e.detail.field,
+                          }
+                          onFilterFieldUpdate(updated, groupIdx, filterIdx)
+                        }}
+                      />
+                    {/if}
                     <Select
                       value={filter.operator}
-                      disabled={!filter.field}
+                      disabled={!filter.field && builderType === "filter"}
                       options={getValidOperatorsForType(filter)}
                       on:change={e => {
                         const updated = { ...filter, operator: e.detail }
@@ -368,11 +405,18 @@
                       }}
                       placeholder={false}
                     />
-
                     <FilterField
                       placeholder="Value"
+                      drawerTitle={builderType === "condition"
+                        ? "Edit binding"
+                        : null}
                       {allowBindings}
-                      {filter}
+                      filter={{
+                        ...filter,
+                        ...(builderType === "condition"
+                          ? { type: FieldType.STRING }
+                          : {}),
+                      }}
                       {schemaFields}
                       {bindings}
                       {panel}
@@ -408,7 +452,7 @@
 
       <div class="filters-footer">
         <Layout noPadding>
-          {#if behaviourFilters && editableFilters?.groups?.length}
+          {#if behaviourFilters && allowOnEmpty && editableFilters?.groups?.length}
             <div class="empty-filter">
               <span>Return</span>
               <span class="empty-filter-picker">
@@ -425,7 +469,7 @@
                   placeholder={false}
                 />
               </span>
-              <span>when all filters are empty</span>
+              <span>when all {builderType}s are empty</span>
             </div>
           {/if}
           <div class="add-group">
@@ -439,17 +483,16 @@
                 })
               }}
             >
-              Add filter group
+              Add {builderType} group
             </Button>
-            <a
-              href="https://docs.budibase.com/docs/searchfilter-data"
-              target="_blank"
-            >
+            {#if docsURL}
+              <a href={docsURL} target="_blank">
                 <Icon
                   name="HelpOutline"
                   color="var(--spectrum-global-color-gray-600)"
                 />
               </a>
+            {/if}
           </div>
         </Layout>
       </div>

packages/frontend-core/src/components/FilterField.svelte

@@ -21,6 +21,7 @@
   export let allowBindings = false
   export let schemaFields
   export let panel
+  export let drawerTitle
   export let toReadable
   export let toRuntime
 
@@ -28,7 +29,6 @@
   const { OperatorOptions, FilterValueType } = Constants
 
   let bindingDrawer
-  let fieldValue
 
   $: fieldValue = filter?.value
   $: readableValue = toReadable ? toReadable(bindings, fieldValue) : fieldValue
@@ -133,7 +133,7 @@
     on:drawerHide
     on:drawerShow
     bind:this={bindingDrawer}
-    title={filter.field}
+    title={drawerTitle || filter.field}
     forceModal
   >
     <Button
@@ -168,7 +168,7 @@
       {#if filter.valueType === FilterValueType.BINDING}
         <Input
           disabled={filter.noValue}
-          readonly={true}
+          readonly={isJS}
           value={isJS ? "(JavaScript function)" : readableValue}
           on:change={onChange}
         />
@@ -231,7 +231,7 @@
 
     <div class="binding-control">
       <!-- needs field, operator -->
-      {#if !disabled && allowBindings && filter.field && !filter.noValue}
+      {#if !disabled && allowBindings && !filter.noValue}
         <!-- svelte-ignore a11y-click-events-have-key-events -->
         <!-- svelte-ignore a11y-no-static-element-interactions -->
         <div

packages/frontend-core/src/components/grid/cells/FormulaCell.svelte

@@ -1,5 +1,21 @@
 <script>
   import TextCell from "./TextCell.svelte"
+  import DateCell from "./DateCell.svelte"
+  import NumberCell from "./NumberCell.svelte"
+  import BooleanCell from "./BooleanCell.svelte"
+  import { FieldType } from "@budibase/types"
+
+  export let schema
+
+  $: responseType = schema.responseType
 </script>
 
+{#if responseType === FieldType.NUMBER}
+  <NumberCell {...$$props} readonly />
+{:else if responseType === FieldType.BOOLEAN}
+  <BooleanCell {...$$props} readonly />
+{:else if responseType === FieldType.DATETIME}
+  <DateCell {...$$props} readonly />
+{:else}
   <TextCell {...$$props} readonly />
+{/if}

packages/frontend-core/src/components/grid/layout/NewColumnButton.svelte

@@ -53,6 +53,7 @@
     on:close={close}
     maxHeight={null}
     resizable
+    minWidth={360}
   >
     <div class="content">
       <slot />
@@ -80,7 +81,6 @@
   }
 
   .content {
-    width: 300px;
     padding: 20px;
     display: flex;
     flex-direction: column;

packages/frontend-core/src/fetch/NestedProviderFetch.js

@@ -5,6 +5,7 @@ export default class NestedProviderFetch extends DataFetch {
     // Nested providers should already have exposed their own schema
     return {
       schema: datasource?.value?.schema,
+      primaryDisplay: datasource?.value?.primaryDisplay,
     }
   }
 

packages/pro

@@ -1 +1 @@
-Subproject commit 2ab8536b6005576684810d774f1ac22239218546
+Subproject commit 25dd40ee12b048307b558ebcedb36548d6e042cd

packages/server/package.json

@@ -13,6 +13,7 @@
     "build": "node ./scripts/build.js",
     "postbuild": "copyfiles -f ../client/dist/budibase-client.js ../client/manifest.json client && copyfiles -f ../../yarn.lock ./dist/",
     "check:types": "tsc -p tsconfig.json --noEmit --paths null --target es2020",
+    "check:dependencies": "node ../../scripts/depcheck.js",
     "build:isolated-vm-lib:snippets": "esbuild --minify --bundle src/jsRunner/bundles/snippets.ts --outfile=src/jsRunner/bundles/snippets.ivm.bundle.js --platform=node --format=iife --global-name=snippets",
     "build:isolated-vm-lib:string-templates": "esbuild --minify --bundle src/jsRunner/bundles/index-helpers.ts --outfile=src/jsRunner/bundles/index-helpers.ivm.bundle.js --platform=node --format=iife --external:handlebars --global-name=helpers",
     "build:isolated-vm-lib:bson": "esbuild --minify --bundle src/jsRunner/bundles/bsonPackage.ts --outfile=src/jsRunner/bundles/bson.ivm.bundle.js --platform=node --format=iife --global-name=bson",
@@ -49,9 +50,11 @@
   "license": "GPL-3.0",
   "dependencies": {
     "@apidevtools/swagger-parser": "10.0.3",
+    "@azure/msal-node": "^2.5.1",
     "@budibase/backend-core": "0.0.0",
     "@budibase/client": "0.0.0",
     "@budibase/frontend-core": "0.0.0",
+    "@budibase/nano": "10.1.5",
     "@budibase/pro": "0.0.0",
     "@budibase/shared-core": "0.0.0",
     "@budibase/string-templates": "0.0.0",
@@ -60,15 +63,17 @@
     "@bull-board/koa": "5.10.2",
     "@elastic/elasticsearch": "7.10.0",
     "@google-cloud/firestore": "7.8.0",
-    "@koa/router": "8.0.8",
+    "@koa/cors": "5.0.0",
+    "@koa/router": "13.1.0",
     "@socket.io/redis-adapter": "^8.2.1",
     "@types/xml2js": "^0.4.14",
     "airtable": "0.12.2",
     "arangojs": "7.2.0",
     "archiver": "7.0.1",
-    "aws-sdk": "2.1030.0",
+    "aws-sdk": "2.1692.0",
     "bcrypt": "5.1.0",
     "bcryptjs": "2.4.3",
+    "bson": "^6.9.0",
     "buffer": "6.0.3",
     "bull": "4.10.1",
     "chokidar": "3.5.3",
@@ -76,17 +81,20 @@
     "cookies": "0.8.0",
     "csvtojson": "2.0.10",
     "curlconverter": "3.21.0",
-    "dd-trace": "5.2.0",
+    "dayjs": "^1.10.8",
+    "dd-trace": "5.26.0",
     "dotenv": "8.2.0",
     "form-data": "4.0.0",
     "global-agent": "3.0.0",
+    "google-auth-library": "^8.0.1",
     "google-spreadsheet": "npm:@budibase/google-spreadsheet@4.1.5",
     "ioredis": "5.3.2",
     "isolated-vm": "^4.7.2",
-    "jimp": "0.22.12",
+    "jimp": "1.1.4",
     "joi": "17.6.0",
     "js-yaml": "4.1.0",
     "jsonschema": "1.4.0",
+    "jsonwebtoken": "9.0.2",
     "knex": "2.4.2",
     "koa": "2.13.4",
     "koa-body": "4.2.0",
@@ -97,7 +105,7 @@
     "lodash": "4.17.21",
     "memorystream": "0.3.1",
     "mongodb": "6.7.0",
-    "mssql": "10.0.1",
+    "mssql": "11.0.1",
     "mysql2": "3.9.8",
     "node-fetch": "2.6.7",
     "object-sizeof": "2.6.1",
@@ -105,24 +113,28 @@
     "openapi-types": "9.3.1",
     "oracledb": "6.5.1",
     "pg": "8.10.0",
-    "pouchdb": "7.3.0",
+    "pouchdb": "9.0.0",
     "pouchdb-all-dbs": "1.1.1",
-    "pouchdb-find": "7.2.2",
+    "pouchdb-find": "9.0.0",
     "redis": "4",
+    "semver": "^7.5.4",
     "serialize-error": "^7.0.1",
     "server-destroy": "1.0.1",
-    "snowflake-promise": "^4.5.0",
-    "socket.io": "4.7.5",
+    "snowflake-sdk": "^1.15.0",
+    "socket.io": "4.8.1",
+    "svelte": "^4.2.10",
     "tar": "6.2.1",
     "tmp": "0.2.3",
     "to-json-schema": "0.2.5",
     "uuid": "^8.3.2",
     "validate.js": "0.13.1",
     "worker-farm": "1.7.0",
-    "xml2js": "0.5.0"
+    "xml2js": "0.6.2"
   },
   "devDependencies": {
+    "@babel/core": "^7.22.5",
     "@babel/preset-env": "7.16.11",
+    "@jest/types": "^29.6.3",
     "@swc/core": "1.3.71",
     "@swc/jest": "0.2.27",
     "@types/archiver": "6.0.2",
@@ -130,19 +142,24 @@
     "@types/jest": "29.5.5",
     "@types/koa": "2.13.4",
     "@types/koa-send": "^4.1.6",
-    "@types/koa__router": "8.0.8",
+    "@types/koa__cors": "5.0.0",
+    "@types/koa__router": "12.0.4",
     "@types/lodash": "4.14.200",
-    "@types/mssql": "9.1.4",
+    "@types/mssql": "9.1.5",
+    "@types/node": "^22.9.0",
     "@types/node-fetch": "2.6.4",
     "@types/oracledb": "6.5.1",
     "@types/pg": "8.6.6",
+    "@types/pouchdb": "6.4.2",
     "@types/server-destroy": "1.0.1",
     "@types/supertest": "2.0.14",
     "@types/tar": "6.1.5",
     "@types/tmp": "0.2.6",
     "@types/uuid": "8.3.4",
+    "chance": "^1.1.12",
     "copyfiles": "2.4.1",
     "docker-compose": "0.23.17",
+    "ioredis-mock": "8.9.0",
     "jest": "29.7.0",
     "jest-extended": "^4.0.2",
     "jest-openapi": "0.14.2",

packages/server/scripts/test.sh

@@ -1,12 +1,12 @@
 #!/bin/bash
-set -e
+set -ex
 
 if [[ -n $CI ]]
 then
   export NODE_OPTIONS="--max-old-space-size=4096 --no-node-snapshot $NODE_OPTIONS"
-  jest --coverage --maxWorkers=4 --forceExit --workerIdleMemoryLimit=2000MB --bail $@
+  jest --coverage --maxWorkers=4 --forceExit --workerIdleMemoryLimit=2000MB --bail "$@"
 else
   # --maxWorkers performs better in development
   export NODE_OPTIONS="--no-node-snapshot $NODE_OPTIONS"
-  jest --coverage --maxWorkers=2 --forceExit $@
+  jest --coverage --maxWorkers=2 --forceExit "$@"
 fi

packages/server/specs/openapi.yaml

@@ -23,6 +23,13 @@ components:
       description: The ID of the table which this request is targeting.
       schema:
         type: string
+    viewId:
+      in: path
+      name: viewId
+      required: true
+      description: The ID of the view which this request is targeting.
+      schema:
+        type: string
     rowId:
       in: path
       name: rowId
@@ -442,6 +449,74 @@ components:
         # TYPE budibase_quota_limit_automations gauge
 
         budibase_quota_limit_automations 9007199254740991
+    view:
+      value:
+        data:
+          name: peopleView
+          tableId: ta_896a325f7e8147d2a2cda93c5d236511
+          schema:
+            name:
+              visible: true
+              readonly: false
+              order: 1
+              width: 300
+            age:
+              visible: true
+              readonly: true
+              order: 2
+              width: 200
+            salary:
+              visible: false
+              readonly: false
+          query:
+            logicalOperator: all
+            onEmptyFilter: none
+            groups:
+              - logicalOperator: any
+                filters:
+                  - operator: string
+                    field: name
+                    value: John
+                  - operator: range
+                    field: age
+                    value:
+                      low: 18
+                      high: 100
+          primaryDisplay: name
+    views:
+      value:
+        data:
+          - name: peopleView
+            tableId: ta_896a325f7e8147d2a2cda93c5d236511
+            schema:
+              name:
+                visible: true
+                readonly: false
+                order: 1
+                width: 300
+              age:
+                visible: true
+                readonly: true
+                order: 2
+                width: 200
+              salary:
+                visible: false
+                readonly: false
+            query:
+              logicalOperator: all
+              onEmptyFilter: none
+              groups:
+                - logicalOperator: any
+                  filters:
+                    - operator: string
+                      field: name
+                      value: John
+                    - operator: range
+                      field: age
+                      value:
+                        low: 18
+                        high: 100
+            primaryDisplay: name
   securitySchemes:
     ApiKeyAuth:
       type: apiKey
@@ -761,7 +836,6 @@ components:
                     enum:
                       - static
                       - dynamic
-                      - ai
                     description: Defines whether this is a static or dynamic formula.
               - type: object
                 properties:
@@ -931,7 +1005,6 @@ components:
                         enum:
                           - static
                           - dynamic
-                          - ai
                         description: Defines whether this is a static or dynamic formula.
                   - type: object
                     properties:
@@ -1108,7 +1181,6 @@ components:
                           enum:
                             - static
                             - dynamic
-                            - ai
                           description: Defines whether this is a static or dynamic formula.
                     - type: object
                       properties:
@@ -1704,6 +1776,644 @@ components:
             - userIds
       required:
         - data
+    view:
+      description: The view to be created/updated.
+      type: object
+      required:
+        - name
+        - schema
+        - tableId
+      properties:
+        name:
+          description: The name of the view.
+          type: string
+        tableId:
+          description: The ID of the table this view is based on.
+          type: string
+        type:
+          description: The type of view - standard (empty value) or calculation.
+          type: string
+          enum:
+            - calculation
+        primaryDisplay:
+          type: string
+          description: A column used to display rows from this view - usually used when
+            rendered in tables.
+        query:
+          description: Search parameters for view
+          type: object
+          required: []
+          properties:
+            logicalOperator:
+              description: When using groups this defines whether all of the filters must
+                match, or only one of them.
+              type: string
+              enum:
+                - all
+                - any
+            onEmptyFilter:
+              description: If no filters match, should the view return all rows, or no rows.
+              type: string
+              enum:
+                - all
+                - none
+            groups:
+              description: A grouping of filters to be applied.
+              type: array
+              items:
+                type: object
+                properties:
+                  logicalOperator:
+                    description: When using groups this defines whether all of the filters must
+                      match, or only one of them.
+                    type: string
+                    enum:
+                      - all
+                      - any
+                  filters:
+                    description: A list of filters to apply
+                    type: array
+                    items:
+                      type: object
+                      properties:
+                        operator:
+                          type: string
+                          description: The type of search operation which is being performed.
+                          enum:
+                            - equal
+                            - notEqual
+                            - empty
+                            - notEmpty
+                            - fuzzy
+                            - string
+                            - contains
+                            - notContains
+                            - containsAny
+                            - oneOf
+                            - range
+                        field:
+                          type: string
+                          description: The field in the view to perform the search on.
+                        value:
+                          description: The value to search for - the type will depend on the operator in
+                            use.
+                          oneOf:
+                            - type: string
+                            - type: number
+                            - type: boolean
+                            - type: object
+                            - type: array
+                  groups:
+                    description: A grouping of filters to be applied.
+                    type: array
+                    items:
+                      type: object
+                      properties:
+                        logicalOperator:
+                          description: When using groups this defines whether all of the filters must
+                            match, or only one of them.
+                          type: string
+                          enum:
+                            - all
+                            - any
+                        filters:
+                          description: A list of filters to apply
+                          type: array
+                          items:
+                            type: object
+                            properties:
+                              operator:
+                                type: string
+                                description: The type of search operation which is being performed.
+                                enum:
+                                  - equal
+                                  - notEqual
+                                  - empty
+                                  - notEmpty
+                                  - fuzzy
+                                  - string
+                                  - contains
+                                  - notContains
+                                  - containsAny
+                                  - oneOf
+                                  - range
+                              field:
+                                type: string
+                                description: The field in the view to perform the search on.
+                              value:
+                                description: The value to search for - the type will depend on the operator in
+                                  use.
+                                oneOf:
+                                  - type: string
+                                  - type: number
+                                  - type: boolean
+                                  - type: object
+                                  - type: array
+        sort:
+          type: object
+          required:
+            - field
+          properties:
+            field:
+              type: string
+              description: The field from the table/view schema to sort on.
+            order:
+              type: string
+              description: The order in which to sort.
+              enum:
+                - ascending
+                - descending
+            type:
+              type: string
+              description: The type of sort to perform (by number, or by alphabetically).
+              enum:
+                - string
+                - number
+        schema:
+          type: object
+          additionalProperties:
+            oneOf:
+              - type: object
+                properties:
+                  visible:
+                    type: boolean
+                    description: Defines whether the column is visible or not - rows
+                      retrieved/updated through this view will not be able to
+                      access it.
+                  readonly:
+                    type: boolean
+                    description: "When used in combination with 'visible: true' the column will be
+                      visible in row responses but cannot be updated."
+                  order:
+                    type: integer
+                    description: A number defining where the column shows up in tables, lowest being
+                      first.
+                  width:
+                    type: integer
+                    description: A width for the column, defined in pixels - this affects rendering
+                      in tables.
+                  column:
+                    type: array
+                    description: If this is a relationship column, we can set the columns we wish to
+                      include
+                    items:
+                      type: object
+                      properties:
+                        readonly:
+                          type: boolean
+              - type: object
+                properties:
+                  calculationType:
+                    type: string
+                    description: This column should be built from a calculation, specifying a type
+                      and field. It is important to note when a calculation is
+                      configured all non-calculation columns will be used for
+                      grouping.
+                    enum:
+                      - sum
+                      - avg
+                      - count
+                      - min
+                      - max
+                  field:
+                    type: string
+                    description: The field from the table to perform the calculation on.
+                  distinct:
+                    type: boolean
+                    description: Can be used in tandem with the count calculation type, to count
+                      unique entries.
+    viewOutput:
+      type: object
+      properties:
+        data:
+          description: The view to be created/updated.
+          type: object
+          required:
+            - name
+            - schema
+            - tableId
+            - id
+          properties:
+            name:
+              description: The name of the view.
+              type: string
+            tableId:
+              description: The ID of the table this view is based on.
+              type: string
+            type:
+              description: The type of view - standard (empty value) or calculation.
+              type: string
+              enum:
+                - calculation
+            primaryDisplay:
+              type: string
+              description: A column used to display rows from this view - usually used when
+                rendered in tables.
+            query:
+              description: Search parameters for view
+              type: object
+              required: []
+              properties:
+                logicalOperator:
+                  description: When using groups this defines whether all of the filters must
+                    match, or only one of them.
+                  type: string
+                  enum:
+                    - all
+                    - any
+                onEmptyFilter:
+                  description: If no filters match, should the view return all rows, or no rows.
+                  type: string
+                  enum:
+                    - all
+                    - none
+                groups:
+                  description: A grouping of filters to be applied.
+                  type: array
+                  items:
+                    type: object
+                    properties:
+                      logicalOperator:
+                        description: When using groups this defines whether all of the filters must
+                          match, or only one of them.
+                        type: string
+                        enum:
+                          - all
+                          - any
+                      filters:
+                        description: A list of filters to apply
+                        type: array
+                        items:
+                          type: object
+                          properties:
+                            operator:
+                              type: string
+                              description: The type of search operation which is being performed.
+                              enum:
+                                - equal
+                                - notEqual
+                                - empty
+                                - notEmpty
+                                - fuzzy
+                                - string
+                                - contains
+                                - notContains
+                                - containsAny
+                                - oneOf
+                                - range
+                            field:
+                              type: string
+                              description: The field in the view to perform the search on.
+                            value:
+                              description: The value to search for - the type will depend on the operator in
+                                use.
+                              oneOf:
+                                - type: string
+                                - type: number
+                                - type: boolean
+                                - type: object
+                                - type: array
+                      groups:
+                        description: A grouping of filters to be applied.
+                        type: array
+                        items:
+                          type: object
+                          properties:
+                            logicalOperator:
+                              description: When using groups this defines whether all of the filters must
+                                match, or only one of them.
+                              type: string
+                              enum:
+                                - all
+                                - any
+                            filters:
+                              description: A list of filters to apply
+                              type: array
+                              items:
+                                type: object
+                                properties:
+                                  operator:
+                                    type: string
+                                    description: The type of search operation which is being performed.
+                                    enum:
+                                      - equal
+                                      - notEqual
+                                      - empty
+                                      - notEmpty
+                                      - fuzzy
+                                      - string
+                                      - contains
+                                      - notContains
+                                      - containsAny
+                                      - oneOf
+                                      - range
+                                  field:
+                                    type: string
+                                    description: The field in the view to perform the search on.
+                                  value:
+                                    description: The value to search for - the type will depend on the operator in
+                                      use.
+                                    oneOf:
+                                      - type: string
+                                      - type: number
+                                      - type: boolean
+                                      - type: object
+                                      - type: array
+            sort:
+              type: object
+              required:
+                - field
+              properties:
+                field:
+                  type: string
+                  description: The field from the table/view schema to sort on.
+                order:
+                  type: string
+                  description: The order in which to sort.
+                  enum:
+                    - ascending
+                    - descending
+                type:
+                  type: string
+                  description: The type of sort to perform (by number, or by alphabetically).
+                  enum:
+                    - string
+                    - number
+            schema:
+              type: object
+              additionalProperties:
+                oneOf:
+                  - type: object
+                    properties:
+                      visible:
+                        type: boolean
+                        description: Defines whether the column is visible or not - rows
+                          retrieved/updated through this view will not be able
+                          to access it.
+                      readonly:
+                        type: boolean
+                        description: "When used in combination with 'visible: true' the column will be
+                          visible in row responses but cannot be updated."
+                      order:
+                        type: integer
+                        description: A number defining where the column shows up in tables, lowest being
+                          first.
+                      width:
+                        type: integer
+                        description: A width for the column, defined in pixels - this affects rendering
+                          in tables.
+                      column:
+                        type: array
+                        description: If this is a relationship column, we can set the columns we wish to
+                          include
+                        items:
+                          type: object
+                          properties:
+                            readonly:
+                              type: boolean
+                  - type: object
+                    properties:
+                      calculationType:
+                        type: string
+                        description: This column should be built from a calculation, specifying a type
+                          and field. It is important to note when a calculation
+                          is configured all non-calculation columns will be used
+                          for grouping.
+                        enum:
+                          - sum
+                          - avg
+                          - count
+                          - min
+                          - max
+                      field:
+                        type: string
+                        description: The field from the table to perform the calculation on.
+                      distinct:
+                        type: boolean
+                        description: Can be used in tandem with the count calculation type, to count
+                          unique entries.
+            id:
+              description: The ID of the view.
+              type: string
+      required:
+        - data
+    viewSearch:
+      type: object
+      properties:
+        data:
+          type: array
+          items:
+            description: The view to be created/updated.
+            type: object
+            required:
+              - name
+              - schema
+              - tableId
+              - id
+            properties:
+              name:
+                description: The name of the view.
+                type: string
+              tableId:
+                description: The ID of the table this view is based on.
+                type: string
+              type:
+                description: The type of view - standard (empty value) or calculation.
+                type: string
+                enum:
+                  - calculation
+              primaryDisplay:
+                type: string
+                description: A column used to display rows from this view - usually used when
+                  rendered in tables.
+              query:
+                description: Search parameters for view
+                type: object
+                required: []
+                properties:
+                  logicalOperator:
+                    description: When using groups this defines whether all of the filters must
+                      match, or only one of them.
+                    type: string
+                    enum:
+                      - all
+                      - any
+                  onEmptyFilter:
+                    description: If no filters match, should the view return all rows, or no rows.
+                    type: string
+                    enum:
+                      - all
+                      - none
+                  groups:
+                    description: A grouping of filters to be applied.
+                    type: array
+                    items:
+                      type: object
+                      properties:
+                        logicalOperator:
+                          description: When using groups this defines whether all of the filters must
+                            match, or only one of them.
+                          type: string
+                          enum:
+                            - all
+                            - any
+                        filters:
+                          description: A list of filters to apply
+                          type: array
+                          items:
+                            type: object
+                            properties:
+                              operator:
+                                type: string
+                                description: The type of search operation which is being performed.
+                                enum:
+                                  - equal
+                                  - notEqual
+                                  - empty
+                                  - notEmpty
+                                  - fuzzy
+                                  - string
+                                  - contains
+                                  - notContains
+                                  - containsAny
+                                  - oneOf
+                                  - range
+                              field:
+                                type: string
+                                description: The field in the view to perform the search on.
+                              value:
+                                description: The value to search for - the type will depend on the operator in
+                                  use.
+                                oneOf:
+                                  - type: string
+                                  - type: number
+                                  - type: boolean
+                                  - type: object
+                                  - type: array
+                        groups:
+                          description: A grouping of filters to be applied.
+                          type: array
+                          items:
+                            type: object
+                            properties:
+                              logicalOperator:
+                                description: When using groups this defines whether all of the filters must
+                                  match, or only one of them.
+                                type: string
+                                enum:
+                                  - all
+                                  - any
+                              filters:
+                                description: A list of filters to apply
+                                type: array
+                                items:
+                                  type: object
+                                  properties:
+                                    operator:
+                                      type: string
+                                      description: The type of search operation which is being performed.
+                                      enum:
+                                        - equal
+                                        - notEqual
+                                        - empty
+                                        - notEmpty
+                                        - fuzzy
+                                        - string
+                                        - contains
+                                        - notContains
+                                        - containsAny
+                                        - oneOf
+                                        - range
+                                    field:
+                                      type: string
+                                      description: The field in the view to perform the search on.
+                                    value:
+                                      description: The value to search for - the type will depend on the operator in
+                                        use.
+                                      oneOf:
+                                        - type: string
+                                        - type: number
+                                        - type: boolean
+                                        - type: object
+                                        - type: array
+              sort:
+                type: object
+                required:
+                  - field
+                properties:
+                  field:
+                    type: string
+                    description: The field from the table/view schema to sort on.
+                  order:
+                    type: string
+                    description: The order in which to sort.
+                    enum:
+                      - ascending
+                      - descending
+                  type:
+                    type: string
+                    description: The type of sort to perform (by number, or by alphabetically).
+                    enum:
+                      - string
+                      - number
+              schema:
+                type: object
+                additionalProperties:
+                  oneOf:
+                    - type: object
+                      properties:
+                        visible:
+                          type: boolean
+                          description: Defines whether the column is visible or not - rows
+                            retrieved/updated through this view will not be able
+                            to access it.
+                        readonly:
+                          type: boolean
+                          description: "When used in combination with 'visible: true' the column will be
+                            visible in row responses but cannot be updated."
+                        order:
+                          type: integer
+                          description: A number defining where the column shows up in tables, lowest being
+                            first.
+                        width:
+                          type: integer
+                          description: A width for the column, defined in pixels - this affects rendering
+                            in tables.
+                        column:
+                          type: array
+                          description: If this is a relationship column, we can set the columns we wish to
+                            include
+                          items:
+                            type: object
+                            properties:
+                              readonly:
+                                type: boolean
+                    - type: object
+                      properties:
+                        calculationType:
+                          type: string
+                          description: This column should be built from a calculation, specifying a type
+                            and field. It is important to note when a
+                            calculation is configured all non-calculation
+                            columns will be used for grouping.
+                          enum:
+                            - sum
+                            - avg
+                            - count
+                            - min
+                            - max
+                        field:
+                          type: string
+                          description: The field from the table to perform the calculation on.
+                        distinct:
+                          type: boolean
+                          description: Can be used in tandem with the count calculation type, to count
+                            unique entries.
+              id:
+                description: The ID of the view.
+                type: string
+      required:
+        - data
 security:
   - ApiKeyAuth: []
 paths:
@@ -2136,6 +2846,32 @@ paths:
               examples:
                 search:
                   $ref: "#/components/examples/rows"
+  "/views/{viewId}/rows/search":
+    post:
+      operationId: rowViewSearch
+      summary: Search for rows in a view
+      tags:
+        - rows
+      parameters:
+        - $ref: "#/components/parameters/viewId"
+        - $ref: "#/components/parameters/appId"
+      requestBody:
+        required: true
+        content:
+          application/json:
+            schema:
+              $ref: "#/components/schemas/rowSearch"
+      responses:
+        "200":
+          description: The response will contain an array of rows that match the search
+            parameters.
+          content:
+            application/json:
+              schema:
+                $ref: "#/components/schemas/searchOutput"
+              examples:
+                search:
+                  $ref: "#/components/examples/rows"
   /tables:
     post:
       operationId: tableCreate
@@ -2359,4 +3095,123 @@ paths:
               examples:
                 users:
                   $ref: "#/components/examples/users"
+  /views:
+    post:
+      operationId: viewCreate
+      summary: Create a view
+      description: Create a view, this can be against an internal or external table.
+      tags:
+        - views
+      parameters:
+        - $ref: "#/components/parameters/appId"
+      requestBody:
+        content:
+          application/json:
+            schema:
+              $ref: "#/components/schemas/view"
+            examples:
+              view:
+                $ref: "#/components/examples/view"
+      responses:
+        "200":
+          description: Returns the created view, including the ID which has been generated
+            for it.
+          content:
+            application/json:
+              schema:
+                $ref: "#/components/schemas/viewOutput"
+              examples:
+                view:
+                  $ref: "#/components/examples/view"
+  "/views/{viewId}":
+    put:
+      operationId: viewUpdate
+      summary: Update a view
+      description: Update a view, this can be against an internal or external table.
+      tags:
+        - views
+      parameters:
+        - $ref: "#/components/parameters/viewId"
+        - $ref: "#/components/parameters/appId"
+      requestBody:
+        content:
+          application/json:
+            schema:
+              $ref: "#/components/schemas/view"
+            examples:
+              view:
+                $ref: "#/components/examples/view"
+      responses:
+        "200":
+          description: Returns the updated view.
+          content:
+            application/json:
+              schema:
+                $ref: "#/components/schemas/viewOutput"
+              examples:
+                view:
+                  $ref: "#/components/examples/view"
+    delete:
+      operationId: viewDestroy
+      summary: Delete a view
+      description: Delete a view, this can be against an internal or external table.
+      tags:
+        - views
+      parameters:
+        - $ref: "#/components/parameters/viewId"
+        - $ref: "#/components/parameters/appId"
+      responses:
+        "200":
+          description: Returns the deleted view.
+          content:
+            application/json:
+              schema:
+                $ref: "#/components/schemas/viewOutput"
+              examples:
+                view:
+                  $ref: "#/components/examples/view"
+    get:
+      operationId: viewGetById
+      summary: Retrieve a view
+      description: Lookup a view, this could be internal or external.
+      tags:
+        - views
+      parameters:
+        - $ref: "#/components/parameters/viewId"
+        - $ref: "#/components/parameters/appId"
+      responses:
+        "200":
+          description: Returns the retrieved view.
+          content:
+            application/json:
+              schema:
+                $ref: "#/components/schemas/viewOutput"
+              examples:
+                view:
+                  $ref: "#/components/examples/view"
+  /views/search:
+    post:
+      operationId: viewSearch
+      summary: Search for views
+      description: Based on view properties (currently only name) search for views.
+      tags:
+        - views
+      parameters:
+        - $ref: "#/components/parameters/appId"
+      requestBody:
+        required: true
+        content:
+          application/json:
+            schema:
+              $ref: "#/components/schemas/nameSearch"
+      responses:
+        "200":
+          description: Returns the found views, based on the search parameters.
+          content:
+            application/json:
+              schema:
+                $ref: "#/components/schemas/viewSearch"
+              examples:
+                views:
+                  $ref: "#/components/examples/views"
 tags: []

packages/server/specs/parameters.ts

@@ -8,6 +8,16 @@ export const tableId = {
   },
 }
 
+export const viewId = {
+  in: "path",
+  name: "viewId",
+  required: true,
+  description: "The ID of the view which this request is targeting.",
+  schema: {
+    type: "string",
+  },
+}
+
 export const rowId = {
   in: "path",
   name: "rowId",

packages/server/specs/resources/index.ts

@@ -6,6 +6,7 @@ import user from "./user"
 import metrics from "./metrics"
 import misc from "./misc"
 import roles from "./roles"
+import view from "./view"
 
 export const examples = {
   ...application.getExamples(),
@@ -16,6 +17,7 @@ export const examples = {
   ...misc.getExamples(),
   ...metrics.getExamples(),
   ...roles.getExamples(),
+  ...view.getExamples(),
 }
 
 export const schemas = {
@@ -26,4 +28,5 @@ export const schemas = {
   ...user.getSchemas(),
   ...misc.getSchemas(),
   ...roles.getSchemas(),
+  ...view.getSchemas(),
 }

packages/server/specs/resources/misc.ts

@@ -1,10 +1,7 @@
 import { object } from "./utils"
 import Resource from "./utils/Resource"
 
-export default new Resource().setSchemas({
-  rowSearch: object(
-    {
-      query: {
+export const searchSchema = {
   type: "object",
   properties: {
     allOr: {
@@ -93,7 +90,12 @@ export default new Resource().setSchemas({
       },
     },
   },
-      },
+}
+
+export default new Resource().setSchemas({
+  rowSearch: object(
+    {
+      query: searchSchema,
       paginate: {
         type: "boolean",
         description: "Enables pagination, by default this is disabled.",

packages/server/specs/resources/view.ts

@@ -0,0 +1,274 @@
+import { object } from "./utils"
+import Resource from "./utils/Resource"
+import {
+  ArrayOperator,
+  BasicOperator,
+  CalculationType,
+  RangeOperator,
+  SortOrder,
+  SortType,
+} from "@budibase/types"
+import { cloneDeep } from "lodash"
+
+const view = {
+  name: "peopleView",
+  tableId: "ta_896a325f7e8147d2a2cda93c5d236511",
+  schema: {
+    name: {
+      visible: true,
+      readonly: false,
+      order: 1,
+      width: 300,
+    },
+    age: {
+      visible: true,
+      readonly: true,
+      order: 2,
+      width: 200,
+    },
+    salary: {
+      visible: false,
+      readonly: false,
+    },
+  },
+  query: {
+    logicalOperator: "all",
+    onEmptyFilter: "none",
+    groups: [
+      {
+        logicalOperator: "any",
+        filters: [
+          { operator: "string", field: "name", value: "John" },
+          { operator: "range", field: "age", value: { low: 18, high: 100 } },
+        ],
+      },
+    ],
+  },
+  primaryDisplay: "name",
+}
+
+const baseColumnDef = {
+  visible: {
+    type: "boolean",
+    description:
+      "Defines whether the column is visible or not - rows retrieved/updated through this view will not be able to access it.",
+  },
+  readonly: {
+    type: "boolean",
+    description:
+      "When used in combination with 'visible: true' the column will be visible in row responses but cannot be updated.",
+  },
+  order: {
+    type: "integer",
+    description:
+      "A number defining where the column shows up in tables, lowest being first.",
+  },
+  width: {
+    type: "integer",
+    description:
+      "A width for the column, defined in pixels - this affects rendering in tables.",
+  },
+  column: {
+    type: "array",
+    description:
+      "If this is a relationship column, we can set the columns we wish to include",
+    items: {
+      type: "object",
+      properties: {
+        readonly: {
+          type: "boolean",
+        },
+      },
+    },
+  },
+}
+
+const logicalOperator = {
+  description:
+    "When using groups this defines whether all of the filters must match, or only one of them.",
+  type: "string",
+  enum: ["all", "any"],
+}
+
+const filterGroup = {
+  description: "A grouping of filters to be applied.",
+  type: "array",
+  items: {
+    type: "object",
+    properties: {
+      logicalOperator,
+      filters: {
+        description: "A list of filters to apply",
+        type: "array",
+        items: {
+          type: "object",
+          properties: {
+            operator: {
+              type: "string",
+              description:
+                "The type of search operation which is being performed.",
+              enum: [
+                ...Object.values(BasicOperator),
+                ...Object.values(ArrayOperator),
+                ...Object.values(RangeOperator),
+              ],
+            },
+            field: {
+              type: "string",
+              description: "The field in the view to perform the search on.",
+            },
+            value: {
+              description:
+                "The value to search for - the type will depend on the operator in use.",
+              oneOf: [
+                { type: "string" },
+                { type: "number" },
+                { type: "boolean" },
+                { type: "object" },
+                { type: "array" },
+              ],
+            },
+          },
+        },
+      },
+    },
+  },
+}
+
+// have to clone to avoid constantly recursive structure - we can't represent this easily
+const layeredFilterGroup: any = cloneDeep(filterGroup)
+layeredFilterGroup.items.properties.groups = filterGroup
+
+const viewQuerySchema = {
+  description: "Search parameters for view",
+  type: "object",
+  required: [],
+  properties: {
+    logicalOperator,
+    onEmptyFilter: {
+      description:
+        "If no filters match, should the view return all rows, or no rows.",
+      type: "string",
+      enum: ["all", "none"],
+    },
+    groups: layeredFilterGroup,
+  },
+}
+
+const viewSchema = {
+  description: "The view to be created/updated.",
+  type: "object",
+  required: ["name", "schema", "tableId"],
+  properties: {
+    name: {
+      description: "The name of the view.",
+      type: "string",
+    },
+    tableId: {
+      description: "The ID of the table this view is based on.",
+      type: "string",
+    },
+    type: {
+      description: "The type of view - standard (empty value) or calculation.",
+      type: "string",
+      enum: ["calculation"],
+    },
+    primaryDisplay: {
+      type: "string",
+      description:
+        "A column used to display rows from this view - usually used when rendered in tables.",
+    },
+    query: viewQuerySchema,
+    sort: {
+      type: "object",
+      required: ["field"],
+      properties: {
+        field: {
+          type: "string",
+          description: "The field from the table/view schema to sort on.",
+        },
+        order: {
+          type: "string",
+          description: "The order in which to sort.",
+          enum: Object.values(SortOrder),
+        },
+        type: {
+          type: "string",
+          description:
+            "The type of sort to perform (by number, or by alphabetically).",
+          enum: Object.values(SortType),
+        },
+      },
+    },
+    schema: {
+      type: "object",
+      additionalProperties: {
+        oneOf: [
+          {
+            type: "object",
+            properties: baseColumnDef,
+          },
+          {
+            type: "object",
+            properties: {
+              calculationType: {
+                type: "string",
+                description:
+                  "This column should be built from a calculation, specifying a type and field. It is important to note when a calculation is configured all non-calculation columns will be used for grouping.",
+                enum: Object.values(CalculationType),
+              },
+              field: {
+                type: "string",
+                description:
+                  "The field from the table to perform the calculation on.",
+              },
+              distinct: {
+                type: "boolean",
+                description:
+                  "Can be used in tandem with the count calculation type, to count unique entries.",
+              },
+            },
+          },
+        ],
+      },
+    },
+  },
+}
+
+const viewOutputSchema = {
+  ...viewSchema,
+  properties: {
+    ...viewSchema.properties,
+    id: {
+      description: "The ID of the view.",
+      type: "string",
+    },
+  },
+  required: [...viewSchema.required, "id"],
+}
+
+export default new Resource()
+  .setExamples({
+    view: {
+      value: {
+        data: view,
+      },
+    },
+    views: {
+      value: {
+        data: [view],
+      },
+    },
+  })
+  .setSchemas({
+    view: viewSchema,
+    viewOutput: object({
+      data: viewOutputSchema,
+    }),
+    viewSearch: object({
+      data: {
+        type: "array",
+        items: viewOutputSchema,
+      },
+    }),
+  })

packages/server/src/api/controllers/application.ts

@@ -153,7 +153,11 @@ async function createInstance(appId: string, template: AppTemplate) {
   await createAllSearchIndex()
 
   if (template && template.useTemplate) {
-    await sdk.backups.importApp(appId, db, template)
+    const opts = {
+      importObjStoreContents: true,
+      updateAttachmentColumns: !template.key, // preserve attachments when using Budibase templates
+    }
+    await sdk.backups.importApp(appId, db, template, opts)
   } else {
     // create the users table
     await db.put(USERS_TABLE_SCHEMA)

packages/server/src/api/controllers/public/mapping/applications.ts

@@ -1,6 +1,7 @@
 import { Application } from "./types"
+import { RequiredKeys } from "@budibase/types"
 
-function application(body: any): Application {
+function application(body: any): RequiredKeys<Application> {
   let app = body?.application ? body.application : body
   return {
     _id: app.appId,

packages/server/src/api/controllers/public/mapping/index.ts

@@ -3,6 +3,7 @@ import applications from "./applications"
 import users from "./users"
 import rows from "./rows"
 import queries from "./queries"
+import views from "./views"
 
 export default {
   ...tables,
@@ -10,4 +11,5 @@ export default {
   ...users,
   ...rows,
   ...queries,
+  ...views,
 }

packages/server/src/api/controllers/public/mapping/queries.ts

@@ -1,6 +1,7 @@
 import { Query, ExecuteQuery } from "./types"
+import { RequiredKeys } from "@budibase/types"
 
-function query(body: any): Query {
+function query(body: any): RequiredKeys<Query> {
   return {
     _id: body._id,
     datasourceId: body.datasourceId,

packages/server/src/api/controllers/public/mapping/rows.ts

@@ -1,6 +1,7 @@
 import { Row, RowSearch } from "./types"
+import { RequiredKeys } from "@budibase/types"
 
-function row(body: any): Row {
+function row(body: any): RequiredKeys<Row> {
   delete body._rev
   // have to input everything, since structure unknown
   return {

packages/server/src/api/controllers/public/mapping/tables.ts

@@ -1,6 +1,7 @@
 import { Table } from "./types"
+import { RequiredKeys } from "@budibase/types"
 
-function table(body: any): Table {
+function table(body: any): RequiredKeys<Table> {
   return {
     _id: body._id,
     name: body.name,

packages/server/src/api/controllers/public/mapping/types.ts

@@ -9,6 +9,9 @@ export type CreateApplicationParams = components["schemas"]["application"]
 export type Table = components["schemas"]["tableOutput"]["data"]
 export type CreateTableParams = components["schemas"]["table"]
 
+export type View = components["schemas"]["viewOutput"]["data"]
+export type CreateViewParams = components["schemas"]["view"]
+
 export type Row = components["schemas"]["rowOutput"]["data"]
 export type RowSearch = components["schemas"]["searchOutput"]
 export type CreateRowParams = components["schemas"]["row"]

packages/server/src/api/controllers/public/mapping/users.ts

@@ -1,6 +1,7 @@
 import { User } from "./types"
+import { RequiredKeys } from "@budibase/types"
 
-function user(body: any): User {
+function user(body: any): RequiredKeys<User> {
   return {
     _id: body._id,
     email: body.email,

packages/server/src/api/controllers/public/mapping/views.ts

@@ -0,0 +1,32 @@
+import { View } from "./types"
+import { ViewV2, Ctx, RequiredKeys } from "@budibase/types"
+import { dataFilters } from "@budibase/shared-core"
+
+function view(body: ViewV2): RequiredKeys<View> {
+  return {
+    id: body.id,
+    tableId: body.tableId,
+    type: body.type,
+    name: body.name,
+    schema: body.schema!,
+    primaryDisplay: body.primaryDisplay,
+    query: dataFilters.buildQuery(body.query),
+    sort: body.sort,
+  }
+}
+
+function mapView(ctx: Ctx<{ data: ViewV2 }>): { data: View } {
+  return {
+    data: view(ctx.body.data),
+  }
+}
+
+function mapViews(ctx: Ctx<{ data: ViewV2[] }>): { data: View[] } {
+  const views = ctx.body.data.map((body: ViewV2) => view(body))
+  return { data: views }
+}
+
+export default {
+  mapView,
+  mapViews,
+}

packages/server/src/api/controllers/public/rows.ts

@@ -22,13 +22,13 @@ export function fixRow(row: Row, params: any) {
   return row
 }
 
-export async function search(ctx: UserCtx, next: Next) {
+function buildSearchRequestBody(ctx: UserCtx) {
   let { sort, paginate, bookmark, limit, query } = ctx.request.body
   // update the body to the correct format of the internal search
   if (!sort) {
     sort = {}
   }
-  ctx.request.body = {
+  return {
     sort: sort.column,
     sortType: sort.type,
     sortOrder: sort.order,
@@ -37,10 +37,23 @@ export async function search(ctx: UserCtx, next: Next) {
     limit,
     query,
   }
+}
+
+export async function search(ctx: UserCtx, next: Next) {
+  ctx.request.body = buildSearchRequestBody(ctx)
   await rowController.search(ctx)
   await next()
 }
 
+export async function viewSearch(ctx: UserCtx, next: Next) {
+  ctx.request.body = buildSearchRequestBody(ctx)
+  ctx.params = {
+    viewId: ctx.params.viewId,
+  }
+  await rowController.views.searchView(ctx)
+  await next()
+}
+
 export async function create(ctx: UserCtx, next: Next) {
   ctx.request.body = fixRow(ctx.request.body, ctx.params)
   await rowController.save(ctx)
@@ -79,4 +92,5 @@ export default {
   update,
   destroy,
   search,
+  viewSearch,
 }

packages/server/src/api/controllers/public/views.ts

@@ -0,0 +1,95 @@
+import { search as stringSearch } from "./utils"
+import * as controller from "../view"
+import { ViewV2, UserCtx, UISearchFilter, PublicAPIView } from "@budibase/types"
+import { Next } from "koa"
+import { merge } from "lodash"
+
+function viewRequest(view: PublicAPIView, params?: { viewId: string }) {
+  const viewV2: ViewV2 = view
+  if (!viewV2) {
+    return viewV2
+  }
+  if (params?.viewId) {
+    viewV2.id = params.viewId
+  }
+  if (!view.query) {
+    viewV2.query = {}
+  } else {
+    // public API only has one form of query
+    viewV2.queryUI = viewV2.query as UISearchFilter
+  }
+  viewV2.version = 2
+  return viewV2
+}
+
+function viewResponse(view: ViewV2): PublicAPIView {
+  // remove our internal structure - always un-necessary
+  delete view.query
+  return {
+    ...view,
+    query: view.queryUI,
+  }
+}
+
+function viewsResponse(views: ViewV2[]): PublicAPIView[] {
+  return views.map(viewResponse)
+}
+
+export async function search(ctx: UserCtx, next: Next) {
+  const { name } = ctx.request.body
+  await controller.v2.fetch(ctx)
+  ctx.body.data = viewsResponse(stringSearch(ctx.body.data, name))
+  await next()
+}
+
+export async function create(ctx: UserCtx, next: Next) {
+  ctx = merge(ctx, {
+    request: {
+      body: viewRequest(ctx.request.body),
+    },
+  })
+  await controller.v2.create(ctx)
+  ctx.body.data = viewResponse(ctx.body.data)
+  await next()
+}
+
+export async function read(ctx: UserCtx, next: Next) {
+  ctx = merge(ctx, {
+    params: {
+      viewId: ctx.params.viewId,
+    },
+  })
+  await controller.v2.get(ctx)
+  ctx.body.data = viewResponse(ctx.body.data)
+  await next()
+}
+
+export async function update(ctx: UserCtx, next: Next) {
+  const viewId = ctx.params.viewId
+  ctx = merge(ctx, {
+    request: {
+      body: {
+        data: viewRequest(ctx.request.body, { viewId }),
+      },
+    },
+    params: {
+      viewId,
+    },
+  })
+  await controller.v2.update(ctx)
+  ctx.body.data = viewResponse(ctx.body.data)
+  await next()
+}
+
+export async function destroy(ctx: UserCtx, next: Next) {
+  await controller.v2.remove(ctx)
+  await next()
+}
+
+export default {
+  create,
+  read,
+  update,
+  destroy,
+  search,
+}