Merge branch 'master' of github.com:Budibase/budibase into labday/sqs

.github/workflows/budibase_ci.yml

@@ -99,11 +99,6 @@ jobs:
           else
             yarn test --ignore=@budibase/worker --ignore=@budibase/server --ignore=@budibase/pro
           fi
-      - uses: codecov/codecov-action@v3
-        with:
-          token: ${{ secrets.CODECOV_TOKEN }} # not required for public repos
-          name: codecov-umbrella
-          verbose: true
 
   test-worker:
     runs-on: ubuntu-latest
@@ -129,12 +124,6 @@ jobs:
             yarn test --scope=@budibase/worker
           fi
 
-      - uses: codecov/codecov-action@v3
-        with:
-          token: ${{ secrets.CODECOV_TOKEN || github.token }} # not required for public repos
-          name: codecov-umbrella
-          verbose: true
-
   test-server:
     runs-on: ubuntu-latest
     steps:
@@ -159,12 +148,6 @@ jobs:
             yarn test --scope=@budibase/server
           fi
 
-      - uses: codecov/codecov-action@v3
-        with:
-          token: ${{ secrets.CODECOV_TOKEN || github.token }} # not required for public repos
-          name: codecov-umbrella
-          verbose: true
-
   test-pro:
     runs-on: ubuntu-latest
     if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == 'Budibase/budibase'

.github/workflows/stale_bot.yml

@@ -10,6 +10,7 @@ jobs:
     steps:
       - uses: actions/stale@v8
         with:
+          days-before-stale: 330
           operations-per-run: 1
           # stale rules for PRs
           days-before-pr-stale: 7

CODEOWNERS

@@ -0,0 +1,3 @@
+/packages/server         @Budibase/backend
+/packages/worker         @Budibase/backend
+/packages/backend-core   @Budibase/backend

LICENSE

@@ -1,7 +1,9 @@
-Copyright 2019-2021, Budibase Ltd.
+Copyright 2019-2023, Budibase Ltd.
 
 Each Budibase package has its own license, please check the license file in each package.
 
 You can consider Budibase to be GPLv3 licensed overall.
 
 The apps that you build with Budibase do not package any GPLv3 licensed code, thus do not fall under those restrictions.
+
+Budibase ships with Structured Query Server, by The Neighbourhoodie Software GmbH. This license for this can be found at ./SQS_LICENSE

SQS_LICENSE

@@ -0,0 +1,31 @@
+FORM OF CUSTOMER LICENCE
+
+Budibase hereby grants the Customer a worldwide, royalty free, non-exclusive,
+perpetual (for the lifetime of the intellectual property rights contained in the Product)
+right and title to utilise the binary code of the The Neighbourhoodie Software GmbH
+Structured Query Server software product (Product) for its own internal business
+purposes (the Purpose) only (the Licence). The Product has the function of bringing a
+CouchDB database (NoSQL database) into an SQL database form (SQLite) and thereby
+making it usable for complex queries - which originally could only be displayed in an
+SQL database. By indexing in SQLite and a server that is tailored to it, the Product
+enables the use of CouchDB with SQL queries.
+The Licence shall not permit sub-licensing, resale or transfer of the Product to third
+parties, other than sub-licensing to the Customer’s direct contractors for the purposes
+of utilizing the Product as contemplated above.
+The Licence shall not permit the adaptation, modification, decompilation, reverse
+engineering or similar activities with respect to the Product.
+This licence is granted to the Customer only, although Customer and its Affiliates’
+employees, servants and agents shall be entitled to utilize the Product within the scope
+of the Licence for the Customer’s Purpose only.
+Reproduction is not permitted to users, except for reproductions that are necessary for
+the use of the product under the licence described above. These conditions apply to the
+product regardless of the form in which we make the product available and on which
+devices it is installed and/or with which devices it is ultimately used. Depending on the
+product variant or intended use, certain technical requirements in the IT infrastructure
+must be satisfied as a prerequisite for use.
+The law of the Northern Ireland applies exclusively to this licence, and the courts of
+Northern Ireland shall have exclusive jurisdiction, save that we reserve a right to sue
+you in the jurisdiction in which you are based. The application of the UN Sales
+Convention (CISG) is excluded.
+The invalidity of any part of this licence does not affect the validity of the remaining
+regulations.

charts/budibase/Chart.lock

@@ -1,9 +1,6 @@
 dependencies:
 - name: couchdb
   repository: https://apache.github.io/couchdb-helm
-  version: 3.3.4
+  version: 4.3.0
-- name: ingress-nginx
+digest: sha256:94449a7f195b186f5af33ec5aa66d58b36bede240fae710f021ca87837b30606
-  repository: https://kubernetes.github.io/ingress-nginx
+generated: "2023-11-20T17:43:02.777596Z"
-  version: 4.0.13
-digest: sha256:20892705c2d8e64c98257d181063a514ac55013e2b43399a6e54868a97f97845
-generated: "2021-12-30T18:55:30.878411Z"

charts/budibase/Chart.yaml

@@ -17,10 +17,6 @@ version: 0.0.0
 appVersion: 0.0.0
 dependencies:
   - name: couchdb
-    version: 3.3.4
+    version: 4.3.0
     repository: https://apache.github.io/couchdb-helm
     condition: services.couchdb.enabled
-  - name: ingress-nginx
-    version: 4.0.13
-    repository: https://kubernetes.github.io/ingress-nginx
-    condition: ingress.nginx

charts/budibase/README.md

@@ -1,39 +1,217 @@
-# Budibase
+# budibase
 
-[Budibase](https://budibase.com/) Budibase is an open source low-code platform, helping thousands of teams build apps for their workplace in minutes.
+Budibase is an open source low-code platform, helping thousands of teams build apps for their workplace in minutes.
-
-## TL;DR;
-```console
-$ cd chart
-$ helm install budibase .
-```
-
-## Introduction
-
-This chart bootstraps a [Budibase](https://budibase.com/) deployment on a [Kubernetes](http://kubernetes.io) cluster using the [Helm](https://helm.sh) package manager.
 
 ## Prerequisites
 
-- helm v3 or above
+- `helm` v3 or above
 - Kubernetes 1.4+
-- PV provisioner support in the underlying infrastructure (with persistence storage enabled)
+- A storage controller (if you want to use persistent storage)
+- An ingress controller (if you want to define an `Ingress` resource)
+- `metrics-server` (if you want to make use of horizontal pod autoscaling)
 
-## Installing the Chart
+## Chart dependencies
 
-To install the chart with the release name `budi-release`:
+This chart depends on the official Apache CouchDB chart. You can see its
+documentation here:
+<https://github.com/apache/couchdb-helm/tree/couchdb-4.3.0/couchdb>.
+
+## Upgrading
+
+### `2.x` to `3.0.0`
+
+We made a number of breaking changes in this release to make the chart more
+idiomatic and easier to use.
+
+1. We no longer bundle `ingress-nginx`. If you were relying on this to supply
+   an ingress controller to your cluster, you will now need to deploy that
+   separately. You'll find guidance for that here:
+   <https://kubernetes.github.io/ingress-nginx/>.
+2. We've upgraded the version of the [CouchDB chart](https://github.com/apache/couchdb-helm)
+   we use from `3.3.4` to `4.3.0`. The primary motivation for this was to align
+   the CouchDB chart used with the CouchDB version used, which has also updated
+   from 3.1.1 to 3.2.1. Additionally, we're moving away from the official CouchDB
+   to one we're building ourselves.
+3. We've separated out the supplied AWS ALB ingress resource for those deploying
+   into EKS. Where previously you enabled this by setting `ingress.enabled: false`
+   and `ingress.aws: true`, you now set `awsAlbIngress.enabled: true` and all
+   configuration for it is under `awsAlbIngress`.
+4. The `HorizontalPodAutoscaler` that was configured at `hpa.enabled: true` has
+   been split into 3 separate HPAs, one for each of `apps`, `worker`, and `proxy`.
+   They are configured at `services.{apps,worker,proxy}.autoscaling`.
+
+## Installing
+
+To install the chart from our repository:
 
 ```console
-$ helm install budi-release .
+$ helm repo add budibase https://budibase.github.io/budibase/
+$ helm repo update
+$ helm install --create-namespace --namespace budibase budibase budibase/budibase
 ```
 
-The command deploys Budibase on the Kubernetes cluster in the default configuration. The [configuration](#configuration) section lists the parameters that can be configured during installation.
+To install the chart from this repo:
-
-> **Tip**: List all releases using `helm list`
-
-## Uninstalling the Chart
-
-To uninstall/delete the `my-release` deployment:
 
 ```console
-$ helm delete my-release
+$ git clone git@github.com:budibase/budibase.git
+$ cd budibase/charts/budibase
+$ helm install --create-namespace --namespace budibase budibase .
 ```
+
+## Example minimal configuration
+
+Here's an example `values.yaml` that would get a Budibase instance running in a home
+cluster using an nginx ingress controller and NFS as cluster storage (basically one of our
+staff's homelabs).
+
+<details>
+
+```yaml
+ingress:
+  enabled: true
+  className: "nginx"
+  hosts:
+    - host: budibase.local # set this to whatever DNS name you'd use
+      paths:
+        - backend:
+            service:
+              name: proxy-service
+              port:
+                number: 10000
+          path: /
+          pathType: Prefix
+
+couchdb:
+  persistentVolume:
+    enabled: true
+    storageClass: "nfs-client"
+  adminPassword: admin
+
+services:
+  objectStore:
+    storageClass: "nfs-client"
+  redis:
+    storageClass: "nfs-client"
+```
+
+If you wanted to use this when bringing up Budibase in your own cluster, you could save it
+to your hard disk and run the following:
+
+```console
+$ helm install --create-namespace --namespace budibase budibase . -f values.yaml
+```
+
+</details>
+
+## Configuring
+
+| Key | Type | Default | Description |
+|-----|------|---------|-------------|
+| affinity | object | `{}` | Sets the affinity for all pods created by this chart. Should not ordinarily need to be changed.  See <https://kubernetes.io/docs/tasks/configure-pod-container/assign-pods-nodes-using-node-affinity/> for more information on affinity. |
+| awsAlbIngress.certificateArn | string | `""` | If you're wanting to use HTTPS, you'll need to create an ACM certificate and specify the ARN here. |
+| awsAlbIngress.enabled | bool | `false` | Whether to create an ALB Ingress resource pointing to the Budibase proxy. Requires the AWS ALB Ingress Controller. |
+| couchdb.clusterSize | int | `1` | The number of replicas to run in the CouchDB cluster. We set this to 1 by default to make things simpler, but you can set it to 3 if you need a high-availability CouchDB cluster. |
+| couchdb.couchdbConfig.couchdb.uuid | string | `"budibase-couchdb"` | Unique identifier for this CouchDB server instance. You shouldn't need to change this. |
+| couchdb.image | object | `{}` | We use a custom CouchDB image for running Budibase and we don't support using any other CouchDB image. You shouldn't change this, and if you do we can't guarantee that Budibase will work. |
+| globals.apiEncryptionKey | string | `""` | Used for encrypting API keys and environment variables when stored in the database. You don't need to set this if `createSecrets` is true. |
+| globals.appVersion | string | `""` | The version of Budibase to deploy. Defaults to what's specified by {{ .Chart.AppVersion }}. Ends up being used as the image version tag for the apps, proxy, and worker images. |
+| globals.automationMaxIterations | string | `"200"` | The maximum number of iterations allows for an automation loop step. You can read more about looping here: <https://docs.budibase.com/docs/looping>. |
+| globals.budibaseEnv | string | `"PRODUCTION"` | Sets the environment variable BUDIBASE_ENVIRONMENT for the apps and worker pods. Should not ordinarily need to be changed. |
+| globals.cookieDomain | string | `""` | Sets the domain attribute of the cookie that Budibase uses to store session information. See <https://developer.mozilla.org/en-US/docs/Web/HTTP/Cookies#define_where_cookies_are_sent> for details on why you might want to set this. |
+| globals.createSecrets | bool | `true` | Create an internal API key, JWT secret, object store access key and secret, and store them in a Kubernetes `Secret`. |
+| globals.enableAnalytics | string | `"1"` | Whether to enable analytics or not. You can read more about our analytics here: <https://docs.budibase.com/docs/analytics>. |
+| globals.google | object | `{"clientId":"","secret":""}` | Google OAuth settings. These can also be set in the Budibase UI, see <https://docs.budibase.com/docs/sso-with-google> for details. |
+| globals.google.clientId | string | `""` | Client ID of your Google OAuth app. |
+| globals.google.secret | string | `""` | Client secret of your Google OAuth app. |
+| globals.httpMigrations | string | `"0"` | Whether or not to enable doing data migrations over the HTTP API. If this is set to "0", migrations are run on startup. You shouldn't ordinarily need to change this. |
+| globals.internalApiKey | string | `""` | API key used for internal Budibase API calls. You don't need to set this if `createSecrets` is true. |
+| globals.internalApiKeyFallback | string | `""` | A fallback value for `internalApiKey`. If you're rotating your encryption key, you can set this to the old value for the duration of the rotation. |
+| globals.jwtSecret | string | `""` | Secret used for signing JWTs. You don't need to set this if `createSecrets` is true. |
+| globals.jwtSecretFallback | string | `""` | A fallback value for `jwtSecret`. If you're rotating your JWT secret, you can set this to the old value for the duration of the rotation. |
+| globals.platformUrl | string | `""` | Set the `platformUrl` binding. You can also do this in Settings > Organisation if you are self-hosting. |
+| globals.smtp.enabled | bool | `false` | Whether to enable SMTP or not. |
+| globals.smtp.from | string | `""` | The email address to use in the "From:" field of emails sent by Budibase. |
+| globals.smtp.host | string | `""` | The hostname of your SMTP server. |
+| globals.smtp.password | string | `""` | The password to use when authenticating with your SMTP server. |
+| globals.smtp.port | string | `"587"` | The port of your SMTP server. |
+| globals.smtp.user | string | `""` | The username to use when authenticating with your SMTP server. |
+| globals.tenantFeatureFlags | string | `"*:LICENSING,*:USER_GROUPS,*:ONBOARDING_TOUR"` | Sets what feature flags are enabled and for which tenants. Should not ordinarily need to be changed. |
+| imagePullSecrets | list | `[]` | Passed to all pods created by this chart. Should not ordinarily need to be changed. |
+| ingress.className | string | `""` | What ingress class to use. |
+| ingress.enabled | bool | `true` | Whether to create an Ingress resource pointing to the Budibase proxy. |
+| ingress.hosts | list | `[]` | Standard hosts block for the Ingress resource. Defaults to pointing to the Budibase proxy. |
+| nameOverride | string | `""` | Override the name of the deploymen. Defaults to {{ .Chart.Name }}. |
+| service.port | int | `10000` | Port to expose on the service. |
+| service.type | string | `"ClusterIP"` | Service type for the service that points to the main Budibase proxy pod. |
+| serviceAccount.annotations | object | `{}` | Annotations to add to the service account |
+| serviceAccount.create | bool | `true` | Specifies whether a service account should be created |
+| serviceAccount.name | string | `""` | The name of the service account to use. If not set and create is true, a name is generated using the fullname template |
+| services.apps.autoscaling.enabled | bool | `false` | Whether to enable horizontal pod autoscaling for the apps service. |
+| services.apps.autoscaling.maxReplicas | int | `10` |  |
+| services.apps.autoscaling.minReplicas | int | `1` |  |
+| services.apps.autoscaling.targetCPUUtilizationPercentage | int | `80` | Target CPU utilization percentage for the apps service. Note that for autoscaling to work, you will need to have metrics-server configured, and resources set for the apps pods. |
+| services.apps.httpLogging | int | `1` | Whether or not to log HTTP requests to the apps service. |
+| services.apps.livenessProbe | object | HTTP health checks. | Liveness probe configuration for apps pods. You shouldn't need to change this, but if you want to you can find more information here: <https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/> |
+| services.apps.logLevel | string | `"info"` | The log level for the apps service. |
+| services.apps.readinessProbe | object | HTTP health checks. | Readiness probe configuration for apps pods. You shouldn't need to change this, but if you want to you can find more information here: <https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/> |
+| services.apps.replicaCount | int | `1` | The number of apps replicas to run. |
+| services.apps.resources | object | `{}` | The resources to use for apps pods. See <https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/> for more information on how to set these. |
+| services.apps.startupProbe | object | HTTP health checks. | Startup probe configuration for apps pods. You shouldn't need to change this, but if you want to you can find more information here: <https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/> |
+| services.couchdb.backup.enabled | bool | `false` | Whether or not to enable periodic CouchDB backups. This works by replicating to another CouchDB instance. |
+| services.couchdb.backup.interval | string | `""` | Backup interval in seconds |
+| services.couchdb.backup.resources | object | `{}` | The resources to use for CouchDB backup pods. See <https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/> for more information on how to set these. |
+| services.couchdb.backup.target | string | `""` | Target couchDB instance to back up to, either a hostname or an IP address. |
+| services.couchdb.enabled | bool | `true` | Whether or not to spin up a CouchDB instance in your cluster. True by default, and the configuration for the CouchDB instance is under the `couchdb` key at the root of this file. You can see what options are available to you by looking at the official CouchDB Helm chart: <https://github.com/apache/couchdb-helm/tree/couchdb-4.3.0/couchdb>. |
+| services.couchdb.port | int | `5984` |  |
+| services.dns | string | `"cluster.local"` | The DNS suffix to use for service discovery. You only need to change this if you've configured your cluster to use a different DNS suffix. |
+| services.objectStore.accessKey | string | `""` | AWS_ACCESS_KEY if using S3 |
+| services.objectStore.browser | bool | `true` | Whether to enable the Minio web console or not. If you're exposing Minio to the Internet (via a custom Ingress record, for example), you should set this to false. If you're only exposing Minio to your cluster, you can leave this as true. |
+| services.objectStore.cloudfront.cdn | string | `""` | Set the url of a distribution to enable cloudfront. |
+| services.objectStore.cloudfront.privateKey64 | string | `""` | Base64 encoded private key for the above public key. |
+| services.objectStore.cloudfront.publicKeyId | string | `""` | ID of public key stored in cloudfront. |
+| services.objectStore.minio | bool | `true` | Set to false if using another object store, such as S3. You will need to set `services.objectStore.url` to point to your bucket if you do this. |
+| services.objectStore.region | string | `""` | AWS_REGION if using S3 |
+| services.objectStore.resources | object | `{}` | The resources to use for Minio pods. See <https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/> for more information on how to set these. |
+| services.objectStore.secretKey | string | `""` | AWS_SECRET_ACCESS_KEY if using S3 |
+| services.objectStore.storage | string | `"100Mi"` | How much storage to give Minio in its PersistentVolumeClaim. |
+| services.objectStore.storageClass | string | `""` | If defined, storageClassName: <storageClass> If set to "-", storageClassName: "", which disables dynamic provisioning If undefined (the default) or set to null, no storageClassName spec is set, choosing the default provisioner. |
+| services.objectStore.url | string | `"http://minio-service:9000"` | URL to use for object storage. Only change this if you're using an external object store, such as S3. Remember to set `minio: false` if you do this. |
+| services.proxy.autoscaling.enabled | bool | `false` | Whether to enable horizontal pod autoscaling for the proxy service. |
+| services.proxy.autoscaling.maxReplicas | int | `10` |  |
+| services.proxy.autoscaling.minReplicas | int | `1` |  |
+| services.proxy.autoscaling.targetCPUUtilizationPercentage | int | `80` | Target CPU utilization percentage for the proxy service. Note that for autoscaling to work, you will need to have metrics-server configured, and resources set for the proxy pods. |
+| services.proxy.livenessProbe | object | HTTP health checks. | Liveness probe configuration for proxy pods. You shouldn't need to change this, but if you want to you can find more information here: <https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/> |
+| services.proxy.readinessProbe | object | HTTP health checks. | Readiness probe configuration for proxy pods. You shouldn't need to change this, but if you want to you can find more information here: <https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/> |
+| services.proxy.replicaCount | int | `1` | The number of proxy replicas to run. |
+| services.proxy.resources | object | `{}` | The resources to use for proxy pods. See <https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/> for more information on how to set these. |
+| services.proxy.startupProbe | object | HTTP health checks. | Startup probe configuration for proxy pods. You shouldn't need to change this, but if you want to you can find more information here: <https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/> |
+| services.redis.enabled | bool | `true` | Whether or not to deploy a Redis pod into your cluster. |
+| services.redis.password | string | `"budibase"` | The password to use when connecting to Redis. It's recommended that you change this from the default if you're running Redis in-cluster. |
+| services.redis.port | int | `6379` | Port to expose Redis on. |
+| services.redis.resources | object | `{}` | The resources to use for Redis pods. See <https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/> for more information on how to set these. |
+| services.redis.storage | string | `"100Mi"` | How much persistent storage to allocate to Redis. |
+| services.redis.storageClass | string | `""` | If defined, storageClassName: <storageClass> If set to "-", storageClassName: "", which disables dynamic provisioning If undefined (the default) or set to null, no storageClassName spec is set, choosing the default provisioner. |
+| services.redis.url | string | `""` | If you choose to run Redis externally to this chart, you can specify the connection details here. |
+| services.worker.autoscaling.enabled | bool | `false` | Whether to enable horizontal pod autoscaling for the worker service. |
+| services.worker.autoscaling.maxReplicas | int | `10` |  |
+| services.worker.autoscaling.minReplicas | int | `1` |  |
+| services.worker.autoscaling.targetCPUUtilizationPercentage | int | `80` | Target CPU utilization percentage for the worker service. Note that for autoscaling to work, you will need to have metrics-server configured, and resources set for the worker pods. |
+| services.worker.httpLogging | int | `1` | Whether or not to log HTTP requests to the worker service. |
+| services.worker.livenessProbe | object | HTTP health checks. | Liveness probe configuration for worker pods. You shouldn't need to change this, but if you want to you can find more information here: <https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/> |
+| services.worker.logLevel | string | `"info"` | The log level for the worker service. |
+| services.worker.readinessProbe | object | HTTP health checks. | Readiness probe configuration for worker pods. You shouldn't need to change this, but if you want to you can find more information here: <https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/> |
+| services.worker.replicaCount | int | `1` | The number of worker replicas to run. |
+| services.worker.resources | object | `{}` | The resources to use for worker pods. See <https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/> for more information on how to set these. |
+| services.worker.startupProbe | object | HTTP health checks. | Startup probe configuration for worker pods. You shouldn't need to change this, but if you want to you can find more information here: <https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/> |
+| tolerations | list | `[]` | Sets the tolerations for all pods created by this chart. Should not ordinarily need to be changed. See <https://kubernetes.io/docs/concepts/scheduling-eviction/taint-and-toleration/> for more information on tolerations. |
+
+## Uninstalling
+
+To uninstall the chart, assuming you named the release `budibase` (both commands in the installation section do so):
+
+```console
+$ helm uninstall --namespace budibase budibase
+```
+
+----------------------------------------------
+Autogenerated from chart metadata using [helm-docs v1.11.3](https://github.com/norwoodj/helm-docs/releases/v1.11.3)

charts/budibase/README.md.gotmpl

@@ -0,0 +1,117 @@
+{{ template "chart.header" . }}
+{{ template "chart.description" . }}
+
+## Prerequisites
+
+- `helm` v3 or above
+- Kubernetes 1.4+
+- A storage controller (if you want to use persistent storage)
+- An ingress controller (if you want to define an `Ingress` resource)
+- `metrics-server` (if you want to make use of horizontal pod autoscaling)
+
+## Chart dependencies
+
+This chart depends on the official Apache CouchDB chart. You can see its
+documentation here:
+<https://github.com/apache/couchdb-helm/tree/couchdb-4.3.0/couchdb>.
+
+## Upgrading
+
+### `2.x` to `3.0.0`
+
+We made a number of breaking changes in this release to make the chart more
+idiomatic and easier to use.
+
+1. We no longer bundle `ingress-nginx`. If you were relying on this to supply
+   an ingress controller to your cluster, you will now need to deploy that
+   separately. You'll find guidance for that here:
+   <https://kubernetes.github.io/ingress-nginx/>.
+2. We've upgraded the version of the [CouchDB chart](https://github.com/apache/couchdb-helm)
+   we use from `3.3.4` to `4.3.0`. The primary motivation for this was to align
+   the CouchDB chart used with the CouchDB version used, which has also updated
+   from 3.1.1 to 3.2.1. Additionally, we're moving away from the official CouchDB
+   to one we're building ourselves.
+3. We've separated out the supplied AWS ALB ingress resource for those deploying
+   into EKS. Where previously you enabled this by setting `ingress.enabled: false`
+   and `ingress.aws: true`, you now set `awsAlbIngress.enabled: true` and all
+   configuration for it is under `awsAlbIngress`.
+4. The `HorizontalPodAutoscaler` that was configured at `hpa.enabled: true` has
+   been split into 3 separate HPAs, one for each of `apps`, `worker`, and `proxy`.
+   They are configured at `services.{apps,worker,proxy}.autoscaling`.
+
+## Installing
+
+To install the chart from our repository:
+
+```console
+$ helm repo add budibase https://budibase.github.io/budibase/
+$ helm repo update
+$ helm install --create-namespace --namespace budibase budibase budibase/budibase
+```
+
+To install the chart from this repo:
+
+```console
+$ git clone git@github.com:budibase/budibase.git
+$ cd budibase/charts/budibase
+$ helm install --create-namespace --namespace budibase budibase .
+```
+
+## Example minimal configuration
+
+Here's an example `values.yaml` that would get a Budibase instance running in a home
+cluster using an nginx ingress controller and NFS as cluster storage (basically one of our
+staff's homelabs).
+
+<details>
+
+```yaml
+ingress:
+  enabled: true
+  className: "nginx"
+  hosts:
+    - host: budibase.local # set this to whatever DNS name you'd use
+      paths:
+        - backend:
+            service:
+              name: proxy-service
+              port:
+                number: 10000
+          path: /
+          pathType: Prefix
+
+couchdb:
+  persistentVolume:
+    enabled: true
+    storageClass: "nfs-client"
+  adminPassword: admin
+
+services:
+  objectStore:
+    storageClass: "nfs-client"
+  redis:
+    storageClass: "nfs-client"
+```
+
+If you wanted to use this when bringing up Budibase in your own cluster, you could save it
+to your hard disk and run the following:
+
+```console
+$ helm install --create-namespace --namespace budibase budibase . -f values.yaml
+```
+
+</details>
+
+## Configuring
+
+{{ template "chart.valuesTable" . }}
+
+## Uninstalling
+
+To uninstall the chart, assuming you named the release `budibase` (both commands in the installation section do so):
+
+```console
+$ helm uninstall --namespace budibase budibase
+```
+
+{{ template "helm-docs.versionFooter" . }}

charts/budibase/templates/alb-ingress.yaml

@@ -1,4 +1,4 @@
-{{- if .Values.ingress.aws }}
+{{- if .Values.awsAlbIngress.enabled }}
 apiVersion: networking.k8s.io/v1
 kind: Ingress
 metadata:
@@ -7,24 +7,24 @@ metadata:
     kubernetes.io/ingress.class: alb
     alb.ingress.kubernetes.io/scheme: internet-facing
     alb.ingress.kubernetes.io/target-type: ip
-    alb.ingress.kubernetes.io/success-codes: 200,301
+    alb.ingress.kubernetes.io/success-codes: '200'
-    alb.ingress.kubernetes.io/healthcheck-path: /
+    alb.ingress.kubernetes.io/healthcheck-path: '/health'
-    {{- if .Values.ingress.certificateArn }}
+    {{- if .Values.awsAlbIngress.certificateArn }}
     alb.ingress.kubernetes.io/actions.ssl-redirect: '{"Type": "redirect", "RedirectConfig": { "Protocol": "HTTPS", "Port": "443", "StatusCode": "HTTP_301"}}'
     alb.ingress.kubernetes.io/listen-ports: '[{"HTTP": 80}, {"HTTPS":443}]'
-    alb.ingress.kubernetes.io/certificate-arn: {{ .Values.ingress.certificateArn }} 
+    alb.ingress.kubernetes.io/certificate-arn: {{ .Values.awsAlbIngress.certificateArn }} 
     {{- end }}
-    {{- if .Values.ingress.sslPolicy }}
+    {{- if .Values.awsAlbIngress.sslPolicy }}
-    alb.ingress.kubernetes.io/actions.ssl-policy: {{ .Values.ingress.sslPolicy }}
+    alb.ingress.kubernetes.io/actions.ssl-policy: {{ .Values.awsAlbIngress.sslPolicy }}
     {{- end }}
-    {{- if .Values.ingress.securityGroups }}
+    {{- if .Values.awsAlbIngress.securityGroups }}
-    alb.ingress.kubernetes.io/security-groups: {{ .Values.ingress.securityGroups }}
+    alb.ingress.kubernetes.io/security-groups: {{ .Values.awsAlbIngress.securityGroups }}
     {{- end }}
 spec:
   rules:
     - http:
         paths:
-        {{- if .Values.ingress.certificateArn }}
+        {{- if .Values.awsAlbIngress.certificateArn }}
         - path: /
           pathType: Prefix
           backend:

charts/budibase/templates/app-service-deployment.yaml

@@ -2,12 +2,9 @@ apiVersion: apps/v1
 kind: Deployment
 metadata:
   annotations:
-    kompose.cmd: kompose convert
-    kompose.version: 1.21.0 (992df58d8)
 {{ if .Values.services.apps.deploymentAnnotations }}
 {{- toYaml .Values.services.apps.deploymentAnnotations | indent 4 -}}
 {{ end }}
-  creationTimestamp: null
   labels:
     io.kompose.service: app-service
 {{ if .Values.services.apps.deploymentLabels }}
@@ -24,12 +21,9 @@ spec:
   template:
     metadata:
       annotations:
-        kompose.cmd: kompose convert
-        kompose.version: 1.21.0 (992df58d8)
 {{ if .Values.services.apps.templateAnnotations }}
 {{- toYaml .Values.services.apps.templateAnnotations | indent 8 -}}
 {{ end }}
-      creationTimestamp: null
       labels:
         io.kompose.service: app-service
 {{ if .Values.services.apps.templateLabels }}

charts/budibase/templates/app-service-hpa.yaml

@@ -0,0 +1,32 @@
+{{- if .Values.services.apps.autoscaling.enabled }}
+apiVersion: {{ ternary "autoscaling/v2" "autoscaling/v2beta2" (.Capabilities.APIVersions.Has "autoscaling/v2") }}
+kind: HorizontalPodAutoscaler
+metadata:
+  name: {{ include "budibase.fullname" . }}-apps
+  labels:
+    {{- include "budibase.labels" . | nindent 4 }}
+spec:
+  scaleTargetRef:
+    apiVersion: apps/v1
+    kind: Deployment
+    name: app-service
+  minReplicas: {{ .Values.services.apps.autoscaling.minReplicas }}
+  maxReplicas: {{ .Values.services.apps.autoscaling.maxReplicas }}
+  metrics:
+    {{- if .Values.services.apps.autoscaling.targetCPUUtilizationPercentage }}
+    - type: Resource
+      resource:
+        name: cpu
+        target:
+          type: Utilization
+          averageUtilization: {{ .Values.services.apps.autoscaling.targetCPUUtilizationPercentage }}
+    {{- end }}
+    {{- if .Values.services.apps.autoscaling.targetMemoryUtilizationPercentage }}
+    - type: Resource
+      resource:
+        name: memory
+        target:
+          type: Utilization
+          averageUtilization: {{ .Values.services.apps.autoscaling.targetMemoryUtilizationPercentage }}
+    {{- end }}
+{{- end }}

charts/budibase/templates/app-service-service.yaml

@@ -1,10 +1,6 @@
 apiVersion: v1
 kind: Service
 metadata:
-  annotations:
-    kompose.cmd: kompose convert
-    kompose.version: 1.21.0 (992df58d8)
-  creationTimestamp: null
   labels:
     io.kompose.service: app-service
   name: app-service

charts/budibase/templates/couchdb-backup.yaml

@@ -2,10 +2,6 @@
 apiVersion: apps/v1
 kind: Deployment
 metadata:
-  annotations:
-    kompose.cmd: kompose convert
-    kompose.version: 1.21.0 (992df58d8)
-  creationTimestamp: null
   labels:
     app.kubernetes.io/name: couchdb-backup
   name: couchdb-backup
@@ -18,10 +14,6 @@ spec:
     type: Recreate
   template:
     metadata:
-      annotations:
-        kompose.cmd: kompose convert
-        kompose.version: 1.21.0 (992df58d8)
-      creationTimestamp: null
       labels:
         app.kubernetes.io/name: couchdb-backup
     spec:

charts/budibase/templates/hpa.yaml

@@ -1,28 +0,0 @@
-{{- if .Values.autoscaling.enabled }}
-apiVersion: autoscaling/v2beta1
-kind: HorizontalPodAutoscaler
-metadata:
-  name: {{ include "budibase.fullname" . }}
-  labels:
-    {{- include "budibase.labels" . | nindent 4 }}
-spec:
-  scaleTargetRef:
-    apiVersion: apps/v1
-    kind: Deployment
-    name: {{ include "budibase.fullname" . }}
-  minReplicas: {{ .Values.autoscaling.minReplicas }}
-  maxReplicas: {{ .Values.autoscaling.maxReplicas }}
-  metrics:
-    {{- if .Values.autoscaling.targetCPUUtilizationPercentage }}
-    - type: Resource
-      resource:
-        name: cpu
-        targetAverageUtilization: {{ .Values.autoscaling.targetCPUUtilizationPercentage }}
-    {{- end }}
-    {{- if .Values.autoscaling.targetMemoryUtilizationPercentage }}
-    - type: Resource
-      resource:
-        name: memory
-        targetAverageUtilization: {{ .Values.autoscaling.targetMemoryUtilizationPercentage }}
-    {{- end }}
-{{- end }}

charts/budibase/templates/minio-data-persistentvolumeclaim.yaml

@@ -2,7 +2,6 @@
 apiVersion: v1
 kind: PersistentVolumeClaim
 metadata:
-  creationTimestamp: null
   labels:
     io.kompose.service: minio-data
   name: minio-data

charts/budibase/templates/minio-service-deployment.yaml

@@ -2,10 +2,6 @@
 apiVersion: apps/v1
 kind: Deployment
 metadata:
-  annotations:
-    kompose.cmd: kompose convert
-    kompose.version: 1.21.0 (992df58d8)
-  creationTimestamp: null
   labels:
     io.kompose.service: minio-service
   name: minio-service
@@ -18,10 +14,6 @@ spec:
     type: Recreate
   template:
     metadata:
-      annotations:
-        kompose.cmd: kompose convert
-        kompose.version: 1.21.0 (992df58d8)
-      creationTimestamp: null
       labels:
         io.kompose.service: minio-service
     spec:

charts/budibase/templates/minio-service-service.yaml

@@ -2,10 +2,6 @@
 apiVersion: v1
 kind: Service
 metadata:
-  annotations:
-    kompose.cmd: kompose convert
-    kompose.version: 1.21.0 (992df58d8)
-  creationTimestamp: null
   labels:
     io.kompose.service: minio-service
   name: minio-service

charts/budibase/templates/proxy-service-deployment.yaml

@@ -2,12 +2,9 @@ apiVersion: apps/v1
 kind: Deployment
 metadata:
   annotations:
-    kompose.cmd: kompose convert
-    kompose.version: 1.21.0 (992df58d8)
 {{ if .Values.services.proxy.deploymentAnnotations }}
 {{- toYaml .Values.services.proxy.deploymentAnnotations | indent 4 -}}
 {{ end }}
-  creationTimestamp: null
   labels:
     app.kubernetes.io/name: budibase-proxy
 {{ if .Values.services.proxy.deploymentLabels }}
@@ -19,17 +16,15 @@ spec:
   selector:
     matchLabels:
       app.kubernetes.io/name: budibase-proxy
+  minReadySeconds: 10
   strategy:
     type: RollingUpdate
   template:
     metadata:
       annotations:
-        kompose.cmd: kompose convert
-        kompose.version: 1.21.0 (992df58d8)
 {{ if .Values.services.proxy.templateAnnotations }}
 {{- toYaml .Values.services.proxy.templateAnnotations | indent 8 -}}
 {{ end }}
-      creationTimestamp: null
       labels:
         app.kubernetes.io/name: budibase-proxy
 {{ if .Values.services.proxy.templateLabels }}

charts/budibase/templates/proxy-service-hpa.yaml

@@ -0,0 +1,32 @@
+{{- if .Values.services.proxy.autoscaling.enabled }}
+apiVersion: {{ ternary "autoscaling/v2" "autoscaling/v2beta2" (.Capabilities.APIVersions.Has "autoscaling/v2") }}
+kind: HorizontalPodAutoscaler
+metadata:
+  name: {{ include "budibase.fullname" . }}-proxy
+  labels:
+    {{- include "budibase.labels" . | nindent 4 }}
+spec:
+  scaleTargetRef:
+    apiVersion: apps/v1
+    kind: Deployment
+    name: proxy-service
+  minReplicas: {{ .Values.services.proxy.autoscaling.minReplicas }}
+  maxReplicas: {{ .Values.services.proxy.autoscaling.maxReplicas }}
+  metrics:
+    {{- if .Values.services.proxy.autoscaling.targetCPUUtilizationPercentage }}
+    - type: Resource
+      resource:
+        name: cpu
+        target:
+          type: Utilization
+          averageUtilization: {{ .Values.services.proxy.autoscaling.targetCPUUtilizationPercentage }}
+    {{- end }}
+    {{- if .Values.services.proxy.autoscaling.targetMemoryUtilizationPercentage }}
+    - type: Resource
+      resource:
+        name: memory
+        target:
+          type: Utilization
+          averageUtilization: {{ .Values.services.proxy.autoscaling.targetMemoryUtilizationPercentage }}
+    {{- end }}
+{{- end }}

charts/budibase/templates/proxy-service-service.yaml

@@ -1,10 +1,6 @@
 apiVersion: v1
 kind: Service
 metadata:
-  annotations:
-    kompose.cmd: kompose convert
-    kompose.version: 1.21.0 (992df58d8)
-  creationTimestamp: null
   labels:
     app.kubernetes.io/name: budibase-proxy
   name: proxy-service
@@ -16,4 +12,4 @@ spec:
   selector:
     app.kubernetes.io/name: budibase-proxy
 status:
-  loadBalancer: {}
+  loadBalancer: {}

charts/budibase/templates/redis-data-persistentvolumeclaim.yaml

@@ -2,7 +2,6 @@
 apiVersion: v1
 kind: PersistentVolumeClaim
 metadata:
-  creationTimestamp: null
   labels:
     io.kompose.service: redis-data
   name: redis-data

charts/budibase/templates/redis-service-deployment.yaml

@@ -2,10 +2,6 @@
 apiVersion: apps/v1
 kind: Deployment
 metadata:
-  annotations:
-    kompose.cmd: kompose convert
-    kompose.version: 1.21.0 (992df58d8)
-  creationTimestamp: null
   labels:
     io.kompose.service: redis-service
   name: redis-service
@@ -18,10 +14,6 @@ spec:
     type: Recreate
   template:
     metadata:
-      annotations:
-        kompose.cmd: kompose convert
-        kompose.version: 1.21.0 (992df58d8)
-      creationTimestamp: null
       labels:
         io.kompose.service: redis-service
     spec:

charts/budibase/templates/redis-service-service.yaml

@@ -2,10 +2,6 @@
 apiVersion: v1
 kind: Service
 metadata:
-  annotations:
-    kompose.cmd: kompose convert
-    kompose.version: 1.21.0 (992df58d8)
-  creationTimestamp: null
   labels:
     io.kompose.service: redis-service
   name: redis-service

charts/budibase/templates/worker-service-deployment.yaml

@@ -2,12 +2,9 @@ apiVersion: apps/v1
 kind: Deployment
 metadata:
   annotations:
-    kompose.cmd: kompose convert
-    kompose.version: 1.21.0 (992df58d8)
 {{ if .Values.services.worker.deploymentAnnotations }}
 {{- toYaml .Values.services.worker.deploymentAnnotations | indent 4 -}}
 {{ end }}
-  creationTimestamp: null
   labels:
     io.kompose.service: worker-service
 {{ if .Values.services.worker.deploymentLabels }}
@@ -24,12 +21,9 @@ spec:
   template:
     metadata:
       annotations:
-        kompose.cmd: kompose convert
-        kompose.version: 1.21.0 (992df58d8)
 {{ if .Values.services.worker.templateAnnotations }}
 {{- toYaml .Values.services.worker.templateAnnotations | indent 8 -}}
 {{ end }}
-      creationTimestamp: null
       labels:
         io.kompose.service: worker-service
 {{ if .Values.services.worker.templateLabels }}

charts/budibase/templates/worker-service-hpa.yaml

@@ -0,0 +1,32 @@
+{{- if .Values.services.worker.autoscaling.enabled }}
+apiVersion: {{ ternary "autoscaling/v2" "autoscaling/v2beta2" (.Capabilities.APIVersions.Has "autoscaling/v2") }}
+kind: HorizontalPodAutoscaler
+metadata:
+  name: {{ include "budibase.fullname" . }}-worker
+  labels:
+    {{- include "budibase.labels" . | nindent 4 }}
+spec:
+  scaleTargetRef:
+    apiVersion: apps/v1
+    kind: Deployment
+    name: worker-service
+  minReplicas: {{ .Values.services.worker.autoscaling.minReplicas }}
+  maxReplicas: {{ .Values.services.worker.autoscaling.maxReplicas }}
+  metrics:
+    {{- if .Values.services.worker.autoscaling.targetCPUUtilizationPercentage }}
+    - type: Resource
+      resource:
+        name: cpu
+        target:
+          type: Utilization
+          averageUtilization: {{ .Values.services.worker.autoscaling.targetCPUUtilizationPercentage }}
+    {{- end }}
+    {{- if .Values.services.worker.autoscaling.targetMemoryUtilizationPercentage }}
+    - type: Resource
+      resource:
+        name: memory
+        target:
+          type: Utilization
+          averageUtilization: {{ .Values.services.worker.autoscaling.targetMemoryUtilizationPercentage }}
+    {{- end }}
+{{- end }}

charts/budibase/templates/worker-service-service.yaml

@@ -1,10 +1,6 @@
 apiVersion: v1
 kind: Service
 metadata:
-  annotations:
-    kompose.cmd: kompose convert
-    kompose.version: 1.21.0 (992df58d8)
-  creationTimestamp: null
   labels:
     io.kompose.service: worker-service
   name: worker-service

charts/budibase/values.yaml

@@ -1,56 +1,32 @@
-# Default values for budibase.
+# -- Passed to all pods created by this chart. Should not ordinarily need to be changed.
-# This is a YAML-formatted file.
-# Declare variables to be passed into your templates.
-
-image:
-  pullPolicy: IfNotPresent
-  # Overrides the image tag whose default is the chart appVersion.
-  tag: ""
-
 imagePullSecrets: []
+# -- Override the name of the deploymen. Defaults to {{ .Chart.Name }}.
 nameOverride: ""
-# fullnameOverride: ""
 
 serviceAccount:
-  # Specifies whether a service account should be created
+  # -- Specifies whether a service account should be created
   create: true
-  # Annotations to add to the service account
+  # -- Annotations to add to the service account
   annotations: {}
-  # The name of the service account to use.
+  # -- The name of the service account to use.
   # If not set and create is true, a name is generated using the fullname template
   name: ""
 
-podAnnotations: {}
-
-podSecurityContext:
-  {}
-  # fsGroup: 2000
-
-securityContext:
-  {}
-  # capabilities:
-  #   drop:
-  #   - ALL
-  # readOnlyRootFilesystem: true
-  # runAsNonRoot: true
-  # runAsUser: 1000
-
 service:
+  # -- Service type for the service that points to the main Budibase proxy pod.
   type: ClusterIP
+  # -- Port to expose on the service.
   port: 10000
 
 ingress:
+  # -- Whether to create an Ingress resource pointing to the Budibase proxy.
   enabled: true
-  aws: false
+  # -- What ingress class to use.
-  nginx: true
-  certificateArn: ""
   className: ""
-  annotations:
+  # -- Standard hosts block for the Ingress resource. Defaults to pointing to the Budibase proxy.
-    kubernetes.io/ingress.class: nginx
-    nginx.ingress.kubernetes.io/client-max-body-size: 150M
-    nginx.ingress.kubernetes.io/proxy-body-size: 50m
   hosts:
-    - host: # change if using custom domain
+    # @ignore
+    - host:
       paths:
         - path: /
           pathType: Prefix
@@ -60,361 +36,426 @@ ingress:
               port:
                 number: 10000
 
-autoscaling:
+awsAlbIngress:
+  # -- Whether to create an ALB Ingress resource pointing to the Budibase proxy. Requires the AWS ALB Ingress Controller.
   enabled: false
-  minReplicas: 1
+  # -- If you're wanting to use HTTPS, you'll need to create an ACM certificate and specify the ARN here.
-  maxReplicas: 100
+  certificateArn: ""
-  targetCPUUtilizationPercentage: 80
-  # targetMemoryUtilizationPercentage: 80
-
-nodeSelector: {}
 
+# -- Sets the tolerations for all pods created by this chart. Should not ordinarily need to be changed.
+# See <https://kubernetes.io/docs/concepts/scheduling-eviction/taint-and-toleration/> for more information
+# on tolerations.
 tolerations: []
 
+# -- Sets the affinity for all pods created by this chart. Should not ordinarily
+# need to be changed.  See
+# <https://kubernetes.io/docs/tasks/configure-pod-container/assign-pods-nodes-using-node-affinity/>
+# for more information on affinity.
 affinity: {}
 
 globals:
-  appVersion: "" # Use as an override to .Chart.AppVersion
+  # -- The version of Budibase to deploy. Defaults to what's specified by {{ .Chart.AppVersion }}.
+  # Ends up being used as the image version tag for the apps, proxy, and worker images.
+  appVersion: ""
+  # -- Sets the environment variable BUDIBASE_ENVIRONMENT for the apps and worker pods. Should not
+  # ordinarily need to be changed.
   budibaseEnv: PRODUCTION
+  # -- Sets what feature flags are enabled and for which tenants. Should not ordinarily need to be
+  # changed.
   tenantFeatureFlags: "*:LICENSING,*:USER_GROUPS,*:ONBOARDING_TOUR"
+  # -- Whether to enable analytics or not. You can read more about our analytics here:
+  # <https://docs.budibase.com/docs/analytics>.
   enableAnalytics: "1"
+  # @ignore (only used if enableAnalytics is set to 1)
   posthogToken: "phc_bIjZL7oh2GEUd2vqvTBH8WvrX0fWTFQMs6H5KQxiUxU"
-  selfHosted: "1" # set to 0 for budibase cloud environment, set to 1 for self-hosted setup
+  # @ignore (should not normally need to be changed, we only set this to "0"
-  multiTenancy: "0" # set to 0 to disable multiple orgs, set to 1 to enable multiple orgs
+  # when deploying to our Cloud environment)
-  offlineMode: "0" # set to 1 to enable offline mode
+  selfHosted: "1"
+  # @ignore (doesn't work out of the box for self-hosted users, only meant for Budicloud)
+  multiTenancy: "0"
+  # @ignore (only currently used to determine whether to fetch licenses offline or not, should
+  # not normally need to be changed, and only applies to Enterprise customers)
+  offlineMode: "0"
+  # @ignore (only needs to be set in our cloud environment)
   accountPortalUrl: ""
+  # @ignore (only needs to be set in our cloud environment)
   accountPortalApiKey: ""
+  # -- Sets the domain attribute of the cookie that Budibase uses to store session information.
+  # See <https://developer.mozilla.org/en-US/docs/Web/HTTP/Cookies#define_where_cookies_are_sent>
+  # for details on why you might want to set this.
   cookieDomain: ""
+  # -- Set the `platformUrl` binding. You can also do this in Settings > Organisation if you are
+  # self-hosting.
   platformUrl: ""
+  # -- Whether or not to enable doing data migrations over the HTTP API. If this is set to "0",
+  # migrations are run on startup. You shouldn't ordinarily need to change this.
   httpMigrations: "0"
+  # -- Google OAuth settings. These can also be set in the Budibase UI, see
+  # <https://docs.budibase.com/docs/sso-with-google> for details.
   google:
+    # -- Client ID of your Google OAuth app.
     clientId: ""
+    # -- Client secret of your Google OAuth app.
     secret: ""
+  # -- The maximum number of iterations allows for an automation loop step. You can read more about
+  # looping here: <https://docs.budibase.com/docs/looping>.
   automationMaxIterations: "200"
 
-  createSecrets: true # creates an internal API key, JWT secrets and redis password for you
+  # -- Create an internal API key, JWT secret, object store access key and
+  # secret, and store them in a Kubernetes `Secret`.
+  createSecrets: true
 
-  # if createSecrets is set to false, you can hard-code your secrets here
+  # -- Used for encrypting API keys and environment variables when stored in the database.
+  # You don't need to set this if `createSecrets` is true.
   apiEncryptionKey: ""
+  # -- API key used for internal Budibase API calls. You don't need to set this
+  # if `createSecrets` is true.
   internalApiKey: ""
+  # -- Secret used for signing JWTs. You don't need to set this if `createSecrets` is true.
   jwtSecret: ""
-  cdnUrl: ""
+
-  # fallback values used during live rotation
+  # -- A fallback value for `internalApiKey`. If you're rotating your encryption key, you can
+  # set this to the old value for the duration of the rotation.
   internalApiKeyFallback: ""
+  # -- A fallback value for `jwtSecret`. If you're rotating your JWT secret, you can set this
+  # to the old value for the duration of the rotation.
   jwtSecretFallback: ""
 
   smtp:
+    # -- Whether to enable SMTP or not.
     enabled: false
-
+    # -- The hostname of your SMTP server.
-#  globalAgentHttpProxy:
+    host: ""
-#  globalAgentHttpsProxy:
+    # -- The port of your SMTP server.
-#  globalAgentNoProxy:
+    port: "587"
+    # -- The email address to use in the "From:" field of emails sent by Budibase.
+    from: ""
+    # -- The username to use when authenticating with your SMTP server.
+    user: ""
+    # -- The password to use when authenticating with your SMTP server.
+    password: ""
 
 services:
-  budibaseVersion: latest
+  # -- The DNS suffix to use for service discovery. You only need to change this
+  # if you've configured your cluster to use a different DNS suffix.
   dns: cluster.local
-  # tlsRejectUnauthorized: 0
 
   proxy:
+    # @ignore (you shouldn't need to change this)
     port: 10000
+    # -- The number of proxy replicas to run.
     replicaCount: 1
+    # @ignore (you should never need to change this)
     upstreams:
       apps: "http://app-service.{{ .Release.Namespace }}.svc.{{ .Values.services.dns }}:{{ .Values.services.apps.port }}"
       worker: "http://worker-service.{{ .Release.Namespace }}.svc.{{ .Values.services.dns }}:{{ .Values.services.worker.port }}"
       minio: "http://minio-service.{{ .Release.Namespace }}.svc.{{ .Values.services.dns }}:{{ .Values.services.objectStore.port }}"
       couchdb: "http://{{ .Release.Name }}-svc-couchdb:{{ .Values.services.couchdb.port }}"
+    # -- The resources to use for proxy pods. See
+    # <https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/>
+    # for more information on how to set these.
     resources: {}
+    # -- Startup probe configuration for proxy pods. You shouldn't need to
+    # change this, but if you want to you can find more information here:
+    # <https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/>
+    # @default -- HTTP health checks.
     startupProbe:
+      # @ignore
       httpGet:
         path: /health
         port: 10000
         scheme: HTTP
+      # @ignore
       failureThreshold: 30
+      # @ignore
       periodSeconds: 3
+    # -- Readiness probe configuration for proxy pods. You shouldn't need to
+    # change this, but if you want to you can find more information here:
+    # <https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/>
+    # @default -- HTTP health checks.
     readinessProbe:
+      # @ignore
       httpGet:
         path: /health
         port: 10000
         scheme: HTTP
+      # @ignore
       periodSeconds: 3
+      # @ignore
       failureThreshold: 1
+    # -- Liveness probe configuration for proxy pods. You shouldn't need to
+    # change this, but if you want to you can find more information here:
+    # <https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/>
+    # @default -- HTTP health checks.
     livenessProbe:
+      # @ignore
       httpGet:
         path: /health
         port: 10000
         scheme: HTTP
+      # @ignore
       failureThreshold: 3
+      # @ignore
       periodSeconds: 5
-  #    annotations:
+    autoscaling:
-  #      co.elastic.logs/module: nginx
+      # -- Whether to enable horizontal pod autoscaling for the proxy service.
-  #      co.elastic.logs/fileset.stdout: access
+      enabled: false
-  #      co.elastic.logs/fileset.stderr: error
+      minReplicas: 1
+      maxReplicas: 10
+      # -- Target CPU utilization percentage for the proxy service. Note that
+      # for autoscaling to work, you will need to have metrics-server
+      # configured, and resources set for the proxy pods.
+      targetCPUUtilizationPercentage: 80
 
   apps:
+    # @ignore (you shouldn't need to change this)
     port: 4002
+    # -- The number of apps replicas to run.
     replicaCount: 1
+    # -- The log level for the apps service.
     logLevel: info
+    # -- Whether or not to log HTTP requests to the apps service.
     httpLogging: 1
+    # -- The resources to use for apps pods. See
+    # <https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/>
+    # for more information on how to set these.
     resources: {}
+    # -- Startup probe configuration for apps pods. You shouldn't need to
+    # change this, but if you want to you can find more information here:
+    # <https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/>
+    # @default -- HTTP health checks.
     startupProbe:
+      # @ignore
       httpGet:
         path: /health
         port: 4002
         scheme: HTTP
+      # @ignore
       failureThreshold: 30
+      # @ignore
       periodSeconds: 3
+    # -- Readiness probe configuration for apps pods. You shouldn't need to
+    # change this, but if you want to you can find more information here:
+    # <https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/>
+    # @default -- HTTP health checks.
     readinessProbe:
+      # @ignore
       httpGet:
         path: /health
         port: 4002
         scheme: HTTP
+      # @ignore
       periodSeconds: 3
+      # @ignore
       failureThreshold: 1
+    # -- Liveness probe configuration for apps pods. You shouldn't need to
+    # change this, but if you want to you can find more information here:
+    # <https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/>
+    # @default -- HTTP health checks.
     livenessProbe:
+      # @ignore
       httpGet:
         path: /health
         port: 4002
         scheme: HTTP
+      # @ignore
       failureThreshold: 3
+      # @ignore
       periodSeconds: 5
-  #    nodeDebug: "" # set the value of NODE_DEBUG
+    autoscaling:
-  #    annotations:
+      # -- Whether to enable horizontal pod autoscaling for the apps service.
-  #      co.elastic.logs/multiline.type: pattern
+      enabled: false
-  #      co.elastic.logs/multiline.pattern: '^[[:space:]]'
+      minReplicas: 1
-  #      co.elastic.logs/multiline.negate: false
+      maxReplicas: 10
-  #      co.elastic.logs/multiline.match: after
+      # -- Target CPU utilization percentage for the apps service. Note that for
+      # autoscaling to work, you will need to have metrics-server configured,
+      # and resources set for the apps pods.
+      targetCPUUtilizationPercentage: 80
+
   worker:
+    # @ignore (you shouldn't need to change this)
     port: 4003
+    # -- The number of worker replicas to run.
     replicaCount: 1
+    # -- The log level for the worker service.
     logLevel: info
+    # -- Whether or not to log HTTP requests to the worker service.
     httpLogging: 1
+    # -- The resources to use for worker pods. See
+    # <https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/>
+    # for more information on how to set these.
     resources: {}
+    # -- Startup probe configuration for worker pods. You shouldn't need to
+    # change this, but if you want to you can find more information here:
+    # <https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/>
+    # @default -- HTTP health checks.
     startupProbe:
+      # @ignore
       httpGet:
         path: /health
         port: 4003
         scheme: HTTP
+      # @ignore
       failureThreshold: 30
+      # @ignore
       periodSeconds: 3
+    # -- Readiness probe configuration for worker pods. You shouldn't need to
+    # change this, but if you want to you can find more information here:
+    # <https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/>
+    # @default -- HTTP health checks.
     readinessProbe:
+      # @ignore
       httpGet:
         path: /health
         port: 4003
         scheme: HTTP
+      # @ignore
       periodSeconds: 3
+      # @ignore
       failureThreshold: 1
+    # -- Liveness probe configuration for worker pods. You shouldn't need to
+    # change this, but if you want to you can find more information here:
+    # <https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/>
+    # @default -- HTTP health checks.
     livenessProbe:
+      # @ignore
       httpGet:
         path: /health
         port: 4003
         scheme: HTTP
+      # @ignore
       failureThreshold: 3
+      # @ignore
       periodSeconds: 5
-  #    annotations:
+    autoscaling:
-  #      co.elastic.logs/multiline.type: pattern
+      # -- Whether to enable horizontal pod autoscaling for the worker service.
-  #      co.elastic.logs/multiline.pattern: '^[[:space:]]'
+      enabled: false
-  #      co.elastic.logs/multiline.negate: false
+      minReplicas: 1
-  #      co.elastic.logs/multiline.match: after
+      maxReplicas: 10
+      # -- Target CPU utilization percentage for the worker service. Note that
+      # for autoscaling to work, you will need to have metrics-server
+      # configured, and resources set for the worker pods.
+      targetCPUUtilizationPercentage: 80
 
   couchdb:
+    # -- Whether or not to spin up a CouchDB instance in your cluster. True by
+    # default, and the configuration for the CouchDB instance is under the
+    # `couchdb` key at the root of this file. You can see what options are
+    # available to you by looking at the official CouchDB Helm chart:
+    # <https://github.com/apache/couchdb-helm/tree/couchdb-4.3.0/couchdb>.
     enabled: true
     # url: "" # only change if pointing to existing couch server
     # user: "" # only change if pointing to existing couch server
     # password: "" # only change if pointing to existing couch server
     port: 5984
     backup:
+      # -- Whether or not to enable periodic CouchDB backups. This works by replicating
+      # to another CouchDB instance.
       enabled: false
-      # target couchDB instance to back up to
+      # -- Target couchDB instance to back up to, either a hostname or an IP address.
       target: ""
-      # backup interval in seconds
+      # -- Backup interval in seconds
       interval: ""
+      # -- The resources to use for CouchDB backup pods. See
+      # <https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/>
+      # for more information on how to set these.
       resources: {}
 
   redis:
-    enabled: true # disable if using external redis
+    # -- Whether or not to deploy a Redis pod into your cluster.
+    enabled: true
+    # -- Port to expose Redis on.
     port: 6379
+    # @ignore (you should leave this as 1, we don't support clustering Redis)
     replicaCount: 1
-    url: "" # only change if pointing to existing redis cluster and enabled: false
+    # -- If you choose to run Redis externally to this chart, you can specify the
-    password: "budibase" # recommended to override if using built-in redis
+    # connection details here.
+    url: ""
+    # -- The password to use when connecting to Redis. It's recommended that you change
+    # this from the default if you're running Redis in-cluster.
+    password: "budibase"
+    # -- How much persistent storage to allocate to Redis.
     storage: 100Mi
-    ## If defined, storageClassName: <storageClass>
+    # -- If defined, storageClassName: <storageClass> If set to "-",
-    ## If set to "-", storageClassName: "", which disables dynamic provisioning
+    # storageClassName: "", which disables dynamic provisioning If undefined
-    ## If undefined (the default) or set to null, no storageClassName spec is
+    # (the default) or set to null, no storageClassName spec is set, choosing
-    ##   set, choosing the default provisioner.
+    # the default provisioner.
     storageClass: ""
+    # -- The resources to use for Redis pods. See
+    # <https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/>
+    # for more information on how to set these.
     resources: {}
 
   objectStore:
-    # Set to false if using another object store such as S3
+    # -- Set to false if using another object store, such as S3. You will need
+    # to set `services.objectStore.url` to point to your bucket if you do this.
     minio: true
+    # -- Whether to enable the Minio web console or not. If you're exposing
+    # Minio to the Internet (via a custom Ingress record, for example), you
+    # should set this to false. If you're only exposing Minio to your cluster,
+    # you can leave this as true.
     browser: true
+    # @ignore
     port: 9000
+    # @ignore (you should leave this as 1, we don't support clustering Minio)
     replicaCount: 1
-    accessKey: "" # AWS_ACCESS_KEY if using S3 or existing minio access key
+    # -- AWS_ACCESS_KEY if using S3
-    secretKey: "" # AWS_SECRET_ACCESS_KEY if using S3 or existing minio secret
+    accessKey: ""
-    region: "" # AWS_REGION if using S3 or existing minio secret
+    # -- AWS_SECRET_ACCESS_KEY if using S3
-    url: "http://minio-service:9000" # only change if pointing to existing minio cluster or S3 and minio: false
+    secretKey: ""
+    # -- AWS_REGION if using S3
+    region: ""
+    # -- URL to use for object storage. Only change this if you're using an
+    # external object store, such as S3. Remember to set `minio: false` if you
+    # do this.
+    url: "http://minio-service:9000"
+    # -- How much storage to give Minio in its PersistentVolumeClaim.
     storage: 100Mi
-    ## If defined, storageClassName: <storageClass>
+    # -- If defined, storageClassName: <storageClass> If set to "-",
-    ## If set to "-", storageClassName: "", which disables dynamic provisioning
+    # storageClassName: "", which disables dynamic provisioning If undefined
-    ## If undefined (the default) or set to null, no storageClassName spec is
+    # (the default) or set to null, no storageClassName spec is set, choosing
-    ##   set, choosing the default provisioner.
+    # the default provisioner.
     storageClass: ""
+    # -- The resources to use for Minio pods. See
+    # <https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/>
+    # for more information on how to set these.
     resources: {}
     cloudfront:
-      # Set the url of a distribution to enable cloudfront
+      # -- Set the url of a distribution to enable cloudfront.
       cdn: ""
-      # ID of public key stored in cloudfront
+      # -- ID of public key stored in cloudfront.
       publicKeyId: ""
-      # Base64 encoded private key for the above public key
+      # -- Base64 encoded private key for the above public key.
       privateKey64: ""
 
-# Override values in couchDB subchart
+# Override values in couchDB subchart. We're only specifying the values we're changing.
+# If you want to see all of the available values, see:
+#   https://github.com/apache/couchdb-helm/tree/couchdb-4.3.0/couchdb
 couchdb:
-  ## clusterSize is the initial size of the CouchDB cluster.
+  # -- The number of replicas to run in the CouchDB cluster. We set this to
+  # 1 by default to make things simpler, but you can set it to 3 if you need
+  # a high-availability CouchDB cluster.
   clusterSize: 1
-  allowAdminParty: false
 
-  # Secret Management
+  # -- We use a custom CouchDB image for running Budibase and we don't support
-  createAdminSecret: true
+  # using any other CouchDB image. You shouldn't change this, and if you do we
-
+  # can't guarantee that Budibase will work.
-  # adminUsername: budibase
-  # adminPassword: budibase
-  # adminHash: -pbkdf2-this_is_not_necessarily_secure_either
-  # cookieAuthSecret: admin
-
-  ## When enabled, will deploy a networkpolicy that allows CouchDB pods to
-  ## communicate with each other for clustering and ingress on port 5984
-  networkPolicy:
-    enabled: true
-
-  # Use a service account
-  serviceAccount:
-    enabled: true
-    create: true
-  # name:
-  # imagePullSecrets:
-  # - name: myimagepullsecret
-
-  ## The storage volume used by each Pod in the StatefulSet. If a
-  ## persistentVolume is not enabled, the Pods will use `emptyDir` ephemeral
-  ## local storage. Setting the storageClass attribute to "-" disables dynamic
-  ## provisioning of Persistent Volumes; leaving it unset will invoke the default
-  ## provisioner.
-  persistentVolume:
-    enabled: false
-    accessModes:
-      - ReadWriteOnce
-    size: 10Gi
-    storageClass: ""
-
-  ## The CouchDB image
   image:
-    repository: couchdb
+    # @ignore
-    tag: 3.1.1
+    repository: budibase/couchdb
-    pullPolicy: IfNotPresent
+    # @ignore
-
+    tag: v3.2.1
-  ## Experimental integration with Lucene-powered fulltext search
+    # @ignore
-  enableSearch: true
-  searchImage:
-    repository: kocolosk/couchdb-search
-    tag: 0.2.0
-    pullPolicy: IfNotPresent
-
-  initImage:
-    repository: busybox
-    tag: latest
     pullPolicy: Always
 
-  ## CouchDB is happy to spin up cluster nodes in parallel, but if you encounter
+  # @ignore
-  ## problems you can try setting podManagementPolicy to the StatefulSet default
+  # This should remain false. We ship Clouseau ourselves as part of the
-  ## `OrderedReady`
+  # budibase/couchdb image, and it's not possible to disable it because it's a
-  podManagementPolicy: Parallel
+  # core part of the Budibase experience.
+  enableSearch: false
 
-  ## Optional pod annotations
-  annotations: {}
-
-  ## Optional tolerations
-  tolerations: []
-
-  affinity: {}
-
-  service:
-    # annotations:
-    enabled: true
-    type: ClusterIP
-    externalPort: 5984
-
-  ## An Ingress resource can provide name-based virtual hosting and TLS
-  ## termination among other things for CouchDB deployments which are accessed
-  ## from outside the Kubernetes cluster.
-  ## ref: https://kubernetes.io/docs/concepts/services-networking/ingress/
-  ingress:
-    enabled: false
-    hosts:
-      - chart-example.local
-    path: /
-    annotations:
-      []
-      # kubernetes.io/ingress.class: nginx
-      # kubernetes.io/tls-acme: "true"
-    tls:
-      # Secrets must be manually created in the namespace.
-      # - secretName: chart-example-tls
-      #   hosts:
-      #     - chart-example.local
-
-  ## Optional resource requests and limits for the CouchDB container
-  ## ref: http://kubernetes.io/docs/user-guide/compute-resources/
-  resources:
-    {}
-    # requests:
-    #  cpu: 100m
-    #  memory: 128Mi
-    # limits:
-    #  cpu: 56
-    #  memory: 256Gi
-
-  ## erlangFlags is a map that is passed to the Erlang VM as flags using the
-  ## ERL_FLAGS env. `name` and `setcookie` flags are minimally required to
-  ## establish connectivity between cluster nodes.
-  ## ref: http://erlang.org/doc/man/erl.html#init_flags
-  erlangFlags:
-    name: couchdb
-    setcookie: monster
-
-  ## couchdbConfig will override default CouchDB configuration settings.
-  ## The contents of this map are reformatted into a .ini file laid down
-  ## by a ConfigMap object.
-  ## ref: http://docs.couchdb.org/en/latest/config/index.html
   couchdbConfig:
     couchdb:
-      uuid: budibase-couchdb # REQUIRED: Unique identifier for this CouchDB server instance
+      # -- Unique identifier for this CouchDB server instance. You shouldn't need
-    # cluster:
+      # to change this.
-    #   q: 8 # Create 8 shards for each database
+      uuid: budibase-couchdb
-    chttpd:
-      bind_address: any
-      # chttpd.require_valid_user disables all the anonymous requests to the port
-      # 5984 when is set to true.
-      require_valid_user: false
-
-  # Kubernetes local cluster domain.
-  # This is used to generate FQDNs for peers when joining the CouchDB cluster.
-  dns:
-    clusterDomainSuffix: cluster.local
-
-  ## Configure liveness and readiness probe values
-  ## Ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-probes/#configure-probes
-  # FOR COUCHDB
-  livenessProbe:
-    failureThreshold: 3
-    initialDelaySeconds: 0
-    periodSeconds: 10
-    successThreshold: 1
-    timeoutSeconds: 1
-  readinessProbe:
-    failureThreshold: 3
-    initialDelaySeconds: 0
-    periodSeconds: 10
-    successThreshold: 1
-    timeoutSeconds: 1

hosting/couchdb/Dockerfile

@@ -33,7 +33,6 @@ WORKDIR /opt/sqs
 ADD sqlite/sqs sqlite/better_sqlite3.node ./
 
 WORKDIR /
-ADD build-target-paths.sh .
 ADD runner.sh ./bbcouch-runner.sh
-RUN chmod +x ./bbcouch-runner.sh /opt/clouseau/bin/clouseau ./build-target-paths.sh /opt/sqs/sqs
+RUN chmod +x ./bbcouch-runner.sh /opt/clouseau/bin/clouseau /opt/sqs/sqs
 CMD ["./bbcouch-runner.sh"]

hosting/couchdb/build-target-paths.sh

@@ -1,24 +0,0 @@
-#!/bin/bash
-
-echo ${TARGETBUILD} > /buildtarget.txt
-if [[ "${TARGETBUILD}" = "aas" ]]; then
-    # Azure AppService uses /home for persistent data & SSH on port 2222
-    DATA_DIR="${DATA_DIR:-/home}"
-    WEBSITES_ENABLE_APP_SERVICE_STORAGE=true
-    mkdir -p $DATA_DIR/{search,minio,couch}
-    mkdir -p $DATA_DIR/couch/{dbs,views}
-    chown -R couchdb:couchdb $DATA_DIR/couch/
-    apt update
-    apt-get install -y openssh-server
-    echo "root:Docker!" | chpasswd
-    mkdir -p /tmp
-    chmod +x /tmp/ssh_setup.sh \
-        && (sleep 1;/tmp/ssh_setup.sh 2>&1 > /dev/null)
-    cp /etc/sshd_config /etc/ssh/sshd_config
-    /etc/init.d/ssh restart
-    sed -i "s#DATA_DIR#/home#g" /opt/clouseau/clouseau.ini
-    sed -i "s#DATA_DIR#/home#g" /opt/couchdb/etc/local.ini
-else
-    sed -i "s#DATA_DIR#/data#g" /opt/clouseau/clouseau.ini
-    sed -i "s#DATA_DIR#/data#g" /opt/couchdb/etc/local.ini
-fi

hosting/couchdb/runner.sh

@@ -1,15 +1,84 @@
 #!/bin/bash
 
 DATA_DIR=${DATA_DIR:-/data}
+
 mkdir -p ${DATA_DIR}
 mkdir -p ${DATA_DIR}/couch/{dbs,views}
 mkdir -p ${DATA_DIR}/search
 chown -R couchdb:couchdb ${DATA_DIR}/couch
-/build-target-paths.sh
+
+echo ${TARGETBUILD} > /buildtarget.txt
+if [[ "${TARGETBUILD}" = "aas" ]]; then
+    # Azure AppService uses /home for persistent data & SSH on port 2222
+    DATA_DIR="${DATA_DIR:-/home}"
+    WEBSITES_ENABLE_APP_SERVICE_STORAGE=true
+    mkdir -p $DATA_DIR/{search,minio,couch}
+    mkdir -p $DATA_DIR/couch/{dbs,views}
+    chown -R couchdb:couchdb $DATA_DIR/couch/
+    apt update
+    apt-get install -y openssh-server
+    echo "root:Docker!" | chpasswd
+    mkdir -p /tmp
+    chmod +x /tmp/ssh_setup.sh \
+        && (sleep 1;/tmp/ssh_setup.sh 2>&1 > /dev/null)
+    cp /etc/sshd_config /etc/ssh/sshd_config
+    /etc/init.d/ssh restart
+    sed -i "s#DATA_DIR#/home#g" /opt/clouseau/clouseau.ini
+    sed -i "s#DATA_DIR#/home#g" /opt/couchdb/etc/local.ini
+elif [[ "${TARGETBUILD}" = "single" ]]; then
+    # In the single image build, the Dockerfile specifies /data as a volume
+    # mount, so we use that for all persistent data.
+    sed -i "s#DATA_DIR#/data#g" /opt/clouseau/clouseau.ini
+    sed -i "s#DATA_DIR#/data#g" /opt/couchdb/etc/local.ini
+elif [[ "${TARGETBUILD}" = "docker-compose" ]]; then
+    # We remove the database_dir and view_index_dir settings from the local.ini
+    # in docker-compose because it will default to /opt/couchdb/data which is what
+    # our docker-compose was using prior to us switching to using our own CouchDB
+    # image.
+    sed -i "s#^database_dir.*\$##g" /opt/couchdb/etc/local.ini
+    sed -i "s#^view_index_dir.*\$##g" /opt/couchdb/etc/local.ini
+    sed -i "s#^dir=.*\$#dir=/opt/couchdb/data#g" /opt/clouseau/clouseau.ini
+elif [[ -n $KUBERNETES_SERVICE_HOST ]]; then
+    # In Kubernetes the directory /opt/couchdb/data has a persistent volume
+    # mount for storing database data.
+    sed -i "s#^dir=.*\$#dir=/opt/couchdb/data#g" /opt/clouseau/clouseau.ini
+
+    # We remove the database_dir and view_index_dir settings from the local.ini
+    # in Kubernetes because it will default to /opt/couchdb/data which is what
+    # our Helm chart was using prior to us switching to using our own CouchDB
+    # image.
+    sed -i "s#^database_dir.*\$##g" /opt/couchdb/etc/local.ini
+    sed -i "s#^view_index_dir.*\$##g" /opt/couchdb/etc/local.ini
+
+    # We remove the -name setting from the vm.args file in Kubernetes because
+    # it will default to the pod FQDN, which is what's required for clustering
+    # to work.
+    sed -i "s/^-name .*$//g" /opt/couchdb/etc/vm.args
+else
+    # For all other builds, we use /data for persistent data.
+    sed -i "s#DATA_DIR#/data#g" /opt/clouseau/clouseau.ini
+    sed -i "s#DATA_DIR#/data#g" /opt/couchdb/etc/local.ini
+fi
+
+# Start Clouseau. Budibase won't function correctly without Clouseau running, it
+# powers the search API endpoints which are used to do all sorts, including
+# populating app grids.
 /opt/clouseau/bin/clouseau > /dev/stdout 2>&1 &
+
+# Start CouchDB.
 /docker-entrypoint.sh /opt/couchdb/bin/couchdb &
+
+# Start SQS.
 /opt/sqs/sqs --server "http://localhost:5984" --data-dir ${DATA_DIR}/sqs --bind-address=0.0.0.0 &
-sleep 10
+
+# Wait for CouchDB to start up.
+while [[ $(curl -s -w "%{http_code}\n" http://localhost:5984/_up -o /dev/null) -ne 200 ]]; do
+    echo 'Waiting for CouchDB to start...';
+    sleep 5;
+done
+
+# CouchDB needs the `_users` and `_replicator` databases to exist before it will
+# function correctly, so we create them here.
 curl -X PUT http://${COUCHDB_USER}:${COUCHDB_PASSWORD}@localhost:5984/_users
 curl -X PUT http://${COUCHDB_USER}:${COUCHDB_PASSWORD}@localhost:5984/_replicator
 sleep infinity

hosting/docker-compose.build.yaml

@@ -6,7 +6,7 @@ services:
   app-service:
     build:
       context: ..
-      dockerfile: packages/server/Dockerfile.v2
+      dockerfile: packages/server/Dockerfile
       args:
         - BUDIBASE_VERSION=0.0.0+dev-docker
     container_name: build-bbapps
@@ -36,7 +36,7 @@ services:
   worker-service:
     build:
       context: ..
-      dockerfile: packages/worker/Dockerfile.v2
+      dockerfile: packages/worker/Dockerfile
       args:
         - BUDIBASE_VERSION=0.0.0+dev-docker
     container_name: build-bbworker

hosting/docker-compose.yaml

@@ -57,7 +57,6 @@ services:
     depends_on:
       - redis-service
       - minio-service
-      - couch-init
 
   minio-service:
     restart: unless-stopped
@@ -70,7 +69,7 @@ services:
       MINIO_BROWSER: "off"
     command: server /data --console-address ":9001"
     healthcheck:
-      test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"]
+      test: "timeout 5s bash -c ':> /dev/tcp/127.0.0.1/9000' || exit 1"
       interval: 30s
       timeout: 20s
       retries: 3
@@ -98,26 +97,15 @@ services:
 
   couchdb-service:
     restart: unless-stopped
-    image: ibmcom/couchdb3
+    image: budibase/couchdb
+    pull_policy: always
     environment:
       - COUCHDB_PASSWORD=${COUCH_DB_PASSWORD}
       - COUCHDB_USER=${COUCH_DB_USER}
+      - TARGETBUILD=docker-compose
     volumes:
       - couchdb3_data:/opt/couchdb/data
 
-  couch-init:
-    image: curlimages/curl
-    environment:
-      PUT_CALL: "curl -u ${COUCH_DB_USER}:${COUCH_DB_PASSWORD} -X PUT couchdb-service:5984"
-    depends_on:
-      - couchdb-service
-    command:
-      [
-        "sh",
-        "-c",
-        "sleep 10 && $${PUT_CALL}/_users && $${PUT_CALL}/_replicator; fg;",
-      ]
-
   redis-service:
     restart: unless-stopped
     image: redis

hosting/nginx.dev.conf

@@ -42,7 +42,7 @@ http {
   server { 
     listen       10000 default_server;
     server_name  _;
-    client_max_body_size 1000m;
+    client_max_body_size 50000m;
     ignore_invalid_headers off;
     proxy_buffering off;
 

hosting/proxy/nginx.prod.conf

@@ -249,4 +249,30 @@ http {
     gzip_comp_level 6;
     gzip_types text/plain text/css text/xml application/json application/javascript application/rss+xml application/atom+xml image/svg+xml;
   }
+
+  # From https://docs.datadoghq.com/integrations/nginx/?tab=kubernetes
+  server {
+    listen 81;
+    server_name localhost;
+
+    access_log off;
+    allow 127.0.0.1;
+    deny all;
+
+    location /nginx_status {
+      # Choose your status module
+
+      # freely available with open source NGINX
+      stub_status;
+
+      # for open source NGINX < version 1.7.5
+      # stub_status on;
+
+      # available only with NGINX Plus
+      # status;
+
+      # ensures the version information can be retrieved
+      server_tokens on;
+    }
+  }
 }

hosting/single/Dockerfile

@@ -1,44 +1,59 @@
 FROM node:18-slim as build
 
 # install node-gyp dependencies
-RUN apt-get update && apt-get upgrade -y && apt-get install -y --no-install-recommends apt-utils cron g++ make python3
+RUN apt-get update && apt-get install -y --no-install-recommends g++ make python3 jq
 
-# add pin script
-WORKDIR /
-ADD scripts/cleanup.sh ./
-RUN chmod +x /cleanup.sh
 
-# build server
+# copy and install dependencies
 WORKDIR /app
-ADD packages/server .
+COPY package.json .
 COPY yarn.lock .
-RUN yarn install --production=true --network-timeout 1000000
+COPY lerna.json .
-RUN /cleanup.sh
+COPY .yarnrc .
 
-# build worker
+COPY packages/server/package.json packages/server/package.json
-WORKDIR /worker
+COPY packages/worker/package.json packages/worker/package.json
-ADD packages/worker .
+# string-templates does not get bundled during the esbuild process, so we want to use the local version
-COPY yarn.lock .
+COPY packages/string-templates/package.json packages/string-templates/package.json
-RUN yarn install --production=true --network-timeout 1000000
-RUN /cleanup.sh
 
-FROM budibase/couchdb
+
+COPY scripts/removeWorkspaceDependencies.sh scripts/removeWorkspaceDependencies.sh
+RUN chmod +x ./scripts/removeWorkspaceDependencies.sh
+RUN ./scripts/removeWorkspaceDependencies.sh packages/server/package.json
+RUN ./scripts/removeWorkspaceDependencies.sh packages/worker/package.json
+
+
+# We will never want to sync pro, but the script is still required
+RUN echo '' > scripts/syncProPackage.js
+RUN jq 'del(.scripts.postinstall)' package.json > temp.json && mv temp.json package.json
+RUN ./scripts/removeWorkspaceDependencies.sh package.json
+RUN --mount=type=cache,target=/root/.yarn YARN_CACHE_FOLDER=/root/.yarn yarn install --production
+
+# copy the actual code
+COPY packages/server/dist packages/server/dist
+COPY packages/server/pm2.config.js packages/server/pm2.config.js
+COPY packages/server/client packages/server/client
+COPY packages/server/builder packages/server/builder
+COPY packages/worker/dist packages/worker/dist
+COPY packages/worker/pm2.config.js packages/worker/pm2.config.js
+COPY packages/string-templates packages/string-templates
+
+
+FROM budibase/couchdb as runner
 ARG TARGETARCH
 ENV TARGETARCH $TARGETARCH
+ENV NODE_MAJOR 18
 #TARGETBUILD can be set to single (for single docker image) or aas (for azure app service)
 # e.g. docker build --build-arg TARGETBUILD=aas ....
 ARG TARGETBUILD=single
 ENV TARGETBUILD $TARGETBUILD
 
-COPY --from=build /app /app
-COPY --from=build /worker /worker
-
 # install base dependencies
 RUN apt-get update && \
-  apt-get install -y --no-install-recommends software-properties-common nginx uuid-runtime redis-server
+  apt-get install -y --no-install-recommends software-properties-common nginx uuid-runtime redis-server libaio1
 
 # Install postgres client for pg_dump utils
-RUN apt install software-properties-common apt-transport-https gpg -y \
+RUN apt install -y software-properties-common apt-transport-https ca-certificates gnupg \
     && curl -fsSl https://www.postgresql.org/media/keys/ACCC4CF8.asc | gpg --dearmor | tee /usr/share/keyrings/postgresql.gpg > /dev/null \
     && echo deb [arch=amd64,arm64,ppc64el signed-by=/usr/share/keyrings/postgresql.gpg] http://apt.postgresql.org/pub/repos/apt/ $(lsb_release -cs)-pgdg main | tee /etc/apt/sources.list.d/postgresql.list \
     && apt update -y \
@@ -47,14 +62,12 @@ RUN apt install software-properties-common apt-transport-https gpg -y \
 
 # install other dependencies, nodejs, oracle requirements, jdk8, redis, nginx
 WORKDIR /nodejs
-RUN curl -sL https://deb.nodesource.com/setup_16.x -o /tmp/nodesource_setup.sh && \
+COPY scripts/install-node.sh ./install.sh
-  bash /tmp/nodesource_setup.sh && \
+RUN chmod +x install.sh && ./install.sh
-  apt-get install -y --no-install-recommends libaio1 nodejs && \
-  npm install --global yarn pm2
 
 # setup nginx
-ADD hosting/single/nginx/nginx.conf /etc/nginx
+COPY hosting/single/nginx/nginx.conf /etc/nginx
-ADD hosting/single/nginx/nginx-default-site.conf /etc/nginx/sites-enabled/default
+COPY hosting/single/nginx/nginx-default-site.conf /etc/nginx/sites-enabled/default
 RUN mkdir -p /var/log/nginx && \
   touch /var/log/nginx/error.log && \
   touch /var/run/nginx.pid && \
@@ -62,29 +75,39 @@ RUN mkdir -p /var/log/nginx && \
 
 WORKDIR /
 RUN mkdir -p scripts/integrations/oracle
-ADD packages/server/scripts/integrations/oracle scripts/integrations/oracle
+COPY packages/server/scripts/integrations/oracle scripts/integrations/oracle
 RUN /bin/bash -e ./scripts/integrations/oracle/instantclient/linux/install.sh
 
 # setup minio
 WORKDIR /minio
-ADD scripts/install-minio.sh ./install.sh
+COPY scripts/install-minio.sh ./install.sh
 RUN chmod +x install.sh && ./install.sh
 
 # setup runner file
 WORKDIR /
-ADD hosting/single/runner.sh .
+COPY hosting/single/runner.sh .
 RUN chmod +x ./runner.sh
-ADD hosting/single/healthcheck.sh .
+COPY hosting/single/healthcheck.sh .
 RUN chmod +x ./healthcheck.sh
 
 # Script below sets the path for storing data based on $DATA_DIR
 # For Azure App Service install SSH & point data locations to /home
-ADD hosting/single/ssh/sshd_config /etc/
+COPY hosting/single/ssh/sshd_config /etc/
-ADD hosting/single/ssh/ssh_setup.sh /tmp
+COPY hosting/single/ssh/ssh_setup.sh /tmp
-RUN /build-target-paths.sh
+
+#  setup letsencrypt certificate
+RUN apt-get install -y certbot python3-certbot-nginx
+COPY hosting/letsencrypt /app/letsencrypt
+RUN chmod +x /app/letsencrypt/certificate-request.sh /app/letsencrypt/certificate-renew.sh
+
+COPY --from=build /app/node_modules /node_modules
+COPY --from=build /app/package.json /package.json
+COPY --from=build /app/packages/server /app
+COPY --from=build /app/packages/worker /worker
+COPY --from=build /app/packages/string-templates /string-templates
+
+RUN cd /string-templates && yarn link && cd ../app && yarn link @budibase/string-templates && cd ../worker && yarn link @budibase/string-templates
 
-# cleanup cache
-RUN yarn cache clean -f
 
 EXPOSE 80
 EXPOSE 443
@@ -92,20 +115,10 @@ EXPOSE 443
 EXPOSE 2222
 VOLUME /data
 
-#  setup letsencrypt certificate
+ARG BUDIBASE_VERSION
-RUN apt-get install -y certbot python3-certbot-nginx
+# Ensuring the version argument is sent
-ADD hosting/letsencrypt /app/letsencrypt
+RUN test -n "$BUDIBASE_VERSION"
-RUN chmod +x /app/letsencrypt/certificate-request.sh /app/letsencrypt/certificate-renew.sh
+ENV BUDIBASE_VERSION=$BUDIBASE_VERSION
-# Remove cached files
-RUN rm -rf \
-  /root/.cache \
-  /root/.npm \
-  /root/.pip \
-  /usr/local/share/doc \
-  /usr/share/doc \
-  /usr/share/man \
-  /var/lib/apt/lists/* \
-  /tmp/*
 
 HEALTHCHECK --interval=15s --timeout=15s --start-period=45s CMD "/healthcheck.sh"
 

hosting/single/Dockerfile.v2

@@ -1,131 +0,0 @@
-FROM node:18-slim as build
-
-# install node-gyp dependencies
-RUN apt-get update && apt-get install -y --no-install-recommends g++ make python3 jq
-
-
-# copy and install dependencies
-WORKDIR /app
-COPY package.json .
-COPY yarn.lock .
-COPY lerna.json .
-COPY .yarnrc .
-
-COPY packages/server/package.json packages/server/package.json
-COPY packages/worker/package.json packages/worker/package.json
-# string-templates does not get bundled during the esbuild process, so we want to use the local version
-COPY packages/string-templates/package.json packages/string-templates/package.json
-
-
-COPY scripts/removeWorkspaceDependencies.sh scripts/removeWorkspaceDependencies.sh
-RUN chmod +x ./scripts/removeWorkspaceDependencies.sh
-RUN ./scripts/removeWorkspaceDependencies.sh packages/server/package.json
-RUN ./scripts/removeWorkspaceDependencies.sh packages/worker/package.json
-
-
-# We will never want to sync pro, but the script is still required
-RUN echo '' > scripts/syncProPackage.js
-RUN jq 'del(.scripts.postinstall)' package.json > temp.json && mv temp.json package.json
-RUN ./scripts/removeWorkspaceDependencies.sh package.json
-RUN --mount=type=cache,target=/root/.yarn YARN_CACHE_FOLDER=/root/.yarn yarn install --production
-
-# copy the actual code
-COPY packages/server/dist packages/server/dist
-COPY packages/server/pm2.config.js packages/server/pm2.config.js
-COPY packages/server/client packages/server/client
-COPY packages/server/builder packages/server/builder
-COPY packages/worker/dist packages/worker/dist
-COPY packages/worker/pm2.config.js packages/worker/pm2.config.js
-COPY packages/string-templates packages/string-templates
-
-
-FROM budibase/couchdb as runner
-ARG TARGETARCH
-ENV TARGETARCH $TARGETARCH
-ENV NODE_MAJOR 18
-#TARGETBUILD can be set to single (for single docker image) or aas (for azure app service)
-# e.g. docker build --build-arg TARGETBUILD=aas ....
-ARG TARGETBUILD=single
-ENV TARGETBUILD $TARGETBUILD
-
-# install base dependencies
-RUN apt-get update && \
-  apt-get install -y --no-install-recommends software-properties-common nginx uuid-runtime redis-server libaio1
-
-# Install postgres client for pg_dump utils
-RUN apt install -y software-properties-common apt-transport-https ca-certificates gnupg \
-    && curl -fsSl https://www.postgresql.org/media/keys/ACCC4CF8.asc | gpg --dearmor | tee /usr/share/keyrings/postgresql.gpg > /dev/null \
-    && echo deb [arch=amd64,arm64,ppc64el signed-by=/usr/share/keyrings/postgresql.gpg] http://apt.postgresql.org/pub/repos/apt/ $(lsb_release -cs)-pgdg main | tee /etc/apt/sources.list.d/postgresql.list \
-    && apt update -y \
-    && apt install postgresql-client-15 -y \
-    && apt remove software-properties-common apt-transport-https gpg -y
-
-# install other dependencies, nodejs, oracle requirements, jdk8, redis, nginx
-WORKDIR /nodejs
-COPY scripts/install-node.sh ./install.sh
-RUN chmod +x install.sh && ./install.sh
-
-# setup nginx
-COPY hosting/single/nginx/nginx.conf /etc/nginx
-COPY hosting/single/nginx/nginx-default-site.conf /etc/nginx/sites-enabled/default
-RUN mkdir -p /var/log/nginx && \
-  touch /var/log/nginx/error.log && \
-  touch /var/run/nginx.pid && \
-  usermod -a -G tty www-data
-
-WORKDIR /
-RUN mkdir -p scripts/integrations/oracle
-COPY packages/server/scripts/integrations/oracle scripts/integrations/oracle
-RUN /bin/bash -e ./scripts/integrations/oracle/instantclient/linux/install.sh
-
-# setup minio
-WORKDIR /minio
-COPY scripts/install-minio.sh ./install.sh
-RUN chmod +x install.sh && ./install.sh
-
-# setup runner file
-WORKDIR /
-COPY hosting/single/runner.sh .
-RUN chmod +x ./runner.sh
-COPY hosting/single/healthcheck.sh .
-RUN chmod +x ./healthcheck.sh
-
-# Script below sets the path for storing data based on $DATA_DIR
-# For Azure App Service install SSH & point data locations to /home
-COPY hosting/single/ssh/sshd_config /etc/
-COPY hosting/single/ssh/ssh_setup.sh /tmp
-RUN /build-target-paths.sh
-
-
-#  setup letsencrypt certificate
-RUN apt-get install -y certbot python3-certbot-nginx
-COPY hosting/letsencrypt /app/letsencrypt
-RUN chmod +x /app/letsencrypt/certificate-request.sh /app/letsencrypt/certificate-renew.sh
-
-COPY --from=build /app/node_modules /node_modules
-COPY --from=build /app/package.json /package.json
-COPY --from=build /app/packages/server /app
-COPY --from=build /app/packages/worker /worker
-COPY --from=build /app/packages/string-templates /string-templates
-
-RUN cd /string-templates && yarn link && cd ../app && yarn link @budibase/string-templates && cd ../worker && yarn link @budibase/string-templates
-
-
-EXPOSE 80
-EXPOSE 443
-# Expose port 2222 for SSH on Azure App Service build
-EXPOSE 2222
-VOLUME /data
-
-ARG BUDIBASE_VERSION
-# Ensuring the version argument is sent
-RUN test -n "$BUDIBASE_VERSION"
-ENV BUDIBASE_VERSION=$BUDIBASE_VERSION
-
-HEALTHCHECK --interval=15s --timeout=15s --start-period=45s CMD "/healthcheck.sh"
-
-# must set this just before running
-ENV NODE_ENV=production
-WORKDIR /
-
-CMD ["./runner.sh"]

hosting/single/runner.sh

@@ -22,11 +22,11 @@ declare -a DOCKER_VARS=("APP_PORT" "APPS_URL" "ARCHITECTURE" "BUDIBASE_ENVIRONME
 
 # Azure App Service customisations
 if [[ "${TARGETBUILD}" = "aas" ]]; then
-    DATA_DIR="${DATA_DIR:-/home}"
+    export DATA_DIR="${DATA_DIR:-/home}"
     WEBSITES_ENABLE_APP_SERVICE_STORAGE=true
     /etc/init.d/ssh start
 else
-    DATA_DIR=${DATA_DIR:-/data}
+    export DATA_DIR=${DATA_DIR:-/data}
 fi
 mkdir -p ${DATA_DIR}
 # Mount NFS or GCP Filestore if env vars exist for it

lerna.json

@@ -1,5 +1,5 @@
 {
-  "version": "2.13.14",
+  "version": "2.13.35",
   "npmClient": "yarn",
   "packages": [
     "packages/*"

packages/backend-core/package.json

@@ -72,7 +72,7 @@
     "@types/tar-fs": "2.0.1",
     "@types/uuid": "8.3.4",
     "chance": "1.1.8",
-    "ioredis-mock": "8.7.0",
+    "ioredis-mock": "8.9.0",
     "jest": "29.6.2",
     "jest-environment-node": "29.6.2",
     "jest-serial-runner": "1.2.1",

packages/backend-core/src/auth/auth.ts

@@ -19,6 +19,7 @@ import {
   GoogleInnerConfig,
   OIDCInnerConfig,
   PlatformLogoutOpts,
+  SessionCookie,
   SSOProviderType,
 } from "@budibase/types"
 import * as events from "../events"
@@ -44,7 +45,6 @@ export const buildAuthMiddleware = authenticated
 export const buildTenancyMiddleware = tenancy
 export const buildCsrfMiddleware = csrf
 export const passport = _passport
-export const jwt = require("jsonwebtoken")
 
 // Strategies
 _passport.use(new LocalStrategy(local.options, local.authenticate))
@@ -191,10 +191,10 @@ export async function platformLogout(opts: PlatformLogoutOpts) {
 
   if (!ctx) throw new Error("Koa context must be supplied to logout.")
 
-  const currentSession = getCookie(ctx, Cookie.Auth)
+  const currentSession = getCookie<SessionCookie>(ctx, Cookie.Auth)
   let sessions = await getSessionsForUser(userId)
 
-  if (keepActiveSession) {
+  if (currentSession && keepActiveSession) {
     sessions = sessions.filter(
       session => session.sessionId !== currentSession.sessionId
     )

packages/backend-core/src/context/mainContext.ts

@@ -99,6 +99,8 @@ function updateContext(updates: ContextMap): ContextMap {
 }
 
 async function newContext<T>(updates: ContextMap, task: () => T) {
+  guardMigration()
+
   // see if there already is a context setup
   let context: ContextMap = updateContext(updates)
   return Context.run(context, task)
@@ -145,23 +147,27 @@ export async function doInTenant<T>(
 }
 
 export async function doInAppContext<T>(
-  appId: string | null,
+  appId: string,
   task: () => T
 ): Promise<T> {
-  if (!appId && !env.isTest()) {
+  return _doInAppContext(appId, task)
+}
+
+async function _doInAppContext<T>(
+  appId: string,
+  task: () => T,
+  extraContextSettings?: ContextMap
+): Promise<T> {
+  if (!appId) {
     throw new Error("appId is required")
   }
 
-  let updates: ContextMap
+  const tenantId = getTenantIDFromAppID(appId)
-  if (!appId) {
+  const updates: ContextMap = { appId, ...extraContextSettings }
-    updates = { appId: "" }
+  if (tenantId) {
-  } else {
+    updates.tenantId = tenantId
-    const tenantId = getTenantIDFromAppID(appId)
-    updates = { appId }
-    if (tenantId) {
-      updates.tenantId = tenantId
-    }
   }
+
   return newContext(updates, task)
 }
 
@@ -182,6 +188,24 @@ export async function doInIdentityContext<T>(
   return newContext(context, task)
 }
 
+function guardMigration() {
+  const context = Context.get()
+  if (context?.isMigrating) {
+    throw new Error(
+      "The context cannot be changed, a migration is currently running"
+    )
+  }
+}
+
+export async function doInAppMigrationContext<T>(
+  appId: string,
+  task: () => T
+): Promise<T> {
+  return _doInAppContext(appId, task, {
+    isMigrating: true,
+  })
+}
+
 export function getIdentity(): IdentityContext | undefined {
   try {
     const context = Context.get()

packages/backend-core/src/context/tests/index.spec.ts

@@ -1,6 +1,11 @@
 import { testEnv } from "../../../tests/extra"
 import * as context from "../"
 import { DEFAULT_TENANT_ID } from "../../constants"
+import { structures } from "../../../tests"
+import { db } from "../.."
+import Context from "../Context"
+import { ContextMap } from "../types"
+import { IdentityType } from "@budibase/types"
 
 describe("context", () => {
   describe("doInTenant", () => {
@@ -144,4 +149,107 @@ describe("context", () => {
       expect(isScim).toBe(false)
     })
   })
+
+  describe("doInAppMigrationContext", () => {
+    it("the context is set correctly", async () => {
+      const appId = db.generateAppID()
+
+      await context.doInAppMigrationContext(appId, () => {
+        const context = Context.get()
+
+        const expected: ContextMap = {
+          appId,
+          isMigrating: true,
+        }
+        expect(context).toEqual(expected)
+      })
+    })
+
+    it("the context is set correctly when running in a tenant id", async () => {
+      const tenantId = structures.tenant.id()
+      const appId = db.generateAppID(tenantId)
+
+      await context.doInAppMigrationContext(appId, () => {
+        const context = Context.get()
+
+        const expected: ContextMap = {
+          appId,
+          isMigrating: true,
+          tenantId,
+        }
+        expect(context).toEqual(expected)
+      })
+    })
+
+    it("the context is not modified outside the delegate", async () => {
+      const appId = db.generateAppID()
+
+      expect(Context.get()).toBeUndefined()
+
+      await context.doInAppMigrationContext(appId, () => {
+        const context = Context.get()
+
+        const expected: ContextMap = {
+          appId,
+          isMigrating: true,
+        }
+        expect(context).toEqual(expected)
+      })
+
+      expect(Context.get()).toBeUndefined()
+    })
+
+    it.each([
+      [
+        "doInAppMigrationContext",
+        () => context.doInAppMigrationContext(db.generateAppID(), () => {}),
+      ],
+      [
+        "doInAppContext",
+        () => context.doInAppContext(db.generateAppID(), () => {}),
+      ],
+      [
+        "doInAutomationContext",
+        () =>
+          context.doInAutomationContext({
+            appId: db.generateAppID(),
+            automationId: structures.generator.guid(),
+            task: () => {},
+          }),
+      ],
+      ["doInContext", () => context.doInContext(db.generateAppID(), () => {})],
+      [
+        "doInEnvironmentContext",
+        () => context.doInEnvironmentContext({}, () => {}),
+      ],
+      [
+        "doInIdentityContext",
+        () =>
+          context.doInIdentityContext(
+            {
+              account: undefined,
+              type: IdentityType.USER,
+              _id: structures.users.user()._id!,
+            },
+            () => {}
+          ),
+      ],
+      ["doInScimContext", () => context.doInScimContext(() => {})],
+      [
+        "doInTenant",
+        () => context.doInTenant(structures.tenant.id(), () => {}),
+      ],
+    ])(
+      "a nested context.%s function cannot run",
+      async (_, otherContextCall: () => Promise<void>) => {
+        await expect(
+          context.doInAppMigrationContext(db.generateAppID(), async () => {
+            await otherContextCall()
+          })
+        ).rejects.toThrowError(
+          "The context cannot be changed, a migration is currently running"
+        )
+      }
+    )
+  })
 })

packages/backend-core/src/context/types.ts

@@ -8,4 +8,5 @@ export type ContextMap = {
   environmentVariables?: Record<string, string>
   isScim?: boolean
   automationId?: string
+  isMigrating?: boolean
 }

packages/backend-core/src/middleware/authenticated.ts

@@ -13,7 +13,7 @@ import { getGlobalDB, doInTenant } from "../context"
 import { decrypt } from "../security/encryption"
 import * as identity from "../context/identity"
 import env from "../environment"
-import { Ctx, EndpointMatcher } from "@budibase/types"
+import { Ctx, EndpointMatcher, SessionCookie } from "@budibase/types"
 import { InvalidAPIKeyError, ErrorCode } from "../errors"
 
 const ONE_MINUTE = env.SESSION_UPDATE_PERIOD
@@ -98,7 +98,9 @@ export default function (
       // check the actual user is authenticated first, try header or cookie
       let headerToken = ctx.request.headers[Header.TOKEN]
 
-      const authCookie = getCookie(ctx, Cookie.Auth) || openJwt(headerToken)
+      const authCookie =
+        getCookie<SessionCookie>(ctx, Cookie.Auth) ||
+        openJwt<SessionCookie>(headerToken)
       let apiKey = ctx.request.headers[Header.API_KEY]
 
       if (!apiKey && ctx.request.headers[Header.AUTHORIZATION]) {

packages/backend-core/src/middleware/passport/datasource/google.ts

@@ -3,7 +3,7 @@ import { Cookie } from "../../../constants"
 import * as configs from "../../../configs"
 import * as cache from "../../../cache"
 import * as utils from "../../../utils"
-import { UserCtx, SSOProfile } from "@budibase/types"
+import { UserCtx, SSOProfile, DatasourceAuthCookie } from "@budibase/types"
 import { ssoSaveUserNoOp } from "../sso/sso"
 
 const GoogleStrategy = require("passport-google-oauth").OAuth2Strategy
@@ -58,7 +58,14 @@ export async function postAuth(
   const platformUrl = await configs.getPlatformUrl({ tenantAware: false })
 
   let callbackUrl = `${platformUrl}/api/global/auth/datasource/google/callback`
-  const authStateCookie = utils.getCookie(ctx, Cookie.DatasourceAuth)
+  const authStateCookie = utils.getCookie<{ appId: string }>(
+    ctx,
+    Cookie.DatasourceAuth
+  )
+
+  if (!authStateCookie) {
+    throw new Error("Unable to fetch datasource auth cookie")
+  }
 
   return passport.authenticate(
     new GoogleStrategy(

packages/backend-core/src/objectStore/objectStore.ts

@@ -260,12 +260,12 @@ export async function listAllObjects(bucketName: string, path: string) {
 }
 
 /**
- * Generate a presigned url with a default TTL of 1 hour
+ * Generate a presigned url with a default TTL of 36 hours
  */
 export function getPresignedUrl(
   bucketName: string,
   key: string,
-  durationSeconds: number = 3600
+  durationSeconds: number = 129600
 ) {
   const objectStore = ObjectStore(bucketName, { presigning: true })
   const params = {
@@ -305,20 +305,33 @@ export async function retrieveDirectory(bucketName: string, path: string) {
   let writePath = join(budibaseTempDir(), v4())
   fs.mkdirSync(writePath)
   const objects = await listAllObjects(bucketName, path)
-  let fullObjects = await Promise.all(
+  let streams = await Promise.all(
-    objects.map(obj => retrieve(bucketName, obj.Key!))
+    objects.map(obj => getReadStream(bucketName, obj.Key!))
   )
   let count = 0
+  const writePromises: Promise<Error>[] = []
   for (let obj of objects) {
     const filename = obj.Key!
-    const data = fullObjects[count++]
+    const stream = streams[count++]
     const possiblePath = filename.split("/")
-    if (possiblePath.length > 1) {
+    const dirs = possiblePath.slice(0, possiblePath.length - 1)
-      const dirs = possiblePath.slice(0, possiblePath.length - 1)
+    const possibleDir = join(writePath, ...dirs)
-      fs.mkdirSync(join(writePath, ...dirs), { recursive: true })
+    if (possiblePath.length > 1 && !fs.existsSync(possibleDir)) {
+      fs.mkdirSync(possibleDir, { recursive: true })
     }
-    fs.writeFileSync(join(writePath, ...possiblePath), data)
+    const writeStream = fs.createWriteStream(join(writePath, ...possiblePath), {
+      mode: 0o644,
+    })
+    stream.pipe(writeStream)
+    writePromises.push(
+      new Promise((resolve, reject) => {
+        stream.on("finish", resolve)
+        stream.on("error", reject)
+        writeStream.on("error", reject)
+      })
+    )
   }
+  await Promise.all(writePromises)
   return writePath
 }
 

packages/backend-core/src/redis/redlockImpl.ts

@@ -2,8 +2,9 @@ import Redlock from "redlock"
 import { getLockClient } from "./init"
 import { LockOptions, LockType } from "@budibase/types"
 import * as context from "../context"
-import env from "../environment"
 import { logWarn } from "../logging"
+import { utils } from "@budibase/shared-core"
+import { Duration } from "../utils"
 
 async function getClient(
   type: LockType,
@@ -12,9 +13,7 @@ async function getClient(
   if (type === LockType.CUSTOM) {
     return newRedlock(opts)
   }
-  if (env.isTest() && type !== LockType.TRY_ONCE) {
+
-    return newRedlock(OPTIONS.TEST)
-  }
   switch (type) {
     case LockType.TRY_ONCE: {
       return newRedlock(OPTIONS.TRY_ONCE)
@@ -28,13 +27,16 @@ async function getClient(
     case LockType.DELAY_500: {
       return newRedlock(OPTIONS.DELAY_500)
     }
+    case LockType.AUTO_EXTEND: {
+      return newRedlock(OPTIONS.AUTO_EXTEND)
+    }
     default: {
-      throw new Error(`Could not get redlock client: ${type}`)
+      throw utils.unreachable(type)
     }
   }
 }
 
-const OPTIONS = {
+const OPTIONS: Record<keyof typeof LockType, Redlock.Options> = {
   TRY_ONCE: {
     // immediately throws an error if the lock is already held
     retryCount: 0,
@@ -42,11 +44,6 @@ const OPTIONS = {
   TRY_TWICE: {
     retryCount: 1,
   },
-  TEST: {
-    // higher retry count in unit tests
-    // due to high contention.
-    retryCount: 100,
-  },
   DEFAULT: {
     // the expected clock drift; for more details
     // see http://redis.io/topics/distlock
@@ -67,10 +64,14 @@ const OPTIONS = {
   DELAY_500: {
     retryDelay: 500,
   },
+  CUSTOM: {},
+  AUTO_EXTEND: {
+    retryCount: -1,
+  },
 }
 
 export async function newRedlock(opts: Redlock.Options = {}) {
-  let options = { ...OPTIONS.DEFAULT, ...opts }
+  const options = { ...OPTIONS.DEFAULT, ...opts }
   const redisWrapper = await getLockClient()
   const client = redisWrapper.getClient()
   return new Redlock([client], options)
@@ -100,17 +101,36 @@ function getLockName(opts: LockOptions) {
   return name
 }
 
+export const AUTO_EXTEND_POLLING_MS = Duration.fromSeconds(10).toMs()
+
 export async function doWithLock<T>(
   opts: LockOptions,
   task: () => Promise<T>
 ): Promise<RedlockExecution<T>> {
   const redlock = await getClient(opts.type, opts.customOptions)
-  let lock
+  let lock: Redlock.Lock | undefined
+  let timeout
   try {
     const name = getLockName(opts)
 
+    const ttl =
+      opts.type === LockType.AUTO_EXTEND ? AUTO_EXTEND_POLLING_MS : opts.ttl
+
     // create the lock
-    lock = await redlock.lock(name, opts.ttl)
+    lock = await redlock.lock(name, ttl)
+
+    if (opts.type === LockType.AUTO_EXTEND) {
+      // We keep extending the lock while the task is running
+      const extendInIntervals = (): void => {
+        timeout = setTimeout(async () => {
+          lock = await lock!.extend(ttl, () => opts.onExtend && opts.onExtend())
+
+          extendInIntervals()
+        }, ttl / 2)
+      }
+
+      extendInIntervals()
+    }
 
     // perform locked task
     // need to await to ensure completion before unlocking
@@ -131,8 +151,7 @@ export async function doWithLock<T>(
       throw e
     }
   } finally {
-    if (lock) {
+    clearTimeout(timeout)
-      await lock.unlock()
+    await lock?.unlock()
-    }
   }
 }

packages/backend-core/src/redis/tests/redlockImpl.spec.ts

@@ -0,0 +1,105 @@
+import { LockName, LockType, LockOptions } from "@budibase/types"
+import { AUTO_EXTEND_POLLING_MS, doWithLock } from "../redlockImpl"
+import { DBTestConfiguration, generator } from "../../../tests"
+
+describe("redlockImpl", () => {
+  beforeEach(() => {
+    jest.useFakeTimers()
+  })
+
+  describe("doWithLock", () => {
+    const config = new DBTestConfiguration()
+    const lockTtl = AUTO_EXTEND_POLLING_MS
+
+    function runLockWithExecutionTime({
+      opts,
+      task,
+      executionTimeMs,
+    }: {
+      opts: LockOptions
+      task: () => Promise<string>
+      executionTimeMs: number
+    }) {
+      return config.doInTenant(() =>
+        doWithLock(opts, async () => {
+          // Run in multiple intervals until hitting the expected time
+          const interval = lockTtl / 10
+          for (let i = executionTimeMs; i > 0; i -= interval) {
+            await jest.advanceTimersByTimeAsync(interval)
+          }
+          return task()
+        })
+      )
+    }
+
+    it.each(Object.values(LockType))(
+      "should return the task value and release the lock",
+      async (lockType: LockType) => {
+        const expectedResult = generator.guid()
+        const mockTask = jest.fn().mockResolvedValue(expectedResult)
+
+        const opts: LockOptions = {
+          name: LockName.PERSIST_WRITETHROUGH,
+          type: lockType,
+          ttl: lockTtl,
+        }
+
+        const result = await runLockWithExecutionTime({
+          opts,
+          task: mockTask,
+          executionTimeMs: 0,
+        })
+
+        expect(result.executed).toBe(true)
+        expect(result.executed && result.result).toBe(expectedResult)
+        expect(mockTask).toHaveBeenCalledTimes(1)
+      }
+    )
+
+    it("should extend when type is autoextend", async () => {
+      const expectedResult = generator.guid()
+      const mockTask = jest.fn().mockResolvedValue(expectedResult)
+      const mockOnExtend = jest.fn()
+
+      const opts: LockOptions = {
+        name: LockName.PERSIST_WRITETHROUGH,
+        type: LockType.AUTO_EXTEND,
+        onExtend: mockOnExtend,
+      }
+
+      const result = await runLockWithExecutionTime({
+        opts,
+        task: mockTask,
+        executionTimeMs: lockTtl * 2.5,
+      })
+
+      expect(result.executed).toBe(true)
+      expect(result.executed && result.result).toBe(expectedResult)
+      expect(mockTask).toHaveBeenCalledTimes(1)
+      expect(mockOnExtend).toHaveBeenCalledTimes(5)
+    })
+
+    it.each(Object.values(LockType).filter(t => t !== LockType.AUTO_EXTEND))(
+      "should timeout when type is %s",
+      async (lockType: LockType) => {
+        const mockTask = jest.fn().mockResolvedValue("mockResult")
+
+        const opts: LockOptions = {
+          name: LockName.PERSIST_WRITETHROUGH,
+          type: lockType,
+          ttl: lockTtl,
+        }
+
+        await expect(
+          runLockWithExecutionTime({
+            opts,
+            task: mockTask,
+            executionTimeMs: lockTtl * 2,
+          })
+        ).rejects.toThrowError(
+          `Unable to fully release the lock on resource \"lock:${config.tenantId}_persist_writethrough\".`
+        )
+      }
+    )
+  })
+})

packages/backend-core/src/security/encryption.ts

@@ -73,6 +73,9 @@ export async function encryptFile(
   const outputFileName = `${filename}.enc`
 
   const filePath = join(dir, filename)
+  if (fs.lstatSync(filePath).isDirectory()) {
+    throw new Error("Unable to encrypt directory")
+  }
   const inputFile = fs.createReadStream(filePath)
   const outputFile = fs.createWriteStream(join(dir, outputFileName))
 
@@ -110,6 +113,9 @@ export async function decryptFile(
   outputPath: string,
   secret: string
 ) {
+  if (fs.lstatSync(inputPath).isDirectory()) {
+    throw new Error("Unable to encrypt directory")
+  }
   const { salt, iv } = await getSaltAndIV(inputPath)
   const inputFile = fs.createReadStream(inputPath, {
     start: SALT_LENGTH + IV_LENGTH,

packages/backend-core/src/security/permissions.ts

@@ -160,4 +160,5 @@ export function isPermissionLevelHigherThanRead(level: PermissionLevel) {
 
 // utility as a lot of things need simply the builder permission
 export const BUILDER = PermissionType.BUILDER
+export const CREATOR = PermissionType.CREATOR
 export const GLOBAL_BUILDER = PermissionType.GLOBAL_BUILDER

packages/backend-core/src/tenancy/tenancy.ts

@@ -93,11 +93,19 @@ export const getTenantIDFromCtx = (
   // subdomain
   if (isAllowed(TenantResolutionStrategy.SUBDOMAIN)) {
     // e.g. budibase.app or local.com:10000
-    const platformHost = new URL(getPlatformURL()).host.split(":")[0]
+    let platformHost
+    try {
+      platformHost = new URL(getPlatformURL()).host.split(":")[0]
+    } catch (err: any) {
+      // if invalid URL, just don't try to process subdomain
+      if (err.code !== "ERR_INVALID_URL") {
+        throw err
+      }
+    }
     // e.g. tenant.budibase.app or tenant.local.com
     const requestHost = ctx.host
     // parse the tenant id from the difference
-    if (requestHost.includes(platformHost)) {
+    if (platformHost && requestHost.includes(platformHost)) {
       const tenantId = requestHost.substring(
         0,
         requestHost.indexOf(`.${platformHost}`)

packages/backend-core/src/users/db.ts

@@ -146,12 +146,12 @@ export class UserDB {
 
   static async allUsers() {
     const db = getGlobalDB()
-    const response = await db.allDocs(
+    const response = await db.allDocs<User>(
       dbUtils.getGlobalUserParams(null, {
         include_docs: true,
       })
     )
-    return response.rows.map((row: any) => row.doc)
+    return response.rows.map(row => row.doc!)
   }
 
   static async countUsersByApp(appId: string) {
@@ -209,13 +209,6 @@ export class UserDB {
       throw new Error("_id or email is required")
     }
 
-    if (
-      user.builder?.apps?.length &&
-      !(await UserDB.features.isAppBuildersEnabled())
-    ) {
-      throw new Error("Unable to update app builders, please check license")
-    }
-
     let dbUser: User | undefined
     if (_id) {
       // try to get existing user from db

packages/backend-core/src/users/users.ts

@@ -25,6 +25,7 @@ import {
 import { getGlobalDB } from "../context"
 import * as context from "../context"
 import { isCreator } from "./utils"
+import { UserDB } from "./db"
 
 type GetOpts = { cleanup?: boolean }
 
@@ -336,3 +337,20 @@ export function cleanseUserObject(user: User | ContextUser, base?: User) {
   }
   return user
 }
+
+export async function addAppBuilder(user: User, appId: string) {
+  const prodAppId = getProdAppID(appId)
+  user.builder ??= {}
+  user.builder.creator = true
+  user.builder.apps ??= []
+  user.builder.apps.push(prodAppId)
+  await UserDB.save(user, { hashPassword: false })
+}
+
+export async function removeAppBuilder(user: User, appId: string) {
+  const prodAppId = getProdAppID(appId)
+  if (user.builder && user.builder.apps?.includes(prodAppId)) {
+    user.builder.apps = user.builder.apps.filter(id => id !== prodAppId)
+  }
+  await UserDB.save(user, { hashPassword: false })
+}

packages/backend-core/src/utils/utils.ts

@@ -11,8 +11,7 @@ import {
   TenantResolutionStrategy,
 } from "@budibase/types"
 import type { SetOption } from "cookies"
-
+import jwt, { Secret } from "jsonwebtoken"
-const jwt = require("jsonwebtoken")
 
 const APP_PREFIX = DocumentType.APP + SEPARATOR
 const PROD_APP_PREFIX = "/app/"
@@ -60,10 +59,7 @@ export function isServingApp(ctx: Ctx) {
     return true
   }
   // prod app
-  if (ctx.path.startsWith(PROD_APP_PREFIX)) {
+  return ctx.path.startsWith(PROD_APP_PREFIX)
-    return true
-  }
-  return false
 }
 
 export function isServingBuilder(ctx: Ctx): boolean {
@@ -138,16 +134,16 @@ function parseAppIdFromUrl(url?: string) {
  * opens the contents of the specified encrypted JWT.
  * @return the contents of the token.
  */
-export function openJwt(token: string) {
+export function openJwt<T>(token?: string): T | undefined {
   if (!token) {
-    return token
+    return undefined
   }
   try {
-    return jwt.verify(token, env.JWT_SECRET)
+    return jwt.verify(token, env.JWT_SECRET as Secret) as T
   } catch (e) {
     if (env.JWT_SECRET_FALLBACK) {
       // fallback to enable rotation
-      return jwt.verify(token, env.JWT_SECRET_FALLBACK)
+      return jwt.verify(token, env.JWT_SECRET_FALLBACK) as T
     } else {
       throw e
     }
@@ -159,13 +155,9 @@ export function isValidInternalAPIKey(apiKey: string) {
     return true
   }
   // fallback to enable rotation
-  if (
+  return !!(
-    env.INTERNAL_API_KEY_FALLBACK &&
+    env.INTERNAL_API_KEY_FALLBACK && env.INTERNAL_API_KEY_FALLBACK === apiKey
-    env.INTERNAL_API_KEY_FALLBACK === apiKey
+  )
-  ) {
-    return true
-  }
-  return false
 }
 
 /**
@@ -173,14 +165,14 @@ export function isValidInternalAPIKey(apiKey: string) {
  * @param ctx The request which is to be manipulated.
  * @param name The name of the cookie to get.
  */
-export function getCookie(ctx: Ctx, name: string) {
+export function getCookie<T>(ctx: Ctx, name: string) {
   const cookie = ctx.cookies.get(name)
 
   if (!cookie) {
-    return cookie
+    return undefined
   }
 
-  return openJwt(cookie)
+  return openJwt<T>(cookie)
 }
 
 /**
@@ -197,7 +189,7 @@ export function setCookie(
   opts = { sign: true }
 ) {
   if (value && opts && opts.sign) {
-    value = jwt.sign(value, env.JWT_SECRET)
+    value = jwt.sign(value, env.JWT_SECRET as Secret)
   }
 
   const config: SetOption = {

packages/bbui/src/ButtonGroup/ButtonGroup.svelte

@@ -2,7 +2,7 @@
   import "@spectrum-css/buttongroup/dist/index-vars.css"
 
   export let vertical = false
-  export let gap = ""
+  export let gap = "M"
 
   $: gapStyle =
     gap === "L"

packages/bbui/src/DetailSummary/DetailSummary.svelte

@@ -1,20 +1,17 @@
 <script>
   import Icon from "../Icon/Icon.svelte"
-  import { createEventDispatcher } from "svelte"
 
   export let name
-  export let show = false
+  export let initiallyShow = false
   export let collapsible = true
 
-  const dispatch = createEventDispatcher()
+  let show = initiallyShow
+
   const onHeaderClick = () => {
     if (!collapsible) {
       return
     }
     show = !show
-    if (show) {
-      dispatch("open")
-    }
   }
 </script>
 

packages/bbui/src/FancyForm/FancySelect.svelte

@@ -12,11 +12,13 @@
   export let error = null
   export let validate = null
   export let options = []
+  export let footer = null
   export let isOptionEnabled = () => true
   export let getOptionLabel = option => extractProperty(option, "label")
   export let getOptionValue = option => extractProperty(option, "value")
   export let getOptionSubtitle = option => extractProperty(option, "subtitle")
   export let getOptionColour = () => null
+
   const dispatch = createEventDispatcher()
 
   let open = false
@@ -100,6 +102,7 @@
     {error}
     {disabled}
     {options}
+    {footer}
     {getOptionLabel}
     {getOptionValue}
     {getOptionSubtitle}

packages/bbui/src/Form/Core/Dropzone.svelte

@@ -53,7 +53,7 @@
   $: {
     if (selectedImage?.url) {
       selectedUrl = selectedImage?.url
-    } else if (selectedImage) {
+    } else if (selectedImage && isImage) {
       try {
         let reader = new FileReader()
         reader.readAsDataURL(selectedImage)

packages/bbui/src/Form/Core/InputDropdown.svelte

@@ -17,7 +17,7 @@
   export let options = []
   export let getOptionLabel = option => extractProperty(option, "label")
   export let getOptionValue = option => extractProperty(option, "value")
-
+  export let getOptionSubtitle = option => option?.subtitle
   export let isOptionSelected = () => false
 
   const dispatch = createEventDispatcher()
@@ -135,7 +135,7 @@
       class="spectrum-Textfield-input spectrum-InputGroup-input"
     />
   </div>
-  <div style="width: 30%">
+  <div style="width: 40%">
     <button
       {id}
       class="spectrum-Picker spectrum-Picker--sizeM override-borders"
@@ -157,38 +157,43 @@
         <use xlink:href="#spectrum-css-icon-Chevron100" />
       </svg>
     </button>
-    {#if open}
-      <div
-        use:clickOutside={handleOutsideClick}
-        transition:fly|local={{ y: -20, duration: 200 }}
-        class="spectrum-Popover spectrum-Popover--bottom spectrum-Picker-popover is-open"
-      >
-        <ul class="spectrum-Menu" role="listbox">
-          {#each options as option, idx}
-            <li
-              class="spectrum-Menu-item"
-              class:is-selected={isOptionSelected(getOptionValue(option, idx))}
-              role="option"
-              aria-selected="true"
-              tabindex="0"
-              on:click={() => onPick(getOptionValue(option, idx))}
-            >
-              <span class="spectrum-Menu-itemLabel">
-                {getOptionLabel(option, idx)}
-              </span>
-              <svg
-                class="spectrum-Icon spectrum-UIIcon-Checkmark100 spectrum-Menu-checkmark spectrum-Menu-itemIcon"
-                focusable="false"
-                aria-hidden="true"
-              >
-                <use xlink:href="#spectrum-css-icon-Checkmark100" />
-              </svg>
-            </li>
-          {/each}
-        </ul>
-      </div>
-    {/if}
   </div>
+  {#if open}
+    <div
+      use:clickOutside={handleOutsideClick}
+      transition:fly|local={{ y: -20, duration: 200 }}
+      class="spectrum-Popover spectrum-Popover--bottom spectrum-Picker-popover is-open"
+    >
+      <ul class="spectrum-Menu" role="listbox">
+        {#each options as option, idx}
+          <li
+            class="spectrum-Menu-item"
+            class:is-selected={isOptionSelected(getOptionValue(option, idx))}
+            role="option"
+            aria-selected="true"
+            tabindex="0"
+            on:click={() => onPick(getOptionValue(option, idx))}
+          >
+            <span class="spectrum-Menu-itemLabel">
+              {getOptionLabel(option, idx)}
+              {#if getOptionSubtitle(option, idx)}
+                <span class="subtitle-text">
+                  {getOptionSubtitle(option, idx)}
+                </span>
+              {/if}
+            </span>
+            <svg
+              class="spectrum-Icon spectrum-UIIcon-Checkmark100 spectrum-Menu-checkmark spectrum-Menu-itemIcon"
+              focusable="false"
+              aria-hidden="true"
+            >
+              <use xlink:href="#spectrum-css-icon-Checkmark100" />
+            </svg>
+          </li>
+        {/each}
+      </ul>
+    </div>
+  {/if}
 </div>
 
 <style>
@@ -196,7 +201,6 @@
     min-width: 0;
     width: 100%;
   }
-
   .spectrum-InputGroup-input {
     border-right-width: 1px;
   }
@@ -206,7 +210,6 @@
   .spectrum-Textfield-input {
     width: 0;
   }
-
   .override-borders {
     border-top-left-radius: 0px;
     border-bottom-left-radius: 0px;
@@ -215,5 +218,18 @@
     max-height: 240px;
     z-index: 999;
     top: 100%;
+    width: 100%;
+  }
+  .subtitle-text {
+    font-size: 12px;
+    line-height: 15px;
+    font-weight: 500;
+    color: var(--spectrum-global-color-gray-600);
+    display: block;
+    margin-top: var(--spacing-s);
+  }
+  .spectrum-Menu-checkmark {
+    align-self: center;
+    margin-top: 0;
   }
 </style>

packages/bbui/src/Form/Core/Picker.svelte

@@ -224,13 +224,12 @@
               </span>
             {/if}
             <span class="spectrum-Menu-itemLabel">
-              {#if getOptionSubtitle(option, idx)}
-                <span class="subtitle-text"
-                  >{getOptionSubtitle(option, idx)}</span
-                >
-              {/if}
-
               {getOptionLabel(option, idx)}
+              {#if getOptionSubtitle(option, idx)}
+                <span class="subtitle-text">
+                  {getOptionSubtitle(option, idx)}
+                </span>
+              {/if}
             </span>
             {#if option.tag}
               <span class="option-tag">
@@ -275,10 +274,9 @@
     font-size: 12px;
     line-height: 15px;
     font-weight: 500;
-    top: 10px;
     color: var(--spectrum-global-color-gray-600);
     display: block;
-    margin-bottom: var(--spacing-s);
+    margin-top: var(--spacing-s);
   }
 
   .spectrum-Picker-label.auto-width {

packages/bbui/src/Form/Core/Select.svelte

@@ -10,8 +10,9 @@
   export let getOptionLabel = option => option
   export let getOptionValue = option => option
   export let getOptionIcon = () => null
-  export let useOptionIconImage = false
   export let getOptionColour = () => null
+  export let getOptionSubtitle = () => null
+  export let useOptionIconImage = false
   export let isOptionEnabled
   export let readonly = false
   export let quiet = false
@@ -82,8 +83,9 @@
   {getOptionLabel}
   {getOptionValue}
   {getOptionIcon}
-  {useOptionIconImage}
   {getOptionColour}
+  {getOptionSubtitle}
+  {useOptionIconImage}
   {isOptionEnabled}
   {autocomplete}
   {sort}

packages/bbui/src/Form/Core/Switch.svelte

@@ -18,6 +18,7 @@
     checked={value}
     {disabled}
     on:change={onChange}
+    on:click
     {id}
     type="checkbox"
     class="spectrum-Switch-input"

packages/bbui/src/Form/Core/TextField.svelte

@@ -20,7 +20,7 @@
   let focus = false
 
   const updateValue = newValue => {
-    if (readonly) {
+    if (readonly || disabled) {
       return
     }
     if (type === "number") {
@@ -31,14 +31,14 @@
   }
 
   const onFocus = () => {
-    if (readonly) {
+    if (readonly || disabled) {
       return
     }
     focus = true
   }
 
   const onBlur = event => {
-    if (readonly) {
+    if (readonly || disabled) {
       return
     }
     focus = false
@@ -46,14 +46,14 @@
   }
 
   const onInput = event => {
-    if (readonly || !updateOnChange) {
+    if (readonly || !updateOnChange || disabled) {
       return
     }
     updateValue(event.target.value)
   }
 
   const updateValueOnEnter = event => {
-    if (readonly) {
+    if (readonly || disabled) {
       return
     }
     if (event.key === "Enter") {
@@ -69,6 +69,7 @@
   }
 
   onMount(() => {
+    if (disabled) return
     focus = autofocus
     if (focus) field.focus()
   })
@@ -108,4 +109,16 @@
   .spectrum-Textfield {
     width: 100%;
   }
+
+  input::placeholder {
+    color: var(--grey-7);
+  }
+
+  input:hover::placeholder {
+    color: var(--grey-7) !important;
+  }
+
+  input:focus::placeholder {
+    color: var(--grey-7) !important;
+  }
 </style>

packages/bbui/src/Form/InputDropdown.svelte

@@ -43,6 +43,7 @@
     {quiet}
     {autofocus}
     {options}
+    isOptionSelected={option => option === dropdownValue}
     on:change={onChange}
     on:pick={onPick}
     on:click

packages/bbui/src/Form/Select.svelte

@@ -13,9 +13,10 @@
   export let options = []
   export let getOptionLabel = option => extractProperty(option, "label")
   export let getOptionValue = option => extractProperty(option, "value")
+  export let getOptionSubtitle = option => option?.subtitle
   export let getOptionIcon = option => option?.icon
-  export let useOptionIconImage = false
   export let getOptionColour = option => option?.colour
+  export let useOptionIconImage = false
   export let isOptionEnabled
   export let quiet = false
   export let autoWidth = false
@@ -58,6 +59,7 @@
     {getOptionValue}
     {getOptionIcon}
     {getOptionColour}
+    {getOptionSubtitle}
     {useOptionIconImage}
     {isOptionEnabled}
     {autocomplete}

packages/bbui/src/Form/Toggle.svelte

@@ -19,5 +19,5 @@
 </script>
 
 <Field {helpText} {label} {labelPosition} {error}>
-  <Switch {error} {disabled} {text} {value} on:change={onChange} />
+  <Switch {error} {disabled} {text} {value} on:change={onChange} on:click />
 </Field>

packages/builder/src/builderStore/componentUtils.js

@@ -1,4 +1,5 @@
 import { store } from "./index"
+import { get } from "svelte/store"
 import { Helpers } from "@budibase/bbui"
 import {
   decodeJSBinding,
@@ -238,6 +239,10 @@ export const makeComponentUnique = component => {
 }
 
 export const getComponentText = component => {
+  if (component == null) {
+    return ""
+  }
+
   if (component?._instanceName) {
     return component._instanceName
   }
@@ -246,3 +251,16 @@ export const getComponentText = component => {
     "component"
   return capitalise(type)
 }
+
+export const getComponentName = component => {
+  if (component == null) {
+    return ""
+  }
+
+  const components = get(store)?.components || {}
+  const componentDefinition = components[component._component] || {}
+  const name =
+    componentDefinition.friendlyName || componentDefinition.name || ""
+
+  return name
+}

packages/builder/src/builderStore/dataBinding.js

@@ -29,6 +29,12 @@ const CAPTURE_VAR_INSIDE_TEMPLATE = /{{([^}]+)}}/g
 const CAPTURE_VAR_INSIDE_JS = /\$\("([^")]+)"\)/g
 const CAPTURE_HBS_TEMPLATE = /{{[\S\s]*?}}/g
 
+const UpdateReferenceAction = {
+  ADD: "add",
+  DELETE: "delete",
+  MOVE: "move",
+}
+
 /**
  * Gets all bindable data context fields and instance fields.
  */
@@ -1090,17 +1096,18 @@ export const removeBindings = (obj, replacement = "Invalid binding") => {
  * When converting from readable to runtime it can sometimes add too many square brackets,
  * this makes sure that doesn't happen.
  */
-const shouldReplaceBinding = (currentValue, convertFrom, convertTo) => {
+const shouldReplaceBinding = (currentValue, from, convertTo, binding) => {
-  if (!currentValue?.includes(convertFrom)) {
+  if (!currentValue?.includes(from)) {
     return false
   }
   if (convertTo === "readableBinding") {
-    return true
+    // Dont replace if the value already matches the readable binding
+    return currentValue.indexOf(binding.readableBinding) === -1
   }
   // remove all the spaces, if the input is surrounded by spaces e.g. [ Auto ID ] then
   // this makes sure it is detected
   const noSpaces = currentValue.replace(/\s+/g, "")
-  const fromNoSpaces = convertFrom.replace(/\s+/g, "")
+  const fromNoSpaces = from.replace(/\s+/g, "")
   const invalids = [
     `[${fromNoSpaces}]`,
     `"${fromNoSpaces}"`,
@@ -1152,8 +1159,11 @@ const bindingReplacement = (
     // in the search, working from longest to shortest so always use best match first
     let searchString = newBoundValue
     for (let from of convertFromProps) {
-      if (isJS || shouldReplaceBinding(newBoundValue, from, convertTo)) {
+      const binding = bindableProperties.find(el => el[convertFrom] === from)
-        const binding = bindableProperties.find(el => el[convertFrom] === from)
+      if (
+        isJS ||
+        shouldReplaceBinding(newBoundValue, from, convertTo, binding)
+      ) {
         let idx
         do {
           // see if any instances of this binding exist in the search string
@@ -1222,3 +1232,81 @@ export const runtimeToReadableBinding = (
     "readableBinding"
   )
 }
+
+/**
+ * Used to update binding references for automation or action steps
+ *
+ * @param obj - The object to be updated
+ * @param originalIndex - The original index of the step being moved. Not applicable to add/delete.
+ * @param modifiedIndex - The new index of the step being modified
+ * @param action - Used to determine if a step is being added, deleted or moved
+ * @param label - The binding text that describes the steps
+ */
+export const updateReferencesInObject = ({
+  obj,
+  modifiedIndex,
+  action,
+  label,
+  originalIndex,
+}) => {
+  const stepIndexRegex = new RegExp(`{{\\s*${label}\\.(\\d+)\\.`, "g")
+  const updateActionStep = (str, index, replaceWith) =>
+    str.replace(`{{ ${label}.${index}.`, `{{ ${label}.${replaceWith}.`)
+  for (const key in obj) {
+    if (typeof obj[key] === "string") {
+      let matches
+      while ((matches = stepIndexRegex.exec(obj[key])) !== null) {
+        const referencedStep = parseInt(matches[1])
+        if (
+          action === UpdateReferenceAction.ADD &&
+          referencedStep >= modifiedIndex
+        ) {
+          obj[key] = updateActionStep(
+            obj[key],
+            referencedStep,
+            referencedStep + 1
+          )
+        } else if (
+          action === UpdateReferenceAction.DELETE &&
+          referencedStep > modifiedIndex
+        ) {
+          obj[key] = updateActionStep(
+            obj[key],
+            referencedStep,
+            referencedStep - 1
+          )
+        } else if (action === UpdateReferenceAction.MOVE) {
+          if (referencedStep === originalIndex) {
+            obj[key] = updateActionStep(obj[key], referencedStep, modifiedIndex)
+          } else if (
+            modifiedIndex <= referencedStep &&
+            modifiedIndex < originalIndex
+          ) {
+            obj[key] = updateActionStep(
+              obj[key],
+              referencedStep,
+              referencedStep + 1
+            )
+          } else if (
+            modifiedIndex >= referencedStep &&
+            modifiedIndex > originalIndex
+          ) {
+            obj[key] = updateActionStep(
+              obj[key],
+              referencedStep,
+              referencedStep - 1
+            )
+          }
+        }
+      }
+    } else if (typeof obj[key] === "object" && obj[key] !== null) {
+      updateReferencesInObject({
+        obj: obj[key],
+        modifiedIndex,
+        action,
+        label,
+        originalIndex,
+      })
+    }
+  }
+}

packages/builder/src/builderStore/index.js

@@ -4,10 +4,11 @@ import { getTemporalStore } from "./store/temporal"
 import { getThemeStore } from "./store/theme"
 import { getUserStore } from "./store/users"
 import { getDeploymentStore } from "./store/deployments"
-import { derived, writable, get } from "svelte/store"
+import { derived, get } from "svelte/store"
 import { findComponent, findComponentPath } from "./componentUtils"
 import { RoleUtils } from "@budibase/frontend-core"
 import { createHistoryStore } from "builderStore/store/history"
+import { cloneDeep } from "lodash/fp"
 
 export const store = getFrontendStore()
 export const automationStore = getAutomationStore()
@@ -69,7 +70,14 @@ export const selectedComponent = derived(
     if (!$selectedScreen || !$store.selectedComponentId) {
       return null
     }
-    return findComponent($selectedScreen?.props, $store.selectedComponentId)
+    const selected = findComponent(
+      $selectedScreen?.props,
+      $store.selectedComponentId
+    )
+
+    const clone = selected ? cloneDeep(selected) : selected
+    store.actions.components.migrateSettings(clone)
+    return clone
   }
 )
 
@@ -146,5 +154,3 @@ export const userSelectedResourceMap = derived(userStore, $userStore => {
 export const isOnlyUser = derived(userStore, $userStore => {
   return $userStore.length < 2
 })
-
-export const screensHeight = writable("210px")

packages/builder/src/builderStore/store/automation/index.js

@@ -4,6 +4,7 @@ import { cloneDeep } from "lodash/fp"
 import { generate } from "shortid"
 import { selectedAutomation } from "builderStore"
 import { notifications } from "@budibase/bbui"
+import { updateReferencesInObject } from "builderStore/dataBinding"
 
 const initialAutomationState = {
   automations: [],
@@ -22,34 +23,14 @@ export const getAutomationStore = () => {
   return store
 }
 
-const updateReferencesInObject = (obj, modifiedIndex, action) => {
-  const regex = /{{\s*steps\.(\d+)\./g
-  for (const key in obj) {
-    if (typeof obj[key] === "string") {
-      let matches
-      while ((matches = regex.exec(obj[key])) !== null) {
-        const referencedStep = parseInt(matches[1])
-        if (action === "add" && referencedStep >= modifiedIndex) {
-          obj[key] = obj[key].replace(
-            `{{ steps.${referencedStep}.`,
-            `{{ steps.${referencedStep + 1}.`
-          )
-        } else if (action === "delete" && referencedStep > modifiedIndex) {
-          obj[key] = obj[key].replace(
-            `{{ steps.${referencedStep}.`,
-            `{{ steps.${referencedStep - 1}.`
-          )
-        }
-      }
-    } else if (typeof obj[key] === "object" && obj[key] !== null) {
-      updateReferencesInObject(obj[key], modifiedIndex, action)
-    }
-  }
-}
-
 const updateStepReferences = (steps, modifiedIndex, action) => {
   steps.forEach(step => {
-    updateReferencesInObject(step.inputs, modifiedIndex, action)
+    updateReferencesInObject({
+      obj: step.inputs,
+      modifiedIndex,
+      action,
+      label: "steps",
+    })
   })
 }
 

packages/builder/src/builderStore/store/frontend.js

@@ -85,6 +85,7 @@ const INITIAL_FRONTEND_STATE = {
   selectedScreenId: null,
   selectedComponentId: null,
   selectedLayoutId: null,
+  hoverComponentId: null,
 
   // Client state
   selectedComponentInstance: null,
@@ -112,7 +113,7 @@ export const getFrontendStore = () => {
     }
     let clone = cloneDeep(screen)
     const result = patchFn(clone)
-
+    // An explicit false result means skip this change
     if (result === false) {
       return
     }
@@ -601,6 +602,36 @@ export const getFrontendStore = () => {
         // Finally try an external table
         return validTables.find(table => table.sourceType === DB_TYPE_EXTERNAL)
       },
+      migrateSettings: enrichedComponent => {
+        const componentPrefix = "@budibase/standard-components"
+        let migrated = false
+
+        if (enrichedComponent?._component == `${componentPrefix}/formblock`) {
+          // Use default config if the 'buttons' prop has never been initialised
+          if (!("buttons" in enrichedComponent)) {
+            enrichedComponent["buttons"] =
+              Utils.buildDynamicButtonConfig(enrichedComponent)
+            migrated = true
+          } else if (enrichedComponent["buttons"] == null) {
+            // Ignore legacy config if 'buttons' has been reset by 'resetOn'
+            const { _id, actionType, dataSource } = enrichedComponent
+            enrichedComponent["buttons"] = Utils.buildDynamicButtonConfig({
+              _id,
+              actionType,
+              dataSource,
+            })
+            migrated = true
+          }
+
+          // Ensure existing Formblocks position their buttons at the top.
+          if (!("buttonPosition" in enrichedComponent)) {
+            enrichedComponent["buttonPosition"] = "top"
+            migrated = true
+          }
+        }
+
+        return migrated
+      },
       enrichEmptySettings: (component, opts) => {
         if (!component?._component) {
           return
@@ -672,7 +703,6 @@ export const getFrontendStore = () => {
               component[setting.key] = setting.defaultValue
             }
           }
-
           // Validate non-empty settings
           else {
             if (setting.type === "dataProvider") {
@@ -722,6 +752,9 @@ export const getFrontendStore = () => {
           useDefaultValues: true,
         })
 
+        // Migrate nested component settings
+        store.actions.components.migrateSettings(instance)
+
         // Add any extra properties the component needs
         let extras = {}
         if (definition.hasChildren) {
@@ -845,7 +878,16 @@ export const getFrontendStore = () => {
           if (!component) {
             return false
           }
-          return patchFn(component, screen)
+
+          // Mutates the fetched component with updates
+          const patchResult = patchFn(component, screen)
+
+          // Mutates the component with any required settings updates
+          const migrated = store.actions.components.migrateSettings(component)
+
+          // Returning an explicit false signifies that we should skip this
+          // update. If we migrated something, ensure we never skip.
+          return migrated ? null : patchResult
         }
         await store.actions.screens.patch(patchScreen, screenId)
       },
@@ -1247,9 +1289,13 @@ export const getFrontendStore = () => {
           const settings = getComponentSettings(component._component)
           const updatedSetting = settings.find(setting => setting.key === name)
 
-          const resetFields = settings.filter(
+          // Can be a single string or array of strings
-            setting => name === setting.resetOn
+          const resetFields = settings.filter(setting => {
-          )
+            return (
+              name === setting.resetOn ||
+              (Array.isArray(setting.resetOn) && setting.resetOn.includes(name))
+            )
+          })
           resetFields?.forEach(setting => {
             component[setting.key] = null
           })
@@ -1271,6 +1317,7 @@ export const getFrontendStore = () => {
             })
           }
           component[name] = value
+          return true
         }
       },
       requestEjectBlock: componentId => {
@@ -1278,7 +1325,6 @@ export const getFrontendStore = () => {
       },
       handleEjectBlock: async (componentId, ejectedDefinition) => {
         let nextSelectedComponentId
-
         await store.actions.screens.patch(screen => {
           const block = findComponent(screen.props, componentId)
           const parent = findComponentParent(screen.props, componentId)

packages/builder/src/builderStore/tests/dataBinding.test.js

@@ -0,0 +1,545 @@
+import { expect, describe, it, vi } from "vitest"
+import {
+  runtimeToReadableBinding,
+  readableToRuntimeBinding,
+  updateReferencesInObject,
+} from "../dataBinding"
+
+vi.mock("@budibase/frontend-core")
+vi.mock("builderStore/componentUtils")
+vi.mock("builderStore/store")
+vi.mock("builderStore/store/theme")
+vi.mock("builderStore/store/temporal")
+
+describe("runtimeToReadableBinding", () => {
+  const bindableProperties = [
+    {
+      category: "Current User",
+      icon: "User",
+      providerId: "user",
+      readableBinding: "Current User.firstName",
+      runtimeBinding: "[user].[firstName]",
+      type: "context",
+    },
+    {
+      category: "Bindings",
+      icon: "Brackets",
+      readableBinding: "Binding.count",
+      runtimeBinding: "count",
+      type: "context",
+    },
+  ]
+  it("should convert a runtime binding to a readable one", () => {
+    const textWithBindings = `Hello {{ [user].[firstName] }}! The count is {{ count }}.`
+    expect(
+      runtimeToReadableBinding(
+        bindableProperties,
+        textWithBindings,
+        "readableBinding"
+      )
+    ).toEqual(
+      `Hello {{ Current User.firstName }}! The count is {{ Binding.count }}.`
+    )
+  })
+
+  it("should not convert to readable binding if it is already readable", () => {
+    const textWithBindings = `Hello {{ [user].[firstName] }}! The count is {{ Binding.count }}.`
+    expect(
+      runtimeToReadableBinding(
+        bindableProperties,
+        textWithBindings,
+        "readableBinding"
+      )
+    ).toEqual(
+      `Hello {{ Current User.firstName }}! The count is {{ Binding.count }}.`
+    )
+  })
+})
+
+describe("readableToRuntimeBinding", () => {
+  const bindableProperties = [
+    {
+      category: "Current User",
+      icon: "User",
+      providerId: "user",
+      readableBinding: "Current User.firstName",
+      runtimeBinding: "[user].[firstName]",
+      type: "context",
+    },
+    {
+      category: "Bindings",
+      icon: "Brackets",
+      readableBinding: "Binding.count",
+      runtimeBinding: "count",
+      type: "context",
+    },
+  ]
+  it("should convert a readable binding to a runtime one", () => {
+    const textWithBindings = `Hello {{ Current User.firstName }}! The count is {{ Binding.count }}.`
+    expect(
+      readableToRuntimeBinding(
+        bindableProperties,
+        textWithBindings,
+        "runtimeBinding"
+      )
+    ).toEqual(`Hello {{ [user].[firstName] }}! The count is {{ count }}.`)
+  })
+})
+
+describe("updateReferencesInObject", () => {
+  it("should increment steps in sequence on 'add'", () => {
+    let obj = [
+      {
+        id: "a0",
+        parameters: {
+          text: "Alpha",
+        },
+      },
+      {
+        id: "a1",
+        parameters: {
+          text: "Apple",
+        },
+      },
+      {
+        id: "b2",
+        parameters: {
+          text: "Banana {{ actions.1.row }}",
+        },
+      },
+      {
+        id: "c3",
+        parameters: {
+          text: "Carrot {{ actions.1.row }}",
+        },
+      },
+      {
+        id: "d4",
+        parameters: {
+          text: "Dog {{ actions.3.row }}",
+        },
+      },
+      {
+        id: "e5",
+        parameters: {
+          text: "Eagle {{ actions.4.row }}",
+        },
+      },
+    ]
+    updateReferencesInObject({
+      obj,
+      modifiedIndex: 0,
+      action: "add",
+      label: "actions",
+    })
+
+    expect(obj).toEqual([
+      {
+        id: "a0",
+        parameters: {
+          text: "Alpha",
+        },
+      },
+      {
+        id: "a1",
+        parameters: {
+          text: "Apple",
+        },
+      },
+      {
+        id: "b2",
+        parameters: {
+          text: "Banana {{ actions.2.row }}",
+        },
+      },
+      {
+        id: "c3",
+        parameters: {
+          text: "Carrot {{ actions.2.row }}",
+        },
+      },
+      {
+        id: "d4",
+        parameters: {
+          text: "Dog {{ actions.4.row }}",
+        },
+      },
+      {
+        id: "e5",
+        parameters: {
+          text: "Eagle {{ actions.5.row }}",
+        },
+      },
+    ])
+  })
+
+  it("should decrement steps in sequence on 'delete'", () => {
+    let obj = [
+      {
+        id: "a1",
+        parameters: {
+          text: "Apple",
+        },
+      },
+      {
+        id: "b2",
+        parameters: {
+          text: "Banana {{ actions.1.row }}",
+        },
+      },
+      {
+        id: "d4",
+        parameters: {
+          text: "Dog {{ actions.3.row }}",
+        },
+      },
+      {
+        id: "e5",
+        parameters: {
+          text: "Eagle {{ actions.4.row }}",
+        },
+      },
+    ]
+    updateReferencesInObject({
+      obj,
+      modifiedIndex: 2,
+      action: "delete",
+      label: "actions",
+    })
+
+    expect(obj).toEqual([
+      {
+        id: "a1",
+        parameters: {
+          text: "Apple",
+        },
+      },
+      {
+        id: "b2",
+        parameters: {
+          text: "Banana {{ actions.1.row }}",
+        },
+      },
+      {
+        id: "d4",
+        parameters: {
+          text: "Dog {{ actions.2.row }}",
+        },
+      },
+      {
+        id: "e5",
+        parameters: {
+          text: "Eagle {{ actions.3.row }}",
+        },
+      },
+    ])
+  })
+
+  it("should handle on 'move' to a lower index", () => {
+    let obj = [
+      {
+        id: "a1",
+        parameters: {
+          text: "Apple",
+        },
+      },
+      {
+        id: "b2",
+        parameters: {
+          text: "Banana {{ actions.0.row }}",
+        },
+      },
+      {
+        id: "e5",
+        parameters: {
+          text: "Eagle {{ actions.3.row }}",
+        },
+      },
+      {
+        id: "c3",
+        parameters: {
+          text: "Carrot {{ actions.0.row }}",
+        },
+      },
+      {
+        id: "d4",
+        parameters: {
+          text: "Dog {{ actions.2.row }}",
+        },
+      },
+    ]
+    updateReferencesInObject({
+      obj,
+      modifiedIndex: 2,
+      action: "move",
+      label: "actions",
+      originalIndex: 4,
+    })
+
+    expect(obj).toEqual([
+      {
+        id: "a1",
+        parameters: {
+          text: "Apple",
+        },
+      },
+      {
+        id: "b2",
+        parameters: {
+          text: "Banana {{ actions.0.row }}",
+        },
+      },
+      {
+        id: "e5",
+        parameters: {
+          text: "Eagle {{ actions.4.row }}",
+        },
+      },
+      {
+        id: "c3",
+        parameters: {
+          text: "Carrot {{ actions.0.row }}",
+        },
+      },
+      {
+        id: "d4",
+        parameters: {
+          text: "Dog {{ actions.3.row }}",
+        },
+      },
+    ])
+  })
+
+  it("should handle on 'move' to a higher index", () => {
+    let obj = [
+      {
+        id: "b2",
+        parameters: {
+          text: "Banana {{ actions.0.row }}",
+        },
+      },
+      {
+        id: "c3",
+        parameters: {
+          text: "Carrot {{ actions.0.row }}",
+        },
+      },
+      {
+        id: "a1",
+        parameters: {
+          text: "Apple",
+        },
+      },
+      {
+        id: "d4",
+        parameters: {
+          text: "Dog {{ actions.2.row }}",
+        },
+      },
+      {
+        id: "e5",
+        parameters: {
+          text: "Eagle {{ actions.3.row }}",
+        },
+      },
+    ]
+    updateReferencesInObject({
+      obj,
+      modifiedIndex: 2,
+      action: "move",
+      label: "actions",
+      originalIndex: 0,
+    })
+
+    expect(obj).toEqual([
+      {
+        id: "b2",
+        parameters: {
+          text: "Banana {{ actions.2.row }}",
+        },
+      },
+      {
+        id: "c3",
+        parameters: {
+          text: "Carrot {{ actions.2.row }}",
+        },
+      },
+      {
+        id: "a1",
+        parameters: {
+          text: "Apple",
+        },
+      },
+      {
+        id: "d4",
+        parameters: {
+          text: "Dog {{ actions.1.row }}",
+        },
+      },
+      {
+        id: "e5",
+        parameters: {
+          text: "Eagle {{ actions.3.row }}",
+        },
+      },
+    ])
+  })
+
+  it("should handle on 'move' of action being referenced, dragged to a higher index", () => {
+    let obj = [
+      {
+        "##eventHandlerType": "Validate Form",
+        id: "cCD0Dwcnq",
+      },
+      {
+        "##eventHandlerType": "Close Screen Modal",
+        id: "3fbbIOfN0H",
+      },
+      {
+        "##eventHandlerType": "Save Row",
+        parameters: {
+          tableId: "ta_bb_employee",
+        },
+        id: "aehg5cTmhR",
+      },
+      {
+        "##eventHandlerType": "Close Side Panel",
+        id: "mzkpf86cxo",
+      },
+      {
+        "##eventHandlerType": "Navigate To",
+        id: "h0uDFeJa8A",
+      },
+      {
+        parameters: {
+          autoDismiss: true,
+          type: "success",
+          message: "{{ actions.1.row }}",
+        },
+        "##eventHandlerType": "Show Notification",
+        id: "JEI5lAyJZ",
+      },
+    ]
+    updateReferencesInObject({
+      obj,
+      modifiedIndex: 2,
+      action: "move",
+      label: "actions",
+      originalIndex: 1,
+    })
+
+    expect(obj).toEqual([
+      {
+        "##eventHandlerType": "Validate Form",
+        id: "cCD0Dwcnq",
+      },
+      {
+        "##eventHandlerType": "Close Screen Modal",
+        id: "3fbbIOfN0H",
+      },
+      {
+        "##eventHandlerType": "Save Row",
+        parameters: {
+          tableId: "ta_bb_employee",
+        },
+        id: "aehg5cTmhR",
+      },
+      {
+        "##eventHandlerType": "Close Side Panel",
+        id: "mzkpf86cxo",
+      },
+      {
+        "##eventHandlerType": "Navigate To",
+        id: "h0uDFeJa8A",
+      },
+      {
+        parameters: {
+          autoDismiss: true,
+          type: "success",
+          message: "{{ actions.2.row }}",
+        },
+        "##eventHandlerType": "Show Notification",
+        id: "JEI5lAyJZ",
+      },
+    ])
+  })
+
+  it("should handle on 'move' of action being referenced, dragged to a lower index", () => {
+    let obj = [
+      {
+        "##eventHandlerType": "Save Row",
+        parameters: {
+          tableId: "ta_bb_employee",
+        },
+        id: "aehg5cTmhR",
+      },
+      {
+        "##eventHandlerType": "Validate Form",
+        id: "cCD0Dwcnq",
+      },
+      {
+        "##eventHandlerType": "Close Screen Modal",
+        id: "3fbbIOfN0H",
+      },
+      {
+        "##eventHandlerType": "Close Side Panel",
+        id: "mzkpf86cxo",
+      },
+      {
+        "##eventHandlerType": "Navigate To",
+        id: "h0uDFeJa8A",
+      },
+      {
+        parameters: {
+          autoDismiss: true,
+          type: "success",
+          message: "{{ actions.4.row }}",
+        },
+        "##eventHandlerType": "Show Notification",
+        id: "JEI5lAyJZ",
+      },
+    ]
+    updateReferencesInObject({
+      obj,
+      modifiedIndex: 0,
+      action: "move",
+      label: "actions",
+      originalIndex: 4,
+    })
+
+    expect(obj).toEqual([
+      {
+        "##eventHandlerType": "Save Row",
+        parameters: {
+          tableId: "ta_bb_employee",
+        },
+        id: "aehg5cTmhR",
+      },
+      {
+        "##eventHandlerType": "Validate Form",
+        id: "cCD0Dwcnq",
+      },
+      {
+        "##eventHandlerType": "Close Screen Modal",
+        id: "3fbbIOfN0H",
+      },
+      {
+        "##eventHandlerType": "Close Side Panel",
+        id: "mzkpf86cxo",
+      },
+      {
+        "##eventHandlerType": "Navigate To",
+        id: "h0uDFeJa8A",
+      },
+      {
+        parameters: {
+          autoDismiss: true,
+          type: "success",
+          message: "{{ actions.0.row }}",
+        },
+        "##eventHandlerType": "Show Notification",
+        id: "JEI5lAyJZ",
+      },
+    ])
+  })
+})

packages/builder/src/components/automation/AutomationBuilder/FlowChart/FlowChart.svelte

@@ -57,16 +57,11 @@
       }}
       class="buttons"
     >
-      <Icon hoverable size="M" name="Play" />
+      <Icon size="M" name="Play" />
       <div>Run test</div>
     </div>
     <div class="buttons">
-      <Icon
+      <Icon disabled={!$automationStore.testResults} size="M" name="Multiple" />
-        disabled={!$automationStore.testResults}
-        hoverable
-        size="M"
-        name="Multiple"
-      />
       <div
         class:disabled={!$automationStore.testResults}
         on:click={() => {

packages/builder/src/components/automation/AutomationBuilder/FlowChart/FlowItemHeader.svelte

@@ -97,6 +97,7 @@
   class:typing={typing && !automationNameError}
   class:typing-error={automationNameError}
   class="blockSection"
+  on:click={() => dispatch("toggle")}
 >
   <div class="splitHeader">
     <div class="center-items">
@@ -138,7 +139,20 @@
             on:input={e => {
               automationName = e.target.value.trim()
             }}
-            on:click={startTyping}
+            on:click={e => {
+              e.stopPropagation()
+              startTyping()
+            }}
+            on:keydown={async e => {
+              if (e.key === "Enter") {
+                typing = false
+                if (automationNameError) {
+                  automationName = stepNames[block.id] || block?.name
+                } else {
+                  await saveName()
+                }
+              }
+            }}
             on:blur={async () => {
               typing = false
               if (automationNameError) {
@@ -168,7 +182,11 @@
             </StatusLight>
           </div>
           <Icon
-            on:click={() => dispatch("toggle")}
+            e.stopPropagation()
+            on:click={e => {
+              e.stopPropagation()
+              dispatch("toggle")
+            }}
             hoverable
             name={open ? "ChevronUp" : "ChevronDown"}
           />
@@ -195,7 +213,10 @@
         {/if}
         {#if !showTestStatus}
           <Icon
-            on:click={() => dispatch("toggle")}
+            on:click={e => {
+              e.stopPropagation()
+              dispatch("toggle")
+            }}
             hoverable
             name={open ? "ChevronUp" : "ChevronDown"}
           />

packages/builder/src/components/automation/AutomationBuilder/FlowChart/TestDataModal.svelte

@@ -1,11 +1,9 @@
 <script>
   import {
     ModalContent,
-    Tabs,
-    Tab,
     TextArea,
-    Label,
     notifications,
+    ActionButton,
   } from "@budibase/bbui"
   import { automationStore, selectedAutomation } from "builderStore"
   import AutomationBlockSetup from "../../SetupPanel/AutomationBlockSetup.svelte"
@@ -55,50 +53,69 @@
       notifications.error(error)
     }
   }
+
+  const toggle = () => {
+    selectedValues = !selectedValues
+    selectedJSON = !selectedJSON
+  }
+  let selectedValues = true
+  let selectedJSON = false
 </script>
 
 <ModalContent
   title="Add test data"
-  confirmText="Test"
+  confirmText="Run test"
-  size="M"
+  size="L"
   showConfirmButton={true}
   disabled={isError}
   onConfirm={testAutomation}
   cancelText="Cancel"
 >
-  <Tabs selected="Form" quiet>
+  <div class="size">
-    <Tab icon="Form" title="Form">
+    <div class="options">
-      <div class="tab-content-padding">
+      <ActionButton quiet selected={selectedValues} on:click={toggle}
-        <AutomationBlockSetup
+        >Use values</ActionButton
-          {testData}
+      >
-          {schemaProperties}
+      <ActionButton quiet selected={selectedJSON} on:click={toggle}
-          isTestModal
+        >Use JSON</ActionButton
-          block={trigger}
+      >
-        />
+    </div>
-      </div></Tab
+  </div>
-    >
+
-    <Tab icon="FileJson" title="JSON">
+  {#if selectedValues}
-      <div class="tab-content-padding">
+    <div class="tab-content-padding">
-        <Label>JSON</Label>
+      <AutomationBlockSetup
-        <div class="text-area-container">
+        {testData}
-          <TextArea
+        {schemaProperties}
-            value={JSON.stringify($selectedAutomation.testData, null, 2)}
+        isTestModal
-            error={failedParse}
+        block={trigger}
-            on:change={e => parseTestJSON(e)}
+      />
-          />
+    </div>
-        </div>
+  {/if}
-      </div>
+  {#if selectedJSON}
-    </Tab>
+    <div class="text-area-container">
-  </Tabs>
+      <TextArea
+        value={JSON.stringify($selectedAutomation.testData, null, 2)}
+        error={failedParse}
+        on:change={e => parseTestJSON(e)}
+      />
+    </div>
+  {/if}
 </ModalContent>
 
 <style>
   .text-area-container :global(textarea) {
-    min-height: 200px;
+    min-height: 300px;
-    height: 200px;
+    height: 300px;
   }
 
   .tab-content-padding {
-    padding: 0 var(--spacing-xl);
+    padding: 0 var(--spacing-s);
+  }
+
+  .options {
+    display: flex;
+    align-items: center;
+    gap: 8px;
   }
 </style>

packages/builder/src/components/automation/AutomationBuilder/TestPanel.svelte

@@ -9,7 +9,7 @@
 <div class="title">
   <div class="title-text">
     <Icon name="MultipleCheck" />
-    <div style="padding-left: var(--spacing-l)">Test Details</div>
+    <div style="padding-left: var(--spacing-l); ">Test Details</div>
   </div>
   <div style="padding-right: var(--spacing-xl)">
     <Icon
@@ -40,6 +40,7 @@
     display: flex;
     flex-direction: row;
     align-items: center;
+    padding-top: var(--spacing-s);
   }
 
   .title :global(h1) {

packages/builder/src/components/automation/AutomationPanel/AutomationPanel.svelte

@@ -1,20 +1,44 @@
 <script>
   import AutomationList from "./AutomationList.svelte"
   import CreateAutomationModal from "./CreateAutomationModal.svelte"
-  import { Modal, Button, Layout } from "@budibase/bbui"
+  import { Modal, Icon } from "@budibase/bbui"
   import Panel from "components/design/Panel.svelte"
 
   export let modal
   export let webhookModal
 </script>
 
-<Panel title="Automations" borderRight>
+<Panel title="Automations" borderRight noHeaderBorder titleCSS={false}>
-  <Layout paddingX="L" paddingY="XL" gap="S">
+  <span class="panel-title-content" slot="panel-title-content">
-    <Button cta on:click={modal.show}>Add automation</Button>
+    <div class="header">
-  </Layout>
+      <div>Automations</div>
+      <div on:click={modal.show} class="add-automation-button">
+        <Icon name="Add" />
+      </div>
+    </div>
+  </span>
   <AutomationList />
 </Panel>
 
 <Modal bind:this={modal}>
   <CreateAutomationModal {webhookModal} />
 </Modal>
+
+<style>
+  .header {
+    display: flex;
+    align-items: center;
+    justify-content: space-between;
+    gap: var(--spacing-m);
+  }
+
+  .add-automation-button {
+    margin-left: 130px;
+    color: var(--grey-7);
+    cursor: pointer;
+  }
+
+  .add-automation-button:hover {
+    color: var(--ink);
+  }
+</style>

packages/builder/src/components/automation/SetupPanel/RowSelectorTypes.svelte

@@ -64,7 +64,7 @@
   </span>
 {:else if schema.type === "link"}
   <LinkedRowSelector
-    bind:linkedRows={value[field]}
+    linkedRows={value[field]}
     {schema}
     on:change={e => onChange(e, field)}
     useLabel={false}

packages/builder/src/components/automation/SetupPanel/TableSelector.svelte

@@ -22,7 +22,7 @@
 <Select
   on:change={onChange}
   bind:value
-  options={filteredTables.filter(table => table._id !== TableNames.USERS)}
+  options={filteredTables}
   getOptionLabel={table => table.name}
   getOptionValue={table => table._id}
 />

packages/builder/src/components/backend/DataTable/RowFieldControl.svelte

@@ -70,7 +70,12 @@
     options={meta.constraints.inclusion}
   />
 {:else if type === "link"}
-  <LinkedRowSelector {error} bind:linkedRows={value} schema={meta} />
+  <LinkedRowSelector
+    {error}
+    linkedRows={value}
+    schema={meta}
+    on:change={e => (value = e.detail)}
+  />
 {:else if type === "longform"}
   {#if meta.useRichText}
     <RichTextField {error} {label} height="150px" bind:value />

packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte

@@ -149,7 +149,6 @@
   }
   const initialiseField = (field, savingColumn) => {
     isCreating = !field
-
     if (field && !savingColumn) {
       editableColumn = cloneDeep(field)
       originalName = editableColumn.name ? editableColumn.name + "" : null
@@ -171,7 +170,8 @@
           relationshipPart2 = part2
         }
       }
-    } else if (!savingColumn) {
+    }
+    if (!savingColumn && !originalName) {
       let highestNumber = 0
       Object.keys(table.schema).forEach(columnName => {
         const columnNumber = extractColumnNumber(columnName)
@@ -307,12 +307,6 @@
       dispatch("updatecolumns")
       gridDispatch("close-edit-column")
 
-      if (saveColumn.type === LINK_TYPE) {
-        // Fetching the new tables
-        tables.fetch()
-        // Fetching the new relationships
-        datasources.fetch()
-      }
       if (originalName) {
         notifications.success("Column updated successfully")
       } else {
@@ -339,11 +333,6 @@
         confirmDeleteDialog.hide()
         dispatch("updatecolumns")
         gridDispatch("close-edit-column")
-
-        if (editableColumn.type === LINK_TYPE) {
-          // Updating the relationships
-          datasources.fetch()
-        }
       }
     } catch (error) {
       notifications.error(`Error deleting column: ${error.message}`)
@@ -540,8 +529,16 @@
 <Layout noPadding gap="S">
   {#if mounted}
     <Input
+      value={editableColumn.name}
       autofocus
-      bind:value={editableColumn.name}
+      on:input={e => {
+        if (
+          !uneditable &&
+          !(linkEditDisabled && editableColumn.type === LINK_TYPE)
+        ) {
+          editableColumn.name = e.target.value
+        }
+      }}
       disabled={uneditable ||
         (linkEditDisabled && editableColumn.type === LINK_TYPE)}
       error={errors?.name}

packages/builder/src/components/backend/DatasourceNavigator/DatasourceNavigator.svelte

@@ -1,5 +1,6 @@
 <script>
   import { goto, isActive, params } from "@roxi/routify"
+  import { Layout } from "@budibase/bbui"
   import { BUDIBASE_INTERNAL_DB_ID } from "constants/backend"
   import {
     database,
@@ -21,8 +22,11 @@
   import IntegrationIcon from "./IntegrationIcon.svelte"
   import { TableNames } from "constants"
   import { userSelectedResourceMap } from "builderStore"
+  import { enrichDatasources } from "./datasourceUtils"
+  import { onMount } from "svelte"
 
-  let openDataSources = []
+  export let searchTerm
+  let toggledDatasources = {}
 
   $: enrichedDataSources = enrichDatasources(
     $datasources,
@@ -32,52 +36,9 @@
     $queries,
     $views,
     $viewsV2,
-    openDataSources
+    toggledDatasources,
+    searchTerm
   )
-  $: openDataSource = enrichedDataSources.find(x => x.open)
-  $: {
-    // Ensure the open datasource is always actually open
-    if (openDataSource) {
-      openNode(openDataSource)
-    }
-  }
-
-  const enrichDatasources = (
-    datasources,
-    params,
-    isActive,
-    tables,
-    queries,
-    views,
-    viewsV2,
-    openDataSources
-  ) => {
-    if (!datasources?.list?.length) {
-      return []
-    }
-    return datasources.list.map(datasource => {
-      const selected =
-        isActive("./datasource") &&
-        datasources.selectedDatasourceId === datasource._id
-      const open = openDataSources.includes(datasource._id)
-      const containsSelected = containsActiveEntity(
-        datasource,
-        params,
-        isActive,
-        tables,
-        queries,
-        views,
-        viewsV2
-      )
-      const onlySource = datasources.list.length === 1
-      return {
-        ...datasource,
-        selected,
-        containsSelected,
-        open: selected || open || containsSelected || onlySource,
-      }
-    })
-  }
 
   function selectDatasource(datasource) {
     openNode(datasource)
@@ -91,102 +52,42 @@
     }
   }
 
-  function closeNode(datasource) {
-    openDataSources = openDataSources.filter(id => datasource._id !== id)
-  }
-
   function openNode(datasource) {
-    if (!openDataSources.includes(datasource._id)) {
+    toggledDatasources[datasource._id] = true
-      openDataSources = [...openDataSources, datasource._id]
-    }
   }
 
   function toggleNode(datasource) {
-    const isOpen = openDataSources.includes(datasource._id)
+    toggledDatasources[datasource._id] = !datasource.open
-    if (isOpen) {
-      closeNode(datasource)
-    } else {
-      openNode(datasource)
-    }
   }
 
-  const containsActiveEntity = (
+  const appUsersTableName = "App users"
-    datasource,
+  $: showAppUsersTable =
-    params,
+    !searchTerm ||
-    isActive,
+    appUsersTableName.toLowerCase().indexOf(searchTerm.toLowerCase()) > -1
-    tables,
-    queries,
-    views,
-    viewsV2
-  ) => {
-    // Check for being on a datasource page
-    if (params.datasourceId === datasource._id) {
-      return true
-    }
 
-    // Check for hardcoded datasource edge cases
+  onMount(() => {
-    if (
+    if ($tables.selected) {
-      isActive("./datasource/bb_internal") &&
+      toggledDatasources[$tables.selected.sourceId] = true
-      datasource._id === "bb_internal"
-    ) {
-      return true
-    }
-    if (
-      isActive("./datasource/datasource_internal_bb_default") &&
-      datasource._id === "datasource_internal_bb_default"
-    ) {
-      return true
     }
+  })
 
-    // Check for a matching query
+  $: showNoResults =
-    if (params.queryId) {
+    searchTerm && !showAppUsersTable && !enrichedDataSources.find(ds => ds.show)
-      const query = queries.list?.find(q => q._id === params.queryId)
-      return datasource._id === query?.datasourceId
-    }
-
-    // If there are no entities it can't contain anything
-    if (!datasource.entities) {
-      return false
-    }
-
-    // Get a list of table options
-    let options = datasource.entities
-    if (!Array.isArray(options)) {
-      options = Object.values(options)
-    }
-
-    // Check for a matching table
-    if (params.tableId) {
-      const selectedTable = tables.selected?._id
-      return options.find(x => x._id === selectedTable) != null
-    }
-
-    // Check for a matching view
-    const selectedView = views.selected?.name
-    const viewTable = options.find(table => {
-      return table.views?.[selectedView] != null
-    })
-    if (viewTable) {
-      return true
-    }
-
-    // Check for a matching viewV2
-    const viewV2Table = options.find(x => x._id === viewsV2.selected?.tableId)
-    return viewV2Table != null
-  }
 </script>
 
 {#if $database?._id}
   <div class="hierarchy-items-container">
-    <NavItem
+    {#if showAppUsersTable}
-      icon="UserGroup"
+      <NavItem
-      text="App users"
+        icon="UserGroup"
-      selected={$isActive("./table/:tableId") &&
+        text={appUsersTableName}
-        $tables.selected?._id === TableNames.USERS}
+        selected={$isActive("./table/:tableId") &&
-      on:click={() => selectTable(TableNames.USERS)}
+          $tables.selected?._id === TableNames.USERS}
-      selectedBy={$userSelectedResourceMap[TableNames.USERS]}
+        on:click={() => selectTable(TableNames.USERS)}
-    />
+        selectedBy={$userSelectedResourceMap[TableNames.USERS]}
-    {#each enrichedDataSources as datasource}
+      />
+    {/if}
+    {#each enrichedDataSources.filter(ds => ds.show) as datasource}
       <NavItem
         border
         text={datasource.name}
@@ -210,8 +111,8 @@
       </NavItem>
 
       {#if datasource.open}
-        <TableNavigator sourceId={datasource._id} {selectTable} />
+        <TableNavigator tables={datasource.tables} {selectTable} />
-        {#each $queries.list.filter(query => query.datasourceId === datasource._id) as query}
+        {#each datasource.queries as query}
           <NavItem
             indentLevel={1}
             icon="SQLQuery"
@@ -228,6 +129,13 @@
         {/each}
       {/if}
     {/each}
+    {#if showNoResults}
+      <Layout paddingY="none" paddingX="L">
+        <div class="no-results">
+          There aren't any datasources matching that name
+        </div>
+      </Layout>
+    {/if}
   </div>
 {/if}
 
@@ -240,4 +148,8 @@
     place-items: center;
     flex: 0 0 24px;
   }
+
+  .no-results {
+    color: var(--spectrum-global-color-gray-600);
+  }
 </style>

packages/builder/src/components/backend/DatasourceNavigator/datasourceUtils.js

@@ -0,0 +1,181 @@
+import { TableNames } from "constants"
+
+const showDatasourceOpen = ({
+  selected,
+  containsSelected,
+  dsToggledStatus,
+  searchTerm,
+  onlyOneSource,
+}) => {
+  // We want to display all the ds expanded while filtering ds
+  if (searchTerm) {
+    return true
+  }
+
+  // If the toggle status has been a value
+  if (dsToggledStatus !== undefined) {
+    return dsToggledStatus
+  }
+
+  if (onlyOneSource) {
+    return true
+  }
+
+  return selected || containsSelected
+}
+
+const containsActiveEntity = (
+  datasource,
+  params,
+  isActive,
+  tables,
+  queries,
+  views,
+  viewsV2
+) => {
+  // Check for being on a datasource page
+  if (params.datasourceId === datasource._id) {
+    return true
+  }
+
+  // Check for hardcoded datasource edge cases
+  if (
+    isActive("./datasource/bb_internal") &&
+    datasource._id === "bb_internal"
+  ) {
+    return true
+  }
+  if (
+    isActive("./datasource/datasource_internal_bb_default") &&
+    datasource._id === "datasource_internal_bb_default"
+  ) {
+    return true
+  }
+
+  // Check for a matching query
+  if (params.queryId) {
+    const query = queries.list?.find(q => q._id === params.queryId)
+    return datasource._id === query?.datasourceId
+  }
+
+  // If there are no entities it can't contain anything
+  if (!datasource.entities) {
+    return false
+  }
+
+  // Get a list of table options
+  let options = datasource.entities
+  if (!Array.isArray(options)) {
+    options = Object.values(options)
+  }
+
+  // Check for a matching table
+  if (params.tableId) {
+    const selectedTable = tables.selected?._id
+    return options.find(x => x._id === selectedTable) != null
+  }
+
+  // Check for a matching view
+  const selectedView = views.selected?.name
+  const viewTable = options.find(table => {
+    return table.views?.[selectedView] != null
+  })
+  if (viewTable) {
+    return true
+  }
+
+  // Check for a matching viewV2
+  const viewV2Table = options.find(x => x._id === viewsV2.selected?.tableId)
+  return viewV2Table != null
+}
+
+export const enrichDatasources = (
+  datasources,
+  params,
+  isActive,
+  tables,
+  queries,
+  views,
+  viewsV2,
+  toggledDatasources,
+  searchTerm
+) => {
+  if (!datasources?.list?.length) {
+    return []
+  }
+
+  const onlySource = datasources.list.length === 1
+  return datasources.list.map(datasource => {
+    const selected =
+      isActive("./datasource") &&
+      datasources.selectedDatasourceId === datasource._id
+    const containsSelected = containsActiveEntity(
+      datasource,
+      params,
+      isActive,
+      tables,
+      queries,
+      views,
+      viewsV2
+    )
+
+    const dsTables = tables.list.filter(
+      table =>
+        table.sourceId === datasource._id && table._id !== TableNames.USERS
+    )
+    const dsQueries = queries.list.filter(
+      query => query.datasourceId === datasource._id
+    )
+
+    const open = showDatasourceOpen({
+      selected,
+      containsSelected,
+      dsToggledStatus: toggledDatasources[datasource._id],
+      searchTerm,
+      onlyOneSource: onlySource,
+    })
+
+    const visibleDsQueries = dsQueries.filter(
+      q =>
+        !searchTerm ||
+        q.name?.toLowerCase()?.indexOf(searchTerm.toLowerCase()) > -1
+    )
+
+    const visibleDsTables = dsTables
+      .map(t => ({
+        ...t,
+        views: !searchTerm
+          ? t.views
+          : Object.keys(t.views || {})
+              .filter(
+                viewName =>
+                  viewName.toLowerCase().indexOf(searchTerm.toLowerCase()) > -1
+              )
+              .reduce(
+                (acc, viewName) => ({ ...acc, [viewName]: t.views[viewName] }),
+                {}
+              ),
+      }))
+      .filter(
+        table =>
+          !searchTerm ||
+          table.name?.toLowerCase()?.indexOf(searchTerm.toLowerCase()) > -1 ||
+          Object.keys(table.views).length
+      )
+
+    const show = !!(
+      !searchTerm ||
+      visibleDsQueries.length ||
+      visibleDsTables.length
+    )
+    return {
+      ...datasource,
+      selected,
+      containsSelected,
+      open,
+      queries: visibleDsQueries,
+      tables: visibleDsTables,
+      show,
+    }
+  })
+}

packages/builder/src/components/backend/DatasourceNavigator/tests/datasourceUtils.spec.js

@@ -0,0 +1,219 @@
+import { enrichDatasources } from "../datasourceUtils"
+
+describe("datasourceUtils", () => {
+  describe("enrichDatasources", () => {
+    it.each([
+      ["undefined", undefined],
+      ["undefined list", {}],
+      ["empty list", { list: [] }],
+    ])("%s datasources will return an empty list", datasources => {
+      const result = enrichDatasources(datasources)
+
+      expect(result).toEqual([])
+    })
+
+    describe("filtering", () => {
+      const internalTables = {
+        _id: "datasource_internal_bb_default",
+        name: "Sample Data",
+      }
+
+      const pgDatasource = {
+        _id: "pg_ds",
+        name: "PostgreSQL local",
+      }
+
+      const mysqlDatasource = {
+        _id: "mysql_ds",
+        name: "My SQL local",
+      }
+
+      const tables = [
+        ...[
+          {
+            _id: "ta_bb_employee",
+            name: "Employees",
+          },
+          {
+            _id: "ta_bb_expenses",
+            name: "Expenses",
+          },
+          {
+            _id: "ta_bb_expenses_2",
+            name: "Expenses 2",
+          },
+          {
+            _id: "ta_bb_inventory",
+            name: "Inventory",
+          },
+          {
+            _id: "ta_bb_jobs",
+            name: "Jobs",
+          },
+        ].map(t => ({
+          ...t,
+          sourceId: internalTables._id,
+        })),
+        ...[
+          {
+            _id: "pg_ds-external_inventory",
+            name: "External Inventory",
+            views: {
+              "External Inventory first view": {
+                name: "External Inventory first view",
+                id: "pg_ds_view_1",
+              },
+              "External Inventory second view": {
+                name: "External Inventory second view",
+                id: "pg_ds_view_2",
+              },
+            },
+          },
+          {
+            _id: "pg_ds-another_table",
+            name: "Another table",
+            views: {
+              view1: {
+                id: "pg_ds-another_table-view1",
+                name: "view1",
+              },
+              ["View 2"]: {
+                id: "pg_ds-another_table-view2",
+                name: "View 2",
+              },
+            },
+          },
+          {
+            _id: "pg_ds_table2",
+            name: "table2",
+            views: {
+              "new 2": {
+                name: "new 2",
+                id: "pg_ds_table2_new_2",
+              },
+              new: {
+                name: "new",
+                id: "pg_ds_table2_new_",
+              },
+            },
+          },
+        ].map(t => ({
+          ...t,
+          sourceId: pgDatasource._id,
+        })),
+        ...[
+          {
+            _id: "mysql_ds-mysql_table",
+            name: "MySQL table",
+          },
+        ].map(t => ({
+          ...t,
+          sourceId: mysqlDatasource._id,
+        })),
+      ]
+
+      const datasources = {
+        list: [internalTables, pgDatasource, mysqlDatasource],
+      }
+      const isActive = vi.fn().mockReturnValue(true)
+
+      it("without a search term, all datasources are returned", () => {
+        const searchTerm = ""
+
+        const result = enrichDatasources(
+          datasources,
+          {},
+          isActive,
+          { list: [] },
+          { list: [] },
+          { list: [] },
+          { list: [] },
+          {},
+          searchTerm
+        )
+
+        expect(result).toEqual(
+          datasources.list.map(d =>
+            expect.objectContaining({
+              _id: d._id,
+              show: true,
+            })
+          )
+        )
+      })
+
+      it("given a valid search term, all tables are correctly filtered", () => {
+        const searchTerm = "ex"
+
+        const result = enrichDatasources(
+          datasources,
+          {},
+          isActive,
+          { list: tables },
+          { list: [] },
+          { list: [] },
+          { list: [] },
+          {},
+          searchTerm
+        )
+
+        expect(result).toEqual([
+          expect.objectContaining({
+            _id: internalTables._id,
+            show: true,
+            tables: [
+              expect.objectContaining({ _id: "ta_bb_expenses" }),
+              expect.objectContaining({ _id: "ta_bb_expenses_2" }),
+            ],
+          }),
+          expect.objectContaining({
+            _id: pgDatasource._id,
+            show: true,
+            tables: [
+              expect.objectContaining({ _id: "pg_ds-external_inventory" }),
+            ],
+          }),
+          expect.objectContaining({
+            _id: mysqlDatasource._id,
+            show: false,
+            tables: [],
+          }),
+        ])
+      })
+
+      it("given a non matching search term, all entities are empty", () => {
+        const searchTerm = "non matching"
+
+        const result = enrichDatasources(
+          datasources,
+          {},
+          isActive,
+          { list: tables },
+          { list: [] },
+          { list: [] },
+          { list: [] },
+          {},
+          searchTerm
+        )
+
+        expect(result).toEqual([
+          expect.objectContaining({
+            _id: internalTables._id,
+            show: false,
+            tables: [],
+          }),
+          expect.objectContaining({
+            _id: pgDatasource._id,
+            show: false,
+            tables: [],
+          }),
+          expect.objectContaining({
+            _id: mysqlDatasource._id,
+            show: false,
+            tables: [],
+          }),
+        ])
+      })
+    })
+  })
+})

packages/builder/src/components/backend/TableNavigator/TableNavigator.svelte

@@ -1,5 +1,10 @@
 <script>
-  import { tables, views, viewsV2, database } from "stores/backend"
+  import {
+    tables as tablesStore,
+    views,
+    viewsV2,
+    database,
+  } from "stores/backend"
   import { TableNames } from "constants"
   import EditTablePopover from "./popovers/EditTablePopover.svelte"
   import EditViewPopover from "./popovers/EditViewPopover.svelte"
@@ -7,14 +12,10 @@
   import { goto, isActive } from "@roxi/routify"
   import { userSelectedResourceMap } from "builderStore"
 
-  export let sourceId
+  export let tables
   export let selectTable
 
-  $: sortedTables = $tables.list
+  $: sortedTables = tables.sort(alphabetical)
-    .filter(
-      table => table.sourceId === sourceId && table._id !== TableNames.USERS
-    )
-    .sort(alphabetical)
 
   const alphabetical = (a, b) => {
     return a.name?.toLowerCase() > b.name?.toLowerCase() ? 1 : -1
@@ -37,7 +38,7 @@
         icon={table._id === TableNames.USERS ? "UserGroup" : "Table"}
         text={table.name}
         selected={$isActive("./table/:tableId") &&
-          $tables.selected?._id === table._id}
+          $tablesStore.selected?._id === table._id}
         on:click={() => selectTable(table._id)}
         selectedBy={$userSelectedResourceMap[table._id]}
       >

packages/builder/src/components/common/LinkedRowSelector.svelte

@@ -56,12 +56,12 @@
   />
 {:else}
   <Multiselect
-    bind:value={linkedIds}
+    value={linkedIds}
     {label}
     options={rows}
     getOptionLabel={getPrettyName}
     getOptionValue={row => row._id}
     sort
-    on:change={() => dispatch("change", linkedIds)}
+    on:change
   />
 {/if}

packages/builder/src/components/common/NavHeader.svelte

@@ -0,0 +1,149 @@
+<script>
+  import { tick } from "svelte"
+  import { Icon, Body } from "@budibase/bbui"
+  import { keyUtils } from "helpers/keyUtils"
+
+  export let title
+  export let placeholder
+  export let value
+  export let onAdd
+  export let search
+
+  let searchInput
+
+  const openSearch = async () => {
+    search = true
+    await tick()
+    searchInput.focus()
+  }
+
+  const closeSearch = async () => {
+    search = false
+    value = ""
+  }
+
+  const onKeyDown = e => {
+    if (e.key === "Escape") {
+      closeSearch()
+    }
+  }
+
+  const handleAddButton = () => {
+    if (search) {
+      closeSearch()
+    } else {
+      onAdd()
+    }
+  }
+</script>
+
+<svelte:window on:keydown={onKeyDown} />
+
+<div class="header" class:search>
+  <input
+    readonly={!search}
+    bind:value
+    bind:this={searchInput}
+    class="searchBox"
+    class:hide={!search}
+    {placeholder}
+  />
+
+  <div class="title" class:hide={search}>
+    <Body size="S">{title}</Body>
+  </div>
+  <div
+    on:click={openSearch}
+    on:keydown={keyUtils.handleEnter(openSearch)}
+    class="searchButton"
+    class:hide={search}
+  >
+    <Icon size="S" name="Search" />
+  </div>
+
+  <div
+    on:click={handleAddButton}
+    on:keydown={keyUtils.handleEnter(handleAddButton)}
+    class="addButton"
+    class:rotate={search}
+  >
+    <Icon name="Add" />
+  </div>
+</div>
+
+<style>
+  .search {
+    transition: height 300ms ease-out;
+    max-height: none;
+  }
+
+  .header {
+    flex-shrink: 0;
+    flex-direction: row;
+    position: relative;
+    height: 50px;
+    width: 100%;
+    box-sizing: border-box;
+    display: flex;
+    align-items: center;
+    justify-content: space-between;
+    border-bottom: 2px solid transparent;
+    transition: border-bottom 130ms ease-out;
+    gap: var(--spacing-l);
+  }
+
+  .searchBox {
+    font-family: var(--font-sans);
+    color: var(--ink);
+    background-color: transparent;
+    border: none;
+    font-size: var(--spectrum-alias-font-size-default);
+    display: flex;
+  }
+  .searchBox:focus {
+    outline: none;
+  }
+  .searchBox::placeholder {
+    color: var(--spectrum-global-color-gray-600);
+  }
+
+  .title {
+    display: flex;
+    align-items: center;
+    height: 100%;
+    box-sizing: border-box;
+    flex: 1;
+    opacity: 1;
+    z-index: 1;
+  }
+
+  .searchButton {
+    color: var(--grey-7);
+    cursor: pointer;
+    opacity: 1;
+  }
+  .searchButton:hover {
+    color: var(--ink);
+  }
+
+  .hide {
+    opacity: 0;
+    pointer-events: none;
+    display: none !important;
+  }
+
+  .addButton {
+    display: flex;
+    transition: transform 300ms ease-out;
+    color: var(--grey-7);
+    cursor: pointer;
+  }
+
+  .addButton:hover {
+    color: var(--ink);
+  }
+
+  .rotate {
+    transform: rotate(45deg);
+  }
+</style>

packages/builder/src/components/common/NavItem.svelte

@@ -1,10 +1,11 @@
 <script>
-  import { Icon } from "@budibase/bbui"
+  import { AbsTooltip, Icon } from "@budibase/bbui"
   import { createEventDispatcher, getContext } from "svelte"
   import { helpers } from "@budibase/shared-core"
   import { UserAvatars } from "@budibase/frontend-core"
 
   export let icon
+  export let iconTooltip
   export let withArrow = false
   export let withActions = true
   export let indentLevel = 0
@@ -22,6 +23,7 @@
   export let showTooltip = false
   export let selectedBy = null
   export let compact = false
+  export let hovering = false
 
   const scrollApi = getContext("scroll")
   const dispatch = createEventDispatcher()
@@ -60,6 +62,7 @@
 
 <div
   class="nav-item"
+  class:hovering
   class:border
   class:selected
   class:withActions
@@ -70,6 +73,8 @@
   on:dragstart
   on:dragover
   on:drop
+  on:mouseenter
+  on:mouseleave
   on:click={onClick}
   ondragover="return false"
   ondragenter="return false"
@@ -77,7 +82,11 @@
   {style}
   {draggable}
 >
-  <div class="nav-item-content" bind:this={contentRef}>
+  <div
+    class="nav-item-content"
+    bind:this={contentRef}
+    class:right={rightAlignIcon}
+  >
     {#if withArrow}
       <div
         class:opened
@@ -98,7 +107,9 @@
       </div>
     {:else if icon}
       <div class="icon" class:right={rightAlignIcon}>
-        <Icon color={iconColor} size="S" name={icon} />
+        <AbsTooltip type="info" position="right" text={iconTooltip}>
+          <Icon color={iconColor} size="S" name={icon} />
+        </AbsTooltip>
       </div>
     {/if}
     <div class="text" title={showTooltip ? text : null}>
@@ -145,15 +156,17 @@
     --avatars-background: var(--spectrum-global-color-gray-200);
   }
   .nav-item.selected {
-    background-color: var(--spectrum-global-color-gray-300);
+    background-color: var(--spectrum-global-color-gray-300) !important;
     --avatars-background: var(--spectrum-global-color-gray-300);
     color: var(--ink);
   }
-  .nav-item:hover {
+  .nav-item:hover,
-    background-color: var(--spectrum-global-color-gray-300);
+  .hovering {
+    background-color: var(--spectrum-global-color-gray-200);
     --avatars-background: var(--spectrum-global-color-gray-300);
   }
-  .nav-item:hover .actions {
+  .nav-item:hover .actions,
+  .hovering .actions {
     visibility: visible;
   }
   .nav-item-content {
@@ -166,6 +179,11 @@
     width: max-content;
     position: relative;
     padding-left: var(--spacing-l);
+    box-sizing: border-box;
+  }
+
+  .nav-item-content.right {
+    width: 100%;
   }
 
   /* Needed to fully display the actions icon */
@@ -189,6 +207,7 @@
     flex: 0 0 20px;
     pointer-events: all;
     order: 0;
+    transition: transform 100ms linear;
   }
   .icon.arrow.absolute {
     position: absolute;
@@ -263,6 +282,7 @@
   }
 
   .right {
+    margin-left: auto;
     order: 10;
   }
 </style>

packages/builder/src/components/common/RoleSelect.svelte

@@ -20,73 +20,91 @@
   export let allowedRoles = null
   export let allowCreator = false
   export let fancySelect = false
+  export let labelPrefix = null
 
   const dispatch = createEventDispatcher()
   const RemoveID = "remove"
 
+  $: enrichLabel = label => (labelPrefix ? `${labelPrefix} ${label}` : label)
   $: options = getOptions(
     $roles,
     allowPublic,
     allowRemove,
     allowedRoles,
-    allowCreator
+    allowCreator,
+    enrichLabel
   )
+
   const getOptions = (
     roles,
     allowPublic,
     allowRemove,
     allowedRoles,
-    allowCreator
+    allowCreator,
+    enrichLabel
   ) => {
+    // Use roles whitelist if specified
     if (allowedRoles?.length) {
-      const filteredRoles = roles.filter(role =>
+      let options = roles
-        allowedRoles.includes(role._id)
+        .filter(role => allowedRoles.includes(role._id))
-      )
+        .map(role => ({
-      return [
+          name: enrichLabel(role.name),
-        ...filteredRoles,
+          _id: role._id,
-        ...(allowedRoles.includes(Constants.Roles.CREATOR)
+        }))
-          ? [{ _id: Constants.Roles.CREATOR, name: "Creator", enabled: false }]
+      if (allowedRoles.includes(Constants.Roles.CREATOR)) {
-          : []),
+        options.push({
-      ]
-    }
-    let newRoles = [...roles]
-
-    if (allowCreator) {
-      newRoles = [
-        {
           _id: Constants.Roles.CREATOR,
-          name: "Creator",
+          name: "Can edit",
-          tag:
+          enabled: false,
-            !$licensing.perAppBuildersEnabled &&
+        })
-            capitalise(Constants.PlanType.BUSINESS),
+      }
-        },
+      return options
-        ...newRoles,
-      ]
     }
+
+    // Allow all core roles
+    let options = roles.map(role => ({
+      name: enrichLabel(role.name),
+      _id: role._id,
+    }))
+
+    // Add creator if required
+    if (allowCreator) {
+      options.unshift({
+        _id: Constants.Roles.CREATOR,
+        name: "Can edit",
+        tag:
+          !$licensing.perAppBuildersEnabled &&
+          capitalise(Constants.PlanType.BUSINESS),
+      })
+    }
+
+    // Add remove option if required
     if (allowRemove) {
-      newRoles = [
+      options.push({
-        ...newRoles,
+        _id: RemoveID,
-        {
+        name: "Remove",
-          _id: RemoveID,
+      })
-          name: "Remove",
-        },
-      ]
     }
-    if (allowPublic) {
+
-      return newRoles
+    // Remove public if not allowed
+    if (!allowPublic) {
+      options = options.filter(role => role._id !== Constants.Roles.PUBLIC)
     }
-    return newRoles.filter(role => role._id !== Constants.Roles.PUBLIC)
+
+    return options
   }
 
   const getColor = role => {
-    if (allowRemove && role._id === RemoveID) {
+    // Creator and remove options have no colors
+    if (role._id === Constants.Roles.CREATOR || role._id === RemoveID) {
       return null
     }
     return RoleUtils.getRoleColour(role._id)
   }
 
   const getIcon = role => {
-    if (allowRemove && role._id === RemoveID) {
+    // Only remove option has an icon
+    if (role._id === RemoveID) {
       return "Close"
     }
     return null

packages/builder/src/components/common/resizable.js

@@ -0,0 +1,119 @@
+const getResizeActions = (
+  cssProperty,
+  mouseMoveEventProperty,
+  elementProperty,
+  initialValue,
+  setValue = () => {}
+) => {
+  let element = null
+
+  const elementAction = node => {
+    element = node
+
+    if (initialValue != null) {
+      element.style[cssProperty] = `${initialValue}px`
+    }
+
+    return {
+      destroy() {
+        element = null
+      },
+    }
+  }
+
+  const dragHandleAction = node => {
+    let startProperty = null
+    let startPosition = null
+
+    const handleMouseMove = e => {
+      e.preventDefault() // Prevent highlighting while dragging
+      const change = e[mouseMoveEventProperty] - startPosition
+      element.style[cssProperty] = `${startProperty + change}px`
+    }
+
+    const handleMouseUp = e => {
+      e.preventDefault() // Prevent highlighting while dragging
+      window.removeEventListener("mousemove", handleMouseMove)
+      window.removeEventListener("mouseup", handleMouseUp)
+
+      element.style.removeProperty("transition") // remove temporary transition override
+      for (let item of document.getElementsByTagName("iframe")) {
+        item.style.removeProperty("pointer-events")
+      }
+
+      setValue(element[elementProperty])
+    }
+
+    const handleMouseDown = e => {
+      if (e.detail > 1) {
+        // e.detail is the number of rapid clicks, so e.detail = 2 is
+        // a double click. We want to prevent default behaviour in
+        // this case as it highlights nearby selectable elements, which
+        // then interferes with the resizing mousemove.
+        // Putting this on the double click handler doesn't seem to
+        // work, so it must go here.
+        e.preventDefault()
+      }
+
+      if (
+        e.target.hasAttribute("disabled") &&
+        e.target.getAttribute("disabled") !== "false"
+      ) {
+        return
+      }
+
+      element.style.transition = `${cssProperty} 0ms` // temporarily override any height transitions
+
+      // iframes swallow mouseup events if your cursor ends up over it during a drag, so make them
+      // temporarily non-interactive
+      for (let item of document.getElementsByTagName("iframe")) {
+        item.style.pointerEvents = "none"
+      }
+
+      startProperty = element[elementProperty]
+      startPosition = e[mouseMoveEventProperty]
+
+      window.addEventListener("mousemove", handleMouseMove)
+      window.addEventListener("mouseup", handleMouseUp)
+    }
+
+    const handleDoubleClick = () => {
+      element.style.removeProperty(cssProperty)
+    }
+
+    node.addEventListener("mousedown", handleMouseDown)
+    node.addEventListener("dblclick", handleDoubleClick)
+
+    return {
+      destroy() {
+        node.removeEventListener("mousedown", handleMouseDown)
+        node.removeEventListener("dblclick", handleDoubleClick)
+      },
+    }
+  }
+
+  return [elementAction, dragHandleAction]
+}
+
+export const getVerticalResizeActions = (initialValue, setValue = () => {}) => {
+  return getResizeActions(
+    "height",
+    "pageY",
+    "clientHeight",
+    initialValue,
+    setValue
+  )
+}
+
+export const getHorizontalResizeActions = (
+  initialValue,
+  setValue = () => {}
+) => {
+  return getResizeActions(
+    "width",
+    "pageX",
+    "clientWidth",
+    initialValue,
+    setValue
+  )
+}

packages/builder/src/components/design/Panel.svelte

@@ -1,8 +1,9 @@
 <script>
-  import { Icon, Body } from "@budibase/bbui"
+  import { AbsTooltip, Icon, Body } from "@budibase/bbui"
 
   export let title
   export let icon
+  export let iconTooltip
   export let showAddButton = false
   export let showBackButton = false
   export let showCloseButton = false
@@ -11,10 +12,12 @@
   export let onClickCloseButton
   export let borderLeft = false
   export let borderRight = false
+  export let borderBottomHeader = true
   export let wide = false
   export let extraWide = false
   export let closeButtonIcon = "Close"
-
+  export let noHeaderBorder = false
+  export let titleCSS = true
   $: customHeaderContent = $$slots["panel-header-content"]
   $: customTitleContent = $$slots["panel-title-content"]
 </script>
@@ -26,14 +29,21 @@
   class:borderLeft
   class:borderRight
 >
-  <div class="header" class:custom={customHeaderContent}>
+  <div
+    class="header"
+    class:custom={customHeaderContent}
+    class:borderBottom={borderBottomHeader}
+    class:noHeaderBorder
+  >
     {#if showBackButton}
       <Icon name="ArrowLeft" hoverable on:click={onClickBackButton} />
     {/if}
     {#if icon}
-      <Icon name={icon} />
+      <AbsTooltip type="info" text={iconTooltip}>
+        <Icon name={icon} />
+      </AbsTooltip>
     {/if}
-    <div class="title">
+    <div class:title={titleCSS}>
       {#if customTitleContent}
         <slot name="panel-title-content" />
       {:else}
@@ -63,6 +73,7 @@
 
 <style>
   .panel {
+    min-width: 260px;
     width: 260px;
     flex: 0 0 260px;
     background: var(--background);
@@ -80,6 +91,7 @@
     border-right: var(--border-light);
   }
   .panel.wide {
+    min-width: 310px;
     width: 310px;
     flex: 0 0 310px;
   }
@@ -94,9 +106,15 @@
     justify-content: space-between;
     align-items: center;
     padding: 0 var(--spacing-l);
-    border-bottom: var(--border-light);
     gap: var(--spacing-m);
   }
+
+  .noHeaderBorder {
+    border-bottom: none !important;
+  }
+  .header.borderBottom {
+    border-bottom: var(--border-light);
+  }
   .title {
     flex: 1 1 auto;
     width: 0;

packages/builder/src/components/design/settings/componentSettings.js

@@ -20,7 +20,7 @@ import ValidationEditor from "./controls/ValidationEditor/ValidationEditor.svelt
 import DrawerBindableInput from "components/common/bindings/DrawerBindableInput.svelte"
 import ColumnEditor from "./controls/ColumnEditor/ColumnEditor.svelte"
 import BasicColumnEditor from "./controls/ColumnEditor/BasicColumnEditor.svelte"
-import GridColumnEditor from "./controls/ColumnEditor/GridColumnEditor.svelte"
+import GridColumnEditor from "./controls/GridColumnConfiguration/GridColumnConfiguration.svelte"
 import BarButtonList from "./controls/BarButtonList.svelte"
 import FieldConfiguration from "./controls/FieldConfiguration/FieldConfiguration.svelte"
 import ButtonConfiguration from "./controls/ButtonConfiguration/ButtonConfiguration.svelte"
@@ -28,6 +28,7 @@ import RelationshipFilterEditor from "./controls/RelationshipFilterEditor.svelte
 
 const componentMap = {
   text: DrawerBindableInput,
+  plainText: Input,
   select: Select,
   radio: RadioGroup,
   dataSource: DataSourceSelect,

packages/builder/src/components/design/settings/controls/ButtonActionEditor/ButtonActionDrawer.svelte

@@ -15,6 +15,7 @@
     getEventContextBindings,
     getActionBindings,
     makeStateBinding,
+    updateReferencesInObject,
   } from "builderStore/dataBinding"
   import { cloneDeep } from "lodash/fp"
 
@@ -30,6 +31,7 @@
 
   let actionQuery
   let selectedAction = actions?.length ? actions[0] : null
+  let originalActionIndex
 
   const setUpdateActions = actions => {
     return actions
@@ -115,6 +117,14 @@
     if (isSelected) {
       selectedAction = actions?.length ? actions[0] : null
     }
+
+    // Update action binding references
+    updateReferencesInObject({
+      obj: actions,
+      modifiedIndex: index,
+      action: "delete",
+      label: "actions",
+    })
   }
 
   const toggleActionList = () => {
@@ -137,6 +147,7 @@
 
   const selectAction = action => () => {
     selectedAction = action
+    originalActionIndex = actions.findIndex(item => item.id === action.id)
   }
 
   const onAddAction = actionType => {
@@ -146,9 +157,29 @@
 
   function handleDndConsider(e) {
     actions = e.detail.items
+
+    // set the initial index of the action being dragged
+    if (e.detail.info.trigger === "draggedEntered") {
+      originalActionIndex = actions.findIndex(
+        action => action.id === e.detail.info.id
+      )
+    }
   }
   function handleDndFinalize(e) {
     actions = e.detail.items
+
+    // Update action binding references
+    updateReferencesInObject({
+      obj: actions,
+      modifiedIndex: actions.findIndex(
+        action => action.id === e.detail.info.id
+      ),
+      action: "move",
+      label: "actions",
+      originalIndex: originalActionIndex,
+    })
+
+    originalActionIndex = -1
   }
 
   const getAllBindings = (actionBindings, eventContextBindings, actions) => {
@@ -289,7 +320,7 @@
   </Layout>
   <Layout noPadding>
     {#if selectedActionComponent && !showAvailableActions}
-      {#key selectedAction.id}
+      {#key (selectedAction.id, originalActionIndex)}
         <div class="selected-action-container">
           <svelte:component
             this={selectedActionComponent}

packages/builder/src/components/design/settings/controls/ButtonConfiguration/ButtonSetting.svelte

@@ -55,7 +55,10 @@
       size="S"
       name="Close"
       hoverable
-      on:click={() => removeButton(item._id)}
+      on:click={e => {
+        e.stopPropagation()
+        removeButton(item._id)
+      }}
     />
   </div>
 </div>

packages/builder/src/components/design/settings/controls/ColumnEditor/GridColumnEditor.svelte

@@ -1,10 +0,0 @@
-<script>
-  import ColumnEditor from "./ColumnEditor.svelte"
-</script>
-
-<ColumnEditor
-  {...$$props}
-  on:change
-  allowCellEditing={false}
-  allowReorder={false}
-/>

packages/builder/src/components/design/settings/controls/DraggableList/DraggableList.svelte

@@ -32,11 +32,14 @@
   }
 
   const dispatch = createEventDispatcher()
-  const flipDurationMs = 150
 
   let anchors = {}
   let draggableItems = []
 
+  // Used for controlling cursor behaviour in order to limit drag behaviour
+  // to the drag handle
+  let inactive = true
+
   const buildDraggable = items => {
     return items
       .map(item => {
@@ -64,6 +67,7 @@
   }
 
   const handleFinalize = e => {
+    inactive = true
     updateRowOrder(e)
     dispatch("change", serialiseUpdate())
   }
@@ -77,24 +81,36 @@
   class="list-wrap"
   use:dndzone={{
     items: draggableItems,
-    flipDurationMs,
     dropTargetStyle: { outline: "none" },
-    dragDisabled: !draggable,
+    dragDisabled: !draggable || inactive,
   }}
   on:finalize={handleFinalize}
   on:consider={updateRowOrder}
 >
-  {#each draggableItems as draggable (draggable.id)}
+  {#each draggableItems as draggableItem (draggableItem.id)}
     <li
+      on:click={() => {
+        get(store).actions.select(draggableItem.id)
+      }}
       on:mousedown={() => {
         get(store).actions.select()
       }}
-      bind:this={anchors[draggable.id]}
+      bind:this={anchors[draggableItem.id]}
-      class:highlighted={draggable.id === $store.selected}
+      class:highlighted={draggableItem.id === $store.selected}
     >
       <div class="left-content">
         {#if showHandle}
-          <div class="handle">
+          <div
+            class="handle"
+            aria-label="drag-handle"
+            style={!inactive ? "cursor:grabbing" : "cursor:grab"}
+            on:mousedown={() => {
+              inactive = false
+            }}
+            on:mouseup={() => {
+              inactive = true
+            }}
+          >
             <DragHandle />
           </div>
         {/if}
@@ -102,8 +118,8 @@
       <div class="right-content">
         <svelte:component
           this={listType}
-          anchor={anchors[draggable.item._id]}
+          anchor={anchors[draggableItem.item._id]}
-          item={draggable.item}
+          item={draggableItem.item}
           {...listTypeProps}
           on:change={onItemChanged}
         />
@@ -143,6 +159,7 @@
       --spectrum-table-row-background-color-hover,
       var(--spectrum-alias-highlight-hover)
     );
+    cursor: pointer;
   }
   .list-wrap > li:first-child {
     border-top-left-radius: 4px;
@@ -165,6 +182,9 @@
     display: flex;
     height: var(--spectrum-global-dimension-size-150);
   }
+  .handle:hover {
+    cursor: grab;
+  }
   .handle :global(svg) {
     fill: var(--spectrum-global-color-gray-500);
     margin-right: var(--spacing-m);