Merge remote-tracking branch 'origin/master' into fix/pc-bug-fixes

.github/workflows/budibase_ci.yml

@@ -99,11 +99,6 @@ jobs:
           else
             yarn test --ignore=@budibase/worker --ignore=@budibase/server --ignore=@budibase/pro
           fi
-      - uses: codecov/codecov-action@v3
-        with:
-          token: ${{ secrets.CODECOV_TOKEN }} # not required for public repos
-          name: codecov-umbrella
-          verbose: true
 
   test-worker:
     runs-on: ubuntu-latest
@@ -129,12 +124,6 @@ jobs:
             yarn test --scope=@budibase/worker
           fi
 
-      - uses: codecov/codecov-action@v3
-        with:
-          token: ${{ secrets.CODECOV_TOKEN || github.token }} # not required for public repos
-          name: codecov-umbrella
-          verbose: true
-
   test-server:
     runs-on: ubuntu-latest
     steps:
@@ -159,12 +148,6 @@ jobs:
             yarn test --scope=@budibase/server
           fi
 
-      - uses: codecov/codecov-action@v3
-        with:
-          token: ${{ secrets.CODECOV_TOKEN || github.token }} # not required for public repos
-          name: codecov-umbrella
-          verbose: true
-
   test-pro:
     runs-on: ubuntu-latest
     if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == 'Budibase/budibase'

.github/workflows/stale_bot.yml

@@ -10,6 +10,7 @@ jobs:
     steps:
       - uses: actions/stale@v8
         with:
+          days-before-stale: 330
           operations-per-run: 1
           # stale rules for PRs
           days-before-pr-stale: 7

CODEOWNERS

@@ -0,0 +1,3 @@
+/packages/server         @Budibase/backend
+/packages/worker         @Budibase/backend
+/packages/backend-core   @Budibase/backend

charts/budibase/Chart.lock

@@ -1,9 +1,6 @@
 dependencies:
 - name: couchdb
   repository: https://apache.github.io/couchdb-helm
-  version: 3.3.4
+  version: 4.3.0
-- name: ingress-nginx
+digest: sha256:94449a7f195b186f5af33ec5aa66d58b36bede240fae710f021ca87837b30606
-  repository: https://kubernetes.github.io/ingress-nginx
+generated: "2023-11-20T17:43:02.777596Z"
-  version: 4.0.13
-digest: sha256:20892705c2d8e64c98257d181063a514ac55013e2b43399a6e54868a97f97845
-generated: "2021-12-30T18:55:30.878411Z"

charts/budibase/Chart.yaml

@@ -17,10 +17,6 @@ version: 0.0.0
 appVersion: 0.0.0
 dependencies:
   - name: couchdb
-    version: 3.3.4
+    version: 4.3.0
     repository: https://apache.github.io/couchdb-helm
     condition: services.couchdb.enabled
-  - name: ingress-nginx
-    version: 4.0.13
-    repository: https://kubernetes.github.io/ingress-nginx
-    condition: ingress.nginx

charts/budibase/README.md

@@ -1,39 +1,216 @@
-# Budibase
+# budibase
 
-[Budibase](https://budibase.com/) Budibase is an open source low-code platform, helping thousands of teams build apps for their workplace in minutes.
+Budibase is an open source low-code platform, helping thousands of teams build apps for their workplace in minutes.
-
-## TL;DR;
-```console
-$ cd chart
-$ helm install budibase .
-```
-
-## Introduction
-
-This chart bootstraps a [Budibase](https://budibase.com/) deployment on a [Kubernetes](http://kubernetes.io) cluster using the [Helm](https://helm.sh) package manager.
 
 ## Prerequisites
 
-- helm v3 or above
+- `helm` v3 or above
 - Kubernetes 1.4+
-- PV provisioner support in the underlying infrastructure (with persistence storage enabled)
+- A storage controller (if you want to use persistent storage)
+- An ingress controller (if you want to define an `Ingress` resource)
+- `metrics-server` (if you want to make use of horizontal pod autoscaling)
 
-## Installing the Chart
+## Chart dependencies
 
-To install the chart with the release name `budi-release`:
+This chart depends on the official Apache CouchDB chart. You can see its
+documentation here:
+<https://github.com/apache/couchdb-helm/tree/couchdb-4.3.0/couchdb>.
+
+## Upgrading
+
+### `2.x` to `3.0.0`
+
+We made a number of breaking changes in this release to make the chart more
+idiomatic and easier to use.
+
+1. We no longer bundle `ingress-nginx`. If you were relying on this to supply
+   an ingress controller to your cluster, you will now need to deploy that
+   separately. You'll find guidance for that here:
+   <https://kubernetes.github.io/ingress-nginx/>.
+2. We've upgraded the version of the [CouchDB chart](https://github.com/apache/couchdb-helm)
+   we use from `3.3.4` to `4.3.0`. The primary motivation for this was to align
+   the CouchDB chart used with the CouchDB version used, which has also updated
+   from 3.1.1 to 3.2.1. Additionally, we're moving away from the official CouchDB
+   to one we're building ourselves.
+3. We've separated out the supplied AWS ALB ingress resource for those deploying
+   into EKS. Where previously you enabled this by setting `ingress.enabled: false`
+   and `ingress.aws: true`, you now set `awsAlbIngress.enabled: true` and all
+   configuration for it is under `awsAlbIngress`.
+4. The `HorizontalPodAutoscaler` that was configured at `hpa.enabled: true` has
+   been split into 3 separate HPAs, one for each of `apps`, `worker`, and `proxy`.
+   They are configured at `services.{apps,worker,proxy}.autoscaling`.
+
+## Installing
+
+To install the chart from our repository:
 
 ```console
-$ helm install budi-release .
+$ helm repo add budibase https://budibase.github.io/budibase/
+$ helm repo update
+$ helm install --create-namespace --namespace budibase budibase budibase/budibase
 ```
 
-The command deploys Budibase on the Kubernetes cluster in the default configuration. The [configuration](#configuration) section lists the parameters that can be configured during installation.
+To install the chart from this repo:
-
-> **Tip**: List all releases using `helm list`
-
-## Uninstalling the Chart
-
-To uninstall/delete the `my-release` deployment:
 
 ```console
-$ helm delete my-release
+$ git clone git@github.com:budibase/budibase.git
+$ cd budibase/charts/budibase
+$ helm install --create-namespace --namespace budibase budibase .
 ```
+
+## Example minimal configuration
+
+Here's an example `values.yaml` that would get a Budibase instance running in a home
+cluster using an nginx ingress controller and NFS as cluster storage (basically one of our
+staff's homelabs).
+
+<details>
+
+```yaml
+ingress:
+  enabled: true
+  className: "nginx"
+  hosts:
+    - host: budibase.local # set this to whatever DNS name you'd use
+      paths:
+        - backend:
+            service:
+              name: proxy-service
+              port:
+                number: 10000
+          path: /
+          pathType: Prefix
+
+couchdb:
+  persistentVolume:
+    enabled: true
+    storageClass: "nfs-client"
+  adminPassword: admin
+
+  objectStore:
+    storageClass: "nfs-client"
+  redis:
+    storageClass: "nfs-client"
+```
+
+If you wanted to use this when bringing up Budibase in your own cluster, you could save it
+to your hard disk and run the following:
+
+```console
+$ helm install --create-namespace --namespace budibase budibase . -f values.yaml
+```
+
+</details>
+
+## Configuring
+
+| Key | Type | Default | Description |
+|-----|------|---------|-------------|
+| affinity | object | `{}` | Sets the affinity for all pods created by this chart. Should not ordinarily need to be changed.  See <https://kubernetes.io/docs/tasks/configure-pod-container/assign-pods-nodes-using-node-affinity/> for more information on affinity. |
+| awsAlbIngress.certificateArn | string | `""` | If you're wanting to use HTTPS, you'll need to create an ACM certificate and specify the ARN here. |
+| awsAlbIngress.enabled | bool | `false` | Whether to create an ALB Ingress resource pointing to the Budibase proxy. Requires the AWS ALB Ingress Controller. |
+| couchdb.clusterSize | int | `1` | The number of replicas to run in the CouchDB cluster. We set this to 1 by default to make things simpler, but you can set it to 3 if you need a high-availability CouchDB cluster. |
+| couchdb.couchdbConfig.couchdb.uuid | string | `"budibase-couchdb"` | Unique identifier for this CouchDB server instance. You shouldn't need to change this. |
+| couchdb.image | object | `{}` | We use a custom CouchDB image for running Budibase and we don't support using any other CouchDB image. You shouldn't change this, and if you do we can't guarantee that Budibase will work. |
+| globals.apiEncryptionKey | string | `""` | Used for encrypting API keys and environment variables when stored in the database. You don't need to set this if `createSecrets` is true. |
+| globals.appVersion | string | `""` | The version of Budibase to deploy. Defaults to what's specified by {{ .Chart.AppVersion }}. Ends up being used as the image version tag for the apps, proxy, and worker images. |
+| globals.automationMaxIterations | string | `"200"` | The maximum number of iterations allows for an automation loop step. You can read more about looping here: <https://docs.budibase.com/docs/looping>. |
+| globals.budibaseEnv | string | `"PRODUCTION"` | Sets the environment variable BUDIBASE_ENVIRONMENT for the apps and worker pods. Should not ordinarily need to be changed. |
+| globals.cookieDomain | string | `""` | Sets the domain attribute of the cookie that Budibase uses to store session information. See <https://developer.mozilla.org/en-US/docs/Web/HTTP/Cookies#define_where_cookies_are_sent> for details on why you might want to set this. |
+| globals.createSecrets | bool | `true` | Create an internal API key, JWT secret, object store access key and secret, and store them in a Kubernetes `Secret`. |
+| globals.enableAnalytics | string | `"1"` | Whether to enable analytics or not. You can read more about our analytics here: <https://docs.budibase.com/docs/analytics>. |
+| globals.google | object | `{"clientId":"","secret":""}` | Google OAuth settings. These can also be set in the Budibase UI, see <https://docs.budibase.com/docs/sso-with-google> for details. |
+| globals.google.clientId | string | `""` | Client ID of your Google OAuth app. |
+| globals.google.secret | string | `""` | Client secret of your Google OAuth app. |
+| globals.httpMigrations | string | `"0"` | Whether or not to enable doing data migrations over the HTTP API. If this is set to "0", migrations are run on startup. You shouldn't ordinarily need to change this. |
+| globals.internalApiKey | string | `""` | API key used for internal Budibase API calls. You don't need to set this if `createSecrets` is true. |
+| globals.internalApiKeyFallback | string | `""` | A fallback value for `internalApiKey`. If you're rotating your encryption key, you can set this to the old value for the duration of the rotation. |
+| globals.jwtSecret | string | `""` | Secret used for signing JWTs. You don't need to set this if `createSecrets` is true. |
+| globals.jwtSecretFallback | string | `""` | A fallback value for `jwtSecret`. If you're rotating your JWT secret, you can set this to the old value for the duration of the rotation. |
+| globals.platformUrl | string | `""` | Set the `platformUrl` binding. You can also do this in Settings > Organisation if you are self-hosting. |
+| globals.smtp.enabled | bool | `false` | Whether to enable SMTP or not. |
+| globals.smtp.from | string | `""` | The email address to use in the "From:" field of emails sent by Budibase. |
+| globals.smtp.host | string | `""` | The hostname of your SMTP server. |
+| globals.smtp.password | string | `""` | The password to use when authenticating with your SMTP server. |
+| globals.smtp.port | string | `"587"` | The port of your SMTP server. |
+| globals.smtp.user | string | `""` | The username to use when authenticating with your SMTP server. |
+| globals.tenantFeatureFlags | string | `"*:LICENSING,*:USER_GROUPS,*:ONBOARDING_TOUR"` | Sets what feature flags are enabled and for which tenants. Should not ordinarily need to be changed. |
+| imagePullSecrets | list | `[]` | Passed to all pods created by this chart. Should not ordinarily need to be changed. |
+| ingress.className | string | `""` | What ingress class to use. |
+| ingress.enabled | bool | `true` | Whether to create an Ingress resource pointing to the Budibase proxy. |
+| ingress.hosts | list | `[]` | Standard hosts block for the Ingress resource. Defaults to pointing to the Budibase proxy. |
+| nameOverride | string | `""` | Override the name of the deploymen. Defaults to {{ .Chart.Name }}. |
+| service.port | int | `10000` | Port to expose on the service. |
+| service.type | string | `"ClusterIP"` | Service type for the service that points to the main Budibase proxy pod. |
+| serviceAccount.annotations | object | `{}` | Annotations to add to the service account |
+| serviceAccount.create | bool | `true` | Specifies whether a service account should be created |
+| serviceAccount.name | string | `""` | The name of the service account to use. If not set and create is true, a name is generated using the fullname template |
+| services.apps.autoscaling.enabled | bool | `false` | Whether to enable horizontal pod autoscaling for the apps service. |
+| services.apps.autoscaling.maxReplicas | int | `10` |  |
+| services.apps.autoscaling.minReplicas | int | `1` |  |
+| services.apps.autoscaling.targetCPUUtilizationPercentage | int | `80` | Target CPU utilization percentage for the apps service. Note that for autoscaling to work, you will need to have metrics-server configured, and resources set for the apps pods. |
+| services.apps.httpLogging | int | `1` | Whether or not to log HTTP requests to the apps service. |
+| services.apps.livenessProbe | object | HTTP health checks. | Liveness probe configuration for apps pods. You shouldn't need to change this, but if you want to you can find more information here: <https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/> |
+| services.apps.logLevel | string | `"info"` | The log level for the apps service. |
+| services.apps.readinessProbe | object | HTTP health checks. | Readiness probe configuration for apps pods. You shouldn't need to change this, but if you want to you can find more information here: <https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/> |
+| services.apps.replicaCount | int | `1` | The number of apps replicas to run. |
+| services.apps.resources | object | `{}` | The resources to use for apps pods. See <https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/> for more information on how to set these. |
+| services.apps.startupProbe | object | HTTP health checks. | Startup probe configuration for apps pods. You shouldn't need to change this, but if you want to you can find more information here: <https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/> |
+| services.couchdb.backup.enabled | bool | `false` | Whether or not to enable periodic CouchDB backups. This works by replicating to another CouchDB instance. |
+| services.couchdb.backup.interval | string | `""` | Backup interval in seconds |
+| services.couchdb.backup.resources | object | `{}` | The resources to use for CouchDB backup pods. See <https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/> for more information on how to set these. |
+| services.couchdb.backup.target | string | `""` | Target couchDB instance to back up to, either a hostname or an IP address. |
+| services.couchdb.enabled | bool | `true` | Whether or not to spin up a CouchDB instance in your cluster. True by default, and the configuration for the CouchDB instance is under the `couchdb` key at the root of this file. You can see what options are available to you by looking at the official CouchDB Helm chart: <https://github.com/apache/couchdb-helm/tree/couchdb-4.3.0/couchdb>. |
+| services.couchdb.port | int | `5984` |  |
+| services.dns | string | `"cluster.local"` | The DNS suffix to use for service discovery. You only need to change this if you've configured your cluster to use a different DNS suffix. |
+| services.objectStore.accessKey | string | `""` | AWS_ACCESS_KEY if using S3 |
+| services.objectStore.browser | bool | `true` | Whether to enable the Minio web console or not. If you're exposing Minio to the Internet (via a custom Ingress record, for example), you should set this to false. If you're only exposing Minio to your cluster, you can leave this as true. |
+| services.objectStore.cloudfront.cdn | string | `""` | Set the url of a distribution to enable cloudfront. |
+| services.objectStore.cloudfront.privateKey64 | string | `""` | Base64 encoded private key for the above public key. |
+| services.objectStore.cloudfront.publicKeyId | string | `""` | ID of public key stored in cloudfront. |
+| services.objectStore.minio | bool | `true` | Set to false if using another object store, such as S3. You will need to set `services.objectStore.url` to point to your bucket if you do this. |
+| services.objectStore.region | string | `""` | AWS_REGION if using S3 |
+| services.objectStore.resources | object | `{}` | The resources to use for Minio pods. See <https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/> for more information on how to set these. |
+| services.objectStore.secretKey | string | `""` | AWS_SECRET_ACCESS_KEY if using S3 |
+| services.objectStore.storage | string | `"100Mi"` | How much storage to give Minio in its PersistentVolumeClaim. |
+| services.objectStore.storageClass | string | `""` | If defined, storageClassName: <storageClass> If set to "-", storageClassName: "", which disables dynamic provisioning If undefined (the default) or set to null, no storageClassName spec is set, choosing the default provisioner. |
+| services.objectStore.url | string | `"http://minio-service:9000"` | URL to use for object storage. Only change this if you're using an external object store, such as S3. Remember to set `minio: false` if you do this. |
+| services.proxy.autoscaling.enabled | bool | `false` | Whether to enable horizontal pod autoscaling for the proxy service. |
+| services.proxy.autoscaling.maxReplicas | int | `10` |  |
+| services.proxy.autoscaling.minReplicas | int | `1` |  |
+| services.proxy.autoscaling.targetCPUUtilizationPercentage | int | `80` | Target CPU utilization percentage for the proxy service. Note that for autoscaling to work, you will need to have metrics-server configured, and resources set for the proxy pods. |
+| services.proxy.livenessProbe | object | HTTP health checks. | Liveness probe configuration for proxy pods. You shouldn't need to change this, but if you want to you can find more information here: <https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/> |
+| services.proxy.readinessProbe | object | HTTP health checks. | Readiness probe configuration for proxy pods. You shouldn't need to change this, but if you want to you can find more information here: <https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/> |
+| services.proxy.replicaCount | int | `1` | The number of proxy replicas to run. |
+| services.proxy.resources | object | `{}` | The resources to use for proxy pods. See <https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/> for more information on how to set these. |
+| services.proxy.startupProbe | object | HTTP health checks. | Startup probe configuration for proxy pods. You shouldn't need to change this, but if you want to you can find more information here: <https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/> |
+| services.redis.enabled | bool | `true` | Whether or not to deploy a Redis pod into your cluster. |
+| services.redis.password | string | `"budibase"` | The password to use when connecting to Redis. It's recommended that you change this from the default if you're running Redis in-cluster. |
+| services.redis.port | int | `6379` | Port to expose Redis on. |
+| services.redis.resources | object | `{}` | The resources to use for Redis pods. See <https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/> for more information on how to set these. |
+| services.redis.storage | string | `"100Mi"` | How much persistent storage to allocate to Redis. |
+| services.redis.storageClass | string | `""` | If defined, storageClassName: <storageClass> If set to "-", storageClassName: "", which disables dynamic provisioning If undefined (the default) or set to null, no storageClassName spec is set, choosing the default provisioner. |
+| services.redis.url | string | `""` | If you choose to run Redis externally to this chart, you can specify the connection details here. |
+| services.worker.autoscaling.enabled | bool | `false` | Whether to enable horizontal pod autoscaling for the worker service. |
+| services.worker.autoscaling.maxReplicas | int | `10` |  |
+| services.worker.autoscaling.minReplicas | int | `1` |  |
+| services.worker.autoscaling.targetCPUUtilizationPercentage | int | `80` | Target CPU utilization percentage for the worker service. Note that for autoscaling to work, you will need to have metrics-server configured, and resources set for the worker pods. |
+| services.worker.httpLogging | int | `1` | Whether or not to log HTTP requests to the worker service. |
+| services.worker.livenessProbe | object | HTTP health checks. | Liveness probe configuration for worker pods. You shouldn't need to change this, but if you want to you can find more information here: <https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/> |
+| services.worker.logLevel | string | `"info"` | The log level for the worker service. |
+| services.worker.readinessProbe | object | HTTP health checks. | Readiness probe configuration for worker pods. You shouldn't need to change this, but if you want to you can find more information here: <https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/> |
+| services.worker.replicaCount | int | `1` | The number of worker replicas to run. |
+| services.worker.resources | object | `{}` | The resources to use for worker pods. See <https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/> for more information on how to set these. |
+| services.worker.startupProbe | object | HTTP health checks. | Startup probe configuration for worker pods. You shouldn't need to change this, but if you want to you can find more information here: <https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/> |
+| tolerations | list | `[]` | Sets the tolerations for all pods created by this chart. Should not ordinarily need to be changed. See <https://kubernetes.io/docs/concepts/scheduling-eviction/taint-and-toleration/> for more information on tolerations. |
+
+## Uninstalling
+
+To uninstall the chart, assuming you named the release `budibase` (both commands in the installation section do so):
+
+```console
+$ helm uninstall --namespace budibase budibase
+```
+
+----------------------------------------------
+Autogenerated from chart metadata using [helm-docs v1.11.3](https://github.com/norwoodj/helm-docs/releases/v1.11.3)

charts/budibase/README.md.gotmpl

@@ -0,0 +1,116 @@
+{{ template "chart.header" . }}
+{{ template "chart.description" . }}
+
+## Prerequisites
+
+- `helm` v3 or above
+- Kubernetes 1.4+
+- A storage controller (if you want to use persistent storage)
+- An ingress controller (if you want to define an `Ingress` resource)
+- `metrics-server` (if you want to make use of horizontal pod autoscaling)
+
+## Chart dependencies
+
+This chart depends on the official Apache CouchDB chart. You can see its
+documentation here:
+<https://github.com/apache/couchdb-helm/tree/couchdb-4.3.0/couchdb>.
+
+## Upgrading
+
+### `2.x` to `3.0.0`
+
+We made a number of breaking changes in this release to make the chart more
+idiomatic and easier to use.
+
+1. We no longer bundle `ingress-nginx`. If you were relying on this to supply
+   an ingress controller to your cluster, you will now need to deploy that
+   separately. You'll find guidance for that here:
+   <https://kubernetes.github.io/ingress-nginx/>.
+2. We've upgraded the version of the [CouchDB chart](https://github.com/apache/couchdb-helm)
+   we use from `3.3.4` to `4.3.0`. The primary motivation for this was to align
+   the CouchDB chart used with the CouchDB version used, which has also updated
+   from 3.1.1 to 3.2.1. Additionally, we're moving away from the official CouchDB
+   to one we're building ourselves.
+3. We've separated out the supplied AWS ALB ingress resource for those deploying
+   into EKS. Where previously you enabled this by setting `ingress.enabled: false`
+   and `ingress.aws: true`, you now set `awsAlbIngress.enabled: true` and all
+   configuration for it is under `awsAlbIngress`.
+4. The `HorizontalPodAutoscaler` that was configured at `hpa.enabled: true` has
+   been split into 3 separate HPAs, one for each of `apps`, `worker`, and `proxy`.
+   They are configured at `services.{apps,worker,proxy}.autoscaling`.
+
+## Installing
+
+To install the chart from our repository:
+
+```console
+$ helm repo add budibase https://budibase.github.io/budibase/
+$ helm repo update
+$ helm install --create-namespace --namespace budibase budibase budibase/budibase
+```
+
+To install the chart from this repo:
+
+```console
+$ git clone git@github.com:budibase/budibase.git
+$ cd budibase/charts/budibase
+$ helm install --create-namespace --namespace budibase budibase .
+```
+
+## Example minimal configuration
+
+Here's an example `values.yaml` that would get a Budibase instance running in a home
+cluster using an nginx ingress controller and NFS as cluster storage (basically one of our
+staff's homelabs).
+
+<details>
+
+```yaml
+ingress:
+  enabled: true
+  className: "nginx"
+  hosts:
+    - host: budibase.local # set this to whatever DNS name you'd use
+      paths:
+        - backend:
+            service:
+              name: proxy-service
+              port:
+                number: 10000
+          path: /
+          pathType: Prefix
+
+couchdb:
+  persistentVolume:
+    enabled: true
+    storageClass: "nfs-client"
+  adminPassword: admin
+
+  objectStore:
+    storageClass: "nfs-client"
+  redis:
+    storageClass: "nfs-client"
+```
+
+If you wanted to use this when bringing up Budibase in your own cluster, you could save it
+to your hard disk and run the following:
+
+```console
+$ helm install --create-namespace --namespace budibase budibase . -f values.yaml
+```
+
+</details>
+
+## Configuring
+
+{{ template "chart.valuesTable" . }}
+
+## Uninstalling
+
+To uninstall the chart, assuming you named the release `budibase` (both commands in the installation section do so):
+
+```console
+$ helm uninstall --namespace budibase budibase
+```
+
+{{ template "helm-docs.versionFooter" . }}

charts/budibase/templates/alb-ingress.yaml

@@ -1,4 +1,4 @@
-{{- if .Values.ingress.aws }}
+{{- if .Values.awsAlbIngress.enabled }}
 apiVersion: networking.k8s.io/v1
 kind: Ingress
 metadata:
@@ -7,24 +7,24 @@ metadata:
     kubernetes.io/ingress.class: alb
     alb.ingress.kubernetes.io/scheme: internet-facing
     alb.ingress.kubernetes.io/target-type: ip
-    alb.ingress.kubernetes.io/success-codes: 200,301
+    alb.ingress.kubernetes.io/success-codes: '200'
-    alb.ingress.kubernetes.io/healthcheck-path: /
+    alb.ingress.kubernetes.io/healthcheck-path: '/health'
-    {{- if .Values.ingress.certificateArn }}
+    {{- if .Values.awsAlbIngress.certificateArn }}
     alb.ingress.kubernetes.io/actions.ssl-redirect: '{"Type": "redirect", "RedirectConfig": { "Protocol": "HTTPS", "Port": "443", "StatusCode": "HTTP_301"}}'
     alb.ingress.kubernetes.io/listen-ports: '[{"HTTP": 80}, {"HTTPS":443}]'
-    alb.ingress.kubernetes.io/certificate-arn: {{ .Values.ingress.certificateArn }} 
+    alb.ingress.kubernetes.io/certificate-arn: {{ .Values.awsAlbIngress.certificateArn }} 
     {{- end }}
-    {{- if .Values.ingress.sslPolicy }}
+    {{- if .Values.awsAlbIngress.sslPolicy }}
-    alb.ingress.kubernetes.io/actions.ssl-policy: {{ .Values.ingress.sslPolicy }}
+    alb.ingress.kubernetes.io/actions.ssl-policy: {{ .Values.awsAlbIngress.sslPolicy }}
     {{- end }}
-    {{- if .Values.ingress.securityGroups }}
+    {{- if .Values.awsAlbIngress.securityGroups }}
-    alb.ingress.kubernetes.io/security-groups: {{ .Values.ingress.securityGroups }}
+    alb.ingress.kubernetes.io/security-groups: {{ .Values.awsAlbIngress.securityGroups }}
     {{- end }}
 spec:
   rules:
     - http:
         paths:
-        {{- if .Values.ingress.certificateArn }}
+        {{- if .Values.awsAlbIngress.certificateArn }}
         - path: /
           pathType: Prefix
           backend:

charts/budibase/templates/app-service-deployment.yaml

@@ -2,12 +2,9 @@ apiVersion: apps/v1
 kind: Deployment
 metadata:
   annotations:
-    kompose.cmd: kompose convert
-    kompose.version: 1.21.0 (992df58d8)
 {{ if .Values.services.apps.deploymentAnnotations }}
 {{- toYaml .Values.services.apps.deploymentAnnotations | indent 4 -}}
 {{ end }}
-  creationTimestamp: null
   labels:
     io.kompose.service: app-service
 {{ if .Values.services.apps.deploymentLabels }}
@@ -24,12 +21,9 @@ spec:
   template:
     metadata:
       annotations:
-        kompose.cmd: kompose convert
-        kompose.version: 1.21.0 (992df58d8)
 {{ if .Values.services.apps.templateAnnotations }}
 {{- toYaml .Values.services.apps.templateAnnotations | indent 8 -}}
 {{ end }}
-      creationTimestamp: null
       labels:
         io.kompose.service: app-service
 {{ if .Values.services.apps.templateLabels }}

charts/budibase/templates/app-service-hpa.yaml

@@ -0,0 +1,32 @@
+{{- if .Values.services.apps.autoscaling.enabled }}
+apiVersion: {{ ternary "autoscaling/v2" "autoscaling/v2beta2" (.Capabilities.APIVersions.Has "autoscaling/v2") }}
+kind: HorizontalPodAutoscaler
+metadata:
+  name: {{ include "budibase.fullname" . }}-apps
+  labels:
+    {{- include "budibase.labels" . | nindent 4 }}
+spec:
+  scaleTargetRef:
+    apiVersion: apps/v1
+    kind: Deployment
+    name: app-service
+  minReplicas: {{ .Values.services.apps.autoscaling.minReplicas }}
+  maxReplicas: {{ .Values.services.apps.autoscaling.maxReplicas }}
+  metrics:
+    {{- if .Values.services.apps.autoscaling.targetCPUUtilizationPercentage }}
+    - type: Resource
+      resource:
+        name: cpu
+        target:
+          type: Utilization
+          averageUtilization: {{ .Values.services.apps.autoscaling.targetCPUUtilizationPercentage }}
+    {{- end }}
+    {{- if .Values.services.apps.autoscaling.targetMemoryUtilizationPercentage }}
+    - type: Resource
+      resource:
+        name: memory
+        target:
+          type: Utilization
+          averageUtilization: {{ .Values.services.apps.autoscaling.targetMemoryUtilizationPercentage }}
+    {{- end }}
+{{- end }}

charts/budibase/templates/app-service-service.yaml

@@ -1,10 +1,6 @@
 apiVersion: v1
 kind: Service
 metadata:
-  annotations:
-    kompose.cmd: kompose convert
-    kompose.version: 1.21.0 (992df58d8)
-  creationTimestamp: null
   labels:
     io.kompose.service: app-service
   name: app-service

charts/budibase/templates/couchdb-backup.yaml

@@ -2,10 +2,6 @@
 apiVersion: apps/v1
 kind: Deployment
 metadata:
-  annotations:
-    kompose.cmd: kompose convert
-    kompose.version: 1.21.0 (992df58d8)
-  creationTimestamp: null
   labels:
     app.kubernetes.io/name: couchdb-backup
   name: couchdb-backup
@@ -18,10 +14,6 @@ spec:
     type: Recreate
   template:
     metadata:
-      annotations:
-        kompose.cmd: kompose convert
-        kompose.version: 1.21.0 (992df58d8)
-      creationTimestamp: null
       labels:
         app.kubernetes.io/name: couchdb-backup
     spec:

charts/budibase/templates/hpa.yaml

@@ -1,28 +0,0 @@
-{{- if .Values.autoscaling.enabled }}
-apiVersion: autoscaling/v2beta1
-kind: HorizontalPodAutoscaler
-metadata:
-  name: {{ include "budibase.fullname" . }}
-  labels:
-    {{- include "budibase.labels" . | nindent 4 }}
-spec:
-  scaleTargetRef:
-    apiVersion: apps/v1
-    kind: Deployment
-    name: {{ include "budibase.fullname" . }}
-  minReplicas: {{ .Values.autoscaling.minReplicas }}
-  maxReplicas: {{ .Values.autoscaling.maxReplicas }}
-  metrics:
-    {{- if .Values.autoscaling.targetCPUUtilizationPercentage }}
-    - type: Resource
-      resource:
-        name: cpu
-        targetAverageUtilization: {{ .Values.autoscaling.targetCPUUtilizationPercentage }}
-    {{- end }}
-    {{- if .Values.autoscaling.targetMemoryUtilizationPercentage }}
-    - type: Resource
-      resource:
-        name: memory
-        targetAverageUtilization: {{ .Values.autoscaling.targetMemoryUtilizationPercentage }}
-    {{- end }}
-{{- end }}

charts/budibase/templates/minio-data-persistentvolumeclaim.yaml

@@ -2,7 +2,6 @@
 apiVersion: v1
 kind: PersistentVolumeClaim
 metadata:
-  creationTimestamp: null
   labels:
     io.kompose.service: minio-data
   name: minio-data

charts/budibase/templates/minio-service-deployment.yaml

@@ -2,10 +2,6 @@
 apiVersion: apps/v1
 kind: Deployment
 metadata:
-  annotations:
-    kompose.cmd: kompose convert
-    kompose.version: 1.21.0 (992df58d8)
-  creationTimestamp: null
   labels:
     io.kompose.service: minio-service
   name: minio-service
@@ -18,10 +14,6 @@ spec:
     type: Recreate
   template:
     metadata:
-      annotations:
-        kompose.cmd: kompose convert
-        kompose.version: 1.21.0 (992df58d8)
-      creationTimestamp: null
       labels:
         io.kompose.service: minio-service
     spec:

charts/budibase/templates/minio-service-service.yaml

@@ -2,10 +2,6 @@
 apiVersion: v1
 kind: Service
 metadata:
-  annotations:
-    kompose.cmd: kompose convert
-    kompose.version: 1.21.0 (992df58d8)
-  creationTimestamp: null
   labels:
     io.kompose.service: minio-service
   name: minio-service

charts/budibase/templates/proxy-service-deployment.yaml

@@ -2,12 +2,9 @@ apiVersion: apps/v1
 kind: Deployment
 metadata:
   annotations:
-    kompose.cmd: kompose convert
-    kompose.version: 1.21.0 (992df58d8)
 {{ if .Values.services.proxy.deploymentAnnotations }}
 {{- toYaml .Values.services.proxy.deploymentAnnotations | indent 4 -}}
 {{ end }}
-  creationTimestamp: null
   labels:
     app.kubernetes.io/name: budibase-proxy
 {{ if .Values.services.proxy.deploymentLabels }}
@@ -24,12 +21,9 @@ spec:
   template:
     metadata:
       annotations:
-        kompose.cmd: kompose convert
-        kompose.version: 1.21.0 (992df58d8)
 {{ if .Values.services.proxy.templateAnnotations }}
 {{- toYaml .Values.services.proxy.templateAnnotations | indent 8 -}}
 {{ end }}
-      creationTimestamp: null
       labels:
         app.kubernetes.io/name: budibase-proxy
 {{ if .Values.services.proxy.templateLabels }}

charts/budibase/templates/proxy-service-hpa.yaml

@@ -0,0 +1,32 @@
+{{- if .Values.services.proxy.autoscaling.enabled }}
+apiVersion: {{ ternary "autoscaling/v2" "autoscaling/v2beta2" (.Capabilities.APIVersions.Has "autoscaling/v2") }}
+kind: HorizontalPodAutoscaler
+metadata:
+  name: {{ include "budibase.fullname" . }}-proxy
+  labels:
+    {{- include "budibase.labels" . | nindent 4 }}
+spec:
+  scaleTargetRef:
+    apiVersion: apps/v1
+    kind: Deployment
+    name: proxy-service
+  minReplicas: {{ .Values.services.proxy.autoscaling.minReplicas }}
+  maxReplicas: {{ .Values.services.proxy.autoscaling.maxReplicas }}
+  metrics:
+    {{- if .Values.services.proxy.autoscaling.targetCPUUtilizationPercentage }}
+    - type: Resource
+      resource:
+        name: cpu
+        target:
+          type: Utilization
+          averageUtilization: {{ .Values.services.proxy.autoscaling.targetCPUUtilizationPercentage }}
+    {{- end }}
+    {{- if .Values.services.proxy.autoscaling.targetMemoryUtilizationPercentage }}
+    - type: Resource
+      resource:
+        name: memory
+        target:
+          type: Utilization
+          averageUtilization: {{ .Values.services.proxy.autoscaling.targetMemoryUtilizationPercentage }}
+    {{- end }}
+{{- end }}

charts/budibase/templates/proxy-service-service.yaml

@@ -1,10 +1,6 @@
 apiVersion: v1
 kind: Service
 metadata:
-  annotations:
-    kompose.cmd: kompose convert
-    kompose.version: 1.21.0 (992df58d8)
-  creationTimestamp: null
   labels:
     app.kubernetes.io/name: budibase-proxy
   name: proxy-service
@@ -16,4 +12,4 @@ spec:
   selector:
     app.kubernetes.io/name: budibase-proxy
 status:
-  loadBalancer: {}
+  loadBalancer: {}

charts/budibase/templates/redis-data-persistentvolumeclaim.yaml

@@ -2,7 +2,6 @@
 apiVersion: v1
 kind: PersistentVolumeClaim
 metadata:
-  creationTimestamp: null
   labels:
     io.kompose.service: redis-data
   name: redis-data

charts/budibase/templates/redis-service-deployment.yaml

@@ -2,10 +2,6 @@
 apiVersion: apps/v1
 kind: Deployment
 metadata:
-  annotations:
-    kompose.cmd: kompose convert
-    kompose.version: 1.21.0 (992df58d8)
-  creationTimestamp: null
   labels:
     io.kompose.service: redis-service
   name: redis-service
@@ -18,10 +14,6 @@ spec:
     type: Recreate
   template:
     metadata:
-      annotations:
-        kompose.cmd: kompose convert
-        kompose.version: 1.21.0 (992df58d8)
-      creationTimestamp: null
       labels:
         io.kompose.service: redis-service
     spec:

charts/budibase/templates/redis-service-service.yaml

@@ -2,10 +2,6 @@
 apiVersion: v1
 kind: Service
 metadata:
-  annotations:
-    kompose.cmd: kompose convert
-    kompose.version: 1.21.0 (992df58d8)
-  creationTimestamp: null
   labels:
     io.kompose.service: redis-service
   name: redis-service

charts/budibase/templates/worker-service-deployment.yaml

@@ -2,12 +2,9 @@ apiVersion: apps/v1
 kind: Deployment
 metadata:
   annotations:
-    kompose.cmd: kompose convert
-    kompose.version: 1.21.0 (992df58d8)
 {{ if .Values.services.worker.deploymentAnnotations }}
 {{- toYaml .Values.services.worker.deploymentAnnotations | indent 4 -}}
 {{ end }}
-  creationTimestamp: null
   labels:
     io.kompose.service: worker-service
 {{ if .Values.services.worker.deploymentLabels }}
@@ -24,12 +21,9 @@ spec:
   template:
     metadata:
       annotations:
-        kompose.cmd: kompose convert
-        kompose.version: 1.21.0 (992df58d8)
 {{ if .Values.services.worker.templateAnnotations }}
 {{- toYaml .Values.services.worker.templateAnnotations | indent 8 -}}
 {{ end }}
-      creationTimestamp: null
       labels:
         io.kompose.service: worker-service
 {{ if .Values.services.worker.templateLabels }}

charts/budibase/templates/worker-service-hpa.yaml

@@ -0,0 +1,32 @@
+{{- if .Values.services.worker.autoscaling.enabled }}
+apiVersion: {{ ternary "autoscaling/v2" "autoscaling/v2beta2" (.Capabilities.APIVersions.Has "autoscaling/v2") }}
+kind: HorizontalPodAutoscaler
+metadata:
+  name: {{ include "budibase.fullname" . }}-worker
+  labels:
+    {{- include "budibase.labels" . | nindent 4 }}
+spec:
+  scaleTargetRef:
+    apiVersion: apps/v1
+    kind: Deployment
+    name: worker-service
+  minReplicas: {{ .Values.services.worker.autoscaling.minReplicas }}
+  maxReplicas: {{ .Values.services.worker.autoscaling.maxReplicas }}
+  metrics:
+    {{- if .Values.services.worker.autoscaling.targetCPUUtilizationPercentage }}
+    - type: Resource
+      resource:
+        name: cpu
+        target:
+          type: Utilization
+          averageUtilization: {{ .Values.services.worker.autoscaling.targetCPUUtilizationPercentage }}
+    {{- end }}
+    {{- if .Values.services.worker.autoscaling.targetMemoryUtilizationPercentage }}
+    - type: Resource
+      resource:
+        name: memory
+        target:
+          type: Utilization
+          averageUtilization: {{ .Values.services.worker.autoscaling.targetMemoryUtilizationPercentage }}
+    {{- end }}
+{{- end }}

charts/budibase/templates/worker-service-service.yaml

@@ -1,10 +1,6 @@
 apiVersion: v1
 kind: Service
 metadata:
-  annotations:
-    kompose.cmd: kompose convert
-    kompose.version: 1.21.0 (992df58d8)
-  creationTimestamp: null
   labels:
     io.kompose.service: worker-service
   name: worker-service

charts/budibase/values.yaml

@@ -1,56 +1,32 @@
-# Default values for budibase.
+# -- Passed to all pods created by this chart. Should not ordinarily need to be changed.
-# This is a YAML-formatted file.
-# Declare variables to be passed into your templates.
-
-image:
-  pullPolicy: IfNotPresent
-  # Overrides the image tag whose default is the chart appVersion.
-  tag: ""
-
 imagePullSecrets: []
+# -- Override the name of the deploymen. Defaults to {{ .Chart.Name }}.
 nameOverride: ""
-# fullnameOverride: ""
 
 serviceAccount:
-  # Specifies whether a service account should be created
+  # -- Specifies whether a service account should be created
   create: true
-  # Annotations to add to the service account
+  # -- Annotations to add to the service account
   annotations: {}
-  # The name of the service account to use.
+  # -- The name of the service account to use.
   # If not set and create is true, a name is generated using the fullname template
   name: ""
 
-podAnnotations: {}
-
-podSecurityContext:
-  {}
-  # fsGroup: 2000
-
-securityContext:
-  {}
-  # capabilities:
-  #   drop:
-  #   - ALL
-  # readOnlyRootFilesystem: true
-  # runAsNonRoot: true
-  # runAsUser: 1000
-
 service:
+  # -- Service type for the service that points to the main Budibase proxy pod.
   type: ClusterIP
+  # -- Port to expose on the service.
   port: 10000
 
 ingress:
+  # -- Whether to create an Ingress resource pointing to the Budibase proxy.
   enabled: true
-  aws: false
+  # -- What ingress class to use.
-  nginx: true
-  certificateArn: ""
   className: ""
-  annotations:
+  # -- Standard hosts block for the Ingress resource. Defaults to pointing to the Budibase proxy.
-    kubernetes.io/ingress.class: nginx
-    nginx.ingress.kubernetes.io/client-max-body-size: 150M
-    nginx.ingress.kubernetes.io/proxy-body-size: 50m
   hosts:
-    - host: # change if using custom domain
+    # @ignore
+    - host:
       paths:
         - path: /
           pathType: Prefix
@@ -60,361 +36,426 @@ ingress:
               port:
                 number: 10000
 
-autoscaling:
+awsAlbIngress:
+  # -- Whether to create an ALB Ingress resource pointing to the Budibase proxy. Requires the AWS ALB Ingress Controller.
   enabled: false
-  minReplicas: 1
+  # -- If you're wanting to use HTTPS, you'll need to create an ACM certificate and specify the ARN here.
-  maxReplicas: 100
+  certificateArn: ""
-  targetCPUUtilizationPercentage: 80
-  # targetMemoryUtilizationPercentage: 80
-
-nodeSelector: {}
 
+# -- Sets the tolerations for all pods created by this chart. Should not ordinarily need to be changed.
+# See <https://kubernetes.io/docs/concepts/scheduling-eviction/taint-and-toleration/> for more information
+# on tolerations.
 tolerations: []
 
+# -- Sets the affinity for all pods created by this chart. Should not ordinarily
+# need to be changed.  See
+# <https://kubernetes.io/docs/tasks/configure-pod-container/assign-pods-nodes-using-node-affinity/>
+# for more information on affinity.
 affinity: {}
 
 globals:
-  appVersion: "" # Use as an override to .Chart.AppVersion
+  # -- The version of Budibase to deploy. Defaults to what's specified by {{ .Chart.AppVersion }}.
+  # Ends up being used as the image version tag for the apps, proxy, and worker images.
+  appVersion: ""
+  # -- Sets the environment variable BUDIBASE_ENVIRONMENT for the apps and worker pods. Should not
+  # ordinarily need to be changed.
   budibaseEnv: PRODUCTION
+  # -- Sets what feature flags are enabled and for which tenants. Should not ordinarily need to be
+  # changed.
   tenantFeatureFlags: "*:LICENSING,*:USER_GROUPS,*:ONBOARDING_TOUR"
+  # -- Whether to enable analytics or not. You can read more about our analytics here:
+  # <https://docs.budibase.com/docs/analytics>.
   enableAnalytics: "1"
+  # @ignore (only used if enableAnalytics is set to 1)
   posthogToken: "phc_bIjZL7oh2GEUd2vqvTBH8WvrX0fWTFQMs6H5KQxiUxU"
-  selfHosted: "1" # set to 0 for budibase cloud environment, set to 1 for self-hosted setup
+  # @ignore (should not normally need to be changed, we only set this to "0"
-  multiTenancy: "0" # set to 0 to disable multiple orgs, set to 1 to enable multiple orgs
+  # when deploying to our Cloud environment)
-  offlineMode: "0" # set to 1 to enable offline mode
+  selfHosted: "1"
+  # @ignore (doesn't work out of the box for self-hosted users, only meant for Budicloud)
+  multiTenancy: "0"
+  # @ignore (only currently used to determine whether to fetch licenses offline or not, should
+  # not normally need to be changed, and only applies to Enterprise customers)
+  offlineMode: "0"
+  # @ignore (only needs to be set in our cloud environment)
   accountPortalUrl: ""
+  # @ignore (only needs to be set in our cloud environment)
   accountPortalApiKey: ""
+  # -- Sets the domain attribute of the cookie that Budibase uses to store session information.
+  # See <https://developer.mozilla.org/en-US/docs/Web/HTTP/Cookies#define_where_cookies_are_sent>
+  # for details on why you might want to set this.
   cookieDomain: ""
+  # -- Set the `platformUrl` binding. You can also do this in Settings > Organisation if you are
+  # self-hosting.
   platformUrl: ""
+  # -- Whether or not to enable doing data migrations over the HTTP API. If this is set to "0",
+  # migrations are run on startup. You shouldn't ordinarily need to change this.
   httpMigrations: "0"
+  # -- Google OAuth settings. These can also be set in the Budibase UI, see
+  # <https://docs.budibase.com/docs/sso-with-google> for details.
   google:
+    # -- Client ID of your Google OAuth app.
     clientId: ""
+    # -- Client secret of your Google OAuth app.
     secret: ""
+  # -- The maximum number of iterations allows for an automation loop step. You can read more about
+  # looping here: <https://docs.budibase.com/docs/looping>.
   automationMaxIterations: "200"
 
-  createSecrets: true # creates an internal API key, JWT secrets and redis password for you
+  # -- Create an internal API key, JWT secret, object store access key and
+  # secret, and store them in a Kubernetes `Secret`.
+  createSecrets: true
 
-  # if createSecrets is set to false, you can hard-code your secrets here
+  # -- Used for encrypting API keys and environment variables when stored in the database.
+  # You don't need to set this if `createSecrets` is true.
   apiEncryptionKey: ""
+  # -- API key used for internal Budibase API calls. You don't need to set this
+  # if `createSecrets` is true.
   internalApiKey: ""
+  # -- Secret used for signing JWTs. You don't need to set this if `createSecrets` is true.
   jwtSecret: ""
-  cdnUrl: ""
+
-  # fallback values used during live rotation
+  # -- A fallback value for `internalApiKey`. If you're rotating your encryption key, you can
+  # set this to the old value for the duration of the rotation.
   internalApiKeyFallback: ""
+  # -- A fallback value for `jwtSecret`. If you're rotating your JWT secret, you can set this
+  # to the old value for the duration of the rotation.
   jwtSecretFallback: ""
 
   smtp:
+    # -- Whether to enable SMTP or not.
     enabled: false
-
+    # -- The hostname of your SMTP server.
-#  globalAgentHttpProxy:
+    host: ""
-#  globalAgentHttpsProxy:
+    # -- The port of your SMTP server.
-#  globalAgentNoProxy:
+    port: "587"
+    # -- The email address to use in the "From:" field of emails sent by Budibase.
+    from: ""
+    # -- The username to use when authenticating with your SMTP server.
+    user: ""
+    # -- The password to use when authenticating with your SMTP server.
+    password: ""
 
 services:
-  budibaseVersion: latest
+  # -- The DNS suffix to use for service discovery. You only need to change this
+  # if you've configured your cluster to use a different DNS suffix.
   dns: cluster.local
-  # tlsRejectUnauthorized: 0
 
   proxy:
+    # @ignore (you shouldn't need to change this)
     port: 10000
+    # -- The number of proxy replicas to run.
     replicaCount: 1
+    # @ignore (you should never need to change this)
     upstreams:
       apps: "http://app-service.{{ .Release.Namespace }}.svc.{{ .Values.services.dns }}:{{ .Values.services.apps.port }}"
       worker: "http://worker-service.{{ .Release.Namespace }}.svc.{{ .Values.services.dns }}:{{ .Values.services.worker.port }}"
       minio: "http://minio-service.{{ .Release.Namespace }}.svc.{{ .Values.services.dns }}:{{ .Values.services.objectStore.port }}"
       couchdb: "http://{{ .Release.Name }}-svc-couchdb:{{ .Values.services.couchdb.port }}"
+    # -- The resources to use for proxy pods. See
+    # <https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/>
+    # for more information on how to set these.
     resources: {}
+    # -- Startup probe configuration for proxy pods. You shouldn't need to
+    # change this, but if you want to you can find more information here:
+    # <https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/>
+    # @default -- HTTP health checks.
     startupProbe:
+      # @ignore
       httpGet:
         path: /health
         port: 10000
         scheme: HTTP
+      # @ignore
       failureThreshold: 30
+      # @ignore
       periodSeconds: 3
+    # -- Readiness probe configuration for proxy pods. You shouldn't need to
+    # change this, but if you want to you can find more information here:
+    # <https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/>
+    # @default -- HTTP health checks.
     readinessProbe:
+      # @ignore
       httpGet:
         path: /health
         port: 10000
         scheme: HTTP
+      # @ignore
       periodSeconds: 3
+      # @ignore
       failureThreshold: 1
+    # -- Liveness probe configuration for proxy pods. You shouldn't need to
+    # change this, but if you want to you can find more information here:
+    # <https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/>
+    # @default -- HTTP health checks.
     livenessProbe:
+      # @ignore
       httpGet:
         path: /health
         port: 10000
         scheme: HTTP
+      # @ignore
       failureThreshold: 3
+      # @ignore
       periodSeconds: 5
-  #    annotations:
+    autoscaling:
-  #      co.elastic.logs/module: nginx
+      # -- Whether to enable horizontal pod autoscaling for the proxy service.
-  #      co.elastic.logs/fileset.stdout: access
+      enabled: false
-  #      co.elastic.logs/fileset.stderr: error
+      minReplicas: 1
+      maxReplicas: 10
+      # -- Target CPU utilization percentage for the proxy service. Note that
+      # for autoscaling to work, you will need to have metrics-server
+      # configured, and resources set for the proxy pods.
+      targetCPUUtilizationPercentage: 80
 
   apps:
+    # @ignore (you shouldn't need to change this)
     port: 4002
+    # -- The number of apps replicas to run.
     replicaCount: 1
+    # -- The log level for the apps service.
     logLevel: info
+    # -- Whether or not to log HTTP requests to the apps service.
     httpLogging: 1
+    # -- The resources to use for apps pods. See
+    # <https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/>
+    # for more information on how to set these.
     resources: {}
+    # -- Startup probe configuration for apps pods. You shouldn't need to
+    # change this, but if you want to you can find more information here:
+    # <https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/>
+    # @default -- HTTP health checks.
     startupProbe:
+      # @ignore
       httpGet:
         path: /health
         port: 4002
         scheme: HTTP
+      # @ignore
       failureThreshold: 30
+      # @ignore
       periodSeconds: 3
+    # -- Readiness probe configuration for apps pods. You shouldn't need to
+    # change this, but if you want to you can find more information here:
+    # <https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/>
+    # @default -- HTTP health checks.
     readinessProbe:
+      # @ignore
       httpGet:
         path: /health
         port: 4002
         scheme: HTTP
+      # @ignore
       periodSeconds: 3
+      # @ignore
       failureThreshold: 1
+    # -- Liveness probe configuration for apps pods. You shouldn't need to
+    # change this, but if you want to you can find more information here:
+    # <https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/>
+    # @default -- HTTP health checks.
     livenessProbe:
+      # @ignore
       httpGet:
         path: /health
         port: 4002
         scheme: HTTP
+      # @ignore
       failureThreshold: 3
+      # @ignore
       periodSeconds: 5
-  #    nodeDebug: "" # set the value of NODE_DEBUG
+    autoscaling:
-  #    annotations:
+      # -- Whether to enable horizontal pod autoscaling for the apps service.
-  #      co.elastic.logs/multiline.type: pattern
+      enabled: false
-  #      co.elastic.logs/multiline.pattern: '^[[:space:]]'
+      minReplicas: 1
-  #      co.elastic.logs/multiline.negate: false
+      maxReplicas: 10
-  #      co.elastic.logs/multiline.match: after
+      # -- Target CPU utilization percentage for the apps service. Note that for
+      # autoscaling to work, you will need to have metrics-server configured,
+      # and resources set for the apps pods.
+      targetCPUUtilizationPercentage: 80
+
   worker:
+    # @ignore (you shouldn't need to change this)
     port: 4003
+    # -- The number of worker replicas to run.
     replicaCount: 1
+    # -- The log level for the worker service.
     logLevel: info
+    # -- Whether or not to log HTTP requests to the worker service.
     httpLogging: 1
+    # -- The resources to use for worker pods. See
+    # <https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/>
+    # for more information on how to set these.
     resources: {}
+    # -- Startup probe configuration for worker pods. You shouldn't need to
+    # change this, but if you want to you can find more information here:
+    # <https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/>
+    # @default -- HTTP health checks.
     startupProbe:
+      # @ignore
       httpGet:
         path: /health
         port: 4003
         scheme: HTTP
+      # @ignore
       failureThreshold: 30
+      # @ignore
       periodSeconds: 3
+    # -- Readiness probe configuration for worker pods. You shouldn't need to
+    # change this, but if you want to you can find more information here:
+    # <https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/>
+    # @default -- HTTP health checks.
     readinessProbe:
+      # @ignore
       httpGet:
         path: /health
         port: 4003
         scheme: HTTP
+      # @ignore
       periodSeconds: 3
+      # @ignore
       failureThreshold: 1
+    # -- Liveness probe configuration for worker pods. You shouldn't need to
+    # change this, but if you want to you can find more information here:
+    # <https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/>
+    # @default -- HTTP health checks.
     livenessProbe:
+      # @ignore
       httpGet:
         path: /health
         port: 4003
         scheme: HTTP
+      # @ignore
       failureThreshold: 3
+      # @ignore
       periodSeconds: 5
-  #    annotations:
+    autoscaling:
-  #      co.elastic.logs/multiline.type: pattern
+      # -- Whether to enable horizontal pod autoscaling for the worker service.
-  #      co.elastic.logs/multiline.pattern: '^[[:space:]]'
+      enabled: false
-  #      co.elastic.logs/multiline.negate: false
+      minReplicas: 1
-  #      co.elastic.logs/multiline.match: after
+      maxReplicas: 10
+      # -- Target CPU utilization percentage for the worker service. Note that
+      # for autoscaling to work, you will need to have metrics-server
+      # configured, and resources set for the worker pods.
+      targetCPUUtilizationPercentage: 80
 
   couchdb:
+    # -- Whether or not to spin up a CouchDB instance in your cluster. True by
+    # default, and the configuration for the CouchDB instance is under the
+    # `couchdb` key at the root of this file. You can see what options are
+    # available to you by looking at the official CouchDB Helm chart:
+    # <https://github.com/apache/couchdb-helm/tree/couchdb-4.3.0/couchdb>.
     enabled: true
     # url: "" # only change if pointing to existing couch server
     # user: "" # only change if pointing to existing couch server
     # password: "" # only change if pointing to existing couch server
     port: 5984
     backup:
+      # -- Whether or not to enable periodic CouchDB backups. This works by replicating
+      # to another CouchDB instance.
       enabled: false
-      # target couchDB instance to back up to
+      # -- Target couchDB instance to back up to, either a hostname or an IP address.
       target: ""
-      # backup interval in seconds
+      # -- Backup interval in seconds
       interval: ""
+      # -- The resources to use for CouchDB backup pods. See
+      # <https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/>
+      # for more information on how to set these.
       resources: {}
 
   redis:
-    enabled: true # disable if using external redis
+    # -- Whether or not to deploy a Redis pod into your cluster.
+    enabled: true
+    # -- Port to expose Redis on.
     port: 6379
+    # @ignore (you should leave this as 1, we don't support clustering Redis)
     replicaCount: 1
-    url: "" # only change if pointing to existing redis cluster and enabled: false
+    # -- If you choose to run Redis externally to this chart, you can specify the
-    password: "budibase" # recommended to override if using built-in redis
+    # connection details here.
+    url: ""
+    # -- The password to use when connecting to Redis. It's recommended that you change
+    # this from the default if you're running Redis in-cluster.
+    password: "budibase"
+    # -- How much persistent storage to allocate to Redis.
     storage: 100Mi
-    ## If defined, storageClassName: <storageClass>
+    # -- If defined, storageClassName: <storageClass> If set to "-",
-    ## If set to "-", storageClassName: "", which disables dynamic provisioning
+    # storageClassName: "", which disables dynamic provisioning If undefined
-    ## If undefined (the default) or set to null, no storageClassName spec is
+    # (the default) or set to null, no storageClassName spec is set, choosing
-    ##   set, choosing the default provisioner.
+    # the default provisioner.
     storageClass: ""
+    # -- The resources to use for Redis pods. See
+    # <https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/>
+    # for more information on how to set these.
     resources: {}
 
   objectStore:
-    # Set to false if using another object store such as S3
+    # -- Set to false if using another object store, such as S3. You will need
+    # to set `services.objectStore.url` to point to your bucket if you do this.
     minio: true
+    # -- Whether to enable the Minio web console or not. If you're exposing
+    # Minio to the Internet (via a custom Ingress record, for example), you
+    # should set this to false. If you're only exposing Minio to your cluster,
+    # you can leave this as true.
     browser: true
+    # @ignore
     port: 9000
+    # @ignore (you should leave this as 1, we don't support clustering Minio)
     replicaCount: 1
-    accessKey: "" # AWS_ACCESS_KEY if using S3 or existing minio access key
+    # -- AWS_ACCESS_KEY if using S3
-    secretKey: "" # AWS_SECRET_ACCESS_KEY if using S3 or existing minio secret
+    accessKey: ""
-    region: "" # AWS_REGION if using S3 or existing minio secret
+    # -- AWS_SECRET_ACCESS_KEY if using S3
-    url: "http://minio-service:9000" # only change if pointing to existing minio cluster or S3 and minio: false
+    secretKey: ""
+    # -- AWS_REGION if using S3
+    region: ""
+    # -- URL to use for object storage. Only change this if you're using an
+    # external object store, such as S3. Remember to set `minio: false` if you
+    # do this.
+    url: "http://minio-service:9000"
+    # -- How much storage to give Minio in its PersistentVolumeClaim.
     storage: 100Mi
-    ## If defined, storageClassName: <storageClass>
+    # -- If defined, storageClassName: <storageClass> If set to "-",
-    ## If set to "-", storageClassName: "", which disables dynamic provisioning
+    # storageClassName: "", which disables dynamic provisioning If undefined
-    ## If undefined (the default) or set to null, no storageClassName spec is
+    # (the default) or set to null, no storageClassName spec is set, choosing
-    ##   set, choosing the default provisioner.
+    # the default provisioner.
     storageClass: ""
+    # -- The resources to use for Minio pods. See
+    # <https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/>
+    # for more information on how to set these.
     resources: {}
     cloudfront:
-      # Set the url of a distribution to enable cloudfront
+      # -- Set the url of a distribution to enable cloudfront.
       cdn: ""
-      # ID of public key stored in cloudfront
+      # -- ID of public key stored in cloudfront.
       publicKeyId: ""
-      # Base64 encoded private key for the above public key
+      # -- Base64 encoded private key for the above public key.
       privateKey64: ""
 
-# Override values in couchDB subchart
+# Override values in couchDB subchart. We're only specifying the values we're changing.
+# If you want to see all of the available values, see:
+#   https://github.com/apache/couchdb-helm/tree/couchdb-4.3.0/couchdb
 couchdb:
-  ## clusterSize is the initial size of the CouchDB cluster.
+  # -- The number of replicas to run in the CouchDB cluster. We set this to
+  # 1 by default to make things simpler, but you can set it to 3 if you need
+  # a high-availability CouchDB cluster.
   clusterSize: 1
-  allowAdminParty: false
 
-  # Secret Management
+  # -- We use a custom CouchDB image for running Budibase and we don't support
-  createAdminSecret: true
+  # using any other CouchDB image. You shouldn't change this, and if you do we
-
+  # can't guarantee that Budibase will work.
-  # adminUsername: budibase
-  # adminPassword: budibase
-  # adminHash: -pbkdf2-this_is_not_necessarily_secure_either
-  # cookieAuthSecret: admin
-
-  ## When enabled, will deploy a networkpolicy that allows CouchDB pods to
-  ## communicate with each other for clustering and ingress on port 5984
-  networkPolicy:
-    enabled: true
-
-  # Use a service account
-  serviceAccount:
-    enabled: true
-    create: true
-  # name:
-  # imagePullSecrets:
-  # - name: myimagepullsecret
-
-  ## The storage volume used by each Pod in the StatefulSet. If a
-  ## persistentVolume is not enabled, the Pods will use `emptyDir` ephemeral
-  ## local storage. Setting the storageClass attribute to "-" disables dynamic
-  ## provisioning of Persistent Volumes; leaving it unset will invoke the default
-  ## provisioner.
-  persistentVolume:
-    enabled: false
-    accessModes:
-      - ReadWriteOnce
-    size: 10Gi
-    storageClass: ""
-
-  ## The CouchDB image
   image:
-    repository: couchdb
+    # @ignore
-    tag: 3.1.1
+    repository: budibase/couchdb
-    pullPolicy: IfNotPresent
+    # @ignore
-
+    tag: v3.2.1
-  ## Experimental integration with Lucene-powered fulltext search
+    # @ignore
-  enableSearch: true
-  searchImage:
-    repository: kocolosk/couchdb-search
-    tag: 0.2.0
-    pullPolicy: IfNotPresent
-
-  initImage:
-    repository: busybox
-    tag: latest
     pullPolicy: Always
 
-  ## CouchDB is happy to spin up cluster nodes in parallel, but if you encounter
+  # @ignore
-  ## problems you can try setting podManagementPolicy to the StatefulSet default
+  # This should remain false. We ship Clouseau ourselves as part of the
-  ## `OrderedReady`
+  # budibase/couchdb image, and it's not possible to disable it because it's a
-  podManagementPolicy: Parallel
+  # core part of the Budibase experience.
+  enableSearch: false
 
-  ## Optional pod annotations
-  annotations: {}
-
-  ## Optional tolerations
-  tolerations: []
-
-  affinity: {}
-
-  service:
-    # annotations:
-    enabled: true
-    type: ClusterIP
-    externalPort: 5984
-
-  ## An Ingress resource can provide name-based virtual hosting and TLS
-  ## termination among other things for CouchDB deployments which are accessed
-  ## from outside the Kubernetes cluster.
-  ## ref: https://kubernetes.io/docs/concepts/services-networking/ingress/
-  ingress:
-    enabled: false
-    hosts:
-      - chart-example.local
-    path: /
-    annotations:
-      []
-      # kubernetes.io/ingress.class: nginx
-      # kubernetes.io/tls-acme: "true"
-    tls:
-      # Secrets must be manually created in the namespace.
-      # - secretName: chart-example-tls
-      #   hosts:
-      #     - chart-example.local
-
-  ## Optional resource requests and limits for the CouchDB container
-  ## ref: http://kubernetes.io/docs/user-guide/compute-resources/
-  resources:
-    {}
-    # requests:
-    #  cpu: 100m
-    #  memory: 128Mi
-    # limits:
-    #  cpu: 56
-    #  memory: 256Gi
-
-  ## erlangFlags is a map that is passed to the Erlang VM as flags using the
-  ## ERL_FLAGS env. `name` and `setcookie` flags are minimally required to
-  ## establish connectivity between cluster nodes.
-  ## ref: http://erlang.org/doc/man/erl.html#init_flags
-  erlangFlags:
-    name: couchdb
-    setcookie: monster
-
-  ## couchdbConfig will override default CouchDB configuration settings.
-  ## The contents of this map are reformatted into a .ini file laid down
-  ## by a ConfigMap object.
-  ## ref: http://docs.couchdb.org/en/latest/config/index.html
   couchdbConfig:
     couchdb:
-      uuid: budibase-couchdb # REQUIRED: Unique identifier for this CouchDB server instance
+      # -- Unique identifier for this CouchDB server instance. You shouldn't need
-    # cluster:
+      # to change this.
-    #   q: 8 # Create 8 shards for each database
+      uuid: budibase-couchdb
-    chttpd:
-      bind_address: any
-      # chttpd.require_valid_user disables all the anonymous requests to the port
-      # 5984 when is set to true.
-      require_valid_user: false
-
-  # Kubernetes local cluster domain.
-  # This is used to generate FQDNs for peers when joining the CouchDB cluster.
-  dns:
-    clusterDomainSuffix: cluster.local
-
-  ## Configure liveness and readiness probe values
-  ## Ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-probes/#configure-probes
-  # FOR COUCHDB
-  livenessProbe:
-    failureThreshold: 3
-    initialDelaySeconds: 0
-    periodSeconds: 10
-    successThreshold: 1
-    timeoutSeconds: 1
-  readinessProbe:
-    failureThreshold: 3
-    initialDelaySeconds: 0
-    periodSeconds: 10
-    successThreshold: 1
-    timeoutSeconds: 1

hosting/couchdb/runner.sh

@@ -26,27 +26,48 @@ if [[ "${TARGETBUILD}" = "aas" ]]; then
     sed -i "s#DATA_DIR#/home#g" /opt/clouseau/clouseau.ini
     sed -i "s#DATA_DIR#/home#g" /opt/couchdb/etc/local.ini
 elif [[ "${TARGETBUILD}" = "single" ]]; then
+    # In the single image build, the Dockerfile specifies /data as a volume
+    # mount, so we use that for all persistent data.
     sed -i "s#DATA_DIR#/data#g" /opt/clouseau/clouseau.ini
     sed -i "s#DATA_DIR#/data#g" /opt/couchdb/etc/local.ini
 elif [[ -n $KUBERNETES_SERVICE_HOST ]]; then
     # In Kubernetes the directory /opt/couchdb/data has a persistent volume
     # mount for storing database data.
     sed -i "s#DATA_DIR#/opt/couchdb/data#g" /opt/clouseau/clouseau.ini
-    sed -i "s#DATA_DIR#/opt/couchdb/data#g" /opt/couchdb/etc/local.ini
+
+    # We remove the database_dir and view_index_dir settings from the local.ini
+    # in Kubernetes because it will default to /opt/couchdb/data which is what
+    # our Helm chart was using prior to us switching to using our own CouchDB
+    # image.
+    sed -i "s#^database_dir.*\$##g" /opt/couchdb/etc/local.ini
+    sed -i "s#^view_index_dir.*\$##g" /opt/couchdb/etc/local.ini
+
+    # We remove the -name setting from the vm.args file in Kubernetes because
+    # it will default to the pod FQDN, which is what's required for clustering
+    # to work.
     sed -i "s/^-name .*$//g" /opt/couchdb/etc/vm.args
 else
+    # For all other builds, we use /data for persistent data.
     sed -i "s#DATA_DIR#/data#g" /opt/clouseau/clouseau.ini
     sed -i "s#DATA_DIR#/data#g" /opt/couchdb/etc/local.ini
 fi
 
+# Start Clouseau. Budibase won't function correctly without Clouseau running, it
+# powers the search API endpoints which are used to do all sorts, including
+# populating app grids.
 /opt/clouseau/bin/clouseau > /dev/stdout 2>&1 &
+
+# Start CouchDB.
 /docker-entrypoint.sh /opt/couchdb/bin/couchdb &
 
+# Wati for CouchDB to start up.
 while [[ $(curl -s -w "%{http_code}\n" http://localhost:5984/_up -o /dev/null) -ne 200 ]]; do
     echo 'Waiting for CouchDB to start...';
     sleep 5;
 done
 
+# CouchDB needs the `_users` and `_replicator` databases to exist before it will
+# function correctly, so we create them here.
 curl -X PUT http://${COUCHDB_USER}:${COUCHDB_PASSWORD}@localhost:5984/_users
 curl -X PUT http://${COUCHDB_USER}:${COUCHDB_PASSWORD}@localhost:5984/_replicator
 sleep infinity

lerna.json

@@ -1,5 +1,5 @@
 {
-  "version": "2.13.17",
+  "version": "2.13.31",
   "npmClient": "yarn",
   "packages": [
     "packages/*"

packages/backend-core/package.json

@@ -72,7 +72,7 @@
     "@types/tar-fs": "2.0.1",
     "@types/uuid": "8.3.4",
     "chance": "1.1.8",
-    "ioredis-mock": "8.7.0",
+    "ioredis-mock": "8.9.0",
     "jest": "29.6.2",
     "jest-environment-node": "29.6.2",
     "jest-serial-runner": "1.2.1",

packages/backend-core/src/context/mainContext.ts

@@ -99,6 +99,8 @@ function updateContext(updates: ContextMap): ContextMap {
 }
 
 async function newContext<T>(updates: ContextMap, task: () => T) {
+  guardMigration()
+
   // see if there already is a context setup
   let context: ContextMap = updateContext(updates)
   return Context.run(context, task)
@@ -145,23 +147,27 @@ export async function doInTenant<T>(
 }
 
 export async function doInAppContext<T>(
-  appId: string | null,
+  appId: string,
   task: () => T
 ): Promise<T> {
-  if (!appId && !env.isTest()) {
+  return _doInAppContext(appId, task)
+}
+
+async function _doInAppContext<T>(
+  appId: string,
+  task: () => T,
+  extraContextSettings?: ContextMap
+): Promise<T> {
+  if (!appId) {
     throw new Error("appId is required")
   }
 
-  let updates: ContextMap
+  const tenantId = getTenantIDFromAppID(appId)
-  if (!appId) {
+  const updates: ContextMap = { appId, ...extraContextSettings }
-    updates = { appId: "" }
+  if (tenantId) {
-  } else {
+    updates.tenantId = tenantId
-    const tenantId = getTenantIDFromAppID(appId)
-    updates = { appId }
-    if (tenantId) {
-      updates.tenantId = tenantId
-    }
   }
+
   return newContext(updates, task)
 }
 
@@ -182,6 +188,24 @@ export async function doInIdentityContext<T>(
   return newContext(context, task)
 }
 
+function guardMigration() {
+  const context = Context.get()
+  if (context?.isMigrating) {
+    throw new Error(
+      "The context cannot be changed, a migration is currently running"
+    )
+  }
+}
+
+export async function doInAppMigrationContext<T>(
+  appId: string,
+  task: () => T
+): Promise<T> {
+  return _doInAppContext(appId, task, {
+    isMigrating: true,
+  })
+}
+
 export function getIdentity(): IdentityContext | undefined {
   try {
     const context = Context.get()

packages/backend-core/src/context/tests/index.spec.ts

@@ -1,6 +1,11 @@
 import { testEnv } from "../../../tests/extra"
 import * as context from "../"
 import { DEFAULT_TENANT_ID } from "../../constants"
+import { structures } from "../../../tests"
+import { db } from "../.."
+import Context from "../Context"
+import { ContextMap } from "../types"
+import { IdentityType } from "@budibase/types"
 
 describe("context", () => {
   describe("doInTenant", () => {
@@ -144,4 +149,107 @@ describe("context", () => {
       expect(isScim).toBe(false)
     })
   })
+
+  describe("doInAppMigrationContext", () => {
+    it("the context is set correctly", async () => {
+      const appId = db.generateAppID()
+
+      await context.doInAppMigrationContext(appId, () => {
+        const context = Context.get()
+
+        const expected: ContextMap = {
+          appId,
+          isMigrating: true,
+        }
+        expect(context).toEqual(expected)
+      })
+    })
+
+    it("the context is set correctly when running in a tenant id", async () => {
+      const tenantId = structures.tenant.id()
+      const appId = db.generateAppID(tenantId)
+
+      await context.doInAppMigrationContext(appId, () => {
+        const context = Context.get()
+
+        const expected: ContextMap = {
+          appId,
+          isMigrating: true,
+          tenantId,
+        }
+        expect(context).toEqual(expected)
+      })
+    })
+
+    it("the context is not modified outside the delegate", async () => {
+      const appId = db.generateAppID()
+
+      expect(Context.get()).toBeUndefined()
+
+      await context.doInAppMigrationContext(appId, () => {
+        const context = Context.get()
+
+        const expected: ContextMap = {
+          appId,
+          isMigrating: true,
+        }
+        expect(context).toEqual(expected)
+      })
+
+      expect(Context.get()).toBeUndefined()
+    })
+
+    it.each([
+      [
+        "doInAppMigrationContext",
+        () => context.doInAppMigrationContext(db.generateAppID(), () => {}),
+      ],
+      [
+        "doInAppContext",
+        () => context.doInAppContext(db.generateAppID(), () => {}),
+      ],
+      [
+        "doInAutomationContext",
+        () =>
+          context.doInAutomationContext({
+            appId: db.generateAppID(),
+            automationId: structures.generator.guid(),
+            task: () => {},
+          }),
+      ],
+      ["doInContext", () => context.doInContext(db.generateAppID(), () => {})],
+      [
+        "doInEnvironmentContext",
+        () => context.doInEnvironmentContext({}, () => {}),
+      ],
+      [
+        "doInIdentityContext",
+        () =>
+          context.doInIdentityContext(
+            {
+              account: undefined,
+              type: IdentityType.USER,
+              _id: structures.users.user()._id!,
+            },
+            () => {}
+          ),
+      ],
+      ["doInScimContext", () => context.doInScimContext(() => {})],
+      [
+        "doInTenant",
+        () => context.doInTenant(structures.tenant.id(), () => {}),
+      ],
+    ])(
+      "a nested context.%s function cannot run",
+      async (_, otherContextCall: () => Promise<void>) => {
+        await expect(
+          context.doInAppMigrationContext(db.generateAppID(), async () => {
+            await otherContextCall()
+          })
+        ).rejects.toThrowError(
+          "The context cannot be changed, a migration is currently running"
+        )
+      }
+    )
+  })
 })

packages/backend-core/src/context/types.ts

@@ -8,4 +8,5 @@ export type ContextMap = {
   environmentVariables?: Record<string, string>
   isScim?: boolean
   automationId?: string
+  isMigrating?: boolean
 }

packages/backend-core/src/objectStore/objectStore.ts

@@ -260,12 +260,12 @@ export async function listAllObjects(bucketName: string, path: string) {
 }
 
 /**
- * Generate a presigned url with a default TTL of 1 hour
+ * Generate a presigned url with a default TTL of 36 hours
  */
 export function getPresignedUrl(
   bucketName: string,
   key: string,
-  durationSeconds: number = 3600
+  durationSeconds: number = 129600
 ) {
   const objectStore = ObjectStore(bucketName, { presigning: true })
   const params = {

packages/backend-core/src/redis/redlockImpl.ts

@@ -2,8 +2,9 @@ import Redlock from "redlock"
 import { getLockClient } from "./init"
 import { LockOptions, LockType } from "@budibase/types"
 import * as context from "../context"
-import env from "../environment"
 import { logWarn } from "../logging"
+import { utils } from "@budibase/shared-core"
+import { Duration } from "../utils"
 
 async function getClient(
   type: LockType,
@@ -12,9 +13,7 @@ async function getClient(
   if (type === LockType.CUSTOM) {
     return newRedlock(opts)
   }
-  if (env.isTest() && type !== LockType.TRY_ONCE) {
+
-    return newRedlock(OPTIONS.TEST)
-  }
   switch (type) {
     case LockType.TRY_ONCE: {
       return newRedlock(OPTIONS.TRY_ONCE)
@@ -28,13 +27,16 @@ async function getClient(
     case LockType.DELAY_500: {
       return newRedlock(OPTIONS.DELAY_500)
     }
+    case LockType.AUTO_EXTEND: {
+      return newRedlock(OPTIONS.AUTO_EXTEND)
+    }
     default: {
-      throw new Error(`Could not get redlock client: ${type}`)
+      throw utils.unreachable(type)
     }
   }
 }
 
-const OPTIONS = {
+const OPTIONS: Record<keyof typeof LockType, Redlock.Options> = {
   TRY_ONCE: {
     // immediately throws an error if the lock is already held
     retryCount: 0,
@@ -42,11 +44,6 @@ const OPTIONS = {
   TRY_TWICE: {
     retryCount: 1,
   },
-  TEST: {
-    // higher retry count in unit tests
-    // due to high contention.
-    retryCount: 100,
-  },
   DEFAULT: {
     // the expected clock drift; for more details
     // see http://redis.io/topics/distlock
@@ -67,10 +64,14 @@ const OPTIONS = {
   DELAY_500: {
     retryDelay: 500,
   },
+  CUSTOM: {},
+  AUTO_EXTEND: {
+    retryCount: -1,
+  },
 }
 
 export async function newRedlock(opts: Redlock.Options = {}) {
-  let options = { ...OPTIONS.DEFAULT, ...opts }
+  const options = { ...OPTIONS.DEFAULT, ...opts }
   const redisWrapper = await getLockClient()
   const client = redisWrapper.getClient()
   return new Redlock([client], options)
@@ -100,17 +101,36 @@ function getLockName(opts: LockOptions) {
   return name
 }
 
+export const AUTO_EXTEND_POLLING_MS = Duration.fromSeconds(10).toMs()
+
 export async function doWithLock<T>(
   opts: LockOptions,
   task: () => Promise<T>
 ): Promise<RedlockExecution<T>> {
   const redlock = await getClient(opts.type, opts.customOptions)
-  let lock
+  let lock: Redlock.Lock | undefined
+  let timeout: NodeJS.Timeout | undefined
   try {
     const name = getLockName(opts)
 
+    const ttl =
+      opts.type === LockType.AUTO_EXTEND ? AUTO_EXTEND_POLLING_MS : opts.ttl
+
     // create the lock
-    lock = await redlock.lock(name, opts.ttl)
+    lock = await redlock.lock(name, ttl)
+
+    if (opts.type === LockType.AUTO_EXTEND) {
+      // We keep extending the lock while the task is running
+      const extendInIntervals = (): void => {
+        timeout = setTimeout(async () => {
+          lock = await lock!.extend(ttl, () => opts.onExtend && opts.onExtend())
+
+          extendInIntervals()
+        }, ttl / 2)
+      }
+
+      extendInIntervals()
+    }
 
     // perform locked task
     // need to await to ensure completion before unlocking
@@ -131,8 +151,7 @@ export async function doWithLock<T>(
       throw e
     }
   } finally {
-    if (lock) {
+    clearTimeout(timeout)
-      await lock.unlock()
+    await lock?.unlock()
-    }
   }
 }

packages/backend-core/src/redis/tests/redlockImpl.spec.ts

@@ -0,0 +1,105 @@
+import { LockName, LockType, LockOptions } from "@budibase/types"
+import { AUTO_EXTEND_POLLING_MS, doWithLock } from "../redlockImpl"
+import { DBTestConfiguration, generator } from "../../../tests"
+
+describe("redlockImpl", () => {
+  beforeEach(() => {
+    jest.useFakeTimers()
+  })
+
+  describe("doWithLock", () => {
+    const config = new DBTestConfiguration()
+    const lockTtl = AUTO_EXTEND_POLLING_MS
+
+    function runLockWithExecutionTime({
+      opts,
+      task,
+      executionTimeMs,
+    }: {
+      opts: LockOptions
+      task: () => Promise<string>
+      executionTimeMs: number
+    }) {
+      return config.doInTenant(() =>
+        doWithLock(opts, async () => {
+          // Run in multiple intervals until hitting the expected time
+          const interval = lockTtl / 10
+          for (let i = executionTimeMs; i > 0; i -= interval) {
+            await jest.advanceTimersByTimeAsync(interval)
+          }
+          return task()
+        })
+      )
+    }
+
+    it.each(Object.values(LockType))(
+      "should return the task value and release the lock",
+      async (lockType: LockType) => {
+        const expectedResult = generator.guid()
+        const mockTask = jest.fn().mockResolvedValue(expectedResult)
+
+        const opts: LockOptions = {
+          name: LockName.PERSIST_WRITETHROUGH,
+          type: lockType,
+          ttl: lockTtl,
+        }
+
+        const result = await runLockWithExecutionTime({
+          opts,
+          task: mockTask,
+          executionTimeMs: 0,
+        })
+
+        expect(result.executed).toBe(true)
+        expect(result.executed && result.result).toBe(expectedResult)
+        expect(mockTask).toHaveBeenCalledTimes(1)
+      }
+    )
+
+    it("should extend when type is autoextend", async () => {
+      const expectedResult = generator.guid()
+      const mockTask = jest.fn().mockResolvedValue(expectedResult)
+      const mockOnExtend = jest.fn()
+
+      const opts: LockOptions = {
+        name: LockName.PERSIST_WRITETHROUGH,
+        type: LockType.AUTO_EXTEND,
+        onExtend: mockOnExtend,
+      }
+
+      const result = await runLockWithExecutionTime({
+        opts,
+        task: mockTask,
+        executionTimeMs: lockTtl * 2.5,
+      })
+
+      expect(result.executed).toBe(true)
+      expect(result.executed && result.result).toBe(expectedResult)
+      expect(mockTask).toHaveBeenCalledTimes(1)
+      expect(mockOnExtend).toHaveBeenCalledTimes(5)
+    })
+
+    it.each(Object.values(LockType).filter(t => t !== LockType.AUTO_EXTEND))(
+      "should timeout when type is %s",
+      async (lockType: LockType) => {
+        const mockTask = jest.fn().mockResolvedValue("mockResult")
+
+        const opts: LockOptions = {
+          name: LockName.PERSIST_WRITETHROUGH,
+          type: lockType,
+          ttl: lockTtl,
+        }
+
+        await expect(
+          runLockWithExecutionTime({
+            opts,
+            task: mockTask,
+            executionTimeMs: lockTtl * 2,
+          })
+        ).rejects.toThrowError(
+          `Unable to fully release the lock on resource \"lock:${config.tenantId}_persist_writethrough\".`
+        )
+      }
+    )
+  })
+})

packages/bbui/src/Form/Core/Switch.svelte

@@ -18,6 +18,7 @@
     checked={value}
     {disabled}
     on:change={onChange}
+    on:click
     {id}
     type="checkbox"
     class="spectrum-Switch-input"

packages/bbui/src/Form/Core/TextField.svelte

@@ -20,7 +20,7 @@
   let focus = false
 
   const updateValue = newValue => {
-    if (readonly) {
+    if (readonly || disabled) {
       return
     }
     if (type === "number") {
@@ -31,14 +31,14 @@
   }
 
   const onFocus = () => {
-    if (readonly) {
+    if (readonly || disabled) {
       return
     }
     focus = true
   }
 
   const onBlur = event => {
-    if (readonly) {
+    if (readonly || disabled) {
       return
     }
     focus = false
@@ -46,14 +46,14 @@
   }
 
   const onInput = event => {
-    if (readonly || !updateOnChange) {
+    if (readonly || !updateOnChange || disabled) {
       return
     }
     updateValue(event.target.value)
   }
 
   const updateValueOnEnter = event => {
-    if (readonly) {
+    if (readonly || disabled) {
       return
     }
     if (event.key === "Enter") {
@@ -69,6 +69,7 @@
   }
 
   onMount(() => {
+    if (disabled) return
     focus = autofocus
     if (focus) field.focus()
   })
@@ -108,4 +109,16 @@
   .spectrum-Textfield {
     width: 100%;
   }
+
+  input::placeholder {
+    color: var(--grey-7);
+  }
+
+  input:hover::placeholder {
+    color: var(--grey-7) !important;
+  }
+
+  input:focus::placeholder {
+    color: var(--grey-7) !important;
+  }
 </style>

packages/bbui/src/Form/Toggle.svelte

@@ -19,5 +19,5 @@
 </script>
 
 <Field {helpText} {label} {labelPosition} {error}>
-  <Switch {error} {disabled} {text} {value} on:change={onChange} />
+  <Switch {error} {disabled} {text} {value} on:change={onChange} on:click />
 </Field>

packages/builder/src/builderStore/componentUtils.js

@@ -1,4 +1,5 @@
 import { store } from "./index"
+import { get } from "svelte/store"
 import { Helpers } from "@budibase/bbui"
 import {
   decodeJSBinding,
@@ -238,6 +239,10 @@ export const makeComponentUnique = component => {
 }
 
 export const getComponentText = component => {
+  if (component == null) {
+    return ""
+  }
+
   if (component?._instanceName) {
     return component._instanceName
   }
@@ -246,3 +251,16 @@ export const getComponentText = component => {
     "component"
   return capitalise(type)
 }
+
+export const getComponentName = component => {
+  if (component == null) {
+    return ""
+  }
+
+  const components = get(store)?.components || {}
+  const componentDefinition = components[component._component] || {}
+  const name =
+    componentDefinition.friendlyName || componentDefinition.name || ""
+
+  return name
+}

packages/builder/src/builderStore/dataBinding.js

@@ -29,6 +29,12 @@ const CAPTURE_VAR_INSIDE_TEMPLATE = /{{([^}]+)}}/g
 const CAPTURE_VAR_INSIDE_JS = /\$\("([^")]+)"\)/g
 const CAPTURE_HBS_TEMPLATE = /{{[\S\s]*?}}/g
 
+const UpdateReferenceAction = {
+  ADD: "add",
+  DELETE: "delete",
+  MOVE: "move",
+}
+
 /**
  * Gets all bindable data context fields and instance fields.
  */
@@ -1226,3 +1232,81 @@ export const runtimeToReadableBinding = (
     "readableBinding"
   )
 }
+
+/**
+ * Used to update binding references for automation or action steps
+ *
+ * @param obj - The object to be updated
+ * @param originalIndex - The original index of the step being moved. Not applicable to add/delete.
+ * @param modifiedIndex - The new index of the step being modified
+ * @param action - Used to determine if a step is being added, deleted or moved
+ * @param label - The binding text that describes the steps
+ */
+export const updateReferencesInObject = ({
+  obj,
+  modifiedIndex,
+  action,
+  label,
+  originalIndex,
+}) => {
+  const stepIndexRegex = new RegExp(`{{\\s*${label}\\.(\\d+)\\.`, "g")
+  const updateActionStep = (str, index, replaceWith) =>
+    str.replace(`{{ ${label}.${index}.`, `{{ ${label}.${replaceWith}.`)
+  for (const key in obj) {
+    if (typeof obj[key] === "string") {
+      let matches
+      while ((matches = stepIndexRegex.exec(obj[key])) !== null) {
+        const referencedStep = parseInt(matches[1])
+        if (
+          action === UpdateReferenceAction.ADD &&
+          referencedStep >= modifiedIndex
+        ) {
+          obj[key] = updateActionStep(
+            obj[key],
+            referencedStep,
+            referencedStep + 1
+          )
+        } else if (
+          action === UpdateReferenceAction.DELETE &&
+          referencedStep > modifiedIndex
+        ) {
+          obj[key] = updateActionStep(
+            obj[key],
+            referencedStep,
+            referencedStep - 1
+          )
+        } else if (action === UpdateReferenceAction.MOVE) {
+          if (referencedStep === originalIndex) {
+            obj[key] = updateActionStep(obj[key], referencedStep, modifiedIndex)
+          } else if (
+            modifiedIndex <= referencedStep &&
+            modifiedIndex < originalIndex
+          ) {
+            obj[key] = updateActionStep(
+              obj[key],
+              referencedStep,
+              referencedStep + 1
+            )
+          } else if (
+            modifiedIndex >= referencedStep &&
+            modifiedIndex > originalIndex
+          ) {
+            obj[key] = updateActionStep(
+              obj[key],
+              referencedStep,
+              referencedStep - 1
+            )
+          }
+        }
+      }
+    } else if (typeof obj[key] === "object" && obj[key] !== null) {
+      updateReferencesInObject({
+        obj: obj[key],
+        modifiedIndex,
+        action,
+        label,
+        originalIndex,
+      })
+    }
+  }
+}

packages/builder/src/builderStore/index.js

@@ -4,7 +4,7 @@ import { getTemporalStore } from "./store/temporal"
 import { getThemeStore } from "./store/theme"
 import { getUserStore } from "./store/users"
 import { getDeploymentStore } from "./store/deployments"
-import { derived, writable, get } from "svelte/store"
+import { derived, get } from "svelte/store"
 import { findComponent, findComponentPath } from "./componentUtils"
 import { RoleUtils } from "@budibase/frontend-core"
 import { createHistoryStore } from "builderStore/store/history"
@@ -146,5 +146,3 @@ export const userSelectedResourceMap = derived(userStore, $userStore => {
 export const isOnlyUser = derived(userStore, $userStore => {
   return $userStore.length < 2
 })
-
-export const screensHeight = writable("210px")

packages/builder/src/builderStore/store/automation/index.js

@@ -4,6 +4,7 @@ import { cloneDeep } from "lodash/fp"
 import { generate } from "shortid"
 import { selectedAutomation } from "builderStore"
 import { notifications } from "@budibase/bbui"
+import { updateReferencesInObject } from "builderStore/dataBinding"
 
 const initialAutomationState = {
   automations: [],
@@ -22,34 +23,14 @@ export const getAutomationStore = () => {
   return store
 }
 
-const updateReferencesInObject = (obj, modifiedIndex, action) => {
-  const regex = /{{\s*steps\.(\d+)\./g
-  for (const key in obj) {
-    if (typeof obj[key] === "string") {
-      let matches
-      while ((matches = regex.exec(obj[key])) !== null) {
-        const referencedStep = parseInt(matches[1])
-        if (action === "add" && referencedStep >= modifiedIndex) {
-          obj[key] = obj[key].replace(
-            `{{ steps.${referencedStep}.`,
-            `{{ steps.${referencedStep + 1}.`
-          )
-        } else if (action === "delete" && referencedStep > modifiedIndex) {
-          obj[key] = obj[key].replace(
-            `{{ steps.${referencedStep}.`,
-            `{{ steps.${referencedStep - 1}.`
-          )
-        }
-      }
-    } else if (typeof obj[key] === "object" && obj[key] !== null) {
-      updateReferencesInObject(obj[key], modifiedIndex, action)
-    }
-  }
-}
-
 const updateStepReferences = (steps, modifiedIndex, action) => {
   steps.forEach(step => {
-    updateReferencesInObject(step.inputs, modifiedIndex, action)
+    updateReferencesInObject({
+      obj: step.inputs,
+      modifiedIndex,
+      action,
+      label: "steps",
+    })
   })
 }
 

packages/builder/src/builderStore/tests/dataBinding.test.js

@@ -2,6 +2,7 @@ import { expect, describe, it, vi } from "vitest"
 import {
   runtimeToReadableBinding,
   readableToRuntimeBinding,
+  updateReferencesInObject,
 } from "../dataBinding"
 
 vi.mock("@budibase/frontend-core")
@@ -84,3 +85,461 @@ describe("readableToRuntimeBinding", () => {
     ).toEqual(`Hello {{ [user].[firstName] }}! The count is {{ count }}.`)
   })
 })
+
+describe("updateReferencesInObject", () => {
+  it("should increment steps in sequence on 'add'", () => {
+    let obj = [
+      {
+        id: "a0",
+        parameters: {
+          text: "Alpha",
+        },
+      },
+      {
+        id: "a1",
+        parameters: {
+          text: "Apple",
+        },
+      },
+      {
+        id: "b2",
+        parameters: {
+          text: "Banana {{ actions.1.row }}",
+        },
+      },
+      {
+        id: "c3",
+        parameters: {
+          text: "Carrot {{ actions.1.row }}",
+        },
+      },
+      {
+        id: "d4",
+        parameters: {
+          text: "Dog {{ actions.3.row }}",
+        },
+      },
+      {
+        id: "e5",
+        parameters: {
+          text: "Eagle {{ actions.4.row }}",
+        },
+      },
+    ]
+    updateReferencesInObject({
+      obj,
+      modifiedIndex: 0,
+      action: "add",
+      label: "actions",
+    })
+
+    expect(obj).toEqual([
+      {
+        id: "a0",
+        parameters: {
+          text: "Alpha",
+        },
+      },
+      {
+        id: "a1",
+        parameters: {
+          text: "Apple",
+        },
+      },
+      {
+        id: "b2",
+        parameters: {
+          text: "Banana {{ actions.2.row }}",
+        },
+      },
+      {
+        id: "c3",
+        parameters: {
+          text: "Carrot {{ actions.2.row }}",
+        },
+      },
+      {
+        id: "d4",
+        parameters: {
+          text: "Dog {{ actions.4.row }}",
+        },
+      },
+      {
+        id: "e5",
+        parameters: {
+          text: "Eagle {{ actions.5.row }}",
+        },
+      },
+    ])
+  })
+
+  it("should decrement steps in sequence on 'delete'", () => {
+    let obj = [
+      {
+        id: "a1",
+        parameters: {
+          text: "Apple",
+        },
+      },
+      {
+        id: "b2",
+        parameters: {
+          text: "Banana {{ actions.1.row }}",
+        },
+      },
+      {
+        id: "d4",
+        parameters: {
+          text: "Dog {{ actions.3.row }}",
+        },
+      },
+      {
+        id: "e5",
+        parameters: {
+          text: "Eagle {{ actions.4.row }}",
+        },
+      },
+    ]
+    updateReferencesInObject({
+      obj,
+      modifiedIndex: 2,
+      action: "delete",
+      label: "actions",
+    })
+
+    expect(obj).toEqual([
+      {
+        id: "a1",
+        parameters: {
+          text: "Apple",
+        },
+      },
+      {
+        id: "b2",
+        parameters: {
+          text: "Banana {{ actions.1.row }}",
+        },
+      },
+      {
+        id: "d4",
+        parameters: {
+          text: "Dog {{ actions.2.row }}",
+        },
+      },
+      {
+        id: "e5",
+        parameters: {
+          text: "Eagle {{ actions.3.row }}",
+        },
+      },
+    ])
+  })
+
+  it("should handle on 'move' to a lower index", () => {
+    let obj = [
+      {
+        id: "a1",
+        parameters: {
+          text: "Apple",
+        },
+      },
+      {
+        id: "b2",
+        parameters: {
+          text: "Banana {{ actions.0.row }}",
+        },
+      },
+      {
+        id: "e5",
+        parameters: {
+          text: "Eagle {{ actions.3.row }}",
+        },
+      },
+      {
+        id: "c3",
+        parameters: {
+          text: "Carrot {{ actions.0.row }}",
+        },
+      },
+      {
+        id: "d4",
+        parameters: {
+          text: "Dog {{ actions.2.row }}",
+        },
+      },
+    ]
+    updateReferencesInObject({
+      obj,
+      modifiedIndex: 2,
+      action: "move",
+      label: "actions",
+      originalIndex: 4,
+    })
+
+    expect(obj).toEqual([
+      {
+        id: "a1",
+        parameters: {
+          text: "Apple",
+        },
+      },
+      {
+        id: "b2",
+        parameters: {
+          text: "Banana {{ actions.0.row }}",
+        },
+      },
+      {
+        id: "e5",
+        parameters: {
+          text: "Eagle {{ actions.4.row }}",
+        },
+      },
+      {
+        id: "c3",
+        parameters: {
+          text: "Carrot {{ actions.0.row }}",
+        },
+      },
+      {
+        id: "d4",
+        parameters: {
+          text: "Dog {{ actions.3.row }}",
+        },
+      },
+    ])
+  })
+
+  it("should handle on 'move' to a higher index", () => {
+    let obj = [
+      {
+        id: "b2",
+        parameters: {
+          text: "Banana {{ actions.0.row }}",
+        },
+      },
+      {
+        id: "c3",
+        parameters: {
+          text: "Carrot {{ actions.0.row }}",
+        },
+      },
+      {
+        id: "a1",
+        parameters: {
+          text: "Apple",
+        },
+      },
+      {
+        id: "d4",
+        parameters: {
+          text: "Dog {{ actions.2.row }}",
+        },
+      },
+      {
+        id: "e5",
+        parameters: {
+          text: "Eagle {{ actions.3.row }}",
+        },
+      },
+    ]
+    updateReferencesInObject({
+      obj,
+      modifiedIndex: 2,
+      action: "move",
+      label: "actions",
+      originalIndex: 0,
+    })
+
+    expect(obj).toEqual([
+      {
+        id: "b2",
+        parameters: {
+          text: "Banana {{ actions.2.row }}",
+        },
+      },
+      {
+        id: "c3",
+        parameters: {
+          text: "Carrot {{ actions.2.row }}",
+        },
+      },
+      {
+        id: "a1",
+        parameters: {
+          text: "Apple",
+        },
+      },
+      {
+        id: "d4",
+        parameters: {
+          text: "Dog {{ actions.1.row }}",
+        },
+      },
+      {
+        id: "e5",
+        parameters: {
+          text: "Eagle {{ actions.3.row }}",
+        },
+      },
+    ])
+  })
+
+  it("should handle on 'move' of action being referenced, dragged to a higher index", () => {
+    let obj = [
+      {
+        "##eventHandlerType": "Validate Form",
+        id: "cCD0Dwcnq",
+      },
+      {
+        "##eventHandlerType": "Close Screen Modal",
+        id: "3fbbIOfN0H",
+      },
+      {
+        "##eventHandlerType": "Save Row",
+        parameters: {
+          tableId: "ta_bb_employee",
+        },
+        id: "aehg5cTmhR",
+      },
+      {
+        "##eventHandlerType": "Close Side Panel",
+        id: "mzkpf86cxo",
+      },
+      {
+        "##eventHandlerType": "Navigate To",
+        id: "h0uDFeJa8A",
+      },
+      {
+        parameters: {
+          autoDismiss: true,
+          type: "success",
+          message: "{{ actions.1.row }}",
+        },
+        "##eventHandlerType": "Show Notification",
+        id: "JEI5lAyJZ",
+      },
+    ]
+    updateReferencesInObject({
+      obj,
+      modifiedIndex: 2,
+      action: "move",
+      label: "actions",
+      originalIndex: 1,
+    })
+
+    expect(obj).toEqual([
+      {
+        "##eventHandlerType": "Validate Form",
+        id: "cCD0Dwcnq",
+      },
+      {
+        "##eventHandlerType": "Close Screen Modal",
+        id: "3fbbIOfN0H",
+      },
+      {
+        "##eventHandlerType": "Save Row",
+        parameters: {
+          tableId: "ta_bb_employee",
+        },
+        id: "aehg5cTmhR",
+      },
+      {
+        "##eventHandlerType": "Close Side Panel",
+        id: "mzkpf86cxo",
+      },
+      {
+        "##eventHandlerType": "Navigate To",
+        id: "h0uDFeJa8A",
+      },
+      {
+        parameters: {
+          autoDismiss: true,
+          type: "success",
+          message: "{{ actions.2.row }}",
+        },
+        "##eventHandlerType": "Show Notification",
+        id: "JEI5lAyJZ",
+      },
+    ])
+  })
+
+  it("should handle on 'move' of action being referenced, dragged to a lower index", () => {
+    let obj = [
+      {
+        "##eventHandlerType": "Save Row",
+        parameters: {
+          tableId: "ta_bb_employee",
+        },
+        id: "aehg5cTmhR",
+      },
+      {
+        "##eventHandlerType": "Validate Form",
+        id: "cCD0Dwcnq",
+      },
+      {
+        "##eventHandlerType": "Close Screen Modal",
+        id: "3fbbIOfN0H",
+      },
+      {
+        "##eventHandlerType": "Close Side Panel",
+        id: "mzkpf86cxo",
+      },
+      {
+        "##eventHandlerType": "Navigate To",
+        id: "h0uDFeJa8A",
+      },
+      {
+        parameters: {
+          autoDismiss: true,
+          type: "success",
+          message: "{{ actions.4.row }}",
+        },
+        "##eventHandlerType": "Show Notification",
+        id: "JEI5lAyJZ",
+      },
+    ]
+    updateReferencesInObject({
+      obj,
+      modifiedIndex: 0,
+      action: "move",
+      label: "actions",
+      originalIndex: 4,
+    })
+
+    expect(obj).toEqual([
+      {
+        "##eventHandlerType": "Save Row",
+        parameters: {
+          tableId: "ta_bb_employee",
+        },
+        id: "aehg5cTmhR",
+      },
+      {
+        "##eventHandlerType": "Validate Form",
+        id: "cCD0Dwcnq",
+      },
+      {
+        "##eventHandlerType": "Close Screen Modal",
+        id: "3fbbIOfN0H",
+      },
+      {
+        "##eventHandlerType": "Close Side Panel",
+        id: "mzkpf86cxo",
+      },
+      {
+        "##eventHandlerType": "Navigate To",
+        id: "h0uDFeJa8A",
+      },
+      {
+        parameters: {
+          autoDismiss: true,
+          type: "success",
+          message: "{{ actions.0.row }}",
+        },
+        "##eventHandlerType": "Show Notification",
+        id: "JEI5lAyJZ",
+      },
+    ])
+  })
+})

packages/builder/src/components/automation/SetupPanel/RowSelectorTypes.svelte

@@ -64,7 +64,7 @@
   </span>
 {:else if schema.type === "link"}
   <LinkedRowSelector
-    bind:linkedRows={value[field]}
+    linkedRows={value[field]}
     {schema}
     on:change={e => onChange(e, field)}
     useLabel={false}

packages/builder/src/components/automation/SetupPanel/TableSelector.svelte

@@ -22,7 +22,7 @@
 <Select
   on:change={onChange}
   bind:value
-  options={filteredTables.filter(table => table._id !== TableNames.USERS)}
+  options={filteredTables}
   getOptionLabel={table => table.name}
   getOptionValue={table => table._id}
 />

packages/builder/src/components/backend/DataTable/RowFieldControl.svelte

@@ -70,7 +70,12 @@
     options={meta.constraints.inclusion}
   />
 {:else if type === "link"}
-  <LinkedRowSelector {error} bind:linkedRows={value} schema={meta} />
+  <LinkedRowSelector
+    {error}
+    linkedRows={value}
+    schema={meta}
+    on:change={e => (value = e.detail)}
+  />
 {:else if type === "longform"}
   {#if meta.useRichText}
     <RichTextField {error} {label} height="150px" bind:value />

packages/builder/src/components/common/LinkedRowSelector.svelte

@@ -56,12 +56,12 @@
   />
 {:else}
   <Multiselect
-    bind:value={linkedIds}
+    value={linkedIds}
     {label}
     options={rows}
     getOptionLabel={getPrettyName}
     getOptionValue={row => row._id}
     sort
-    on:change={() => dispatch("change", linkedIds)}
+    on:change
   />
 {/if}

packages/builder/src/components/common/NavItem.svelte

@@ -1,10 +1,11 @@
 <script>
-  import { Icon } from "@budibase/bbui"
+  import { AbsTooltip, Icon } from "@budibase/bbui"
   import { createEventDispatcher, getContext } from "svelte"
   import { helpers } from "@budibase/shared-core"
   import { UserAvatars } from "@budibase/frontend-core"
 
   export let icon
+  export let iconTooltip
   export let withArrow = false
   export let withActions = true
   export let indentLevel = 0
@@ -77,7 +78,11 @@
   {style}
   {draggable}
 >
-  <div class="nav-item-content" bind:this={contentRef}>
+  <div
+    class="nav-item-content"
+    bind:this={contentRef}
+    class:right={rightAlignIcon}
+  >
     {#if withArrow}
       <div
         class:opened
@@ -98,7 +103,9 @@
       </div>
     {:else if icon}
       <div class="icon" class:right={rightAlignIcon}>
-        <Icon color={iconColor} size="S" name={icon} />
+        <AbsTooltip type="info" position="right" text={iconTooltip}>
+          <Icon color={iconColor} size="S" name={icon} />
+        </AbsTooltip>
       </div>
     {/if}
     <div class="text" title={showTooltip ? text : null}>
@@ -166,6 +173,11 @@
     width: max-content;
     position: relative;
     padding-left: var(--spacing-l);
+    box-sizing: border-box;
+  }
+
+  .nav-item-content.right {
+    width: 100%;
   }
 
   /* Needed to fully display the actions icon */
@@ -264,6 +276,7 @@
   }
 
   .right {
+    margin-left: auto;
     order: 10;
   }
 </style>

packages/builder/src/components/common/resizable.js

@@ -0,0 +1,119 @@
+const getResizeActions = (
+  cssProperty,
+  mouseMoveEventProperty,
+  elementProperty,
+  initialValue,
+  setValue = () => {}
+) => {
+  let element = null
+
+  const elementAction = node => {
+    element = node
+
+    if (initialValue != null) {
+      element.style[cssProperty] = `${initialValue}px`
+    }
+
+    return {
+      destroy() {
+        element = null
+      },
+    }
+  }
+
+  const dragHandleAction = node => {
+    let startProperty = null
+    let startPosition = null
+
+    const handleMouseMove = e => {
+      e.preventDefault() // Prevent highlighting while dragging
+      const change = e[mouseMoveEventProperty] - startPosition
+      element.style[cssProperty] = `${startProperty + change}px`
+    }
+
+    const handleMouseUp = e => {
+      e.preventDefault() // Prevent highlighting while dragging
+      window.removeEventListener("mousemove", handleMouseMove)
+      window.removeEventListener("mouseup", handleMouseUp)
+
+      element.style.removeProperty("transition") // remove temporary transition override
+      for (let item of document.getElementsByTagName("iframe")) {
+        item.style.removeProperty("pointer-events")
+      }
+
+      setValue(element[elementProperty])
+    }
+
+    const handleMouseDown = e => {
+      if (e.detail > 1) {
+        // e.detail is the number of rapid clicks, so e.detail = 2 is
+        // a double click. We want to prevent default behaviour in
+        // this case as it highlights nearby selectable elements, which
+        // then interferes with the resizing mousemove.
+        // Putting this on the double click handler doesn't seem to
+        // work, so it must go here.
+        e.preventDefault()
+      }
+
+      if (
+        e.target.hasAttribute("disabled") &&
+        e.target.getAttribute("disabled") !== "false"
+      ) {
+        return
+      }
+
+      element.style.transition = `${cssProperty} 0ms` // temporarily override any height transitions
+
+      // iframes swallow mouseup events if your cursor ends up over it during a drag, so make them
+      // temporarily non-interactive
+      for (let item of document.getElementsByTagName("iframe")) {
+        item.style.pointerEvents = "none"
+      }
+
+      startProperty = element[elementProperty]
+      startPosition = e[mouseMoveEventProperty]
+
+      window.addEventListener("mousemove", handleMouseMove)
+      window.addEventListener("mouseup", handleMouseUp)
+    }
+
+    const handleDoubleClick = () => {
+      element.style.removeProperty(cssProperty)
+    }
+
+    node.addEventListener("mousedown", handleMouseDown)
+    node.addEventListener("dblclick", handleDoubleClick)
+
+    return {
+      destroy() {
+        node.removeEventListener("mousedown", handleMouseDown)
+        node.removeEventListener("dblclick", handleDoubleClick)
+      },
+    }
+  }
+
+  return [elementAction, dragHandleAction]
+}
+
+export const getVerticalResizeActions = (initialValue, setValue = () => {}) => {
+  return getResizeActions(
+    "height",
+    "pageY",
+    "clientHeight",
+    initialValue,
+    setValue
+  )
+}
+
+export const getHorizontalResizeActions = (
+  initialValue,
+  setValue = () => {}
+) => {
+  return getResizeActions(
+    "width",
+    "pageX",
+    "clientWidth",
+    initialValue,
+    setValue
+  )
+}

packages/builder/src/components/design/Panel.svelte

@@ -1,8 +1,9 @@
 <script>
-  import { Icon, Body } from "@budibase/bbui"
+  import { AbsTooltip, Icon, Body } from "@budibase/bbui"
 
   export let title
   export let icon
+  export let iconTooltip
   export let showAddButton = false
   export let showBackButton = false
   export let showCloseButton = false
@@ -38,7 +39,9 @@
       <Icon name="ArrowLeft" hoverable on:click={onClickBackButton} />
     {/if}
     {#if icon}
-      <Icon name={icon} />
+      <AbsTooltip type="info" text={iconTooltip}>
+        <Icon name={icon} />
+      </AbsTooltip>
     {/if}
     <div class:title={titleCSS}>
       {#if customTitleContent}
@@ -70,6 +73,7 @@
 
 <style>
   .panel {
+    min-width: 260px;
     width: 260px;
     flex: 0 0 260px;
     background: var(--background);
@@ -87,6 +91,7 @@
     border-right: var(--border-light);
   }
   .panel.wide {
+    min-width: 310px;
     width: 310px;
     flex: 0 0 310px;
   }

packages/builder/src/components/design/settings/controls/ButtonActionEditor/ButtonActionDrawer.svelte

@@ -15,6 +15,7 @@
     getEventContextBindings,
     getActionBindings,
     makeStateBinding,
+    updateReferencesInObject,
   } from "builderStore/dataBinding"
   import { cloneDeep } from "lodash/fp"
 
@@ -30,6 +31,7 @@
 
   let actionQuery
   let selectedAction = actions?.length ? actions[0] : null
+  let originalActionIndex
 
   const setUpdateActions = actions => {
     return actions
@@ -115,6 +117,14 @@
     if (isSelected) {
       selectedAction = actions?.length ? actions[0] : null
     }
+
+    // Update action binding references
+    updateReferencesInObject({
+      obj: actions,
+      modifiedIndex: index,
+      action: "delete",
+      label: "actions",
+    })
   }
 
   const toggleActionList = () => {
@@ -137,6 +147,7 @@
 
   const selectAction = action => () => {
     selectedAction = action
+    originalActionIndex = actions.findIndex(item => item.id === action.id)
   }
 
   const onAddAction = actionType => {
@@ -146,9 +157,29 @@
 
   function handleDndConsider(e) {
     actions = e.detail.items
+
+    // set the initial index of the action being dragged
+    if (e.detail.info.trigger === "draggedEntered") {
+      originalActionIndex = actions.findIndex(
+        action => action.id === e.detail.info.id
+      )
+    }
   }
   function handleDndFinalize(e) {
     actions = e.detail.items
+
+    // Update action binding references
+    updateReferencesInObject({
+      obj: actions,
+      modifiedIndex: actions.findIndex(
+        action => action.id === e.detail.info.id
+      ),
+      action: "move",
+      label: "actions",
+      originalIndex: originalActionIndex,
+    })
+
+    originalActionIndex = -1
   }
 
   const getAllBindings = (actionBindings, eventContextBindings, actions) => {
@@ -289,7 +320,7 @@
   </Layout>
   <Layout noPadding>
     {#if selectedActionComponent && !showAvailableActions}
-      {#key selectedAction.id}
+      {#key (selectedAction.id, originalActionIndex)}
         <div class="selected-action-container">
           <svelte:component
             this={selectedActionComponent}

packages/builder/src/components/design/settings/controls/ButtonConfiguration/ButtonSetting.svelte

@@ -55,7 +55,10 @@
       size="S"
       name="Close"
       hoverable
-      on:click={() => removeButton(item._id)}
+      on:click={e => {
+        e.stopPropagation()
+        removeButton(item._id)
+      }}
     />
   </div>
 </div>

packages/builder/src/components/design/settings/controls/DraggableList/DraggableList.svelte

@@ -32,11 +32,14 @@
   }
 
   const dispatch = createEventDispatcher()
-  const flipDurationMs = 150
 
   let anchors = {}
   let draggableItems = []
 
+  // Used for controlling cursor behaviour in order to limit drag behaviour
+  // to the drag handle
+  let inactive = true
+
   const buildDraggable = items => {
     return items
       .map(item => {
@@ -64,6 +67,7 @@
   }
 
   const handleFinalize = e => {
+    inactive = true
     updateRowOrder(e)
     dispatch("change", serialiseUpdate())
   }
@@ -77,24 +81,36 @@
   class="list-wrap"
   use:dndzone={{
     items: draggableItems,
-    flipDurationMs,
     dropTargetStyle: { outline: "none" },
-    dragDisabled: !draggable,
+    dragDisabled: !draggable || inactive,
   }}
   on:finalize={handleFinalize}
   on:consider={updateRowOrder}
 >
-  {#each draggableItems as draggable (draggable.id)}
+  {#each draggableItems as draggableItem (draggableItem.id)}
     <li
+      on:click={() => {
+        get(store).actions.select(draggableItem.id)
+      }}
       on:mousedown={() => {
         get(store).actions.select()
       }}
-      bind:this={anchors[draggable.id]}
+      bind:this={anchors[draggableItem.id]}
-      class:highlighted={draggable.id === $store.selected}
+      class:highlighted={draggableItem.id === $store.selected}
     >
       <div class="left-content">
         {#if showHandle}
-          <div class="handle">
+          <div
+            class="handle"
+            aria-label="drag-handle"
+            style={!inactive ? "cursor:grabbing" : "cursor:grab"}
+            on:mousedown={() => {
+              inactive = false
+            }}
+            on:mouseup={() => {
+              inactive = true
+            }}
+          >
             <DragHandle />
           </div>
         {/if}
@@ -102,8 +118,8 @@
       <div class="right-content">
         <svelte:component
           this={listType}
-          anchor={anchors[draggable.item._id]}
+          anchor={anchors[draggableItem.item._id]}
-          item={draggable.item}
+          item={draggableItem.item}
           {...listTypeProps}
           on:change={onItemChanged}
         />
@@ -143,6 +159,7 @@
       --spectrum-table-row-background-color-hover,
       var(--spectrum-alias-highlight-hover)
     );
+    cursor: pointer;
   }
   .list-wrap > li:first-child {
     border-top-left-radius: 4px;
@@ -165,6 +182,9 @@
     display: flex;
     height: var(--spectrum-global-dimension-size-150);
   }
+  .handle:hover {
+    cursor: grab;
+  }
   .handle :global(svg) {
     fill: var(--spectrum-global-color-gray-500);
     margin-right: var(--spacing-m);

packages/builder/src/components/design/settings/controls/FieldConfiguration/FieldConfiguration.svelte

@@ -156,7 +156,7 @@
 
 <div class="field-configuration">
   <div class="toggle-all">
-    <span />
+    <span>Fields</span>
     <Toggle
       on:change={() => {
         let update = fieldList.map(field => ({
@@ -186,6 +186,9 @@
 </div>
 
 <style>
+  .field-configuration {
+    padding-top: 8px;
+  }
   .field-configuration :global(.spectrum-ActionButton) {
     width: 100%;
   }
@@ -204,6 +207,5 @@
   .toggle-all span {
     color: var(--spectrum-global-color-gray-700);
     font-size: 12px;
-    margin-left: calc(var(--spacing-s) - 1px);
   }
 </style>

packages/builder/src/components/design/settings/controls/FieldConfiguration/FieldSetting.svelte

@@ -58,7 +58,15 @@
     <div class="field-label">{readableText}</div>
   </div>
   <div class="list-item-right">
-    <Toggle on:change={onToggle(item)} text="" value={item.active} thin />
+    <Toggle
+      on:change={onToggle(item)}
+      on:click={e => {
+        e.stopPropagation()
+      }}
+      text=""
+      value={item.active}
+      thin
+    />
   </div>
 </div>
 

packages/builder/src/pages/builder/app/[application]/design/[screenId]/[componentId]/_components/Component/ComponentSettingsPanel.svelte

@@ -1,7 +1,7 @@
 <script>
   import Panel from "components/design/Panel.svelte"
   import { store, selectedComponent, selectedScreen } from "builderStore"
-  import { getComponentText } from "builderStore/componentUtils"
+  import { getComponentName } from "builderStore/componentUtils"
   import ComponentSettingsSection from "./ComponentSettingsSection.svelte"
   import DesignSection from "./DesignSection.svelte"
   import CustomStylesSection from "./CustomStylesSection.svelte"
@@ -43,17 +43,25 @@
 
   $: id = $selectedComponent?._id
   $: id, (section = tabs[0])
+
+  $: componentName = getComponentName(componentInstance)
 </script>
 
 {#if $selectedComponent}
   {#key $selectedComponent._id}
-    <Panel {title} icon={componentDefinition?.icon} borderLeft wide>
+    <Panel
+      {title}
+      icon={componentDefinition?.icon}
+      iconTooltip={componentName}
+      borderLeft
+      wide
+    >
       <span class="panel-title-content" slot="panel-title-content">
         <input
           class="input"
           value={title}
           {title}
-          placeholder={getComponentText(componentInstance)}
+          placeholder={componentName}
           on:keypress={e => {
             if (e.key.toLowerCase() === "enter") {
               e.target.blur()

packages/builder/src/pages/builder/app/[application]/design/[screenId]/_components/AppPanel.svelte

@@ -25,6 +25,7 @@
 
 <style>
   .app-panel {
+    min-width: 410px;
     flex: 1 1 auto;
     overflow-y: auto;
     display: flex;

packages/builder/src/pages/builder/app/[application]/design/[screenId]/_components/ComponentList/ComponentTree.svelte

@@ -12,6 +12,7 @@
   import {
     findComponentPath,
     getComponentText,
+    getComponentName,
   } from "builderStore/componentUtils"
   import { get } from "svelte/store"
   import { dndStore } from "./dndStore"
@@ -110,6 +111,7 @@
         on:drop={onDrop}
         text={getComponentText(component)}
         icon={getComponentIcon(component)}
+        iconTooltip={getComponentName(component)}
         withArrow={componentHasChildren(component)}
         indentLevel={level}
         selected={$store.selectedComponentId === component._id}

packages/builder/src/pages/builder/app/[application]/design/[screenId]/_components/LeftPanel.svelte

@@ -1,21 +1,55 @@
 <script>
   import ScreenList from "./ScreenList/index.svelte"
   import ComponentList from "./ComponentList/index.svelte"
+  import { getHorizontalResizeActions } from "components/common/resizable"
+
+  const [resizable, resizableHandle] = getHorizontalResizeActions()
 </script>
 
-<div class="panel">
+<div class="panel" use:resizable>
-  <ScreenList />
+  <div class="content">
-  <ComponentList />
+    <ScreenList />
+    <ComponentList />
+  </div>
+  <div class="divider">
+    <div class="dividerClickExtender" role="separator" use:resizableHandle />
+  </div>
 </div>
 
 <style>
   .panel {
+    display: flex;
+    min-width: 270px;
     width: 310px;
     height: 100%;
-    border-right: var(--border-light);
+  }
+
+  .content {
+    overflow: hidden;
+    flex-grow: 1;
+    height: 100%;
     display: flex;
     flex-direction: column;
     background: var(--background);
     position: relative;
   }
+
+  .divider {
+    position: relative;
+    height: 100%;
+    width: 2px;
+    background: var(--spectrum-global-color-gray-200);
+    transition: background 130ms ease-out;
+  }
+  .divider:hover {
+    background: var(--spectrum-global-color-gray-300);
+    cursor: row-resize;
+  }
+
+  .dividerClickExtender {
+    position: absolute;
+    cursor: col-resize;
+    height: 100%;
+    width: 12px;
+  }
 </style>

packages/builder/src/pages/builder/app/[application]/design/[screenId]/_components/ScreenList/index.svelte

@@ -1,108 +1,50 @@
 <script>
   import { Layout } from "@budibase/bbui"
-  import {
+  import { store, sortedScreens, userSelectedResourceMap } from "builderStore"
-    store,
-    sortedScreens,
-    userSelectedResourceMap,
-    screensHeight,
-  } from "builderStore"
   import NavItem from "components/common/NavItem.svelte"
   import RoleIndicator from "./RoleIndicator.svelte"
   import DropdownMenu from "./DropdownMenu.svelte"
-  import { onMount } from "svelte"
   import { goto } from "@roxi/routify"
+  import { getVerticalResizeActions } from "components/common/resizable"
   import NavHeader from "components/common/NavHeader.svelte"
 
-  let search = false
+  const [resizable, resizableHandle] = getVerticalResizeActions()
-  let resizing = false
-  let searchValue = ""
 
-  let container
+  let searching = false
+  let searchValue = ""
   let screensContainer
   let scrolling = false
-  let previousHeight = null
-  let dragOffset
 
   $: filteredScreens = getFilteredScreens($sortedScreens, searchValue)
 
-  const sleep = ms => new Promise(resolve => setTimeout(resolve, ms))
+  const handleOpenSearch = async () => {
-
-  $: search ? openSearch() : closeSearch()
-
-  const openSearch = async () => {
     screensContainer.scroll({ top: 0, behavior: "smooth" })
-    previousHeight = $screensHeight
-    $screensHeight = "calc(100% + 1px)"
   }
 
-  const closeSearch = async () => {
+  $: {
-    if (previousHeight) {
+    if (searching) {
-      // Restore previous height and wait for animation
+      handleOpenSearch()
-      $screensHeight = previousHeight
-      previousHeight = null
-      await sleep(300)
     }
   }
 
-  const getFilteredScreens = (screens, search) => {
+  const getFilteredScreens = (screens, searchValue) => {
     return screens.filter(screen => {
-      return !search || screen.routing.route.includes(search)
+      return !searchValue || screen.routing.route.includes(searchValue)
     })
   }
 
   const handleScroll = e => {
     scrolling = e.target.scrollTop !== 0
   }
-
-  const startResizing = e => {
-    // Reset the height store to match the true height
-    $screensHeight = `${container.getBoundingClientRect().height}px`
-
-    // Store an offset to easily compute new height when moving the mouse
-    dragOffset = parseInt($screensHeight) - e.clientY
-
-    // Add event listeners
-    resizing = true
-    document.addEventListener("mousemove", resize)
-    document.addEventListener("mouseup", stopResizing)
-  }
-
-  const resize = e => {
-    // Prevent negative heights as this screws with layout
-    const newHeight = Math.max(0, e.clientY + dragOffset)
-    if (newHeight == null || isNaN(newHeight)) {
-      return
-    }
-    $screensHeight = `${newHeight}px`
-  }
-
-  const stopResizing = () => {
-    resizing = false
-    document.removeEventListener("mousemove", resize)
-  }
-
-  onMount(() => {
-    // Ensure we aren't stuck at 100% height from leaving while searching
-    if ($screensHeight == null || isNaN(parseInt($screensHeight))) {
-      $screensHeight = "210px"
-    }
-  })
 </script>
 
-<svelte:window />
+<div class="screens" class:searching use:resizable>
-<div
-  class="screens"
-  class:search
-  class:resizing
-  style={`height:${$screensHeight};`}
-  bind:this={container}
->
   <div class="header" class:scrolling>
     <NavHeader
       title="Screens"
       placeholder="Search for screens"
       bind:value={searchValue}
-      bind:search
+      bind:search={searching}
       onAdd={() => $goto("../new")}
     />
   </div>
@@ -110,6 +52,7 @@
     {#if filteredScreens?.length}
       {#each filteredScreens as screen (screen._id)}
         <NavItem
+          scrollable
           icon={screen.routing.homeScreen ? "Home" : null}
           indentLevel={0}
           selected={$store.selectedScreenId === screen._id}
@@ -135,9 +78,11 @@
   </div>
 
   <div
+    role="separator"
+    disabled={searching}
     class="divider"
-    on:mousedown={startResizing}
+    class:disabled={searching}
-    on:dblclick={() => screensHeight.set("210px")}
+    use:resizableHandle
   />
 </div>
 
@@ -148,14 +93,12 @@
     min-height: 147px;
     max-height: calc(100% - 147px);
     position: relative;
-    transition: height 300ms ease-out;
+    transition: height 300ms ease-out, max-height 300ms ease-out;
+    height: 210px;
   }
-  .screens.search {
+  .screens.searching {
-    max-height: none;
+    max-height: 100%;
-  }
+    height: 100% !important;
-  .screens.resizing {
-    user-select: none;
-    cursor: row-resize;
   }
 
   .header {
@@ -177,9 +120,6 @@
     overflow: auto;
     flex-grow: 1;
   }
-  .screens.resizing .content {
-    pointer-events: none;
-  }
 
   .screens :global(.nav-item) {
     padding-right: 8px !important;
@@ -217,4 +157,10 @@
   .divider:hover:after {
     background: var(--spectrum-global-color-gray-300);
   }
+  .divider.disabled {
+    cursor: auto;
+  }
+  .divider.disabled:after {
+    background: var(--spectrum-global-color-gray-200);
+  }
 </style>

packages/builder/src/pages/builder/app/[application]/design/[screenId]/_layout.svelte

@@ -40,6 +40,7 @@
   }
 
   .content {
+    width: 100vw;
     display: flex;
     flex-direction: row;
     justify-content: flex-start;

packages/builder/src/pages/builder/portal/apps/_layout.svelte

@@ -14,7 +14,7 @@
   import PortalSideBar from "./_components/PortalSideBar.svelte"
 
   // Don't block loading if we've already hydrated state
-  let loaded = $apps.length != null
+  let loaded = !!$apps?.length
 
   onMount(async () => {
     try {

packages/builder/src/pages/builder/portal/apps/index.svelte

@@ -1,5 +1,6 @@
 <script>
   import {
+    banner,
     Heading,
     Layout,
     Button,
@@ -10,6 +11,7 @@
     Notification,
     Body,
     Search,
+    BANNER_TYPES,
   } from "@budibase/bbui"
   import Spinner from "components/common/Spinner.svelte"
   import CreateAppModal from "components/start/CreateAppModal.svelte"
@@ -198,6 +200,20 @@
       if (usersLimitLockAction) {
         usersLimitLockAction()
       }
+      if (!$admin.isDev) {
+        await banner.show({
+          messages: [
+            {
+              message:
+                "We've updated our pricing - see our website to learn more.",
+              type: BANNER_TYPES.NEUTRAL,
+              extraButtonText: "Learn More",
+              extraButtonAction: () =>
+                window.open("https://budibase.com/pricing"),
+            },
+          ],
+        })
+      }
     } catch (error) {
       notifications.error("Error getting init info")
     }

packages/builder/src/pages/builder/portal/users/users/_components/RoleTableRenderer.svelte

@@ -8,7 +8,7 @@
     x => x.value === users.getUserRole(row)
   )
   $: value = role?.label || "Not available"
-  $: tooltip = role.subtitle || ""
+  $: tooltip = role?.subtitle || ""
 </script>
 
 <div on:click|stopPropagation title={tooltip}>

packages/client/manifest.json

@@ -6056,18 +6056,6 @@
         "options": ["Create", "Update", "View"],
         "defaultValue": "Create"
       },
-      {
-        "type": "text",
-        "label": "Title",
-        "key": "title",
-        "nested": true
-      },
-      {
-        "type": "text",
-        "label": "Description",
-        "key": "description",
-        "nested": true
-      },
       {
         "section": true,
         "dependsOn": {
@@ -6075,7 +6063,7 @@
           "value": "Create",
           "invert": true
         },
-        "name": "Row details",
+        "name": "Row ID",
         "info": "<a href='https://docs.budibase.com/docs/form-block' target='_blank'>How to pass a row ID using bindings</a>",
         "settings": [
           {
@@ -6095,8 +6083,20 @@
       },
       {
         "section": true,
-        "name": "Fields",
+        "name": "Details",
         "settings": [
+          {
+            "type": "text",
+            "label": "Title",
+            "key": "title",
+            "nested": true
+          },
+          {
+            "type": "text",
+            "label": "Description",
+            "key": "description",
+            "nested": true
+          },
           {
             "type": "fieldConfiguration",
             "key": "fields",

packages/client/src/components/app/forms/RelationshipField.svelte

@@ -1,6 +1,6 @@
 <script>
   import { CoreSelect, CoreMultiselect } from "@budibase/bbui"
-  import { fetchData } from "@budibase/frontend-core"
+  import { fetchData, Utils } from "@budibase/frontend-core"
   import { getContext } from "svelte"
   import Field from "./Field.svelte"
   import { FieldTypes } from "../../../constants"
@@ -108,7 +108,7 @@
     }
   }
 
-  $: fetchRows(searchTerm, primaryDisplay, defaultValue)
+  $: debouncedFetchRows(searchTerm, primaryDisplay, defaultValue)
 
   const fetchRows = async (searchTerm, primaryDisplay, defaultVal) => {
     const allRowsFetched =
@@ -124,10 +124,22 @@
         query: { equal: { _id: defaultVal } },
       })
     }
+
+    // Ensure we match all filters, rather than any
+    const baseFilter = (filter || []).filter(x => x.operator !== "allOr")
     await fetch.update({
-      query: { string: { [primaryDisplay]: searchTerm } },
+      filter: [
+        ...baseFilter,
+        {
+          // Use a big numeric prefix to avoid clashing with an existing filter
+          field: `999:${primaryDisplay}`,
+          operator: "string",
+          value: searchTerm,
+        },
+      ],
     })
   }
+  const debouncedFetchRows = Utils.debounce(fetchRows, 250)
 
   const flatten = values => {
     if (!values) {

packages/pro

@@ -1 +1 @@
-Subproject commit 618613f3575b01f74940d9f58fdb53a9a5b2dc1a
+Subproject commit 1037b032d49244678204704d1bca779a29e395eb

packages/server/specs/openapi.json

@@ -2152,7 +2152,7 @@
     "/applications/{appId}/publish": {
       "post": {
         "operationId": "appPublish",
-        "summary": "Unpublish an application",
+        "summary": "Publish an application",
         "tags": [
           "applications"
         ],

packages/server/specs/openapi.yaml

@@ -1761,7 +1761,7 @@ paths:
   "/applications/{appId}/publish":
     post:
       operationId: appPublish
-      summary: Unpublish an application
+      summary: Publish an application
       tags:
         - applications
       parameters:

packages/server/src/api/controllers/static/index.ts

@@ -24,7 +24,7 @@ import AWS from "aws-sdk"
 import fs from "fs"
 import sdk from "../../../sdk"
 import * as pro from "@budibase/pro"
-import { App, Ctx, ProcessAttachmentResponse, Upload } from "@budibase/types"
+import { App, Ctx, ProcessAttachmentResponse } from "@budibase/types"
 
 const send = require("koa-send")
 
@@ -212,7 +212,9 @@ export const serveBuilderPreview = async function (ctx: Ctx) {
 
   if (!env.isJest()) {
     let appId = context.getAppId()
-    const previewHbs = loadHandlebarsFile(`${__dirname}/preview.hbs`)
+    const templateLoc = join(__dirname, "templates")
+    const previewLoc = fs.existsSync(templateLoc) ? templateLoc : __dirname
+    const previewHbs = loadHandlebarsFile(join(previewLoc, "preview.hbs"))
     ctx.body = await processString(previewHbs, {
       clientLibPath: objectStore.clientLibraryUrl(appId!, appInfo.version),
     })

packages/server/src/api/routes/tests/row.spec.ts

@@ -517,9 +517,24 @@ describe.each([
   })
 
   describe("patch", () => {
+    let otherTable: Table
+
     beforeAll(async () => {
       const tableConfig = generateTableConfig()
       table = await createTable(tableConfig)
+      const otherTableConfig = generateTableConfig()
+      // need a short name of table here - for relationship tests
+      otherTableConfig.name = "a"
+      otherTableConfig.schema.relationship = {
+        name: "relationship",
+        relationshipType: RelationshipType.ONE_TO_MANY,
+        type: FieldType.LINK,
+        tableId: table._id!,
+        fieldName: "relationship",
+      }
+      otherTable = await createTable(otherTableConfig)
+      // need to set the config back to the original table
+      config.table = table
     })
 
     it("should update only the fields that are supplied", async () => {
@@ -615,6 +630,28 @@ describe.each([
       expect(getResp.body.user1[0]._id).toEqual(user2._id)
       expect(getResp.body.user2[0]._id).toEqual(user2._id)
     })
+
+    it("should be able to update relationships when both columns are same name", async () => {
+      let row = await config.api.row.save(table._id!, {
+        name: "test",
+        description: "test",
+      })
+      let row2 = await config.api.row.save(otherTable._id!, {
+        name: "test",
+        description: "test",
+        relationship: [row._id],
+      })
+      row = (await config.api.row.get(table._id!, row._id!)).body
+      expect(row.relationship.length).toBe(1)
+      const resp = await config.api.row.patch(table._id!, {
+        _id: row._id!,
+        _rev: row._rev!,
+        tableId: row.tableId!,
+        name: "test2",
+        relationship: [row2._id],
+      })
+      expect(resp.relationship.length).toBe(1)
+    })
   })
 
   describe("destroy", () => {

packages/server/src/db/linkedRows/LinkController.ts

@@ -251,9 +251,19 @@ class LinkController {
         // find the docs that need to be deleted
         let toDeleteDocs = thisFieldLinkDocs
           .filter(doc => {
-            let correctDoc =
+            let correctDoc
-              doc.doc1.fieldName === fieldName ? doc.doc2 : doc.doc1
+            if (
-            return rowField.indexOf(correctDoc.rowId) === -1
+              doc.doc1.tableId === table._id! &&
+              doc.doc1.fieldName === fieldName
+            ) {
+              correctDoc = doc.doc2
+            } else if (
+              doc.doc2.tableId === table._id! &&
+              doc.doc2.fieldName === fieldName
+            ) {
+              correctDoc = doc.doc1
+            }
+            return correctDoc && rowField.indexOf(correctDoc.rowId) === -1
           })
           .map(doc => {
             return { ...doc, _deleted: true }

packages/server/src/integration-test/postgres.spec.ts

@@ -934,25 +934,43 @@ describe("postgres integrations", () => {
               },
             ],
           })
+          const m2oRel = {
+            [m2oFieldName]: [
+              {
+                _id: row._id,
+              },
+            ],
+          }
           expect(res.body[m2oFieldName]).toEqual([
             {
+              ...m2oRel,
               ...foreignRowsByType[RelationshipType.MANY_TO_ONE][0].row,
               [`fk_${manyToOneRelationshipInfo.table.name}_${manyToOneRelationshipInfo.fieldName}`]:
                 row.id,
             },
             {
+              ...m2oRel,
               ...foreignRowsByType[RelationshipType.MANY_TO_ONE][1].row,
               [`fk_${manyToOneRelationshipInfo.table.name}_${manyToOneRelationshipInfo.fieldName}`]:
                 row.id,
             },
             {
+              ...m2oRel,
               ...foreignRowsByType[RelationshipType.MANY_TO_ONE][2].row,
               [`fk_${manyToOneRelationshipInfo.table.name}_${manyToOneRelationshipInfo.fieldName}`]:
                 row.id,
             },
           ])
+          const o2mRel = {
+            [o2mFieldName]: [
+              {
+                _id: row._id,
+              },
+            ],
+          }
           expect(res.body[o2mFieldName]).toEqual([
             {
+              ...o2mRel,
               ...foreignRowsByType[RelationshipType.ONE_TO_MANY][0].row,
               _id: expect.any(String),
               _rev: expect.any(String),

packages/server/src/sdk/app/rows/search/external.ts

@@ -133,9 +133,14 @@ export async function exportRows(
 
   let result = await search({ tableId, query: requestQuery, sort, sortOrder })
   let rows: Row[] = []
+  let headers
+
+  if (!tableName) {
+    throw new HTTPError("Could not find table name.", 400)
+  }
+  const schema = datasource.entities[tableName].schema
 
   // Filter data to only specified columns if required
-
   if (columns && columns.length) {
     for (let i = 0; i < result.rows.length; i++) {
       rows[i] = {}
@@ -143,22 +148,17 @@ export async function exportRows(
         rows[i][column] = result.rows[i][column]
       }
     }
+    headers = columns
   } else {
     rows = result.rows
   }
 
-  if (!tableName) {
-    throw new HTTPError("Could not find table name.", 400)
-  }
-  const schema = datasource.entities[tableName].schema
   let exportRows = cleanExportRows(rows, schema, format, columns)
 
-  let headers = Object.keys(schema)
-
   let content: string
   switch (format) {
     case exporters.Format.CSV:
-      content = exporters.csv(headers, exportRows)
+      content = exporters.csv(headers ?? Object.keys(schema), exportRows)
       break
     case exporters.Format.JSON:
       content = exporters.json(exportRows)

packages/server/src/sdk/app/rows/search/internal.ts

@@ -110,7 +110,7 @@ export async function exportRows(
 
   let rows: Row[] = []
   let schema = table.schema
-
+  let headers
   // Filter data to only specified columns if required
   if (columns && columns.length) {
     for (let i = 0; i < result.length; i++) {
@@ -119,6 +119,7 @@ export async function exportRows(
         rows[i][column] = result[i][column]
       }
     }
+    headers = columns
   } else {
     rows = result
   }
@@ -127,7 +128,7 @@ export async function exportRows(
   if (format === Format.CSV) {
     return {
       fileName: "export.csv",
-      content: csv(Object.keys(rows[0]), exportRows),
+      content: csv(headers ?? Object.keys(rows[0]), exportRows),
     }
   } else if (format === Format.JSON) {
     return {

packages/server/src/sdk/app/tables/external/index.ts

@@ -136,6 +136,8 @@ export async function save(
     schema.main = true
   }
 
+  // add in the new table for relationship purposes
+  tables[tableToSave.name] = tableToSave
   cleanupRelationships(tableToSave, tables, oldTable)
 
   const operation = tableId ? Operation.UPDATE_TABLE : Operation.CREATE_TABLE

packages/server/src/sdk/app/tables/external/utils.ts

@@ -1,5 +1,6 @@
 import {
   Datasource,
+  FieldType,
   ManyToManyRelationshipFieldMetadata,
   ManyToOneRelationshipFieldMetadata,
   OneToManyRelationshipFieldMetadata,
@@ -42,10 +43,13 @@ export function cleanupRelationships(
       for (let [relatedKey, relatedSchema] of Object.entries(
         relatedTable.schema
       )) {
-        if (
+        if (relatedSchema.type !== FieldType.LINK) {
-          relatedSchema.type === FieldTypes.LINK &&
+          continue
-          relatedSchema.fieldName === foreignKey
+        }
-        ) {
+        // if they both have the same field name it will appear as if it needs to be removed,
+        // don't cleanup in this scenario
+        const sameFieldNameForBoth = relatedSchema.name === schema.name
+        if (relatedSchema.fieldName === foreignKey && !sameFieldNameForBoth) {
           delete relatedTable.schema[relatedKey]
         }
       }

packages/server/src/sdk/tests/rows/row.spec.ts

@@ -18,7 +18,6 @@ jest.mock("../../../utilities/rowProcessor", () => ({
 
 jest.mock("../../../api/controllers/view/exporters", () => ({
   ...jest.requireActual("../../../api/controllers/view/exporters"),
-  csv: jest.fn(),
   Format: {
     CSV: "csv",
   },
@@ -102,5 +101,32 @@ describe("external row sdk", () => {
         new HTTPError("Could not find table name.", 400)
       )
     })
+
+    it("should only export specified columns", async () => {
+      mockDatasourcesGet.mockImplementation(async () => ({
+        entities: {
+          tablename: {
+            schema: {
+              name: {},
+              age: {},
+              dob: {},
+            },
+          },
+        },
+      }))
+      const headers = ["name", "dob"]
+
+      const result = await exportRows({
+        tableId: "datasource__tablename",
+        format: Format.CSV,
+        query: {},
+        columns: headers,
+      })
+
+      expect(result).toEqual({
+        fileName: "export.csv",
+        content: `"name","dob"`,
+      })
+    })
   })
 })

packages/server/src/tests/utilities/TestConfiguration.ts

@@ -137,6 +137,10 @@ class TestConfiguration {
   }
 
   getAppId() {
+    if (!this.appId) {
+      throw "appId has not been initialised properly"
+    }
+
     return this.appId
   }
 
@@ -510,7 +514,7 @@ class TestConfiguration {
     // create dev app
     // clear any old app
     this.appId = null
-    this.app = await context.doInAppContext(null, async () => {
+    this.app = await context.doInTenant(this.tenantId!, async () => {
       const app = await this._req(
         { name: appName },
         null,
@@ -519,7 +523,7 @@ class TestConfiguration {
       this.appId = app.appId!
       return app
     })
-    return await context.doInAppContext(this.appId, async () => {
+    return await context.doInAppContext(this.getAppId(), async () => {
       // create production app
       this.prodApp = await this.publish()
 
@@ -817,7 +821,7 @@ class TestConfiguration {
   }
 
   async getAutomationLogs() {
-    return context.doInAppContext(this.appId, async () => {
+    return context.doInAppContext(this.getAppId(), async () => {
       const now = new Date()
       return await pro.sdk.automations.logs.logSearch({
         startDate: new Date(now.getTime() - 100000).toISOString(),

packages/shared-core/src/filters.ts

@@ -315,7 +315,7 @@ export const runLuceneQuery = (docs: any[], query?: SearchQuery) => {
           new Date(docValue).getTime() > new Date(testValue.high).getTime()
         )
       }
-      throw "Cannot perform range filter - invalid type."
+      return false
     }
   )
 

packages/shared-core/src/tests/filters.test.ts

@@ -130,32 +130,28 @@ describe("runLuceneQuery", () => {
     expect(runLuceneQuery(docs, query).map(row => row.order_id)).toEqual([2])
   })
 
-  it("should throw an error is an invalid doc value is passed into a range filter", async () => {
+  it("should return return all docs if an invalid doc value is passed into a range filter", async () => {
+    const docs = [
+      {
+        order_id: 4,
+        customer_id: 1758,
+        order_status: 5,
+        order_date: "{{ Binding.INVALID }}",
+        required_date: "2017-03-05T00:00:00.000Z",
+        shipped_date: "2017-03-03T00:00:00.000Z",
+        store_id: 2,
+        staff_id: 7,
+        description: undefined,
+        label: "",
+      },
+    ]
     const query = buildQuery("range", {
       order_date: {
         low: "2016-01-04T00:00:00.000Z",
         high: "2016-01-11T00:00:00.000Z",
       },
     })
-    expect(() =>
+    expect(runLuceneQuery(docs, query)).toEqual(docs)
-      runLuceneQuery(
-        [
-          {
-            order_id: 4,
-            customer_id: 1758,
-            order_status: 5,
-            order_date: "INVALID",
-            required_date: "2017-03-05T00:00:00.000Z",
-            shipped_date: "2017-03-03T00:00:00.000Z",
-            store_id: 2,
-            staff_id: 7,
-            description: undefined,
-            label: "",
-          },
-        ],
-        query
-      )
-    ).toThrowError("Cannot perform range filter - invalid type.")
   })
 
   it("should return rows with matches on empty filter", () => {

packages/types/src/sdk/locks.ts

@@ -10,6 +10,7 @@ export enum LockType {
   DEFAULT = "default",
   DELAY_500 = "delay_500",
   CUSTOM = "custom",
+  AUTO_EXTEND = "auto_extend",
 }
 
 export enum LockName {
@@ -21,7 +22,7 @@ export enum LockName {
   QUOTA_USAGE_EVENT = "quota_usage_event",
 }
 
-export interface LockOptions {
+export type LockOptions = {
   /**
    * The lock type determines which client to use
    */
@@ -35,10 +36,6 @@ export interface LockOptions {
    * The name for the lock
    */
   name: LockName
-  /**
-   * The ttl to auto-expire the lock if not unlocked manually
-   */
-  ttl: number
   /**
    * The individual resource to lock. This is useful for locking around very specific identifiers, e.g. a document that is prone to conflicts
    */
@@ -47,4 +44,16 @@ export interface LockOptions {
    * This is a system-wide lock - don't use tenancy in lock key
    */
   systemLock?: boolean
-}
+} & (
+  | {
+      /**
+       * The ttl to auto-expire the lock if not unlocked manually
+       */
+      ttl: number
+      type: Exclude<LockType, LockType.AUTO_EXTEND>
+    }
+  | {
+      type: LockType.AUTO_EXTEND
+      onExtend?: () => void
+    }
+)

yarn.lock

@@ -12667,16 +12667,16 @@ invert-kv@^2.0.0:
   resolved "https://registry.yarnpkg.com/invert-kv/-/invert-kv-2.0.0.tgz#7393f5afa59ec9ff5f67a27620d11c226e3eec02"
   integrity sha512-wPVv/y/QQ/Uiirj/vh3oP+1Ww+AWehmi1g5fFWGPF6IpCBCDVrhgHRMvrLfdYcwDh3QJbGXDW4JAuzxElLSqKA==
 
-ioredis-mock@8.7.0:
+ioredis-mock@8.9.0:
-  version "8.7.0"
+  version "8.9.0"
-  resolved "https://registry.yarnpkg.com/ioredis-mock/-/ioredis-mock-8.7.0.tgz#9877a85e0d233e1b49123d1c6e320df01e9a1d36"
+  resolved "https://registry.yarnpkg.com/ioredis-mock/-/ioredis-mock-8.9.0.tgz#5d694c4b81d3835e4291e0b527f947e260981779"
-  integrity sha512-BJcSjkR3sIMKbH93fpFzwlWi/jl1kd5I3vLvGQxnJ/W/6bD2ksrxnyQN186ljAp3Foz4p1ivViDE3rZeKEAluA==
+  integrity sha512-yIglcCkI1lvhwJVoMsR51fotZVsPsSk07ecTCgRTRlicG0Vq3lke6aAaHklyjmRNRsdYAgswqC2A0bPtQK4LSw==
   dependencies:
     "@ioredis/as-callback" "^3.0.0"
     "@ioredis/commands" "^1.2.0"
     fengari "^0.1.4"
     fengari-interop "^0.1.3"
-    semver "^7.3.8"
+    semver "^7.5.4"
 
 ioredis@5.3.2:
   version "5.3.2"