merge with master

.github/workflows/budibase_ci.yml

@@ -76,6 +76,18 @@ jobs:
             yarn check:types
           fi
 
+  helm-lint:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout repo
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+
+      - name: Use Node.js 18.x
+        uses: azure/setup-helm@v3
+      - run: cd charts/budibase && helm lint .
+
   test-libraries:
     runs-on: ubuntu-latest
     steps:
@@ -204,7 +216,7 @@ jobs:
 
   check-pro-submodule:
     runs-on: ubuntu-latest
-    if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == 'Budibase/budibase'
+    if: inputs.run_as_oss != true  && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == 'Budibase/budibase')
     steps:
       - name: Checkout repo and submodules
         uses: actions/checkout@v3
@@ -254,7 +266,7 @@ jobs:
 
   check-accountportal-submodule:
     runs-on: ubuntu-latest
-    if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == 'Budibase/budibase'
+    if: inputs.run_as_oss != true  && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == 'Budibase/budibase')
     steps:
       - name: Checkout repo and submodules
         uses: actions/checkout@v3

.github/workflows/close-featurebranch.yml

@@ -2,9 +2,7 @@ name: close-featurebranch
 
 on:
   pull_request:
-    types: [closed]
+    types: [closed, unlabeled]
-    branches:
-      - master
   workflow_dispatch:
     inputs:
       BRANCH:
@@ -14,6 +12,9 @@ on:
 
 jobs:
   release:
+    if: |
+      (github.event.action == 'closed' && contains(github.event.pull_request.labels.*.name, 'feature-branch')) ||
+       github.event.label.name == 'feature-branch'
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3

.github/workflows/deploy-featurebranch.yml

@@ -2,12 +2,19 @@ name: deploy-featurebranch
 
 on:
   pull_request:
-    branches:
+    types: [
-      - master
+        labeled,
+        # default types below (https://docs.github.com/en/actions/using-workflows/events-that-trigger-workflows#pull_request)
+        opened,
+        synchronize,
+        reopened,
+      ]
 
 jobs:
   release:
-    if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == 'Budibase/budibase'
+    if: |
+      (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == 'Budibase/budibase') &&
+      contains(github.event.pull_request.labels.*.name, 'feature-branch')
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3

.vscode/settings.json

@@ -1,7 +1,7 @@
 {
   "editor.formatOnSave": true,
   "editor.codeActionsOnSave": {
-    "source.fixAll": true
+    "source.fixAll": "explicit"
   },
   "editor.defaultFormatter": "esbenp.prettier-vscode",
   "[json]": {

charts/budibase/README.md

@@ -157,6 +157,17 @@ $ helm install --create-namespace --namespace budibase budibase . -f values.yaml
 | services.apps.replicaCount | int | `1` | The number of apps replicas to run. |
 | services.apps.resources | object | `{}` | The resources to use for apps pods. See <https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/> for more information on how to set these. |
 | services.apps.startupProbe | object | HTTP health checks. | Startup probe configuration for apps pods. You shouldn't need to change this, but if you want to you can find more information here: <https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/> |
+| services.automationWorkers.autoscaling.enabled | bool | `false` | Whether to enable horizontal pod autoscaling for the apps service. |
+| services.automationWorkers.autoscaling.maxReplicas | int | `10` |  |
+| services.automationWorkers.autoscaling.minReplicas | int | `1` |  |
+| services.automationWorkers.autoscaling.targetCPUUtilizationPercentage | int | `80` | Target CPU utilization percentage for the automation worker service. Note that for autoscaling to work, you will need to have metrics-server configured, and resources set for the automation worker pods. |
+| services.automationWorkers.enabled | bool | `true` | Whether or not to enable the automation worker service. If you disable this, automations will be processed by the apps service. |
+| services.automationWorkers.livenessProbe | object | HTTP health checks. | Liveness probe configuration for automation worker pods. You shouldn't need to change this, but if you want to you can find more information here: <https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/> |
+| services.automationWorkers.logLevel | string | `"info"` | The log level for the automation worker service. |
+| services.automationWorkers.readinessProbe | object | HTTP health checks. | Readiness probe configuration for automation worker pods. You shouldn't need to change this, but if you want to you can find more information here: <https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/> |
+| services.automationWorkers.replicaCount | int | `1` | The number of automation worker replicas to run. |
+| services.automationWorkers.resources | object | `{}` | The resources to use for automation worker pods. See <https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/> for more information on how to set these. |
+| services.automationWorkers.startupProbe | object | HTTP health checks. | Startup probe configuration for automation worker pods. You shouldn't need to change this, but if you want to you can find more information here: <https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/> |
 | services.couchdb.backup.enabled | bool | `false` | Whether or not to enable periodic CouchDB backups. This works by replicating to another CouchDB instance. |
 | services.couchdb.backup.interval | string | `""` | Backup interval in seconds |
 | services.couchdb.backup.resources | object | `{}` | The resources to use for CouchDB backup pods. See <https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/> for more information on how to set these. |

charts/budibase/templates/app-service-deployment.yaml

@@ -192,7 +192,14 @@ spec:
         - name: NODE_TLS_REJECT_UNAUTHORIZED
           value: {{ .Values.services.tlsRejectUnauthorized }}
         {{ end }}
-
+        {{- if .Values.services.automationWorkers.enabled }}
+        - name: APP_FEATURES
+          value: "api"
+        {{- end }}
+        {{- range .Values.services.apps.extraEnv }}
+        - name: {{ .name }}
+          value: {{ .value | quote }}
+        {{- end }}
         image: budibase/apps:{{ .Values.globals.appVersion | default .Chart.AppVersion }}
         imagePullPolicy: Always
         {{- if .Values.services.apps.startupProbe }}
@@ -220,6 +227,14 @@ spec:
         resources:
         {{- toYaml . | nindent 10 }}
         {{ end }}
+        {{ if .Values.services.apps.command }}
+        command:
+        {{- toYaml .Values.services.apps.command | nindent 10 }}
+        {{ end }}
+        {{ if .Values.services.apps.args }}
+        args:
+        {{- toYaml .Values.services.apps.args | nindent 10 }}
+        {{ end }}
     {{- with .Values.affinity }}
       affinity:
         {{- toYaml . | nindent 8 }}

charts/budibase/templates/automation-worker-service-deployment.yaml

@@ -0,0 +1,256 @@
+{{- if .Values.services.automationWorkers.enabled }}
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  annotations:
+{{ if .Values.services.automationWorkers.deploymentAnnotations }}
+{{- toYaml .Values.services.automationWorkers.deploymentAnnotations | indent 4 -}}
+{{ end }}
+  labels:
+    io.kompose.service: automation-worker-service
+{{ if .Values.services.automationWorkers.deploymentLabels }}
+{{- toYaml .Values.services.automationWorkers.deploymentLabels | indent 4 -}}
+{{ end }}
+  name: automation-worker-service
+spec:
+  replicas: {{ .Values.services.automationWorkers.replicaCount }}
+  selector:
+    matchLabels:
+      io.kompose.service: automation-worker-service
+  strategy:
+    type: RollingUpdate
+  template:
+    metadata:
+      annotations:
+{{ if .Values.services.automationWorkers.templateAnnotations }}
+{{- toYaml .Values.services.automationWorkers.templateAnnotations | indent 8 -}}
+{{ end }}
+      labels:
+        io.kompose.service: automation-worker-service
+{{ if .Values.services.automationWorkers.templateLabels }}
+{{- toYaml .Values.services.automationWorkers.templateLabels | indent 8 -}}
+{{ end }}
+    spec:
+      containers:
+      - env:
+        - name: BUDIBASE_ENVIRONMENT
+          value: {{ .Values.globals.budibaseEnv }}
+        - name: DEPLOYMENT_ENVIRONMENT
+          value: "kubernetes"
+        - name: COUCH_DB_URL
+          {{ if .Values.services.couchdb.url }}
+          value: {{ .Values.services.couchdb.url }}
+          {{ else }}
+          value: http://{{ .Release.Name }}-svc-couchdb:{{ .Values.services.couchdb.port }}
+          {{ end }}
+        {{ if .Values.services.couchdb.enabled }}
+        - name: COUCH_DB_USER
+          valueFrom:
+            secretKeyRef:
+              name: {{ template "couchdb.fullname" . }}
+              key: adminUsername
+        - name: COUCH_DB_PASSWORD
+          valueFrom:
+            secretKeyRef:
+              name: {{ template "couchdb.fullname" . }}
+              key: adminPassword
+        {{ end }}
+        - name: ENABLE_ANALYTICS
+          value: {{ .Values.globals.enableAnalytics | quote }}
+        - name: API_ENCRYPTION_KEY
+          value: {{ .Values.globals.apiEncryptionKey | quote }}
+        - name: HTTP_LOGGING
+          value: {{ .Values.services.automationWorkers.httpLogging | quote }}
+        - name: INTERNAL_API_KEY
+          valueFrom:
+            secretKeyRef:
+              name: {{ template "budibase.fullname" . }}
+              key: internalApiKey
+        - name: INTERNAL_API_KEY_FALLBACK
+          value: {{ .Values.globals.internalApiKeyFallback | quote }}
+        - name: JWT_SECRET
+          valueFrom:
+            secretKeyRef:
+              name: {{ template "budibase.fullname" . }}
+              key: jwtSecret
+        - name: JWT_SECRET_FALLBACK
+          value: {{ .Values.globals.jwtSecretFallback | quote }}
+        {{ if .Values.services.objectStore.region }}
+        - name: AWS_REGION
+          value: {{ .Values.services.objectStore.region }}
+        {{ end }}
+        - name: MINIO_ENABLED
+          value: {{ .Values.services.objectStore.minio | quote }}
+        - name: MINIO_ACCESS_KEY
+          valueFrom:
+            secretKeyRef:
+              name: {{ template "budibase.fullname" . }}
+              key: objectStoreAccess
+        - name: MINIO_SECRET_KEY
+          valueFrom:
+            secretKeyRef:
+              name: {{ template "budibase.fullname" . }}
+              key: objectStoreSecret
+        - name: CLOUDFRONT_CDN
+          value: {{ .Values.services.objectStore.cloudfront.cdn | quote }}
+        - name: CLOUDFRONT_PUBLIC_KEY_ID
+          value: {{ .Values.services.objectStore.cloudfront.publicKeyId | quote }}
+        - name: CLOUDFRONT_PRIVATE_KEY_64
+          value: {{ .Values.services.objectStore.cloudfront.privateKey64 | quote }}
+        - name: MINIO_URL
+          value: {{ .Values.services.objectStore.url }}
+        - name: PLUGIN_BUCKET_NAME
+          value: {{ .Values.services.objectStore.pluginBucketName | quote }}
+        - name: APPS_BUCKET_NAME
+          value: {{ .Values.services.objectStore.appsBucketName | quote }}
+        - name: GLOBAL_BUCKET_NAME
+          value: {{ .Values.services.objectStore.globalBucketName | quote }}
+        - name: BACKUPS_BUCKET_NAME
+          value: {{ .Values.services.objectStore.backupsBucketName | quote }}
+        - name: PORT
+          value: {{ .Values.services.automationWorkers.port | quote }}
+        {{ if .Values.services.worker.publicApiRateLimitPerSecond }}
+        - name: API_REQ_LIMIT_PER_SEC
+          value: {{ .Values.globals.automationWorkers.publicApiRateLimitPerSecond | quote }}
+        {{ end }}
+        - name: MULTI_TENANCY
+          value: {{ .Values.globals.multiTenancy | quote }}
+        - name: OFFLINE_MODE
+          value: {{ .Values.globals.offlineMode | quote }}
+        - name: LOG_LEVEL
+          value: {{ .Values.services.automationWorkers.logLevel | quote }}
+        - name: REDIS_PASSWORD
+          value: {{ .Values.services.redis.password }}
+        - name: REDIS_URL
+          {{ if .Values.services.redis.url }}
+          value: {{ .Values.services.redis.url }}
+          {{ else }}
+          value: redis-service:{{ .Values.services.redis.port }}
+          {{ end }}
+        - name: SELF_HOSTED
+          value: {{ .Values.globals.selfHosted | quote }}
+        - name: POSTHOG_TOKEN
+          value: {{ .Values.globals.posthogToken | quote }}
+        - name: WORKER_URL
+          value: http://worker-service:{{ .Values.services.worker.port }}
+        - name: PLATFORM_URL
+          value: {{ .Values.globals.platformUrl | quote }}
+        - name: ACCOUNT_PORTAL_URL
+          value: {{ .Values.globals.accountPortalUrl | quote }}
+        - name: ACCOUNT_PORTAL_API_KEY
+          value: {{ .Values.globals.accountPortalApiKey | quote }}
+        - name: COOKIE_DOMAIN
+          value: {{ .Values.globals.cookieDomain | quote }}
+        - name: HTTP_MIGRATIONS
+          value: {{ .Values.globals.httpMigrations | quote }}
+        - name: GOOGLE_CLIENT_ID
+          value: {{ .Values.globals.google.clientId | quote }}
+        - name: GOOGLE_CLIENT_SECRET
+          value: {{ .Values.globals.google.secret | quote }}
+        - name: AUTOMATION_MAX_ITERATIONS
+          value: {{ .Values.globals.automationMaxIterations | quote }}
+        - name: TENANT_FEATURE_FLAGS
+          value: {{ .Values.globals.tenantFeatureFlags | quote }}
+        - name: ENCRYPTION_KEY
+          value: {{ .Values.globals.bbEncryptionKey | quote }}
+        {{ if .Values.globals.bbAdminUserEmail }}
+        - name: BB_ADMIN_USER_EMAIL
+          value: {{ .Values.globals.bbAdminUserEmail | quote }}
+        {{ end }}
+        {{ if .Values.globals.bbAdminUserPassword }}
+        - name: BB_ADMIN_USER_PASSWORD
+          value: {{ .Values.globals.bbAdminUserPassword | quote }}
+        {{ end }}
+        {{ if .Values.globals.pluginsDir }}
+        - name: PLUGINS_DIR
+          value: {{ .Values.globals.pluginsDir | quote }}
+        {{ end }}
+        {{ if .Values.services.automationWorkers.nodeDebug }}
+        - name: NODE_DEBUG
+          value: {{ .Values.services.automationWorkers.nodeDebug | quote }}
+        {{ end }}
+        {{ if .Values.globals.datadogApmEnabled }}
+        - name: DD_LOGS_INJECTION
+          value: {{ .Values.globals.datadogApmEnabled | quote }}
+        - name: DD_APM_ENABLED
+          value: {{ .Values.globals.datadogApmEnabled | quote }}
+        - name: DD_APM_DD_URL
+          value: https://trace.agent.datadoghq.eu
+        {{ end }}
+        {{ if .Values.globals.globalAgentHttpProxy }}
+        - name: GLOBAL_AGENT_HTTP_PROXY
+          value: {{ .Values.globals.globalAgentHttpProxy | quote }}
+        {{ end }}
+        {{ if .Values.globals.globalAgentHttpsProxy }}
+        - name: GLOBAL_AGENT_HTTPS_PROXY
+          value: {{ .Values.globals.globalAgentHttpsProxy | quote }}
+        {{ end }}
+        {{ if .Values.globals.globalAgentNoProxy }}
+        - name: GLOBAL_AGENT_NO_PROXY
+          value: {{ .Values.globals.globalAgentNoProxy | quote }}
+        {{ end }}
+        {{ if .Values.services.tlsRejectUnauthorized }}
+        - name: NODE_TLS_REJECT_UNAUTHORIZED
+          value: {{ .Values.services.tlsRejectUnauthorized }}
+        {{ end }}
+        - name: APP_FEATURES
+          value: "automations"
+        {{- range .Values.services.automationWorkers.extraEnv }}
+        - name: {{ .name }}
+          value: {{ .value | quote }}
+        {{- end }}
+
+        image: budibase/apps:{{ .Values.globals.appVersion | default .Chart.AppVersion }}
+        imagePullPolicy: Always
+        {{- if .Values.services.automationWorkers.startupProbe }}
+        {{- with .Values.services.automationWorkers.startupProbe }}
+        startupProbe:
+            {{- toYaml . | nindent 10 }}
+        {{- end }}
+        {{- end }}
+        {{- if .Values.services.automationWorkers.livenessProbe }}
+        {{- with .Values.services.automationWorkers.livenessProbe }}
+        livenessProbe:
+            {{- toYaml . | nindent 10 }}
+        {{- end }}
+        {{- end }}
+        {{- if .Values.services.automationWorkers.readinessProbe }}
+        {{- with .Values.services.automationWorkers.readinessProbe }}
+        readinessProbe:
+            {{- toYaml . | nindent 10 }}
+        {{- end }}
+        {{- end }}
+        name: bbautomationworker
+        ports:
+        - containerPort: {{ .Values.services.automationWorkers.port }}
+        {{ with .Values.services.automationWorkers.resources }}
+        resources:
+        {{- toYaml . | nindent 10 }}
+        {{ end }}
+        command:
+        {{- toYaml .Values.services.automationWorkers.command | nindent 10 }}
+        {{ end }}
+        {{ if .Values.services.automationWorkers.args }}
+        args:
+        {{- toYaml .Values.services.automationWorkers.args | nindent 10 }}
+        {{ end }}
+    {{- with .Values.affinity }}
+      affinity:
+        {{- toYaml . | nindent 8 }}
+    {{- end }}
+    {{- with .Values.tolerations }}
+      tolerations:
+        {{- toYaml . | nindent 8 }}
+    {{- end }}
+      {{ if .Values.schedulerName }}
+      schedulerName: {{ .Values.schedulerName | quote }}
+      {{ end }}    
+      {{ if .Values.imagePullSecrets }}
+      imagePullSecrets:
+      {{- toYaml .Values.imagePullSecrets | nindent 6 }}
+      {{ end }}
+      restartPolicy: Always
+      serviceAccountName: ""
+      {{ if .Values.services.automationWorkers.command }}}
+status: {}
+{{- end }}

charts/budibase/templates/automation-worker-service-hpa.yaml

@@ -0,0 +1,32 @@
+{{- if .Values.services.automationWorkers.autoscaling.enabled }}
+apiVersion: {{ ternary "autoscaling/v2" "autoscaling/v2beta2" (.Capabilities.APIVersions.Has "autoscaling/v2") }}
+kind: HorizontalPodAutoscaler
+metadata:
+  name: {{ include "budibase.fullname" . }}-apps
+  labels:
+    {{- include "budibase.labels" . | nindent 4 }}
+spec:
+  scaleTargetRef:
+    apiVersion: apps/v1
+    kind: Deployment
+    name: automation-worker-service
+  minReplicas: {{ .Values.services.automationWorkers.autoscaling.minReplicas }}
+  maxReplicas: {{ .Values.services.automationWorkers.autoscaling.maxReplicas }}
+  metrics:
+    {{- if .Values.services.automationWorkers.autoscaling.targetCPUUtilizationPercentage }}
+    - type: Resource
+      resource:
+        name: cpu
+        target:
+          type: Utilization
+          averageUtilization: {{ .Values.services.automationWorkers.autoscaling.targetCPUUtilizationPercentage }}
+    {{- end }}
+    {{- if .Values.services.automationWorkers.autoscaling.targetMemoryUtilizationPercentage }}
+    - type: Resource
+      resource:
+        name: memory
+        target:
+          type: Utilization
+          averageUtilization: {{ .Values.services.automationWorkers.autoscaling.targetMemoryUtilizationPercentage }}
+    {{- end }}
+{{- end }}

charts/budibase/templates/proxy-service-deployment.yaml

@@ -100,5 +100,13 @@ spec:
       {{ end }}
       restartPolicy: Always
       serviceAccountName: ""
+      {{ if .Values.services.proxy.command }}
+      command:
+      {{- toYaml .Values.services.proxy.command | nindent 8 }}
+      {{ end }}
+      {{ if .Values.services.proxy.args }}
+      args:
+      {{- toYaml .Values.services.proxy.args | nindent 8 }}
+      {{ end }}
       volumes:
 status: {}

charts/budibase/templates/worker-service-deployment.yaml

@@ -182,6 +182,10 @@ spec:
         - name: NODE_TLS_REJECT_UNAUTHORIZED
           value: {{ .Values.services.tlsRejectUnauthorized }}
         {{ end }}
+        {{- range .Values.services.worker.extraEnv }}
+        - name: {{ .name }}
+          value: {{ .value | quote }}
+        {{- end }}
         image: budibase/worker:{{ .Values.globals.appVersion | default .Chart.AppVersion }}
         imagePullPolicy: Always
         {{- if .Values.services.worker.startupProbe }}
@@ -209,6 +213,14 @@ spec:
         resources:
         {{- toYaml . | nindent 10 }}
         {{ end }}
+        {{ if .Values.services.worker.command }}
+        command:
+        {{- toYaml .Values.services.worker.command | nindent 10 }}
+        {{ end }}
+        {{ if .Values.services.worker.args }}
+        args:
+        {{- toYaml .Values.services.worker.args | nindent 10 }}
+        {{ end }}
     {{- with .Values.affinity }}
       affinity:
         {{- toYaml . | nindent 8 }}

charts/budibase/values.yaml

@@ -220,6 +220,9 @@ services:
     # <https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/>
     # for more information on how to set these.
     resources: {}
+    # -- Extra environment variables to set for apps pods. Takes a list of
+    # name=value pairs.
+    extraEnv: []
     # -- Startup probe configuration for apps pods. You shouldn't need to
     # change this, but if you want to you can find more information here:
     # <https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/>
@@ -272,6 +275,78 @@ services:
       # and resources set for the apps pods.
       targetCPUUtilizationPercentage: 80
 
+  automationWorkers:
+    # -- Whether or not to enable the automation worker service. If you disable this,
+    # automations will be processed by the apps service.
+    enabled: true
+    # @ignore (you shouldn't need to change this)
+    port: 4002
+    # -- The number of automation worker replicas to run.
+    replicaCount: 1
+    # -- The log level for the automation worker service.
+    logLevel: info
+    # -- The resources to use for automation worker pods. See
+    # <https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/>
+    # for more information on how to set these.
+    resources: {}
+    # -- Extra environment variables to set for automation worker pods. Takes a list of
+    # name=value pairs.
+    extraEnv: []
+    # -- Startup probe configuration for automation worker pods. You shouldn't
+    # need to change this, but if you want to you can find more information
+    # here:
+    # <https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/>
+    # @default -- HTTP health checks.
+    startupProbe:
+      # @ignore
+      httpGet:
+        path: /health
+        port: 4002
+        scheme: HTTP
+      # @ignore
+      failureThreshold: 30
+      # @ignore
+      periodSeconds: 3
+    # -- Readiness probe configuration for automation worker pods. You shouldn't
+    # need to change this, but if you want to you can find more information
+    # here:
+    # <https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/>
+    # @default -- HTTP health checks.
+    readinessProbe:
+      # @ignore
+      httpGet:
+        path: /health
+        port: 4002
+        scheme: HTTP
+      # @ignore
+      periodSeconds: 3
+      # @ignore
+      failureThreshold: 1
+    # -- Liveness probe configuration for automation worker pods. You shouldn't
+    # need to change this, but if you want to you can find more information
+    # here:
+    # <https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/>
+    # @default -- HTTP health checks.
+    livenessProbe:
+      # @ignore
+      httpGet:
+        path: /health
+        port: 4002
+        scheme: HTTP
+      # @ignore
+      failureThreshold: 3
+      # @ignore
+      periodSeconds: 30
+    autoscaling:
+      # -- Whether to enable horizontal pod autoscaling for the apps service.
+      enabled: false
+      minReplicas: 1
+      maxReplicas: 10
+      # -- Target CPU utilization percentage for the automation worker service.
+      # Note that for autoscaling to work, you will need to have metrics-server
+      # configured, and resources set for the automation worker pods.
+      targetCPUUtilizationPercentage: 80
+
   worker:
     # @ignore (you shouldn't need to change this)
     port: 4003
@@ -285,6 +360,9 @@ services:
     # <https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/>
     # for more information on how to set these.
     resources: {}
+    # -- Extra environment variables to set for worker pods. Takes a list of
+    # name=value pairs.
+    extraEnv: []
     # -- Startup probe configuration for worker pods. You shouldn't need to
     # change this, but if you want to you can find more information here:
     # <https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/>

hosting/proxy/nginx.prod.conf

@@ -257,6 +257,7 @@ http {
 
     access_log off;
     allow 127.0.0.1;
+    allow 10.0.0.0/8;
     deny all;
 
     location /nginx_status {

hosting/single/runner.sh

@@ -7,7 +7,7 @@ declare -a DOCKER_VARS=("APP_PORT" "APPS_URL" "ARCHITECTURE" "BUDIBASE_ENVIRONME
 [[ -z "${BUDIBASE_ENVIRONMENT}" ]] && export BUDIBASE_ENVIRONMENT=PRODUCTION
 [[ -z "${CLUSTER_PORT}" ]] && export CLUSTER_PORT=80
 [[ -z "${DEPLOYMENT_ENVIRONMENT}" ]] && export DEPLOYMENT_ENVIRONMENT=docker
-[[ -z "${MINIO_URL}" ]] && export MINIO_URL=http://127.0.0.1:9000
+[[ -z "${MINIO_URL}" ]] && [[ -z "${USE_S3}" ]] && export MINIO_URL=http://127.0.0.1:9000
 [[ -z "${NODE_ENV}" ]] && export NODE_ENV=production
 [[ -z "${POSTHOG_TOKEN}" ]] && export POSTHOG_TOKEN=phc_bIjZL7oh2GEUd2vqvTBH8WvrX0fWTFQMs6H5KQxiUxU
 [[ -z "${TENANT_FEATURE_FLAGS}" ]] && export TENANT_FEATURE_FLAGS="*:LICENSING,*:USER_GROUPS,*:ONBOARDING_TOUR"
@@ -77,7 +77,12 @@ mkdir -p ${DATA_DIR}/minio
 chown -R couchdb:couchdb ${DATA_DIR}/couch
 redis-server --requirepass $REDIS_PASSWORD > /dev/stdout 2>&1 &
 /bbcouch-runner.sh &
-/minio/minio server --console-address ":9001" ${DATA_DIR}/minio > /dev/stdout 2>&1 &
+
+# only start minio if use s3 isn't passed
+if [[ -z "${USE_S3}" ]]; then
+  /minio/minio server --console-address ":9001" ${DATA_DIR}/minio > /dev/stdout 2>&1 &
+fi
+
 /etc/init.d/nginx restart
 if [[ ! -z "${CUSTOM_DOMAIN}" ]]; then
     # Add monthly cron job to renew certbot certificate

lerna.json

@@ -1,5 +1,5 @@
 {
-  "version": "2.13.37",
+  "version": "2.13.50",
   "npmClient": "yarn",
   "packages": [
     "packages/*",
@@ -7,6 +7,7 @@
     "packages/account-portal/packages/*"
   ],
   "useNx": true,
+  "concurrency": 20,
   "command": {
     "publish": {
       "ignoreChanges": [

packages/backend-core/package.json

@@ -32,6 +32,7 @@
     "bcryptjs": "2.4.3",
     "bull": "4.10.1",
     "correlation-id": "4.0.0",
+    "dd-trace": "3.13.2",
     "dotenv": "16.0.1",
     "ioredis": "5.3.2",
     "joi": "17.6.0",

packages/backend-core/src/cache/tests/writethrough.spec.ts

@@ -1,15 +1,16 @@
 import { DBTestConfiguration } from "../../../tests/extra"
-import {
+import { structures } from "../../../tests"
-  structures,
-  expectFunctionWasCalledTimesWith,
-  mocks,
-} from "../../../tests"
 import { Writethrough } from "../writethrough"
 import { getDB } from "../../db"
+import { Document } from "@budibase/types"
 import tk from "timekeeper"
 
 tk.freeze(Date.now())
 
+interface ValueDoc extends Document {
+  value: any
+}
+
 const DELAY = 5000
 
 describe("writethrough", () => {
@@ -117,7 +118,7 @@ describe("writethrough", () => {
   describe("get", () => {
     it("should be able to retrieve", async () => {
       await config.doInTenant(async () => {
-        const response = await writethrough.get(docId)
+        const response = await writethrough.get<ValueDoc>(docId)
         expect(response.value).toBe(4)
       })
     })

packages/backend-core/src/cache/writethrough.ts

@@ -7,7 +7,7 @@ import * as locks from "../redis/redlockImpl"
 const DEFAULT_WRITE_RATE_MS = 10000
 let CACHE: BaseCache | null = null
 
-interface CacheItem {
+interface CacheItem<T extends Document> {
   doc: any
   lastWrite: number
 }
@@ -24,7 +24,10 @@ function makeCacheKey(db: Database, key: string) {
   return db.name + key
 }
 
-function makeCacheItem(doc: any, lastWrite: number | null = null): CacheItem {
+function makeCacheItem<T extends Document>(
+  doc: T,
+  lastWrite: number | null = null
+): CacheItem<T> {
   return { doc, lastWrite: lastWrite || Date.now() }
 }
 
@@ -35,7 +38,7 @@ async function put(
 ) {
   const cache = await getCache()
   const key = doc._id
-  let cacheItem: CacheItem | undefined
+  let cacheItem: CacheItem<any> | undefined
   if (key) {
     cacheItem = await cache.get(makeCacheKey(db, key))
   }
@@ -53,11 +56,8 @@ async function put(
         const writeDb = async (toWrite: any) => {
           // doc should contain the _id and _rev
           const response = await db.put(toWrite, { force: true })
-          output = {
+          output._id = response.id
-            ...doc,
+          output._rev = response.rev
-            _id: response.id,
-            _rev: response.rev,
-          }
         }
         try {
           await writeDb(doc)
@@ -84,12 +84,12 @@ async function put(
   return { ok: true, id: output._id, rev: output._rev }
 }
 
-async function get(db: Database, id: string): Promise<any> {
+async function get<T extends Document>(db: Database, id: string): Promise<T> {
   const cache = await getCache()
   const cacheKey = makeCacheKey(db, id)
-  let cacheItem: CacheItem = await cache.get(cacheKey)
+  let cacheItem: CacheItem<T> = await cache.get(cacheKey)
   if (!cacheItem) {
-    const doc = await db.get(id)
+    const doc = await db.get<T>(id)
     cacheItem = makeCacheItem(doc)
     await cache.store(cacheKey, cacheItem)
   }
@@ -123,8 +123,8 @@ export class Writethrough {
     return put(this.db, doc, writeRateMs)
   }
 
-  async get(id: string) {
+  async get<T extends Document>(id: string) {
-    return get(this.db, id)
+    return get<T>(this.db, id)
   }
 
   async remove(docOrId: any, rev?: any) {

packages/backend-core/src/constants/misc.ts

@@ -11,24 +11,7 @@ export enum Cookie {
   OIDC_CONFIG = "budibase:oidc:config",
 }
 
-export enum Header {
+export { Header } from "@budibase/shared-core"
-  API_KEY = "x-budibase-api-key",
-  LICENSE_KEY = "x-budibase-license-key",
-  API_VER = "x-budibase-api-version",
-  APP_ID = "x-budibase-app-id",
-  SESSION_ID = "x-budibase-session-id",
-  TYPE = "x-budibase-type",
-  PREVIEW_ROLE = "x-budibase-role",
-  TENANT_ID = "x-budibase-tenant-id",
-  VERIFICATION_CODE = "x-budibase-verification-code",
-  RETURN_VERIFICATION_CODE = "x-budibase-return-verification-code",
-  RESET_PASSWORD_CODE = "x-budibase-reset-password-code",
-  RETURN_RESET_PASSWORD_CODE = "x-budibase-return-reset-password-code",
-  TOKEN = "x-budibase-token",
-  CSRF_TOKEN = "x-csrf-token",
-  CORRELATION_ID = "x-budibase-correlation-id",
-  AUTHORIZATION = "authorization",
-}
 
 export enum GlobalRole {
   OWNER = "owner",

packages/backend-core/src/context/mainContext.ts

@@ -335,3 +335,11 @@ export function isScim(): boolean {
   const scimCall = context?.isScim
   return !!scimCall
 }
+
+export function getCurrentContext(): ContextMap | undefined {
+  try {
+    return Context.get()
+  } catch (e) {
+    return undefined
+  }
+}

packages/backend-core/src/context/types.ts

@@ -1,4 +1,5 @@
 import { IdentityContext } from "@budibase/types"
+import { ExecutionTimeTracker } from "../timers"
 
 // keep this out of Budibase types, don't want to expose context info
 export type ContextMap = {
@@ -9,4 +10,5 @@ export type ContextMap = {
   isScim?: boolean
   automationId?: string
   isMigrating?: boolean
+  jsExecutionTracker?: ExecutionTimeTracker
 }

packages/backend-core/src/db/couch/DatabaseImpl.ts

@@ -35,9 +35,6 @@ export function DatabaseWithConnection(
   connection: string,
   opts?: DatabaseOpts
 ) {
-  if (!connection) {
-    throw new Error("Must provide connection details")
-  }
   return new DatabaseImpl(dbName, opts, connection)
 }
 

packages/backend-core/src/environment.ts

@@ -107,6 +107,7 @@ const environment = {
   ENCRYPTION_KEY: process.env.ENCRYPTION_KEY,
   API_ENCRYPTION_KEY: getAPIEncryptionKey(),
   COUCH_DB_URL: process.env.COUCH_DB_URL || "http://localhost:4005",
+  COUCH_DB_SQL_URL: process.env.COUCH_DB_SQL_URL || "http://localhost:4984",
   COUCH_DB_USERNAME: process.env.COUCH_DB_USER,
   COUCH_DB_PASSWORD: process.env.COUCH_DB_PASSWORD,
   GOOGLE_CLIENT_ID: process.env.GOOGLE_CLIENT_ID,

packages/backend-core/src/logging/pino/logger.ts

@@ -5,6 +5,7 @@ import { IdentityType } from "@budibase/types"
 import env from "../../environment"
 import * as context from "../../context"
 import * as correlation from "../correlation"
+import { formats } from "dd-trace/ext"
 
 import { localFileDestination } from "../system"
 

packages/backend-core/src/queue/inMemoryQueue.ts

@@ -15,6 +15,7 @@ function newJob(queue: string, message: any) {
     timestamp: Date.now(),
     queue: queue,
     data: message,
+    opts: {},
   }
 }
 
@@ -68,6 +69,10 @@ class InMemoryQueue {
     })
   }
 
+  async isReady() {
+    return true
+  }
+
   // simply puts a message to the queue and emits to the queue for processing
   /**
    * Simple function to replicate the add message functionality of Bull, putting

packages/backend-core/src/redis/redlockImpl.ts

@@ -137,7 +137,6 @@ export async function doWithLock<T>(
     const result = await task()
     return { executed: true, result }
   } catch (e: any) {
-    logWarn(`lock type: ${opts.type} error`, e)
     // lock limit exceeded
     if (e.name === "LockError") {
       if (opts.type === LockType.TRY_ONCE) {

packages/backend-core/src/timers/timers.ts

@@ -20,3 +20,41 @@ export function cleanup() {
   }
   intervals = []
 }
+
+export class ExecutionTimeoutError extends Error {
+  public readonly name = "ExecutionTimeoutError"
+}
+
+export class ExecutionTimeTracker {
+  static withLimit(limitMs: number) {
+    return new ExecutionTimeTracker(limitMs)
+  }
+
+  constructor(readonly limitMs: number) {}
+
+  private totalTimeMs = 0
+
+  track<T>(f: () => T): T {
+    this.checkLimit()
+    const start = process.hrtime.bigint()
+    try {
+      return f()
+    } finally {
+      const end = process.hrtime.bigint()
+      this.totalTimeMs += Number(end - start) / 1e6
+      this.checkLimit()
+    }
+  }
+
+  get elapsedMS() {
+    return this.totalTimeMs
+  }
+
+  private checkLimit() {
+    if (this.totalTimeMs > this.limitMs) {
+      throw new ExecutionTimeoutError(
+        `Execution time limit of ${this.limitMs}ms exceeded: ${this.totalTimeMs}ms`
+      )
+    }
+  }
+}

packages/backend-core/src/utils/Duration.ts

@@ -49,4 +49,8 @@ export class Duration {
   static fromDays(duration: number) {
     return Duration.from(DurationType.DAYS, duration)
   }
+
+  static fromMilliseconds(duration: number) {
+    return Duration.from(DurationType.MILLISECONDS, duration)
+  }
 }

packages/backend-core/src/utils/utils.ts

@@ -96,7 +96,7 @@ export async function getAppIdFromCtx(ctx: Ctx) {
   }
 
   // look in the path
-  const pathId = parseAppIdFromUrl(ctx.path)
+  const pathId = parseAppIdFromUrlPath(ctx.path)
   if (!appId && pathId) {
     appId = confirmAppId(pathId)
   }
@@ -116,18 +116,21 @@ export async function getAppIdFromCtx(ctx: Ctx) {
   // referer header is present from a builder redirect
   const referer = ctx.request.headers.referer
   if (!appId && referer?.includes(BUILDER_APP_PREFIX)) {
-    const refererId = parseAppIdFromUrl(ctx.request.headers.referer)
+    const refererId = parseAppIdFromUrlPath(ctx.request.headers.referer)
     appId = confirmAppId(refererId)
   }
 
   return appId
 }
 
-function parseAppIdFromUrl(url?: string) {
+function parseAppIdFromUrlPath(url?: string) {
   if (!url) {
     return
   }
-  return url.split("/").find(subPath => subPath.startsWith(APP_PREFIX))
+  return url
+    .split("?")[0] // Remove any possible query string
+    .split("/")
+    .find(subPath => subPath.startsWith(APP_PREFIX))
 }
 
 /**

packages/builder/src/components/automation/SetupPanel/RowSelectorTypes.svelte

@@ -69,7 +69,15 @@
     on:change={e => onChange(e, field)}
     useLabel={false}
   />
-{:else if schema.type === "string" || schema.type === "number"}
+{:else if schema.type === "bb_reference"}
+  <LinkedRowSelector
+    linkedRows={value[field]}
+    {schema}
+    linkedTableId={"ta_users"}
+    on:change={e => onChange(e, field)}
+    useLabel={false}
+  />
+{:else if ["string", "number", "bigint", "barcodeqr"].includes(schema.type)}
   <svelte:component
     this={isTestModal ? ModalBindableInput : DrawerBindableInput}
     panel={AutomationBindingPanel}

packages/builder/src/components/common/LinkedRowSelector.svelte

@@ -8,6 +8,8 @@
   export let schema
   export let linkedRows = []
   export let useLabel = true
+  export let linkedTableId
+  export let label
   const dispatch = createEventDispatcher()
 
   let rows = []
@@ -16,8 +18,8 @@
   $: linkedIds = (Array.isArray(linkedRows) ? linkedRows : [])?.map(
     row => row?._id || row
   )
-  $: label = capitalise(schema.name)
+  $: label = label || capitalise(schema.name)
-  $: linkedTableId = schema.tableId
+  $: linkedTableId = linkedTableId || schema.tableId
   $: linkedTable = $tables.list.find(table => table._id === linkedTableId)
   $: fetchRows(linkedTableId)
 
@@ -57,7 +59,7 @@
 {:else}
   <Multiselect
     value={linkedIds}
-    {label}
+    label={useLabel ? label : null}
     options={rows}
     getOptionLabel={getPrettyName}
     getOptionValue={row => row._id}

packages/builder/src/components/common/bindings/BindingPicker.svelte

@@ -79,6 +79,7 @@
     bind:this={popover}
     anchor={popoverAnchor}
     maxWidth={300}
+    maxHeight={300}
     dismissible={false}
   >
     <Layout gap="S">

packages/builder/src/components/common/bindings/DrawerBindableSlot.svelte

@@ -113,7 +113,7 @@
     if (type === "json" && !isJSBinding(value)) {
       return "json-slot-icon"
     }
-    if (type !== "string" && type !== "number") {
+    if (!["string", "number", "bigint", "barcodeqr"].includes(type)) {
       return "slot-icon"
     }
     return ""

packages/builder/src/components/design/settings/controls/ValidationEditor/ValidationDrawer.svelte

@@ -164,7 +164,8 @@
     // Required constraint
     if (
       field === dataSourceSchema?.table?.primaryDisplay ||
-      constraints.presence?.allowEmpty === false
+      constraints.presence?.allowEmpty === false ||
+      constraints.presence === true
     ) {
       rules.push({
         constraint: "required",

packages/builder/src/pages/builder/portal/account/auditLogs/index.svelte

@@ -257,7 +257,7 @@
 
 <LockedFeature
   title={"Audit Logs"}
-  planType={"Business plan"}
+  planType={"Enterprise plan"}
   description={"View all events that have occurred in your Budibase installation"}
   enabled={$licensing.auditLogsEnabled}
   upgradeButtonClick={async () => {

packages/builder/src/pages/builder/portal/account/usage.svelte

@@ -15,6 +15,7 @@
   import { DashCard, Usage } from "components/usage"
   import { PlanModel } from "constants"
   import { sdk } from "@budibase/shared-core"
+  import { PlanType } from "@budibase/types"
 
   let staticUsage = []
   let monthlyUsage = []
@@ -106,7 +107,14 @@
   }
 
   const planTitle = () => {
-    return `${capitalise(license?.plan.type)} Plan`
+    const planType = license?.plan.type
+    let planName = license?.plan.type
+    if (planType === PlanType.PREMIUM_PLUS) {
+      planName = "Premium"
+    } else if (planType === PlanType.ENTERPRISE_BASIC) {
+      planName = "Enterprise"
+    }
+    return `${capitalise(planName)} Plan`
   }
 
   const getDaysRemaining = timestamp => {

packages/builder/src/pages/builder/portal/settings/auth/index.svelte

@@ -283,7 +283,7 @@
         </div>
         {#if !$licensing.enforceableSSO}
           <Tags>
-            <Tag icon="LockClosed">Enterprise</Tag>
+            <Tag icon="LockClosed">Enterprise plan</Tag>
           </Tags>
         {/if}
       </div>

packages/builder/src/pages/builder/portal/settings/environment/index.svelte

@@ -59,7 +59,7 @@
 
 <LockedFeature
   title={"Environment Variables"}
-  planType={"Business plan"}
+  planType={"Enterprise plan"}
   description={"Add and manage environment variables for development and production"}
   enabled={$licensing.environmentVariablesEnabled}
   upgradeButtonClick={async () => {

packages/client/src/components/app/forms/validation.js

@@ -23,7 +23,8 @@ export const createValidatorFromConstraints = (
     // Required constraint
     if (
       field === table?.primaryDisplay ||
-      schemaConstraints.presence?.allowEmpty === false
+      schemaConstraints.presence?.allowEmpty === false ||
+      schemaConstraints.presence === true
     ) {
       rules.push({
         type: schemaConstraints.type == "array" ? "array" : "string",

packages/frontend-core/src/api/index.js

@@ -1,4 +1,5 @@
 import { Helpers } from "@budibase/bbui"
+import { Header } from "@budibase/shared-core"
 import { ApiVersion } from "../constants"
 import { buildAnalyticsEndpoints } from "./analytics"
 import { buildAppEndpoints } from "./app"
@@ -62,6 +63,11 @@ const defaultAPIClientConfig = {
    * invoked before the actual JS error is thrown up the stack.
    */
   onError: null,
+
+  /**
+   * A function can be passed to be called when an API call returns info about a migration running for a specific app
+   */
+  onMigrationDetected: null,
 }
 
 /**
@@ -133,9 +139,9 @@ export const createAPIClient = config => {
 
     // Build headers
     let headers = { Accept: "application/json" }
-    headers["x-budibase-session-id"] = APISessionID
+    headers[Header.SESSION_ID] = APISessionID
     if (!external) {
-      headers["x-budibase-api-version"] = ApiVersion
+      headers[Header.API_VER] = ApiVersion
     }
     if (json) {
       headers["Content-Type"] = "application/json"
@@ -170,6 +176,7 @@ export const createAPIClient = config => {
 
     // Handle response
     if (response.status >= 200 && response.status < 400) {
+      handleMigrations(response)
       try {
         if (parseResponse) {
           return await parseResponse(response)
@@ -186,7 +193,18 @@ export const createAPIClient = config => {
     }
   }
 
-  // Performs an API call to the server and caches the response.
+  const handleMigrations = response => {
+    if (!config.onMigrationDetected) {
+      return
+    }
+    const migration = response.headers.get(Header.MIGRATING_APP)
+
+    if (migration) {
+      config.onMigrationDetected(migration)
+    }
+  }
+
+  // Performs an API call to the server  and caches the response.
   // Future invocation for this URL will return the cached result instead of
   // hitting the server again.
   const makeCachedApiCall = async params => {
@@ -242,7 +260,7 @@ export const createAPIClient = config => {
     getAppID: () => {
       let headers = {}
       config?.attachHeaders(headers)
-      return headers?.["x-budibase-app-id"]
+      return headers?.[Header.APP_ID]
     },
   }
 

packages/server/Dockerfile

@@ -68,9 +68,13 @@ COPY packages/server/builder/ builder/
 COPY packages/server/client/ client/
 
 ARG BUDIBASE_VERSION
+ARG GIT_COMMIT_SHA
 # Ensuring the version argument is sent
 RUN test -n "$BUDIBASE_VERSION"
 ENV BUDIBASE_VERSION=$BUDIBASE_VERSION
+ENV DD_GIT_REPOSITORY_URL=https://github.com/budibase/budibase
+ENV DD_GIT_COMMIT_SHA=$GIT_COMMIT_SHA
+ENV DD_VERSION=$BUDIBASE_VERSION
 
 EXPOSE 4001
 

packages/server/scripts/dev/manage.js

@@ -1,7 +1,8 @@
 #!/usr/bin/env node
 const compose = require("docker-compose")
 const path = require("path")
-const fs = require("fs")
+const { parsed: existingConfig } = require("dotenv").config()
+const updateDotEnv = require("update-dotenv")
 
 // This script wraps docker-compose allowing you to manage your dev infrastructure with simple commands.
 const CONFIG = {
@@ -17,45 +18,41 @@ const Commands = {
 }
 
 async function init() {
-  const envFilePath = path.join(process.cwd(), ".env")
+  let config = {
-  if (!fs.existsSync(envFilePath)) {
+    PORT: "4001",
-    const envFileJson = {
+    MINIO_URL: "http://localhost:4004",
-      PORT: 4001,
+    COUCH_DB_URL: "http://budibase:budibase@localhost:4005",
-      MINIO_URL: "http://localhost:4004",
+    REDIS_URL: "localhost:6379",
-      COUCH_DB_URL: "http://budibase:budibase@localhost:4005",
+    WORKER_URL: "http://localhost:4002",
-      REDIS_URL: "localhost:6379",
+    INTERNAL_API_KEY: "budibase",
-      WORKER_URL: "http://localhost:4002",
+    ACCOUNT_PORTAL_URL: "http://localhost:10001",
-      INTERNAL_API_KEY: "budibase",
+    ACCOUNT_PORTAL_API_KEY: "budibase",
-      ACCOUNT_PORTAL_URL: "http://localhost:10001",
+    PLATFORM_URL: "http://localhost:10000",
-      ACCOUNT_PORTAL_API_KEY: "budibase",
+    JWT_SECRET: "testsecret",
-      PLATFORM_URL: "http://localhost:10000",
+    ENCRYPTION_KEY: "testsecret",
-      JWT_SECRET: "testsecret",
+    REDIS_PASSWORD: "budibase",
-      ENCRYPTION_KEY: "testsecret",
+    MINIO_ACCESS_KEY: "budibase",
-      REDIS_PASSWORD: "budibase",
+    MINIO_SECRET_KEY: "budibase",
-      MINIO_ACCESS_KEY: "budibase",
+    COUCH_DB_PASSWORD: "budibase",
-      MINIO_SECRET_KEY: "budibase",
+    COUCH_DB_USER: "budibase",
-      COUCH_DB_PASSWORD: "budibase",
+    SELF_HOSTED: "1",
-      COUCH_DB_USER: "budibase",
+    DISABLE_ACCOUNT_PORTAL: "1",
-      SELF_HOSTED: 1,
+    MULTI_TENANCY: "",
-      DISABLE_ACCOUNT_PORTAL: 1,
+    DISABLE_THREADING: "1",
-      MULTI_TENANCY: "",
+    SERVICE: "app-service",
-      DISABLE_THREADING: 1,
+    DEPLOYMENT_ENVIRONMENT: "development",
-      SERVICE: "app-service",
+    BB_ADMIN_USER_EMAIL: "",
-      DEPLOYMENT_ENVIRONMENT: "development",
+    BB_ADMIN_USER_PASSWORD: "",
-      BB_ADMIN_USER_EMAIL: "",
+    PLUGINS_DIR: "",
-      BB_ADMIN_USER_PASSWORD: "",
+    TENANT_FEATURE_FLAGS: "*:LICENSING,*:USER_GROUPS,*:ONBOARDING_TOUR",
-      PLUGINS_DIR: "",
+    HTTP_MIGRATIONS: "0",
-      TENANT_FEATURE_FLAGS: "*:LICENSING,*:USER_GROUPS,*:ONBOARDING_TOUR",
+    HTTP_LOGGING: "0",
-      HTTP_MIGRATIONS: "0",
+    VERSION: "0.0.0+local",
-      HTTP_LOGGING: "0",
-      VERSION: "0.0.0+local",
-    }
-    let envFile = ""
-    Object.keys(envFileJson).forEach(key => {
-      envFile += `${key}=${envFileJson[key]}\n`
-    })
-    fs.writeFileSync(envFilePath, envFile)
   }
+
+  config = { ...config, ...existingConfig }
+
+  await updateDotEnv(config)
 }
 
 async function up() {

packages/server/src/api/controllers/application.ts

@@ -340,7 +340,7 @@ async function performAppCreate(ctx: UserCtx) {
     // Initialise the app migration version as the latest one
     await appMigrations.updateAppMigrationMetadata({
       appId,
-      version: appMigrations.latestMigration,
+      version: appMigrations.getLatestMigrationId(),
     })
 
     await cache.app.invalidateAppMetadata(appId, newApplication)

packages/server/src/api/controllers/migrations.ts

@@ -1,14 +1,34 @@
+import { context } from "@budibase/backend-core"
 import { migrate as migrationImpl, MIGRATIONS } from "../../migrations"
-import { BBContext } from "@budibase/types"
+import { Ctx } from "@budibase/types"
+import {
+  getAppMigrationVersion,
+  getLatestMigrationId,
+} from "../../appMigrations"
 
-export async function migrate(ctx: BBContext) {
+export async function migrate(ctx: Ctx) {
   const options = ctx.request.body
   // don't await as can take a while, just return
   migrationImpl(options)
   ctx.status = 200
 }
 
-export async function fetchDefinitions(ctx: BBContext) {
+export async function fetchDefinitions(ctx: Ctx) {
   ctx.body = MIGRATIONS
   ctx.status = 200
 }
+
+export async function getMigrationStatus(ctx: Ctx) {
+  const appId = context.getAppId()
+
+  if (!appId) {
+    ctx.throw("AppId could not be found")
+  }
+
+  const latestAppliedMigration = await getAppMigrationVersion(appId)
+
+  const migrated = latestAppliedMigration === getLatestMigrationId()
+
+  ctx.body = { migrated }
+  ctx.status = 200
+}

packages/server/src/api/index.ts

@@ -4,62 +4,75 @@ import currentApp from "../middleware/currentapp"
 import zlib from "zlib"
 import { mainRoutes, staticRoutes, publicRoutes } from "./routes"
 import { middleware as pro } from "@budibase/pro"
+import { apiEnabled, automationsEnabled } from "../features"
 import migrations from "../middleware/appMigrations"
+import { automationQueue } from "../automations"
 
 export { shutdown } from "./routes/public"
 const compress = require("koa-compress")
 
 export const router: Router = new Router()
 
-router.get("/health", ctx => (ctx.status = 200))
+router.get("/health", async ctx => {
+  if (automationsEnabled()) {
+    if (!(await automationQueue.isReady())) {
+      ctx.status = 503
+      return
+    }
+  }
+  ctx.status = 200
+})
 router.get("/version", ctx => (ctx.body = envCore.VERSION))
 
 router.use(middleware.errorHandling)
 
-router
+// only add the routes if they are enabled
-  .use(
+if (apiEnabled()) {
-    compress({
+  router
-      threshold: 2048,
+    .use(
-      gzip: {
+      compress({
-        flush: zlib.constants.Z_SYNC_FLUSH,
+        threshold: 2048,
-      },
+        gzip: {
-      deflate: {
+          flush: zlib.constants.Z_SYNC_FLUSH,
-        flush: zlib.constants.Z_SYNC_FLUSH,
+        },
-      },
+        deflate: {
-      br: false,
+          flush: zlib.constants.Z_SYNC_FLUSH,
-    })
+        },
-  )
+        br: false,
-  // re-direct before any middlewares occur
+      })
-  .redirect("/", "/builder")
+    )
-  .use(
+    // re-direct before any middlewares occur
-    auth.buildAuthMiddleware([], {
+    .redirect("/", "/builder")
-      publicAllowed: true,
+    .use(
-    })
+      auth.buildAuthMiddleware([], {
-  )
+        publicAllowed: true,
-  // nothing in the server should allow query string tenants
+      })
-  // the server can be public anywhere, so nowhere should throw errors
+    )
-  // if the tenancy has not been set, it'll have to be discovered at application layer
+    // nothing in the server should allow query string tenants
-  .use(
+    // the server can be public anywhere, so nowhere should throw errors
-    auth.buildTenancyMiddleware([], [], {
+    // if the tenancy has not been set, it'll have to be discovered at application layer
-      noTenancyRequired: true,
+    .use(
-    })
+      auth.buildTenancyMiddleware([], [], {
-  )
+        noTenancyRequired: true,
-  .use(pro.licensing())
+      })
-  // @ts-ignore
+    )
-  .use(currentApp)
+    .use(pro.licensing())
-  .use(auth.auditLog)
+    // @ts-ignore
-  // @ts-ignore
+    .use(currentApp)
-  .use(migrations)
+    .use(auth.auditLog)
+    // @ts-ignore
+    .use(migrations)
 
-// authenticated routes
+  // authenticated routes
-for (let route of mainRoutes) {
+  for (let route of mainRoutes) {
-  router.use(route.routes())
+    router.use(route.routes())
-  router.use(route.allowedMethods())
+    router.use(route.allowedMethods())
+  }
+
+  router.use(publicRoutes.routes())
+  router.use(publicRoutes.allowedMethods())
+
+  // WARNING - static routes will catch everything else after them this must be last
+  router.use(staticRoutes.routes())
+  router.use(staticRoutes.allowedMethods())
 }
-
-router.use(publicRoutes.routes())
-router.use(publicRoutes.allowedMethods())
-
-// WARNING - static routes will catch everything else after them this must be last
-router.use(staticRoutes.routes())
-router.use(staticRoutes.allowedMethods())

packages/server/src/api/routes/migrations.ts

@@ -11,4 +11,6 @@ router
     auth.internalApi,
     migrationsController.fetchDefinitions
   )
+  .get("/api/migrations/status", migrationsController.getMigrationStatus)
+
 export default router

packages/server/src/api/routes/tests/row.spec.ts

@@ -2086,4 +2086,112 @@ describe.each([
       expect(row.formula).toBe(relatedRow.name)
     })
   })
+
+  describe("Formula JS protection", () => {
+    it("should time out JS execution if a single cell takes too long", async () => {
+      await config.withEnv({ JS_PER_EXECUTION_TIME_LIMIT_MS: 20 }, async () => {
+        const js = Buffer.from(
+          `
+              let i = 0;
+              while (true) {
+                i++;
+              }
+              return i;
+            `
+        ).toString("base64")
+
+        const table = await config.createTable({
+          name: "table",
+          type: "table",
+          schema: {
+            text: {
+              name: "text",
+              type: FieldType.STRING,
+            },
+            formula: {
+              name: "formula",
+              type: FieldType.FORMULA,
+              formula: `{{ js "${js}"}}`,
+              formulaType: FormulaTypes.DYNAMIC,
+            },
+          },
+        })
+
+        await config.api.row.save(table._id!, { text: "foo" })
+        const { rows } = await config.api.row.search(table._id!)
+        expect(rows).toHaveLength(1)
+        const row = rows[0]
+        expect(row.text).toBe("foo")
+        expect(row.formula).toBe("Timed out while executing JS")
+      })
+    })
+
+    it("should time out JS execution if a multiple cells take too long", async () => {
+      await config.withEnv(
+        {
+          JS_PER_EXECUTION_TIME_LIMIT_MS: 20,
+          JS_PER_REQUEST_TIME_LIMIT_MS: 40,
+        },
+        async () => {
+          const js = Buffer.from(
+            `
+              let i = 0;
+              while (true) {
+                i++;
+              }
+              return i;
+            `
+          ).toString("base64")
+
+          const table = await config.createTable({
+            name: "table",
+            type: "table",
+            schema: {
+              text: {
+                name: "text",
+                type: FieldType.STRING,
+              },
+              formula: {
+                name: "formula",
+                type: FieldType.FORMULA,
+                formula: `{{ js "${js}"}}`,
+                formulaType: FormulaTypes.DYNAMIC,
+              },
+            },
+          })
+
+          for (let i = 0; i < 10; i++) {
+            await config.api.row.save(table._id!, { text: "foo" })
+          }
+
+          // Run this test 3 times to make sure that there's no cross-request
+          // pollution of the execution time tracking.
+          for (let reqs = 0; reqs < 3; reqs++) {
+            const { rows } = await config.api.row.search(table._id!)
+            expect(rows).toHaveLength(10)
+
+            let i = 0
+            for (; i < 10; i++) {
+              const row = rows[i]
+              if (row.formula !== "Timed out while executing JS") {
+                break
+              }
+            }
+
+            // Given the execution times are not deterministic, we can't be sure
+            // of the exact number of rows that were executed before the timeout
+            // but it should absolutely be at least 1.
+            expect(i).toBeGreaterThan(0)
+            expect(i).toBeLessThan(5)
+
+            for (; i < 10; i++) {
+              const row = rows[i]
+              expect(row.text).toBe("foo")
+              expect(row.formula).toBe("Request JS execution limit hit")
+            }
+          }
+        }
+      )
+    })
+  })
 })

packages/server/src/app.ts

@@ -9,7 +9,6 @@ import { ServiceType } from "@budibase/types"
 import { env as coreEnv } from "@budibase/backend-core"
 
 coreEnv._set("SERVICE_TYPE", ServiceType.APPS)
-import { apiEnabled } from "./features"
 import createKoaApp from "./koa"
 import Koa from "koa"
 import { Server } from "http"
@@ -18,12 +17,9 @@ import { startup } from "./startup"
 let app: Koa, server: Server
 
 async function start() {
-  // if API disabled, could run automations instead
+  const koa = createKoaApp()
-  if (apiEnabled()) {
+  app = koa.app
-    const koa = createKoaApp()
+  server = koa.server
-    app = koa.app
-    server = koa.server
-  }
   // startup includes automation runner - if enabled
   await startup(app, server)
 }

packages/server/src/appMigrations/index.ts

@@ -1,6 +1,9 @@
 import queue from "./queue"
+import { Next } from "koa"
 import { getAppMigrationVersion } from "./appMigrationMetadata"
 import { MIGRATIONS } from "./migrations"
+import { UserCtx } from "@budibase/types"
+import { Header } from "@budibase/backend-core"
 
 export * from "./appMigrationMetadata"
 
@@ -9,14 +12,20 @@ export type AppMigration = {
   func: () => Promise<void>
 }
 
-export const latestMigration = MIGRATIONS.map(m => m.id)
+export const getLatestMigrationId = () =>
-  .sort()
+  MIGRATIONS.map(m => m.id)
-  .reverse()[0]
+    .sort()
+    .reverse()[0]
 
 const getTimestamp = (versionId: string) => versionId?.split("_")[0]
 
-export async function checkMissingMigrations(appId: string) {
+export async function checkMissingMigrations(
+  ctx: UserCtx,
+  next: Next,
+  appId: string
+) {
   const currentVersion = await getAppMigrationVersion(appId)
+  const latestMigration = getLatestMigrationId()
 
   if (getTimestamp(currentVersion) < getTimestamp(latestMigration)) {
     await queue.add(
@@ -29,5 +38,9 @@ export async function checkMissingMigrations(appId: string) {
         removeOnFail: true,
       }
     )
+
+    ctx.response.set(Header.MIGRATING_APP, appId)
   }
+
+  return next()
 }

packages/server/src/appMigrations/tests/migrations.integrity.spec.ts

@@ -0,0 +1,25 @@
+import { context } from "@budibase/backend-core"
+import * as setup from "../../api/routes/tests/utilities"
+import * as migrations from "../migrations"
+
+describe("migration integrity", () => {
+  // These test is checking that each migration is "idempotent".
+  // We should be able to rerun any migration, with any rerun not modifiying anything. The code should be aware that the migration already ran
+  it("each migration can rerun safely", async () => {
+    const config = setup.getConfig()
+    await config.init()
+
+    await config.doInContext(config.getAppId(), async () => {
+      const db = context.getAppDB()
+      for (const migration of migrations.MIGRATIONS) {
+        await migration.func()
+        const docs = await db.allDocs({ include_docs: true })
+
+        await migration.func()
+        const latestDocs = await db.allDocs({ include_docs: true })
+
+        expect(docs).toEqual(latestDocs)
+      }
+    })
+  })
+})

packages/server/src/appMigrations/tests/migrations.spec.ts

@@ -1,25 +1,53 @@
-import { context } from "@budibase/backend-core"
+import { Header } from "@budibase/backend-core"
 import * as setup from "../../api/routes/tests/utilities"
-import { MIGRATIONS } from "../migrations"
+import * as migrations from "../migrations"
+import { getAppMigrationVersion } from "../appMigrationMetadata"
 
-describe("migration", () => {
+jest.mock<typeof migrations>("../migrations", () => ({
-  // These test is checking that each migration is "idempotent".
+  MIGRATIONS: [
-  // We should be able to rerun any migration, with any rerun not modifiying anything. The code should be aware that the migration already ran
+    {
-  it("each migration can rerun safely", async () => {
+      id: "20231211101320_test",
+      func: async () => {},
+    },
+  ],
+}))
+
+describe("migrations", () => {
+  it("new apps are created with the latest app migration version set", async () => {
     const config = setup.getConfig()
     await config.init()
 
     await config.doInContext(config.getAppId(), async () => {
-      const db = context.getAppDB()
+      const migrationVersion = await getAppMigrationVersion(config.getAppId())
-      for (const migration of MIGRATIONS) {
-        await migration.func()
-        const docs = await db.allDocs({ include_docs: true })
 
-        await migration.func()
+      expect(migrationVersion).toEqual("20231211101320_test")
-        const latestDocs = await db.allDocs({ include_docs: true })
-
-        expect(docs).toEqual(latestDocs)
-      }
     })
   })
+
+  it("accessing an app that has no pending migrations will not attach the migrating header", async () => {
+    const config = setup.getConfig()
+    await config.init()
+
+    const appId = config.getAppId()
+
+    const response = await config.api.application.getRaw(appId)
+
+    expect(response.headers[Header.MIGRATING_APP]).toBeUndefined()
+  })
+
+  it("accessing an app that has pending migrations will attach the migrating header", async () => {
+    const config = setup.getConfig()
+    await config.init()
+
+    const appId = config.getAppId()
+
+    migrations.MIGRATIONS.push({
+      id: "20231211105812_new-test",
+      func: async () => {},
+    })
+
+    const response = await config.api.application.getRaw(appId)
+
+    expect(response.headers[Header.MIGRATING_APP]).toEqual(appId)
+  })
 })

packages/server/src/appMigrations/tests/migrationsProcessor.spec.ts

@@ -4,12 +4,14 @@ import { getAppMigrationVersion } from "../appMigrationMetadata"
 import { context } from "@budibase/backend-core"
 import { AppMigration } from ".."
 
+const futureTimestamp = `20500101174029`
+
 describe("migrationsProcessor", () => {
   it("running migrations will update the latest applied migration", async () => {
     const testMigrations: AppMigration[] = [
-      { id: "123", func: async () => {} },
+      { id: `${futureTimestamp}_123`, func: async () => {} },
-      { id: "124", func: async () => {} },
+      { id: `${futureTimestamp}_124`, func: async () => {} },
-      { id: "125", func: async () => {} },
+      { id: `${futureTimestamp}_125`, func: async () => {} },
     ]
 
     const config = setup.getConfig()
@@ -23,13 +25,13 @@ describe("migrationsProcessor", () => {
 
     expect(
       await config.doInContext(appId, () => getAppMigrationVersion(appId))
-    ).toBe("125")
+    ).toBe(`${futureTimestamp}_125`)
   })
 
   it("no context can be initialised within a migration", async () => {
     const testMigrations: AppMigration[] = [
       {
-        id: "123",
+        id: `${futureTimestamp}_123`,
         func: async () => {
           await context.doInAppMigrationContext("any", () => {})
         },

packages/server/src/automations/steps/updateRow.ts

@@ -84,9 +84,11 @@ export async function run({ inputs, appId, emitter }: AutomationStepInput) {
 
   // clear any undefined, null or empty string properties so that they aren't updated
   for (let propKey of Object.keys(inputs.row)) {
+    const clearRelationships =
+      inputs.meta?.fields?.[propKey]?.clearRelationships
     if (
-      (inputs.row[propKey] == null || inputs.row[propKey] === "") &&
+      (inputs.row[propKey] == null || inputs.row[propKey]?.length === 0) &&
-      !inputs.meta?.fields?.[propKey]?.clearRelationships
+      !clearRelationships
     ) {
       delete inputs.row[propKey]
     }

packages/server/src/automations/utils.ts

@@ -41,6 +41,7 @@ function loggingArgs(job: AutomationJob) {
 export async function processEvent(job: AutomationJob) {
   const appId = job.data.event.appId!
   const automationId = job.data.automation._id!
+
   const task = async () => {
     try {
       // need to actually await these so that an error can be captured properly

packages/server/src/db/linkedRows/linkUtils.ts

@@ -56,7 +56,7 @@ export async function getLinkDocuments(args: {
   try {
     let linkRows = (await db.query(getQueryIndex(ViewName.LINK), params)).rows
     // filter to get unique entries
-    const foundIds: string[] = []
+    const foundIds = new Set()
     linkRows = linkRows.filter(link => {
       // make sure anything unique is the correct key
       if (
@@ -65,9 +65,9 @@ export async function getLinkDocuments(args: {
       ) {
         return false
       }
-      const unique = foundIds.indexOf(link.id) === -1
+      const unique = !foundIds.has(link.id)
       if (unique) {
-        foundIds.push(link.id)
+        foundIds.add(link.id)
       }
       return unique
     })
@@ -99,9 +99,15 @@ export async function getLinkDocuments(args: {
 }
 
 export function getUniqueByProp(array: any[], prop: string) {
-  return array.filter((obj, pos, arr) => {
+  const seen = new Set()
-    return arr.map(mapObj => mapObj[prop]).indexOf(obj[prop]) === pos
+  const filteredArray = []
-  })
+  for (const item of array) {
+    if (!seen.has(item[prop])) {
+      seen.add(item[prop])
+      filteredArray.push(item)
+    }
+  }
+  return filteredArray
 }
 
 export function getLinkedTableIDs(table: Table): string[] {

packages/server/src/environment.ts

@@ -70,6 +70,11 @@ const environment = {
   SELF_HOSTED: process.env.SELF_HOSTED,
   HTTP_MB_LIMIT: process.env.HTTP_MB_LIMIT,
   FORKED_PROCESS_NAME: process.env.FORKED_PROCESS_NAME || "main",
+  JS_PER_EXECUTION_TIME_LIMIT_MS:
+    parseIntSafe(process.env.JS_PER_EXECUTION_TIME_LIMIT_MS) || 1000,
+  JS_PER_REQUEST_TIME_LIMIT_MS: parseIntSafe(
+    process.env.JS_PER_REQUEST_TIME_LIMIT_MS
+  ),
   // old
   CLIENT_ID: process.env.CLIENT_ID,
   _set(key: string, value: any) {

packages/server/src/features.ts

@@ -22,3 +22,10 @@ export function automationsEnabled() {
 export function apiEnabled() {
   return featureList.includes(AppFeature.API)
 }
+
+export function printFeatures() {
+  if (!env.APP_FEATURES) {
+    return
+  }
+  console.log(`**** APP FEATURES SET: ${featureList.join(", ")} ****`)
+}

packages/server/src/integrations/couchdb.ts

@@ -1,5 +1,6 @@
 import {
   ConnectionInfo,
+  Database,
   DatasourceFeature,
   DatasourceFieldType,
   Document,
@@ -66,7 +67,7 @@ const SCHEMA: Integration = {
 }
 
 class CouchDBIntegration implements IntegrationBase {
-  private readonly client: dbCore.DatabaseImpl
+  private readonly client: Database
 
   constructor(config: CouchDBConfig) {
     this.client = dbCore.DatabaseWithConnection(config.database, config.url)

packages/server/src/integrations/rest.ts

@@ -131,7 +131,10 @@ class RestIntegration implements IntegrationBase {
     let data, raw, headers
     const contentType = response.headers.get("content-type") || ""
     try {
-      if (contentType.includes("application/json")) {
+      if (response.status === 204) {
+        data = []
+        raw = []
+      } else if (contentType.includes("application/json")) {
         data = await response.json()
         raw = JSON.stringify(data)
       } else if (

packages/server/src/integrations/tests/rest.spec.ts

@@ -186,9 +186,15 @@ describe("REST Integration", () => {
   })
 
   describe("response", () => {
-    function buildInput(json: any, text: any, header: any) {
+    const contentTypes = ["application/json", "text/plain", "application/xml"]
+    function buildInput(
+      json: any,
+      text: any,
+      header: any,
+      status: number = 200
+    ) {
       return {
-        status: 200,
+        status,
         json: json ? async () => json : undefined,
         text: text ? async () => text : undefined,
         headers: {
@@ -225,6 +231,18 @@ describe("REST Integration", () => {
       expect(output.extra.raw).toEqual(text)
       expect(output.extra.headers["content-type"]).toEqual("application/xml")
     })
+
+    test.each(contentTypes)(
+      "should not throw an error on 204 no content",
+      async contentType => {
+        const input = buildInput(undefined, null, contentType, 204)
+        const output = await config.integration.parseResponse(input)
+        expect(output.data).toEqual([])
+        expect(output.extra.raw).toEqual([])
+        expect(output.info.code).toEqual(204)
+        expect(output.extra.headers["content-type"]).toEqual(contentType)
+      }
+    )
   })
 
   describe("authentication", () => {

packages/server/src/jsRunner.ts

@@ -0,0 +1,36 @@
+import vm from "vm"
+import env from "./environment"
+import { setJSRunner } from "@budibase/string-templates"
+import { context, timers } from "@budibase/backend-core"
+
+type TrackerFn = <T>(f: () => T) => T
+
+export function init() {
+  setJSRunner((js: string, ctx: vm.Context) => {
+    const perRequestLimit = env.JS_PER_REQUEST_TIME_LIMIT_MS
+    let track: TrackerFn = f => f()
+    if (perRequestLimit) {
+      const bbCtx = context.getCurrentContext()
+      if (bbCtx) {
+        if (!bbCtx.jsExecutionTracker) {
+          bbCtx.jsExecutionTracker =
+            timers.ExecutionTimeTracker.withLimit(perRequestLimit)
+        }
+        track = bbCtx.jsExecutionTracker.track.bind(bbCtx.jsExecutionTracker)
+      }
+    }
+
+    ctx = {
+      ...ctx,
+      alert: undefined,
+      setInterval: undefined,
+      setTimeout: undefined,
+    }
+    vm.createContext(ctx)
+    return track(() =>
+      vm.runInNewContext(js, ctx, {
+        timeout: env.JS_PER_EXECUTION_TIME_LIMIT_MS,
+      })
+    )
+  })
+}

packages/server/src/middleware/appMigrations.ts

@@ -8,7 +8,5 @@ export default async (ctx: UserCtx, next: any) => {
     return next()
   }
 
-  await checkMissingMigrations(appId)
+  return checkMissingMigrations(ctx, next, appId)
-
-  return next()
 }

packages/server/src/startup.ts

@@ -19,11 +19,15 @@ import * as pro from "@budibase/pro"
 import * as api from "./api"
 import sdk from "./sdk"
 import { initialise as initialiseWebsockets } from "./websockets"
-import { automationsEnabled } from "./features"
+import { automationsEnabled, printFeatures } from "./features"
+import Koa from "koa"
+import { Server } from "http"
+import { AddressInfo } from "net"
+import * as jsRunner from "./jsRunner"
 
 let STARTUP_RAN = false
 
-async function initRoutes(app: any) {
+async function initRoutes(app: Koa) {
   if (!env.isTest()) {
     const plugin = await bullboard.init()
     app.use(plugin)
@@ -48,27 +52,31 @@ async function initPro() {
   })
 }
 
-function shutdown(server?: any) {
+function shutdown(server?: Server) {
   if (server) {
     server.close()
     server.destroy()
   }
 }
 
-export async function startup(app?: any, server?: any) {
+export async function startup(app?: Koa, server?: Server) {
   if (STARTUP_RAN) {
     return
   }
+  printFeatures()
   STARTUP_RAN = true
-  if (server && !env.CLUSTER_MODE) {
+  if (app && server && !env.CLUSTER_MODE) {
     console.log(`Budibase running on ${JSON.stringify(server.address())}`)
-    env._set("PORT", server.address().port)
+    const address = server.address() as AddressInfo
+    env._set("PORT", address.port)
   }
   eventEmitter.emitPort(env.PORT)
   fileSystem.init()
   await redis.init()
   eventInit()
-  initialiseWebsockets(app, server)
+  if (app && server) {
+    initialiseWebsockets(app, server)
+  }
 
   // run migrations on startup if not done via http
   // not recommended in a clustered environment
@@ -145,4 +153,6 @@ export async function startup(app?: any, server?: any) {
       }
     })
   }
+
+  jsRunner.init()
 }

packages/server/src/tests/utilities/TestConfiguration.ts

@@ -17,7 +17,6 @@ import {
   basicWebhook,
 } from "./structures"
 import {
-  auth,
   cache,
   constants,
   context,

packages/server/src/tests/utilities/api/application.ts

@@ -1,3 +1,4 @@
+import { Response } from "supertest"
 import { App } from "@budibase/types"
 import TestConfiguration from "../TestConfiguration"
 import { TestAPI } from "./base"
@@ -7,12 +8,17 @@ export class ApplicationAPI extends TestAPI {
     super(config)
   }
 
-  get = async (appId: string): Promise<App> => {
+  getRaw = async (appId: string): Promise<Response> => {
     const result = await this.request
       .get(`/api/applications/${appId}/appPackage`)
       .set(this.config.defaultHeaders())
       .expect("Content-Type", /json/)
       .expect(200)
+    return result
+  }
+
+  get = async (appId: string): Promise<App> => {
+    const result = await this.getRaw(appId)
     return result.body.application as App
   }
 }

packages/server/src/threads/automation.ts

@@ -265,6 +265,10 @@ class Orchestrator {
     }
     const start = performance.now()
     for (let step of automation.definition.steps) {
+      let input: any,
+        iterations = 1,
+        iterationCount = 0
+
       if (timeoutFlag) {
         break
       }
@@ -276,10 +280,6 @@ class Orchestrator {
       }
 
       stepCount++
-      let input: any,
-        iterations = 1,
-        iterationCount = 0
-
       if (step.stepId === LOOP_STEP_ID) {
         loopStep = step
         loopStepNumber = stepCount
@@ -294,7 +294,10 @@ class Orchestrator {
         let originalStepInput = cloneDeep(step.inputs)
         // Handle if the user has set a max iteration count or if it reaches the max limit set by us
         if (loopStep && input.binding) {
-          let tempOutput = { items: loopSteps, iterations: iterationCount }
+          let tempOutput = {
+            items: loopSteps,
+            iterations: iterationCount,
+          }
           try {
             loopStep.inputs.binding = automationUtils.typecastForLooping(
               loopStep as LoopStep,

packages/server/src/utilities/rowProcessor/utils.ts

@@ -51,7 +51,7 @@ export function processFormulas<T extends Row | Row[]>(
   { dynamic, contextRows }: FormulaOpts = { dynamic: true }
 ): T {
   const rows = Array.isArray(inputRows) ? inputRows : [inputRows]
-  if (rows)
+  if (rows) {
     for (let [column, schema] of Object.entries(table.schema)) {
       if (schema.type !== FieldTypes.FORMULA) {
         continue
@@ -70,12 +70,14 @@ export function processFormulas<T extends Row | Row[]>(
       for (let i = 0; i < rows.length; i++) {
         let row = rows[i]
         let context = contextRows ? contextRows[i] : row
+        let formula = schema.formula
         rows[i] = {
           ...row,
-          [column]: processStringSync(schema.formula, context),
+          [column]: processStringSync(formula, context),
         }
       }
     }
+  }
   return Array.isArray(inputRows) ? rows : rows[0]
 }
 

packages/shared-core/src/constants/api.ts

@@ -0,0 +1,19 @@
+export enum Header {
+  API_KEY = "x-budibase-api-key",
+  LICENSE_KEY = "x-budibase-license-key",
+  API_VER = "x-budibase-api-version",
+  APP_ID = "x-budibase-app-id",
+  SESSION_ID = "x-budibase-session-id",
+  TYPE = "x-budibase-type",
+  PREVIEW_ROLE = "x-budibase-role",
+  TENANT_ID = "x-budibase-tenant-id",
+  VERIFICATION_CODE = "x-budibase-verification-code",
+  RETURN_VERIFICATION_CODE = "x-budibase-return-verification-code",
+  RESET_PASSWORD_CODE = "x-budibase-reset-password-code",
+  RETURN_RESET_PASSWORD_CODE = "x-budibase-return-reset-password-code",
+  TOKEN = "x-budibase-token",
+  CSRF_TOKEN = "x-csrf-token",
+  CORRELATION_ID = "x-budibase-correlation-id",
+  AUTHORIZATION = "authorization",
+  MIGRATING_APP = "x-budibase-migrating-app",
+}

packages/shared-core/src/constants/index.ts

@@ -1,3 +1,5 @@
+export * from "./api"
+
 export const OperatorOptions = {
   Equals: {
     value: "equal",

packages/string-templates/manifest.json

@@ -1180,6 +1180,14 @@
       "description": "<p>Stringify an object using <code>JSON.stringify</code>.</p>\n"
     }
   },
+  "uuid": {
+    "uuid": {
+      "args": [],
+      "numArgs": 0,
+      "example": "{{ uuid }} -> f34ebc66-93bd-4f7c-b79b-92b5569138bc",
+      "description": "<p>Generates a UUID, using the V4 method (identical to the browser crypto.randomUUID function).</p>\n"
+    }
+  },
   "date": {
     "date": {
       "args": [

packages/string-templates/package.json

@@ -24,7 +24,7 @@
     "manifest": "node ./scripts/gen-collection-info.js"
   },
   "dependencies": {
-    "@budibase/handlebars-helpers": "^0.11.9",
+    "@budibase/handlebars-helpers": "^0.11.11",
     "dayjs": "^1.10.8",
     "handlebars": "^4.7.6",
     "isolated-vm": "^4.6.0",

packages/string-templates/scripts/gen-collection-info.js

@@ -20,6 +20,7 @@ const COLLECTIONS = [
   "string",
   "comparison",
   "object",
+  "uuid",
 ]
 const FILENAME = join(__dirname, "..", "manifest.json")
 const outputJSON = {}

packages/string-templates/src/helpers/external.js

@@ -16,6 +16,7 @@ const EXTERNAL_FUNCTION_COLLECTIONS = [
   "comparison",
   "object",
   "regex",
+  "uuid",
 ]
 
 const ADDED_HELPERS = {

packages/string-templates/src/helpers/javascript.js

@@ -56,6 +56,12 @@ module.exports.processJS = (handlebars, context) => {
     const res = { data: runJS(js, sandboxContext) }
     return `{{${LITERAL_MARKER} js_result-${JSON.stringify(res)}}}`
   } catch (error) {
+    if (error.code === "ERR_SCRIPT_EXECUTION_TIMEOUT") {
+      return "Timed out while executing JS"
+    }
+    if (error.name === "ExecutionTimeoutError") {
+      return "Request JS execution limit hit"
+    }
     return "Error while executing JS"
   }
 }

packages/string-templates/src/index.js

@@ -9,6 +9,7 @@ const {
   findDoubleHbsInstances,
 } = require("./utilities")
 const { convertHBSBlock } = require("./conversion")
+const javascript = require("./helpers/javascript")
 
 const hbsInstance = handlebars.create()
 registerAll(hbsInstance)
@@ -362,6 +363,8 @@ module.exports.doesContainString = (template, string) => {
   return exports.doesContainStrings(template, [string])
 }
 
+module.exports.setJSRunner = javascript.setJSRunner
+
 module.exports.convertToJS = hbs => {
   const blocks = exports.findHBSBlocks(hbs)
   let js = "return `",

packages/string-templates/src/index.mjs

@@ -1,6 +1,5 @@
 import vm from "vm"
 import templates from "./index.js"
-import { setJSRunner } from "./helpers/javascript"
 
 /**
  * ES6 entrypoint for rollup
@@ -20,6 +19,7 @@ export const doesContainString = templates.doesContainString
 export const disableEscaping = templates.disableEscaping
 export const findHBSBlocks = templates.findHBSBlocks
 export const convertToJS = templates.convertToJS
+export const setJSRunner = templates.setJSRunner
 export const FIND_ANY_HBS_REGEX = templates.FIND_ANY_HBS_REGEX
 
 if (process && !process.env.NO_JS) {

packages/string-templates/test/constants.js

@@ -0,0 +1,2 @@
+module.exports.UUID_REGEX =
+  /^[0-9A-F]{8}-[0-9A-F]{4}-4[0-9A-F]{3}-[89AB][0-9A-F]{3}-[0-9A-F]{12}$/i

packages/string-templates/test/helpers.spec.js

@@ -1,6 +1,7 @@
 const { processString, processObject, isValid } = require("../src/index.cjs")
 const tableJson = require("./examples/table.json")
 const dayjs = require("dayjs")
+const { UUID_REGEX } = require("./constants")
 
 describe("test the custom helpers we have applied", () => {
   it("should be able to use the object helper", async () => {
@@ -477,3 +478,10 @@ describe("Cover a few complex use cases", () => {
     expect(output.dataProvider).toBe("%5B%221%22%2C%221%22%5D")
   })
 })
+
+describe("uuid", () => {
+  it("should be able to generate a UUID", async () => {
+    const output = await processString("{{ uuid }}", {})
+    expect(output).toMatch(UUID_REGEX)
+  })
+})

packages/string-templates/test/javascript.spec.js

@@ -1,4 +1,5 @@
 const { processStringSync, encodeJSBinding } = require("../src/index.cjs")
+const { UUID_REGEX } = require("./constants")
 
 const processJS = (js, context) => {
   return processStringSync(encodeJSBinding(js), context)
@@ -114,7 +115,7 @@ describe("Test the JavaScript helper", () => {
 
   it("should timeout after one second", () => {
     const output = processJS(`while (true) {}`)
-    expect(output).toBe("Error while executing JS")
+    expect(output).toBe("Timed out while executing JS")
   })
 
   it("should prevent access to the process global", () => {
@@ -140,4 +141,9 @@ describe("check JS helpers", () => {
     const output = processJS(`return helpers.toInt(4.3)`)
     expect(output).toBe(4)
   })
+
+  it("should be able to use uuid", () => {
+    const output = processJS(`return helpers.uuid()`)
+    expect(output).toMatch(UUID_REGEX)
+  })
 })

packages/worker/Dockerfile

@@ -51,8 +51,12 @@ ENV TENANT_FEATURE_FLAGS=*:LICENSING,*:USER_GROUPS,*:ONBOARDING_TOUR
 ENV ACCOUNT_PORTAL_URL=https://account.budibase.app
 
 ARG BUDIBASE_VERSION
+ARG GIT_COMMIT_SHA
 # Ensuring the version argument is sent
 RUN test -n "$BUDIBASE_VERSION"
 ENV BUDIBASE_VERSION=$BUDIBASE_VERSION
+ENV DD_GIT_REPOSITORY_URL=https://github.com/budibase/budibase
+ENV DD_GIT_COMMIT_SHA=$GIT_COMMIT_SHA
+ENV DD_VERSION=$BUDIBASE_VERSION
 
 CMD ["./docker_run.sh"]

packages/worker/scripts/dev/manage.js

@@ -1,44 +1,40 @@
 #!/usr/bin/env node
-const path = require("path")
+const { parsed: existingConfig } = require("dotenv").config()
-const fs = require("fs")
+const updateDotEnv = require("update-dotenv")
 
 async function init() {
-  const envFilePath = path.join(process.cwd(), ".env")
+  let config = {
-  if (!fs.existsSync(envFilePath)) {
+    SELF_HOSTED: "1",
-    const envFileJson = {
+    PORT: "4002",
-      SELF_HOSTED: 1,
+    CLUSTER_PORT: "10000",
-      PORT: 4002,
+    JWT_SECRET: "testsecret",
-      CLUSTER_PORT: 10000,
+    INTERNAL_API_KEY: "budibase",
-      JWT_SECRET: "testsecret",
+    MINIO_ACCESS_KEY: "budibase",
-      INTERNAL_API_KEY: "budibase",
+    MINIO_SECRET_KEY: "budibase",
-      MINIO_ACCESS_KEY: "budibase",
+    REDIS_URL: "localhost:6379",
-      MINIO_SECRET_KEY: "budibase",
+    REDIS_PASSWORD: "budibase",
-      REDIS_URL: "localhost:6379",
+    MINIO_URL: "http://localhost:4004",
-      REDIS_PASSWORD: "budibase",
+    COUCH_DB_URL: "http://budibase:budibase@localhost:4005",
-      MINIO_URL: "http://localhost:4004",
+    COUCH_DB_USERNAME: "budibase",
-      COUCH_DB_URL: "http://budibase:budibase@localhost:4005",
+    COUCH_DB_PASSWORD: "budibase",
-      COUCH_DB_USERNAME: "budibase",
+    // empty string is false
-      COUCH_DB_PASSWORD: "budibase",
+    MULTI_TENANCY: "",
-      // empty string is false
+    DISABLE_ACCOUNT_PORTAL: "1",
-      MULTI_TENANCY: "",
+    ACCOUNT_PORTAL_URL: "http://localhost:10001",
-      DISABLE_ACCOUNT_PORTAL: 1,
+    ACCOUNT_PORTAL_API_KEY: "budibase",
-      ACCOUNT_PORTAL_URL: "http://localhost:10001",
+    PLATFORM_URL: "http://localhost:10000",
-      ACCOUNT_PORTAL_API_KEY: "budibase",
+    APPS_URL: "http://localhost:4001",
-      PLATFORM_URL: "http://localhost:10000",
+    SERVICE: "worker-service",
-      APPS_URL: "http://localhost:4001",
+    DEPLOYMENT_ENVIRONMENT: "development",
-      SERVICE: "worker-service",
+    TENANT_FEATURE_FLAGS: "*:LICENSING,*:USER_GROUPS,*:ONBOARDING_TOUR",
-      DEPLOYMENT_ENVIRONMENT: "development",
+    ENABLE_EMAIL_TEST_MODE: "1",
-      TENANT_FEATURE_FLAGS: "*:LICENSING,*:USER_GROUPS,*:ONBOARDING_TOUR",
+    HTTP_LOGGING: "0",
-      ENABLE_EMAIL_TEST_MODE: 1,
+    VERSION: "0.0.0+local",
-      HTTP_LOGGING: 0,
-      VERSION: "0.0.0+local",
-    }
-    let envFile = ""
-    Object.keys(envFileJson).forEach(key => {
-      envFile += `${key}=${envFileJson[key]}\n`
-    })
-    fs.writeFileSync(envFilePath, envFile)
   }
+
+  config = { ...config, ...existingConfig }
+
+  await updateDotEnv(config)
 }
 
 // if more than init required use this to determine the command type

packages/worker/src/api/controllers/system/environment.ts

@@ -1,6 +1,25 @@
 import { Ctx } from "@budibase/types"
 import env from "../../../environment"
 import { env as coreEnv } from "@budibase/backend-core"
+import nodeFetch from "node-fetch"
+
+let sqsAvailable: boolean
+async function isSqsAvailable() {
+  if (sqsAvailable !== undefined) {
+    return sqsAvailable
+  }
+
+  try {
+    await nodeFetch(coreEnv.COUCH_DB_SQL_URL, {
+      timeout: 1000,
+    })
+    sqsAvailable = true
+    return true
+  } catch (e) {
+    sqsAvailable = false
+    return false
+  }
+}
 
 export const fetch = async (ctx: Ctx) => {
   ctx.body = {
@@ -12,4 +31,10 @@ export const fetch = async (ctx: Ctx) => {
     baseUrl: env.PLATFORM_URL,
     isDev: env.isDev() && !env.isTest(),
   }
+
+  if (env.SELF_HOSTED) {
+    ctx.body.infrastructure = {
+      sqs: await isSqsAvailable(),
+    }
+  }
 }

packages/worker/src/api/routes/system/tests/environment.spec.ts

@@ -1,5 +1,7 @@
 import { TestConfiguration } from "../../../../tests"
 
+jest.unmock("node-fetch")
+
 describe("/api/system/environment", () => {
   const config = new TestConfiguration()
 
@@ -27,5 +29,22 @@ describe("/api/system/environment", () => {
         offlineMode: false,
       })
     })
+
+    it("returns the expected environment for self hosters", async () => {
+      await config.withEnv({ SELF_HOSTED: true }, async () => {
+        const env = await config.api.environment.getEnvironment()
+        expect(env.body).toEqual({
+          cloud: false,
+          disableAccountPortal: 0,
+          isDev: false,
+          multiTenancy: true,
+          baseUrl: "http://localhost:10000",
+          offlineMode: false,
+          infrastructure: {
+            sqs: false,
+          },
+        })
+      })
+    })
   })
 })

packages/worker/src/tests/TestConfiguration.ts

@@ -36,6 +36,7 @@ import {
 } from "@budibase/types"
 import API from "./api"
 import jwt, { Secret } from "jsonwebtoken"
+import cloneDeep from "lodash/fp/cloneDeep"
 
 class TestConfiguration {
   server: any
@@ -240,6 +241,34 @@ class TestConfiguration {
     return { message: "Admin user only endpoint.", status: 403 }
   }
 
+  async withEnv(newEnvVars: Partial<typeof env>, f: () => Promise<void>) {
+    let cleanup = this.setEnv(newEnvVars)
+    try {
+      await f()
+    } finally {
+      cleanup()
+    }
+  }
+
+  /*
+   * Sets the environment variables to the given values and returns a function
+   * that can be called to reset the environment variables to their original values.
+   */
+  setEnv(newEnvVars: Partial<typeof env>): () => void {
+    const oldEnv = cloneDeep(env)
+
+    let key: keyof typeof newEnvVars
+    for (key in newEnvVars) {
+      env._set(key, newEnvVars[key])
+    }
+
+    return () => {
+      for (const [key, value] of Object.entries(oldEnv)) {
+        env._set(key, value)
+      }
+    }
+  }
+
   // USERS
 
   async createDefaultUser() {

scripts/build.js

@@ -59,6 +59,7 @@ function runBuild(entry, outfile) {
       "pouchdb",
       "bcrypt",
       "bcryptjs",
+      "graphql/*",
     ],
   }