Merge branch 'master' into feature/rm-ff-code-per-user-per-creator

charts/budibase/README.md

@@ -157,6 +157,17 @@ $ helm install --create-namespace --namespace budibase budibase . -f values.yaml
 | services.apps.replicaCount | int | `1` | The number of apps replicas to run. |
 | services.apps.resources | object | `{}` | The resources to use for apps pods. See <https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/> for more information on how to set these. |
 | services.apps.startupProbe | object | HTTP health checks. | Startup probe configuration for apps pods. You shouldn't need to change this, but if you want to you can find more information here: <https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/> |
+| services.automationWorkers.autoscaling.enabled | bool | `false` | Whether to enable horizontal pod autoscaling for the apps service. |
+| services.automationWorkers.autoscaling.maxReplicas | int | `10` |  |
+| services.automationWorkers.autoscaling.minReplicas | int | `1` |  |
+| services.automationWorkers.autoscaling.targetCPUUtilizationPercentage | int | `80` | Target CPU utilization percentage for the automation worker service. Note that for autoscaling to work, you will need to have metrics-server configured, and resources set for the automation worker pods. |
+| services.automationWorkers.enabled | bool | `true` | Whether or not to enable the automation worker service. If you disable this, automations will be processed by the apps service. |
+| services.automationWorkers.livenessProbe | object | HTTP health checks. | Liveness probe configuration for automation worker pods. You shouldn't need to change this, but if you want to you can find more information here: <https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/> |
+| services.automationWorkers.logLevel | string | `"info"` | The log level for the automation worker service. |
+| services.automationWorkers.readinessProbe | object | HTTP health checks. | Readiness probe configuration for automation worker pods. You shouldn't need to change this, but if you want to you can find more information here: <https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/> |
+| services.automationWorkers.replicaCount | int | `1` | The number of automation worker replicas to run. |
+| services.automationWorkers.resources | object | `{}` | The resources to use for automation worker pods. See <https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/> for more information on how to set these. |
+| services.automationWorkers.startupProbe | object | HTTP health checks. | Startup probe configuration for automation worker pods. You shouldn't need to change this, but if you want to you can find more information here: <https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/> |
 | services.couchdb.backup.enabled | bool | `false` | Whether or not to enable periodic CouchDB backups. This works by replicating to another CouchDB instance. |
 | services.couchdb.backup.interval | string | `""` | Backup interval in seconds |
 | services.couchdb.backup.resources | object | `{}` | The resources to use for CouchDB backup pods. See <https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/> for more information on how to set these. |

charts/budibase/templates/app-service-deployment.yaml

@@ -192,7 +192,14 @@ spec:
         - name: NODE_TLS_REJECT_UNAUTHORIZED
           value: {{ .Values.services.tlsRejectUnauthorized }}
         {{ end }}
-
+        {{- if .Values.services.automationWorkers.enabled }}
+        - name: APP_FEATURES
+          value: "api"
+        {{- end }}
+        {{- range .Values.services.apps.extraEnv }}
+        - name: {{ .name }}
+          value: {{ .value | quote }}
+        {{- end }}
         image: budibase/apps:{{ .Values.globals.appVersion | default .Chart.AppVersion }}
         imagePullPolicy: Always
         {{- if .Values.services.apps.startupProbe }}

charts/budibase/templates/automation-worker-service-deployment.yaml

@@ -0,0 +1,248 @@
+{{- if .Values.services.automationWorkers.enabled }}
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  annotations:
+{{ if .Values.services.automationWorkers.deploymentAnnotations }}
+{{- toYaml .Values.services.automationWorkers.deploymentAnnotations | indent 4 -}}
+{{ end }}
+  labels:
+    io.kompose.service: automation-worker-service
+{{ if .Values.services.automationWorkers.deploymentLabels }}
+{{- toYaml .Values.services.automationWorkers.deploymentLabels | indent 4 -}}
+{{ end }}
+  name: automation-worker-service
+spec:
+  replicas: {{ .Values.services.automationWorkers.replicaCount }}
+  selector:
+    matchLabels:
+      io.kompose.service: automation-worker-service
+  strategy:
+    type: RollingUpdate
+  template:
+    metadata:
+      annotations:
+{{ if .Values.services.automationWorkers.templateAnnotations }}
+{{- toYaml .Values.services.automationWorkers.templateAnnotations | indent 8 -}}
+{{ end }}
+      labels:
+        io.kompose.service: automation-worker-service
+{{ if .Values.services.automationWorkers.templateLabels }}
+{{- toYaml .Values.services.automationWorkers.templateLabels | indent 8 -}}
+{{ end }}
+    spec:
+      containers:
+      - env:
+        - name: BUDIBASE_ENVIRONMENT
+          value: {{ .Values.globals.budibaseEnv }}
+        - name: DEPLOYMENT_ENVIRONMENT
+          value: "kubernetes"
+        - name: COUCH_DB_URL
+          {{ if .Values.services.couchdb.url }}
+          value: {{ .Values.services.couchdb.url }}
+          {{ else }}
+          value: http://{{ .Release.Name }}-svc-couchdb:{{ .Values.services.couchdb.port }}
+          {{ end }}
+        {{ if .Values.services.couchdb.enabled }}
+        - name: COUCH_DB_USER
+          valueFrom:
+            secretKeyRef:
+              name: {{ template "couchdb.fullname" . }}
+              key: adminUsername
+        - name: COUCH_DB_PASSWORD
+          valueFrom:
+            secretKeyRef:
+              name: {{ template "couchdb.fullname" . }}
+              key: adminPassword
+        {{ end }}
+        - name: ENABLE_ANALYTICS
+          value: {{ .Values.globals.enableAnalytics | quote }}
+        - name: API_ENCRYPTION_KEY
+          value: {{ .Values.globals.apiEncryptionKey | quote }}
+        - name: HTTP_LOGGING
+          value: {{ .Values.services.automationWorkers.httpLogging | quote }}
+        - name: INTERNAL_API_KEY
+          valueFrom:
+            secretKeyRef:
+              name: {{ template "budibase.fullname" . }}
+              key: internalApiKey
+        - name: INTERNAL_API_KEY_FALLBACK
+          value: {{ .Values.globals.internalApiKeyFallback | quote }}
+        - name: JWT_SECRET
+          valueFrom:
+            secretKeyRef:
+              name: {{ template "budibase.fullname" . }}
+              key: jwtSecret
+        - name: JWT_SECRET_FALLBACK
+          value: {{ .Values.globals.jwtSecretFallback | quote }}
+        {{ if .Values.services.objectStore.region }}
+        - name: AWS_REGION
+          value: {{ .Values.services.objectStore.region }}
+        {{ end }}
+        - name: MINIO_ENABLED
+          value: {{ .Values.services.objectStore.minio | quote }}
+        - name: MINIO_ACCESS_KEY
+          valueFrom:
+            secretKeyRef:
+              name: {{ template "budibase.fullname" . }}
+              key: objectStoreAccess
+        - name: MINIO_SECRET_KEY
+          valueFrom:
+            secretKeyRef:
+              name: {{ template "budibase.fullname" . }}
+              key: objectStoreSecret
+        - name: CLOUDFRONT_CDN
+          value: {{ .Values.services.objectStore.cloudfront.cdn | quote }}
+        - name: CLOUDFRONT_PUBLIC_KEY_ID
+          value: {{ .Values.services.objectStore.cloudfront.publicKeyId | quote }}
+        - name: CLOUDFRONT_PRIVATE_KEY_64
+          value: {{ .Values.services.objectStore.cloudfront.privateKey64 | quote }}
+        - name: MINIO_URL
+          value: {{ .Values.services.objectStore.url }}
+        - name: PLUGIN_BUCKET_NAME
+          value: {{ .Values.services.objectStore.pluginBucketName | quote }}
+        - name: APPS_BUCKET_NAME
+          value: {{ .Values.services.objectStore.appsBucketName | quote }}
+        - name: GLOBAL_BUCKET_NAME
+          value: {{ .Values.services.objectStore.globalBucketName | quote }}
+        - name: BACKUPS_BUCKET_NAME
+          value: {{ .Values.services.objectStore.backupsBucketName | quote }}
+        - name: PORT
+          value: {{ .Values.services.automationWorkers.port | quote }}
+        {{ if .Values.services.worker.publicApiRateLimitPerSecond }}
+        - name: API_REQ_LIMIT_PER_SEC
+          value: {{ .Values.globals.automationWorkers.publicApiRateLimitPerSecond | quote }}
+        {{ end }}
+        - name: MULTI_TENANCY
+          value: {{ .Values.globals.multiTenancy | quote }}
+        - name: OFFLINE_MODE
+          value: {{ .Values.globals.offlineMode | quote }}
+        - name: LOG_LEVEL
+          value: {{ .Values.services.automationWorkers.logLevel | quote }}
+        - name: REDIS_PASSWORD
+          value: {{ .Values.services.redis.password }}
+        - name: REDIS_URL
+          {{ if .Values.services.redis.url }}
+          value: {{ .Values.services.redis.url }}
+          {{ else }}
+          value: redis-service:{{ .Values.services.redis.port }}
+          {{ end }}
+        - name: SELF_HOSTED
+          value: {{ .Values.globals.selfHosted | quote }}
+        - name: POSTHOG_TOKEN
+          value: {{ .Values.globals.posthogToken | quote }}
+        - name: WORKER_URL
+          value: http://worker-service:{{ .Values.services.worker.port }}
+        - name: PLATFORM_URL
+          value: {{ .Values.globals.platformUrl | quote }}
+        - name: ACCOUNT_PORTAL_URL
+          value: {{ .Values.globals.accountPortalUrl | quote }}
+        - name: ACCOUNT_PORTAL_API_KEY
+          value: {{ .Values.globals.accountPortalApiKey | quote }}
+        - name: COOKIE_DOMAIN
+          value: {{ .Values.globals.cookieDomain | quote }}
+        - name: HTTP_MIGRATIONS
+          value: {{ .Values.globals.httpMigrations | quote }}
+        - name: GOOGLE_CLIENT_ID
+          value: {{ .Values.globals.google.clientId | quote }}
+        - name: GOOGLE_CLIENT_SECRET
+          value: {{ .Values.globals.google.secret | quote }}
+        - name: AUTOMATION_MAX_ITERATIONS
+          value: {{ .Values.globals.automationMaxIterations | quote }}
+        - name: TENANT_FEATURE_FLAGS
+          value: {{ .Values.globals.tenantFeatureFlags | quote }}
+        - name: ENCRYPTION_KEY
+          value: {{ .Values.globals.bbEncryptionKey | quote }}
+        {{ if .Values.globals.bbAdminUserEmail }}
+        - name: BB_ADMIN_USER_EMAIL
+          value: {{ .Values.globals.bbAdminUserEmail | quote }}
+        {{ end }}
+        {{ if .Values.globals.bbAdminUserPassword }}
+        - name: BB_ADMIN_USER_PASSWORD
+          value: {{ .Values.globals.bbAdminUserPassword | quote }}
+        {{ end }}
+        {{ if .Values.globals.pluginsDir }}
+        - name: PLUGINS_DIR
+          value: {{ .Values.globals.pluginsDir | quote }}
+        {{ end }}
+        {{ if .Values.services.automationWorkers.nodeDebug }}
+        - name: NODE_DEBUG
+          value: {{ .Values.services.automationWorkers.nodeDebug | quote }}
+        {{ end }}
+        {{ if .Values.globals.datadogApmEnabled }}
+        - name: DD_LOGS_INJECTION
+          value: {{ .Values.globals.datadogApmEnabled | quote }}
+        - name: DD_APM_ENABLED
+          value: {{ .Values.globals.datadogApmEnabled | quote }}
+        - name: DD_APM_DD_URL
+          value: https://trace.agent.datadoghq.eu
+        {{ end }}
+        {{ if .Values.globals.globalAgentHttpProxy }}
+        - name: GLOBAL_AGENT_HTTP_PROXY
+          value: {{ .Values.globals.globalAgentHttpProxy | quote }}
+        {{ end }}
+        {{ if .Values.globals.globalAgentHttpsProxy }}
+        - name: GLOBAL_AGENT_HTTPS_PROXY
+          value: {{ .Values.globals.globalAgentHttpsProxy | quote }}
+        {{ end }}
+        {{ if .Values.globals.globalAgentNoProxy }}
+        - name: GLOBAL_AGENT_NO_PROXY
+          value: {{ .Values.globals.globalAgentNoProxy | quote }}
+        {{ end }}
+        {{ if .Values.services.tlsRejectUnauthorized }}
+        - name: NODE_TLS_REJECT_UNAUTHORIZED
+          value: {{ .Values.services.tlsRejectUnauthorized }}
+        {{ end }}
+        - name: APP_FEATURES
+          value: "automations"
+        {{- range .Values.services.automationWorkers.extraEnv }}
+        - name: {{ .name }}
+          value: {{ .value | quote }}
+        {{- end }}
+
+        image: budibase/apps:{{ .Values.globals.appVersion | default .Chart.AppVersion }}
+        imagePullPolicy: Always
+        {{- if .Values.services.automationWorkers.startupProbe }}
+        {{- with .Values.services.automationWorkers.startupProbe }}
+        startupProbe:
+            {{- toYaml . | nindent 10 }}
+        {{- end }}
+        {{- end }}
+        {{- if .Values.services.automationWorkers.livenessProbe }}
+        {{- with .Values.services.automationWorkers.livenessProbe }}
+        livenessProbe:
+            {{- toYaml . | nindent 10 }}
+        {{- end }}
+        {{- end }}
+        {{- if .Values.services.automationWorkers.readinessProbe }}
+        {{- with .Values.services.automationWorkers.readinessProbe }}
+        readinessProbe:
+            {{- toYaml . | nindent 10 }}
+        {{- end }}
+        {{- end }}
+        name: bbautomationworker
+        ports:
+        - containerPort: {{ .Values.services.automationWorkers.port }}
+        {{ with .Values.services.automationWorkers.resources }}
+        resources:
+        {{- toYaml . | nindent 10 }}
+        {{ end }}
+    {{- with .Values.affinity }}
+      affinity:
+        {{- toYaml . | nindent 8 }}
+    {{- end }}
+    {{- with .Values.tolerations }}
+      tolerations:
+        {{- toYaml . | nindent 8 }}
+    {{- end }}
+      {{ if .Values.schedulerName }}
+      schedulerName: {{ .Values.schedulerName | quote }}
+      {{ end }}    
+      {{ if .Values.imagePullSecrets }}
+      imagePullSecrets:
+      {{- toYaml .Values.imagePullSecrets | nindent 6 }}
+      {{ end }}
+      restartPolicy: Always
+      serviceAccountName: ""
+status: {}
+{{- end }}

charts/budibase/templates/automation-worker-service-hpa.yaml

@@ -0,0 +1,32 @@
+{{- if .Values.services.automationWorkers.autoscaling.enabled }}
+apiVersion: {{ ternary "autoscaling/v2" "autoscaling/v2beta2" (.Capabilities.APIVersions.Has "autoscaling/v2") }}
+kind: HorizontalPodAutoscaler
+metadata:
+  name: {{ include "budibase.fullname" . }}-apps
+  labels:
+    {{- include "budibase.labels" . | nindent 4 }}
+spec:
+  scaleTargetRef:
+    apiVersion: apps/v1
+    kind: Deployment
+    name: automation-worker-service
+  minReplicas: {{ .Values.services.automationWorkers.autoscaling.minReplicas }}
+  maxReplicas: {{ .Values.services.automationWorkers.autoscaling.maxReplicas }}
+  metrics:
+    {{- if .Values.services.automationWorkers.autoscaling.targetCPUUtilizationPercentage }}
+    - type: Resource
+      resource:
+        name: cpu
+        target:
+          type: Utilization
+          averageUtilization: {{ .Values.services.automationWorkers.autoscaling.targetCPUUtilizationPercentage }}
+    {{- end }}
+    {{- if .Values.services.automationWorkers.autoscaling.targetMemoryUtilizationPercentage }}
+    - type: Resource
+      resource:
+        name: memory
+        target:
+          type: Utilization
+          averageUtilization: {{ .Values.services.automationWorkers.autoscaling.targetMemoryUtilizationPercentage }}
+    {{- end }}
+{{- end }}

charts/budibase/templates/worker-service-deployment.yaml

@@ -182,6 +182,10 @@ spec:
         - name: NODE_TLS_REJECT_UNAUTHORIZED
           value: {{ .Values.services.tlsRejectUnauthorized }}
         {{ end }}
+        {{- range .Values.services.worker.extraEnv }}
+        - name: {{ .name }}
+          value: {{ .value | quote }}
+        {{- end }}
         image: budibase/worker:{{ .Values.globals.appVersion | default .Chart.AppVersion }}
         imagePullPolicy: Always
         {{- if .Values.services.worker.startupProbe }}

charts/budibase/values.yaml

@@ -220,6 +220,9 @@ services:
     # <https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/>
     # for more information on how to set these.
     resources: {}
+    # -- Extra environment variables to set for apps pods. Takes a list of
+    # name=value pairs.
+    extraEnv: []
     # -- Startup probe configuration for apps pods. You shouldn't need to
     # change this, but if you want to you can find more information here:
     # <https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/>
@@ -272,6 +275,78 @@ services:
       # and resources set for the apps pods.
       targetCPUUtilizationPercentage: 80
 
+  automationWorkers:
+    # -- Whether or not to enable the automation worker service. If you disable this,
+    # automations will be processed by the apps service.
+    enabled: true
+    # @ignore (you shouldn't need to change this)
+    port: 4002
+    # -- The number of automation worker replicas to run.
+    replicaCount: 1
+    # -- The log level for the automation worker service.
+    logLevel: info
+    # -- The resources to use for automation worker pods. See
+    # <https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/>
+    # for more information on how to set these.
+    resources: {}
+    # -- Extra environment variables to set for automation worker pods. Takes a list of
+    # name=value pairs.
+    extraEnv: []
+    # -- Startup probe configuration for automation worker pods. You shouldn't
+    # need to change this, but if you want to you can find more information
+    # here:
+    # <https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/>
+    # @default -- HTTP health checks.
+    startupProbe:
+      # @ignore
+      httpGet:
+        path: /health
+        port: 4002
+        scheme: HTTP
+      # @ignore
+      failureThreshold: 30
+      # @ignore
+      periodSeconds: 3
+    # -- Readiness probe configuration for automation worker pods. You shouldn't
+    # need to change this, but if you want to you can find more information
+    # here:
+    # <https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/>
+    # @default -- HTTP health checks.
+    readinessProbe:
+      # @ignore
+      httpGet:
+        path: /health
+        port: 4002
+        scheme: HTTP
+      # @ignore
+      periodSeconds: 3
+      # @ignore
+      failureThreshold: 1
+    # -- Liveness probe configuration for automation worker pods. You shouldn't
+    # need to change this, but if you want to you can find more information
+    # here:
+    # <https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/>
+    # @default -- HTTP health checks.
+    livenessProbe:
+      # @ignore
+      httpGet:
+        path: /health
+        port: 4002
+        scheme: HTTP
+      # @ignore
+      failureThreshold: 3
+      # @ignore
+      periodSeconds: 30
+    autoscaling:
+      # -- Whether to enable horizontal pod autoscaling for the apps service.
+      enabled: false
+      minReplicas: 1
+      maxReplicas: 10
+      # -- Target CPU utilization percentage for the automation worker service.
+      # Note that for autoscaling to work, you will need to have metrics-server
+      # configured, and resources set for the automation worker pods.
+      targetCPUUtilizationPercentage: 80
+
   worker:
     # @ignore (you shouldn't need to change this)
     port: 4003
@@ -285,6 +360,9 @@ services:
     # <https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/>
     # for more information on how to set these.
     resources: {}
+    # -- Extra environment variables to set for worker pods. Takes a list of
+    # name=value pairs.
+    extraEnv: []
     # -- Startup probe configuration for worker pods. You shouldn't need to
     # change this, but if you want to you can find more information here:
     # <https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/>

lerna.json

@@ -1,5 +1,5 @@
 {
-  "version": "2.13.37",
+  "version": "2.13.39",
   "npmClient": "yarn",
   "packages": [
     "packages/*",

packages/backend-core/src/cache/tests/writethrough.spec.ts

@@ -1,15 +1,16 @@
 import { DBTestConfiguration } from "../../../tests/extra"
-import {
+import { structures } from "../../../tests"
-  structures,
-  expectFunctionWasCalledTimesWith,
-  mocks,
-} from "../../../tests"
 import { Writethrough } from "../writethrough"
 import { getDB } from "../../db"
+import { Document } from "@budibase/types"
 import tk from "timekeeper"
 
 tk.freeze(Date.now())
 
+interface ValueDoc extends Document {
+  value: any
+}
+
 const DELAY = 5000
 
 describe("writethrough", () => {
@@ -117,7 +118,7 @@ describe("writethrough", () => {
   describe("get", () => {
     it("should be able to retrieve", async () => {
       await config.doInTenant(async () => {
-        const response = await writethrough.get(docId)
+        const response = await writethrough.get<ValueDoc>(docId)
         expect(response.value).toBe(4)
       })
     })

packages/backend-core/src/cache/writethrough.ts

@@ -7,7 +7,7 @@ import * as locks from "../redis/redlockImpl"
 const DEFAULT_WRITE_RATE_MS = 10000
 let CACHE: BaseCache | null = null
 
-interface CacheItem {
+interface CacheItem<T extends Document> {
   doc: any
   lastWrite: number
 }
@@ -24,7 +24,10 @@ function makeCacheKey(db: Database, key: string) {
   return db.name + key
 }
 
-function makeCacheItem(doc: any, lastWrite: number | null = null): CacheItem {
+function makeCacheItem<T extends Document>(
+  doc: T,
+  lastWrite: number | null = null
+): CacheItem<T> {
   return { doc, lastWrite: lastWrite || Date.now() }
 }
 
@@ -35,7 +38,7 @@ async function put(
 ) {
   const cache = await getCache()
   const key = doc._id
-  let cacheItem: CacheItem | undefined
+  let cacheItem: CacheItem<any> | undefined
   if (key) {
     cacheItem = await cache.get(makeCacheKey(db, key))
   }
@@ -84,12 +87,12 @@ async function put(
   return { ok: true, id: output._id, rev: output._rev }
 }
 
-async function get(db: Database, id: string): Promise<any> {
+async function get<T extends Document>(db: Database, id: string): Promise<T> {
   const cache = await getCache()
   const cacheKey = makeCacheKey(db, id)
-  let cacheItem: CacheItem = await cache.get(cacheKey)
+  let cacheItem: CacheItem<T> = await cache.get(cacheKey)
   if (!cacheItem) {
-    const doc = await db.get(id)
+    const doc = await db.get<T>(id)
     cacheItem = makeCacheItem(doc)
     await cache.store(cacheKey, cacheItem)
   }
@@ -123,8 +126,8 @@ export class Writethrough {
     return put(this.db, doc, writeRateMs)
   }
 
-  async get(id: string) {
+  async get<T extends Document>(id: string) {
-    return get(this.db, id)
+    return get<T>(this.db, id)
   }
 
   async remove(docOrId: any, rev?: any) {

packages/backend-core/src/environment.ts

@@ -107,6 +107,7 @@ const environment = {
   ENCRYPTION_KEY: process.env.ENCRYPTION_KEY,
   API_ENCRYPTION_KEY: getAPIEncryptionKey(),
   COUCH_DB_URL: process.env.COUCH_DB_URL || "http://localhost:4005",
+  COUCH_DB_SQL_URL: process.env.COUCH_DB_SQL_URL || "http://localhost:4984",
   COUCH_DB_USERNAME: process.env.COUCH_DB_USER,
   COUCH_DB_PASSWORD: process.env.COUCH_DB_PASSWORD,
   GOOGLE_CLIENT_ID: process.env.GOOGLE_CLIENT_ID,

packages/backend-core/src/queue/inMemoryQueue.ts

@@ -68,6 +68,10 @@ class InMemoryQueue {
     })
   }
 
+  async isReady() {
+    return true
+  }
+
   // simply puts a message to the queue and emits to the queue for processing
   /**
    * Simple function to replicate the add message functionality of Bull, putting

packages/backend-core/src/redis/redlockImpl.ts

@@ -137,7 +137,6 @@ export async function doWithLock<T>(
     const result = await task()
     return { executed: true, result }
   } catch (e: any) {
-    logWarn(`lock type: ${opts.type} error`, e)
     // lock limit exceeded
     if (e.name === "LockError") {
       if (opts.type === LockType.TRY_ONCE) {

packages/pro

@@ -1 +1 @@
-Subproject commit 056c2093dbc93d9a10ea9f5050c84a84edd8100c
+Subproject commit 992486c10044a7495496b97bdf5f454d4020bfba

packages/server/scripts/dev/manage.js

@@ -1,7 +1,8 @@
 #!/usr/bin/env node
 const compose = require("docker-compose")
 const path = require("path")
-const fs = require("fs")
+const { parsed: existingConfig } = require("dotenv").config()
+const updateDotEnv = require("update-dotenv")
 
 // This script wraps docker-compose allowing you to manage your dev infrastructure with simple commands.
 const CONFIG = {
@@ -17,10 +18,8 @@ const Commands = {
 }
 
 async function init() {
-  const envFilePath = path.join(process.cwd(), ".env")
+  let config = {
-  if (!fs.existsSync(envFilePath)) {
+    PORT: "4001",
-    const envFileJson = {
-      PORT: 4001,
     MINIO_URL: "http://localhost:4004",
     COUCH_DB_URL: "http://budibase:budibase@localhost:4005",
     REDIS_URL: "localhost:6379",
@@ -36,10 +35,10 @@ async function init() {
     MINIO_SECRET_KEY: "budibase",
     COUCH_DB_PASSWORD: "budibase",
     COUCH_DB_USER: "budibase",
-      SELF_HOSTED: 1,
+    SELF_HOSTED: "1",
-      DISABLE_ACCOUNT_PORTAL: 1,
+    DISABLE_ACCOUNT_PORTAL: "1",
     MULTI_TENANCY: "",
-      DISABLE_THREADING: 1,
+    DISABLE_THREADING: "1",
     SERVICE: "app-service",
     DEPLOYMENT_ENVIRONMENT: "development",
     BB_ADMIN_USER_EMAIL: "",
@@ -50,12 +49,10 @@ async function init() {
     HTTP_LOGGING: "0",
     VERSION: "0.0.0+local",
   }
-    let envFile = ""
+
-    Object.keys(envFileJson).forEach(key => {
+  config = { ...config, ...existingConfig }
-      envFile += `${key}=${envFileJson[key]}\n`
+
-    })
+  await updateDotEnv(config)
-    fs.writeFileSync(envFilePath, envFile)
-  }
 }
 
 async function up() {

packages/server/src/api/index.ts

@@ -4,19 +4,31 @@ import currentApp from "../middleware/currentapp"
 import zlib from "zlib"
 import { mainRoutes, staticRoutes, publicRoutes } from "./routes"
 import { middleware as pro } from "@budibase/pro"
+import { apiEnabled, automationsEnabled } from "../features"
 import migrations from "../middleware/appMigrations"
+import { automationQueue } from "../automations"
 
 export { shutdown } from "./routes/public"
 const compress = require("koa-compress")
 
 export const router: Router = new Router()
 
-router.get("/health", ctx => (ctx.status = 200))
+router.get("/health", async ctx => {
+  if (automationsEnabled()) {
+    if (!(await automationQueue.isReady())) {
+      ctx.status = 503
+      return
+    }
+  }
+  ctx.status = 200
+})
 router.get("/version", ctx => (ctx.body = envCore.VERSION))
 
 router.use(middleware.errorHandling)
 
-router
+// only add the routes if they are enabled
+if (apiEnabled()) {
+  router
     .use(
       compress({
         threshold: 2048,
@@ -51,15 +63,16 @@ router
     // @ts-ignore
     .use(migrations)
 
-// authenticated routes
+  // authenticated routes
-for (let route of mainRoutes) {
+  for (let route of mainRoutes) {
     router.use(route.routes())
     router.use(route.allowedMethods())
+  }
+
+  router.use(publicRoutes.routes())
+  router.use(publicRoutes.allowedMethods())
+
+  // WARNING - static routes will catch everything else after them this must be last
+  router.use(staticRoutes.routes())
+  router.use(staticRoutes.allowedMethods())
 }
-
-router.use(publicRoutes.routes())
-router.use(publicRoutes.allowedMethods())
-
-// WARNING - static routes will catch everything else after them this must be last
-router.use(staticRoutes.routes())
-router.use(staticRoutes.allowedMethods())

packages/server/src/app.ts

@@ -9,7 +9,6 @@ import { ServiceType } from "@budibase/types"
 import { env as coreEnv } from "@budibase/backend-core"
 
 coreEnv._set("SERVICE_TYPE", ServiceType.APPS)
-import { apiEnabled } from "./features"
 import createKoaApp from "./koa"
 import Koa from "koa"
 import { Server } from "http"
@@ -18,12 +17,9 @@ import { startup } from "./startup"
 let app: Koa, server: Server
 
 async function start() {
-  // if API disabled, could run automations instead
-  if (apiEnabled()) {
   const koa = createKoaApp()
   app = koa.app
   server = koa.server
-  }
   // startup includes automation runner - if enabled
   await startup(app, server)
 }

packages/server/src/features.ts

@@ -22,3 +22,10 @@ export function automationsEnabled() {
 export function apiEnabled() {
   return featureList.includes(AppFeature.API)
 }
+
+export function printFeatures() {
+  if (!env.APP_FEATURES) {
+    return
+  }
+  console.log(`**** APP FEATURES SET: ${featureList.join(", ")} ****`)
+}

packages/server/src/startup.ts

@@ -19,11 +19,14 @@ import * as pro from "@budibase/pro"
 import * as api from "./api"
 import sdk from "./sdk"
 import { initialise as initialiseWebsockets } from "./websockets"
-import { automationsEnabled } from "./features"
+import { automationsEnabled, printFeatures } from "./features"
+import Koa from "koa"
+import { Server } from "http"
+import { AddressInfo } from "net"
 
 let STARTUP_RAN = false
 
-async function initRoutes(app: any) {
+async function initRoutes(app: Koa) {
   if (!env.isTest()) {
     const plugin = await bullboard.init()
     app.use(plugin)
@@ -48,27 +51,31 @@ async function initPro() {
   })
 }
 
-function shutdown(server?: any) {
+function shutdown(server?: Server) {
   if (server) {
     server.close()
     server.destroy()
   }
 }
 
-export async function startup(app?: any, server?: any) {
+export async function startup(app?: Koa, server?: Server) {
   if (STARTUP_RAN) {
     return
   }
+  printFeatures()
   STARTUP_RAN = true
-  if (server && !env.CLUSTER_MODE) {
+  if (app && server && !env.CLUSTER_MODE) {
     console.log(`Budibase running on ${JSON.stringify(server.address())}`)
-    env._set("PORT", server.address().port)
+    const address = server.address() as AddressInfo
+    env._set("PORT", address.port)
   }
   eventEmitter.emitPort(env.PORT)
   fileSystem.init()
   await redis.init()
   eventInit()
+  if (app && server) {
     initialiseWebsockets(app, server)
+  }
 
   // run migrations on startup if not done via http
   // not recommended in a clustered environment

packages/server/src/tests/utilities/TestConfiguration.ts

@@ -17,7 +17,6 @@ import {
   basicWebhook,
 } from "./structures"
 import {
-  auth,
   cache,
   constants,
   context,

packages/worker/scripts/dev/manage.js

@@ -1,14 +1,12 @@
 #!/usr/bin/env node
-const path = require("path")
+const { parsed: existingConfig } = require("dotenv").config()
-const fs = require("fs")
+const updateDotEnv = require("update-dotenv")
 
 async function init() {
-  const envFilePath = path.join(process.cwd(), ".env")
+  let config = {
-  if (!fs.existsSync(envFilePath)) {
+    SELF_HOSTED: "1",
-    const envFileJson = {
+    PORT: "4002",
-      SELF_HOSTED: 1,
+    CLUSTER_PORT: "10000",
-      PORT: 4002,
-      CLUSTER_PORT: 10000,
     JWT_SECRET: "testsecret",
     INTERNAL_API_KEY: "budibase",
     MINIO_ACCESS_KEY: "budibase",
@@ -21,7 +19,7 @@ async function init() {
     COUCH_DB_PASSWORD: "budibase",
     // empty string is false
     MULTI_TENANCY: "",
-      DISABLE_ACCOUNT_PORTAL: 1,
+    DISABLE_ACCOUNT_PORTAL: "1",
     ACCOUNT_PORTAL_URL: "http://localhost:10001",
     ACCOUNT_PORTAL_API_KEY: "budibase",
     PLATFORM_URL: "http://localhost:10000",
@@ -29,16 +27,14 @@ async function init() {
     SERVICE: "worker-service",
     DEPLOYMENT_ENVIRONMENT: "development",
     TENANT_FEATURE_FLAGS: "*:LICENSING,*:USER_GROUPS,*:ONBOARDING_TOUR",
-      ENABLE_EMAIL_TEST_MODE: 1,
+    ENABLE_EMAIL_TEST_MODE: "1",
-      HTTP_LOGGING: 0,
+    HTTP_LOGGING: "0",
     VERSION: "0.0.0+local",
   }
-    let envFile = ""
+
-    Object.keys(envFileJson).forEach(key => {
+  config = { ...config, ...existingConfig }
-      envFile += `${key}=${envFileJson[key]}\n`
+
-    })
+  await updateDotEnv(config)
-    fs.writeFileSync(envFilePath, envFile)
-  }
 }
 
 // if more than init required use this to determine the command type

packages/worker/src/api/controllers/system/environment.ts

@@ -1,6 +1,25 @@
 import { Ctx } from "@budibase/types"
 import env from "../../../environment"
 import { env as coreEnv } from "@budibase/backend-core"
+import nodeFetch from "node-fetch"
+
+let sqsAvailable: boolean
+async function isSqsAvailable() {
+  if (sqsAvailable !== undefined) {
+    return sqsAvailable
+  }
+
+  try {
+    await nodeFetch(coreEnv.COUCH_DB_SQL_URL, {
+      timeout: 1000,
+    })
+    sqsAvailable = true
+    return true
+  } catch (e) {
+    sqsAvailable = false
+    return false
+  }
+}
 
 export const fetch = async (ctx: Ctx) => {
   ctx.body = {
@@ -12,4 +31,10 @@ export const fetch = async (ctx: Ctx) => {
     baseUrl: env.PLATFORM_URL,
     isDev: env.isDev() && !env.isTest(),
   }
+
+  if (env.SELF_HOSTED) {
+    ctx.body.infrastructure = {
+      sqs: await isSqsAvailable(),
+    }
+  }
 }

packages/worker/src/api/routes/system/tests/environment.spec.ts

@@ -1,5 +1,7 @@
 import { TestConfiguration } from "../../../../tests"
 
+jest.unmock("node-fetch")
+
 describe("/api/system/environment", () => {
   const config = new TestConfiguration()
 
@@ -27,5 +29,22 @@ describe("/api/system/environment", () => {
         offlineMode: false,
       })
     })
+
+    it("returns the expected environment for self hosters", async () => {
+      await config.withEnv({ SELF_HOSTED: true }, async () => {
+        const env = await config.api.environment.getEnvironment()
+        expect(env.body).toEqual({
+          cloud: false,
+          disableAccountPortal: 0,
+          isDev: false,
+          multiTenancy: true,
+          baseUrl: "http://localhost:10000",
+          offlineMode: false,
+          infrastructure: {
+            sqs: false,
+          },
+        })
+      })
+    })
   })
 })

packages/worker/src/tests/TestConfiguration.ts

@@ -36,6 +36,7 @@ import {
 } from "@budibase/types"
 import API from "./api"
 import jwt, { Secret } from "jsonwebtoken"
+import cloneDeep from "lodash/fp/cloneDeep"
 
 class TestConfiguration {
   server: any
@@ -240,6 +241,34 @@ class TestConfiguration {
     return { message: "Admin user only endpoint.", status: 403 }
   }
 
+  async withEnv(newEnvVars: Partial<typeof env>, f: () => Promise<void>) {
+    let cleanup = this.setEnv(newEnvVars)
+    try {
+      await f()
+    } finally {
+      cleanup()
+    }
+  }
+
+  /*
+   * Sets the environment variables to the given values and returns a function
+   * that can be called to reset the environment variables to their original values.
+   */
+  setEnv(newEnvVars: Partial<typeof env>): () => void {
+    const oldEnv = cloneDeep(env)
+
+    let key: keyof typeof newEnvVars
+    for (key in newEnvVars) {
+      env._set(key, newEnvVars[key])
+    }
+
+    return () => {
+      for (const [key, value] of Object.entries(oldEnv)) {
+        env._set(key, value)
+      }
+    }
+  }
+
   // USERS
 
   async createDefaultUser() {