diff --git a/charts/budibase/README.md b/charts/budibase/README.md index d8191026ce..342011bdb1 100644 --- a/charts/budibase/README.md +++ b/charts/budibase/README.md @@ -157,6 +157,17 @@ $ helm install --create-namespace --namespace budibase budibase . -f values.yaml | services.apps.replicaCount | int | `1` | The number of apps replicas to run. | | services.apps.resources | object | `{}` | The resources to use for apps pods. See for more information on how to set these. | | services.apps.startupProbe | object | HTTP health checks. | Startup probe configuration for apps pods. You shouldn't need to change this, but if you want to you can find more information here: | +| services.automationWorkers.autoscaling.enabled | bool | `false` | Whether to enable horizontal pod autoscaling for the apps service. | +| services.automationWorkers.autoscaling.maxReplicas | int | `10` | | +| services.automationWorkers.autoscaling.minReplicas | int | `1` | | +| services.automationWorkers.autoscaling.targetCPUUtilizationPercentage | int | `80` | Target CPU utilization percentage for the automation worker service. Note that for autoscaling to work, you will need to have metrics-server configured, and resources set for the automation worker pods. | +| services.automationWorkers.enabled | bool | `true` | Whether or not to enable the automation worker service. If you disable this, automations will be processed by the apps service. | +| services.automationWorkers.livenessProbe | object | HTTP health checks. | Liveness probe configuration for automation worker pods. You shouldn't need to change this, but if you want to you can find more information here: | +| services.automationWorkers.logLevel | string | `"info"` | The log level for the automation worker service. | +| services.automationWorkers.readinessProbe | object | HTTP health checks. | Readiness probe configuration for automation worker pods. You shouldn't need to change this, but if you want to you can find more information here: | +| services.automationWorkers.replicaCount | int | `1` | The number of automation worker replicas to run. | +| services.automationWorkers.resources | object | `{}` | The resources to use for automation worker pods. See for more information on how to set these. | +| services.automationWorkers.startupProbe | object | HTTP health checks. | Startup probe configuration for automation worker pods. You shouldn't need to change this, but if you want to you can find more information here: | | services.couchdb.backup.enabled | bool | `false` | Whether or not to enable periodic CouchDB backups. This works by replicating to another CouchDB instance. | | services.couchdb.backup.interval | string | `""` | Backup interval in seconds | | services.couchdb.backup.resources | object | `{}` | The resources to use for CouchDB backup pods. See for more information on how to set these. | diff --git a/charts/budibase/templates/app-service-deployment.yaml b/charts/budibase/templates/app-service-deployment.yaml index 7358e474ca..9fb435c2a3 100644 --- a/charts/budibase/templates/app-service-deployment.yaml +++ b/charts/budibase/templates/app-service-deployment.yaml @@ -192,7 +192,14 @@ spec: - name: NODE_TLS_REJECT_UNAUTHORIZED value: {{ .Values.services.tlsRejectUnauthorized }} {{ end }} - + {{- if .Values.services.automationWorkers.enabled }} + - name: APP_FEATURES + value: "api" + {{- end }} + {{- range .Values.services.apps.extraEnv }} + - name: {{ .name }} + value: {{ .value | quote }} + {{- end }} image: budibase/apps:{{ .Values.globals.appVersion | default .Chart.AppVersion }} imagePullPolicy: Always {{- if .Values.services.apps.startupProbe }} diff --git a/charts/budibase/templates/automation-worker-service-deployment.yaml b/charts/budibase/templates/automation-worker-service-deployment.yaml new file mode 100644 index 0000000000..46be6a4435 --- /dev/null +++ b/charts/budibase/templates/automation-worker-service-deployment.yaml @@ -0,0 +1,248 @@ +{{- if .Values.services.automationWorkers.enabled }} +apiVersion: apps/v1 +kind: Deployment +metadata: + annotations: +{{ if .Values.services.automationWorkers.deploymentAnnotations }} +{{- toYaml .Values.services.automationWorkers.deploymentAnnotations | indent 4 -}} +{{ end }} + labels: + io.kompose.service: automation-worker-service +{{ if .Values.services.automationWorkers.deploymentLabels }} +{{- toYaml .Values.services.automationWorkers.deploymentLabels | indent 4 -}} +{{ end }} + name: automation-worker-service +spec: + replicas: {{ .Values.services.automationWorkers.replicaCount }} + selector: + matchLabels: + io.kompose.service: automation-worker-service + strategy: + type: RollingUpdate + template: + metadata: + annotations: +{{ if .Values.services.automationWorkers.templateAnnotations }} +{{- toYaml .Values.services.automationWorkers.templateAnnotations | indent 8 -}} +{{ end }} + labels: + io.kompose.service: automation-worker-service +{{ if .Values.services.automationWorkers.templateLabels }} +{{- toYaml .Values.services.automationWorkers.templateLabels | indent 8 -}} +{{ end }} + spec: + containers: + - env: + - name: BUDIBASE_ENVIRONMENT + value: {{ .Values.globals.budibaseEnv }} + - name: DEPLOYMENT_ENVIRONMENT + value: "kubernetes" + - name: COUCH_DB_URL + {{ if .Values.services.couchdb.url }} + value: {{ .Values.services.couchdb.url }} + {{ else }} + value: http://{{ .Release.Name }}-svc-couchdb:{{ .Values.services.couchdb.port }} + {{ end }} + {{ if .Values.services.couchdb.enabled }} + - name: COUCH_DB_USER + valueFrom: + secretKeyRef: + name: {{ template "couchdb.fullname" . }} + key: adminUsername + - name: COUCH_DB_PASSWORD + valueFrom: + secretKeyRef: + name: {{ template "couchdb.fullname" . }} + key: adminPassword + {{ end }} + - name: ENABLE_ANALYTICS + value: {{ .Values.globals.enableAnalytics | quote }} + - name: API_ENCRYPTION_KEY + value: {{ .Values.globals.apiEncryptionKey | quote }} + - name: HTTP_LOGGING + value: {{ .Values.services.automationWorkers.httpLogging | quote }} + - name: INTERNAL_API_KEY + valueFrom: + secretKeyRef: + name: {{ template "budibase.fullname" . }} + key: internalApiKey + - name: INTERNAL_API_KEY_FALLBACK + value: {{ .Values.globals.internalApiKeyFallback | quote }} + - name: JWT_SECRET + valueFrom: + secretKeyRef: + name: {{ template "budibase.fullname" . }} + key: jwtSecret + - name: JWT_SECRET_FALLBACK + value: {{ .Values.globals.jwtSecretFallback | quote }} + {{ if .Values.services.objectStore.region }} + - name: AWS_REGION + value: {{ .Values.services.objectStore.region }} + {{ end }} + - name: MINIO_ENABLED + value: {{ .Values.services.objectStore.minio | quote }} + - name: MINIO_ACCESS_KEY + valueFrom: + secretKeyRef: + name: {{ template "budibase.fullname" . }} + key: objectStoreAccess + - name: MINIO_SECRET_KEY + valueFrom: + secretKeyRef: + name: {{ template "budibase.fullname" . }} + key: objectStoreSecret + - name: CLOUDFRONT_CDN + value: {{ .Values.services.objectStore.cloudfront.cdn | quote }} + - name: CLOUDFRONT_PUBLIC_KEY_ID + value: {{ .Values.services.objectStore.cloudfront.publicKeyId | quote }} + - name: CLOUDFRONT_PRIVATE_KEY_64 + value: {{ .Values.services.objectStore.cloudfront.privateKey64 | quote }} + - name: MINIO_URL + value: {{ .Values.services.objectStore.url }} + - name: PLUGIN_BUCKET_NAME + value: {{ .Values.services.objectStore.pluginBucketName | quote }} + - name: APPS_BUCKET_NAME + value: {{ .Values.services.objectStore.appsBucketName | quote }} + - name: GLOBAL_BUCKET_NAME + value: {{ .Values.services.objectStore.globalBucketName | quote }} + - name: BACKUPS_BUCKET_NAME + value: {{ .Values.services.objectStore.backupsBucketName | quote }} + - name: PORT + value: {{ .Values.services.automationWorkers.port | quote }} + {{ if .Values.services.worker.publicApiRateLimitPerSecond }} + - name: API_REQ_LIMIT_PER_SEC + value: {{ .Values.globals.automationWorkers.publicApiRateLimitPerSecond | quote }} + {{ end }} + - name: MULTI_TENANCY + value: {{ .Values.globals.multiTenancy | quote }} + - name: OFFLINE_MODE + value: {{ .Values.globals.offlineMode | quote }} + - name: LOG_LEVEL + value: {{ .Values.services.automationWorkers.logLevel | quote }} + - name: REDIS_PASSWORD + value: {{ .Values.services.redis.password }} + - name: REDIS_URL + {{ if .Values.services.redis.url }} + value: {{ .Values.services.redis.url }} + {{ else }} + value: redis-service:{{ .Values.services.redis.port }} + {{ end }} + - name: SELF_HOSTED + value: {{ .Values.globals.selfHosted | quote }} + - name: POSTHOG_TOKEN + value: {{ .Values.globals.posthogToken | quote }} + - name: WORKER_URL + value: http://worker-service:{{ .Values.services.worker.port }} + - name: PLATFORM_URL + value: {{ .Values.globals.platformUrl | quote }} + - name: ACCOUNT_PORTAL_URL + value: {{ .Values.globals.accountPortalUrl | quote }} + - name: ACCOUNT_PORTAL_API_KEY + value: {{ .Values.globals.accountPortalApiKey | quote }} + - name: COOKIE_DOMAIN + value: {{ .Values.globals.cookieDomain | quote }} + - name: HTTP_MIGRATIONS + value: {{ .Values.globals.httpMigrations | quote }} + - name: GOOGLE_CLIENT_ID + value: {{ .Values.globals.google.clientId | quote }} + - name: GOOGLE_CLIENT_SECRET + value: {{ .Values.globals.google.secret | quote }} + - name: AUTOMATION_MAX_ITERATIONS + value: {{ .Values.globals.automationMaxIterations | quote }} + - name: TENANT_FEATURE_FLAGS + value: {{ .Values.globals.tenantFeatureFlags | quote }} + - name: ENCRYPTION_KEY + value: {{ .Values.globals.bbEncryptionKey | quote }} + {{ if .Values.globals.bbAdminUserEmail }} + - name: BB_ADMIN_USER_EMAIL + value: {{ .Values.globals.bbAdminUserEmail | quote }} + {{ end }} + {{ if .Values.globals.bbAdminUserPassword }} + - name: BB_ADMIN_USER_PASSWORD + value: {{ .Values.globals.bbAdminUserPassword | quote }} + {{ end }} + {{ if .Values.globals.pluginsDir }} + - name: PLUGINS_DIR + value: {{ .Values.globals.pluginsDir | quote }} + {{ end }} + {{ if .Values.services.automationWorkers.nodeDebug }} + - name: NODE_DEBUG + value: {{ .Values.services.automationWorkers.nodeDebug | quote }} + {{ end }} + {{ if .Values.globals.datadogApmEnabled }} + - name: DD_LOGS_INJECTION + value: {{ .Values.globals.datadogApmEnabled | quote }} + - name: DD_APM_ENABLED + value: {{ .Values.globals.datadogApmEnabled | quote }} + - name: DD_APM_DD_URL + value: https://trace.agent.datadoghq.eu + {{ end }} + {{ if .Values.globals.globalAgentHttpProxy }} + - name: GLOBAL_AGENT_HTTP_PROXY + value: {{ .Values.globals.globalAgentHttpProxy | quote }} + {{ end }} + {{ if .Values.globals.globalAgentHttpsProxy }} + - name: GLOBAL_AGENT_HTTPS_PROXY + value: {{ .Values.globals.globalAgentHttpsProxy | quote }} + {{ end }} + {{ if .Values.globals.globalAgentNoProxy }} + - name: GLOBAL_AGENT_NO_PROXY + value: {{ .Values.globals.globalAgentNoProxy | quote }} + {{ end }} + {{ if .Values.services.tlsRejectUnauthorized }} + - name: NODE_TLS_REJECT_UNAUTHORIZED + value: {{ .Values.services.tlsRejectUnauthorized }} + {{ end }} + - name: APP_FEATURES + value: "automations" + {{- range .Values.services.automationWorkers.extraEnv }} + - name: {{ .name }} + value: {{ .value | quote }} + {{- end }} + + image: budibase/apps:{{ .Values.globals.appVersion | default .Chart.AppVersion }} + imagePullPolicy: Always + {{- if .Values.services.automationWorkers.startupProbe }} + {{- with .Values.services.automationWorkers.startupProbe }} + startupProbe: + {{- toYaml . | nindent 10 }} + {{- end }} + {{- end }} + {{- if .Values.services.automationWorkers.livenessProbe }} + {{- with .Values.services.automationWorkers.livenessProbe }} + livenessProbe: + {{- toYaml . | nindent 10 }} + {{- end }} + {{- end }} + {{- if .Values.services.automationWorkers.readinessProbe }} + {{- with .Values.services.automationWorkers.readinessProbe }} + readinessProbe: + {{- toYaml . | nindent 10 }} + {{- end }} + {{- end }} + name: bbautomationworker + ports: + - containerPort: {{ .Values.services.automationWorkers.port }} + {{ with .Values.services.automationWorkers.resources }} + resources: + {{- toYaml . | nindent 10 }} + {{ end }} + {{- with .Values.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} + {{ if .Values.schedulerName }} + schedulerName: {{ .Values.schedulerName | quote }} + {{ end }} + {{ if .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml .Values.imagePullSecrets | nindent 6 }} + {{ end }} + restartPolicy: Always + serviceAccountName: "" +status: {} +{{- end }} \ No newline at end of file diff --git a/charts/budibase/templates/automation-worker-service-hpa.yaml b/charts/budibase/templates/automation-worker-service-hpa.yaml new file mode 100644 index 0000000000..f29223b61b --- /dev/null +++ b/charts/budibase/templates/automation-worker-service-hpa.yaml @@ -0,0 +1,32 @@ +{{- if .Values.services.automationWorkers.autoscaling.enabled }} +apiVersion: {{ ternary "autoscaling/v2" "autoscaling/v2beta2" (.Capabilities.APIVersions.Has "autoscaling/v2") }} +kind: HorizontalPodAutoscaler +metadata: + name: {{ include "budibase.fullname" . }}-apps + labels: + {{- include "budibase.labels" . | nindent 4 }} +spec: + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: automation-worker-service + minReplicas: {{ .Values.services.automationWorkers.autoscaling.minReplicas }} + maxReplicas: {{ .Values.services.automationWorkers.autoscaling.maxReplicas }} + metrics: + {{- if .Values.services.automationWorkers.autoscaling.targetCPUUtilizationPercentage }} + - type: Resource + resource: + name: cpu + target: + type: Utilization + averageUtilization: {{ .Values.services.automationWorkers.autoscaling.targetCPUUtilizationPercentage }} + {{- end }} + {{- if .Values.services.automationWorkers.autoscaling.targetMemoryUtilizationPercentage }} + - type: Resource + resource: + name: memory + target: + type: Utilization + averageUtilization: {{ .Values.services.automationWorkers.autoscaling.targetMemoryUtilizationPercentage }} + {{- end }} +{{- end }} diff --git a/charts/budibase/templates/worker-service-deployment.yaml b/charts/budibase/templates/worker-service-deployment.yaml index 6427aa70e8..1d90aaf954 100644 --- a/charts/budibase/templates/worker-service-deployment.yaml +++ b/charts/budibase/templates/worker-service-deployment.yaml @@ -182,6 +182,10 @@ spec: - name: NODE_TLS_REJECT_UNAUTHORIZED value: {{ .Values.services.tlsRejectUnauthorized }} {{ end }} + {{- range .Values.services.worker.extraEnv }} + - name: {{ .name }} + value: {{ .value | quote }} + {{- end }} image: budibase/worker:{{ .Values.globals.appVersion | default .Chart.AppVersion }} imagePullPolicy: Always {{- if .Values.services.worker.startupProbe }} diff --git a/charts/budibase/values.yaml b/charts/budibase/values.yaml index 13054e75fc..09262df463 100644 --- a/charts/budibase/values.yaml +++ b/charts/budibase/values.yaml @@ -220,6 +220,9 @@ services: # # for more information on how to set these. resources: {} + # -- Extra environment variables to set for apps pods. Takes a list of + # name=value pairs. + extraEnv: [] # -- Startup probe configuration for apps pods. You shouldn't need to # change this, but if you want to you can find more information here: # @@ -272,6 +275,78 @@ services: # and resources set for the apps pods. targetCPUUtilizationPercentage: 80 + automationWorkers: + # -- Whether or not to enable the automation worker service. If you disable this, + # automations will be processed by the apps service. + enabled: true + # @ignore (you shouldn't need to change this) + port: 4002 + # -- The number of automation worker replicas to run. + replicaCount: 1 + # -- The log level for the automation worker service. + logLevel: info + # -- The resources to use for automation worker pods. See + # + # for more information on how to set these. + resources: {} + # -- Extra environment variables to set for automation worker pods. Takes a list of + # name=value pairs. + extraEnv: [] + # -- Startup probe configuration for automation worker pods. You shouldn't + # need to change this, but if you want to you can find more information + # here: + # + # @default -- HTTP health checks. + startupProbe: + # @ignore + httpGet: + path: /health + port: 4002 + scheme: HTTP + # @ignore + failureThreshold: 30 + # @ignore + periodSeconds: 3 + # -- Readiness probe configuration for automation worker pods. You shouldn't + # need to change this, but if you want to you can find more information + # here: + # + # @default -- HTTP health checks. + readinessProbe: + # @ignore + httpGet: + path: /health + port: 4002 + scheme: HTTP + # @ignore + periodSeconds: 3 + # @ignore + failureThreshold: 1 + # -- Liveness probe configuration for automation worker pods. You shouldn't + # need to change this, but if you want to you can find more information + # here: + # + # @default -- HTTP health checks. + livenessProbe: + # @ignore + httpGet: + path: /health + port: 4002 + scheme: HTTP + # @ignore + failureThreshold: 3 + # @ignore + periodSeconds: 30 + autoscaling: + # -- Whether to enable horizontal pod autoscaling for the apps service. + enabled: false + minReplicas: 1 + maxReplicas: 10 + # -- Target CPU utilization percentage for the automation worker service. + # Note that for autoscaling to work, you will need to have metrics-server + # configured, and resources set for the automation worker pods. + targetCPUUtilizationPercentage: 80 + worker: # @ignore (you shouldn't need to change this) port: 4003 @@ -285,6 +360,9 @@ services: # # for more information on how to set these. resources: {} + # -- Extra environment variables to set for worker pods. Takes a list of + # name=value pairs. + extraEnv: [] # -- Startup probe configuration for worker pods. You shouldn't need to # change this, but if you want to you can find more information here: # diff --git a/lerna.json b/lerna.json index caae7fbaaa..ed49a4267d 100644 --- a/lerna.json +++ b/lerna.json @@ -1,5 +1,5 @@ { - "version": "2.13.37", + "version": "2.13.39", "npmClient": "yarn", "packages": [ "packages/*", @@ -22,4 +22,4 @@ "loadEnvFiles": false } } -} +} \ No newline at end of file diff --git a/packages/backend-core/src/cache/tests/writethrough.spec.ts b/packages/backend-core/src/cache/tests/writethrough.spec.ts index 97d3ece7a6..37887b4bd9 100644 --- a/packages/backend-core/src/cache/tests/writethrough.spec.ts +++ b/packages/backend-core/src/cache/tests/writethrough.spec.ts @@ -1,15 +1,16 @@ import { DBTestConfiguration } from "../../../tests/extra" -import { - structures, - expectFunctionWasCalledTimesWith, - mocks, -} from "../../../tests" +import { structures } from "../../../tests" import { Writethrough } from "../writethrough" import { getDB } from "../../db" +import { Document } from "@budibase/types" import tk from "timekeeper" tk.freeze(Date.now()) +interface ValueDoc extends Document { + value: any +} + const DELAY = 5000 describe("writethrough", () => { @@ -117,7 +118,7 @@ describe("writethrough", () => { describe("get", () => { it("should be able to retrieve", async () => { await config.doInTenant(async () => { - const response = await writethrough.get(docId) + const response = await writethrough.get(docId) expect(response.value).toBe(4) }) }) diff --git a/packages/backend-core/src/cache/writethrough.ts b/packages/backend-core/src/cache/writethrough.ts index c331d791a6..24e519dc7f 100644 --- a/packages/backend-core/src/cache/writethrough.ts +++ b/packages/backend-core/src/cache/writethrough.ts @@ -7,7 +7,7 @@ import * as locks from "../redis/redlockImpl" const DEFAULT_WRITE_RATE_MS = 10000 let CACHE: BaseCache | null = null -interface CacheItem { +interface CacheItem { doc: any lastWrite: number } @@ -24,7 +24,10 @@ function makeCacheKey(db: Database, key: string) { return db.name + key } -function makeCacheItem(doc: any, lastWrite: number | null = null): CacheItem { +function makeCacheItem( + doc: T, + lastWrite: number | null = null +): CacheItem { return { doc, lastWrite: lastWrite || Date.now() } } @@ -35,7 +38,7 @@ async function put( ) { const cache = await getCache() const key = doc._id - let cacheItem: CacheItem | undefined + let cacheItem: CacheItem | undefined if (key) { cacheItem = await cache.get(makeCacheKey(db, key)) } @@ -84,12 +87,12 @@ async function put( return { ok: true, id: output._id, rev: output._rev } } -async function get(db: Database, id: string): Promise { +async function get(db: Database, id: string): Promise { const cache = await getCache() const cacheKey = makeCacheKey(db, id) - let cacheItem: CacheItem = await cache.get(cacheKey) + let cacheItem: CacheItem = await cache.get(cacheKey) if (!cacheItem) { - const doc = await db.get(id) + const doc = await db.get(id) cacheItem = makeCacheItem(doc) await cache.store(cacheKey, cacheItem) } @@ -123,8 +126,8 @@ export class Writethrough { return put(this.db, doc, writeRateMs) } - async get(id: string) { - return get(this.db, id) + async get(id: string) { + return get(this.db, id) } async remove(docOrId: any, rev?: any) { diff --git a/packages/backend-core/src/environment.ts b/packages/backend-core/src/environment.ts index ed882fe96a..138dbbd9e0 100644 --- a/packages/backend-core/src/environment.ts +++ b/packages/backend-core/src/environment.ts @@ -107,6 +107,7 @@ const environment = { ENCRYPTION_KEY: process.env.ENCRYPTION_KEY, API_ENCRYPTION_KEY: getAPIEncryptionKey(), COUCH_DB_URL: process.env.COUCH_DB_URL || "http://localhost:4005", + COUCH_DB_SQL_URL: process.env.COUCH_DB_SQL_URL || "http://localhost:4984", COUCH_DB_USERNAME: process.env.COUCH_DB_USER, COUCH_DB_PASSWORD: process.env.COUCH_DB_PASSWORD, GOOGLE_CLIENT_ID: process.env.GOOGLE_CLIENT_ID, diff --git a/packages/backend-core/src/queue/inMemoryQueue.ts b/packages/backend-core/src/queue/inMemoryQueue.ts index a8add7ecb6..ac7cdf550b 100644 --- a/packages/backend-core/src/queue/inMemoryQueue.ts +++ b/packages/backend-core/src/queue/inMemoryQueue.ts @@ -68,6 +68,10 @@ class InMemoryQueue { }) } + async isReady() { + return true + } + // simply puts a message to the queue and emits to the queue for processing /** * Simple function to replicate the add message functionality of Bull, putting diff --git a/packages/backend-core/src/redis/redlockImpl.ts b/packages/backend-core/src/redis/redlockImpl.ts index 4de2516ab2..e57a3721b5 100644 --- a/packages/backend-core/src/redis/redlockImpl.ts +++ b/packages/backend-core/src/redis/redlockImpl.ts @@ -137,7 +137,6 @@ export async function doWithLock( const result = await task() return { executed: true, result } } catch (e: any) { - logWarn(`lock type: ${opts.type} error`, e) // lock limit exceeded if (e.name === "LockError") { if (opts.type === LockType.TRY_ONCE) { diff --git a/packages/pro b/packages/pro index 056c2093db..992486c100 160000 --- a/packages/pro +++ b/packages/pro @@ -1 +1 @@ -Subproject commit 056c2093dbc93d9a10ea9f5050c84a84edd8100c +Subproject commit 992486c10044a7495496b97bdf5f454d4020bfba diff --git a/packages/server/scripts/dev/manage.js b/packages/server/scripts/dev/manage.js index b469c1ffc7..6dc0966f78 100644 --- a/packages/server/scripts/dev/manage.js +++ b/packages/server/scripts/dev/manage.js @@ -1,7 +1,8 @@ #!/usr/bin/env node const compose = require("docker-compose") const path = require("path") -const fs = require("fs") +const { parsed: existingConfig } = require("dotenv").config() +const updateDotEnv = require("update-dotenv") // This script wraps docker-compose allowing you to manage your dev infrastructure with simple commands. const CONFIG = { @@ -17,45 +18,41 @@ const Commands = { } async function init() { - const envFilePath = path.join(process.cwd(), ".env") - if (!fs.existsSync(envFilePath)) { - const envFileJson = { - PORT: 4001, - MINIO_URL: "http://localhost:4004", - COUCH_DB_URL: "http://budibase:budibase@localhost:4005", - REDIS_URL: "localhost:6379", - WORKER_URL: "http://localhost:4002", - INTERNAL_API_KEY: "budibase", - ACCOUNT_PORTAL_URL: "http://localhost:10001", - ACCOUNT_PORTAL_API_KEY: "budibase", - PLATFORM_URL: "http://localhost:10000", - JWT_SECRET: "testsecret", - ENCRYPTION_KEY: "testsecret", - REDIS_PASSWORD: "budibase", - MINIO_ACCESS_KEY: "budibase", - MINIO_SECRET_KEY: "budibase", - COUCH_DB_PASSWORD: "budibase", - COUCH_DB_USER: "budibase", - SELF_HOSTED: 1, - DISABLE_ACCOUNT_PORTAL: 1, - MULTI_TENANCY: "", - DISABLE_THREADING: 1, - SERVICE: "app-service", - DEPLOYMENT_ENVIRONMENT: "development", - BB_ADMIN_USER_EMAIL: "", - BB_ADMIN_USER_PASSWORD: "", - PLUGINS_DIR: "", - TENANT_FEATURE_FLAGS: "*:LICENSING,*:USER_GROUPS,*:ONBOARDING_TOUR", - HTTP_MIGRATIONS: "0", - HTTP_LOGGING: "0", - VERSION: "0.0.0+local", - } - let envFile = "" - Object.keys(envFileJson).forEach(key => { - envFile += `${key}=${envFileJson[key]}\n` - }) - fs.writeFileSync(envFilePath, envFile) + let config = { + PORT: "4001", + MINIO_URL: "http://localhost:4004", + COUCH_DB_URL: "http://budibase:budibase@localhost:4005", + REDIS_URL: "localhost:6379", + WORKER_URL: "http://localhost:4002", + INTERNAL_API_KEY: "budibase", + ACCOUNT_PORTAL_URL: "http://localhost:10001", + ACCOUNT_PORTAL_API_KEY: "budibase", + PLATFORM_URL: "http://localhost:10000", + JWT_SECRET: "testsecret", + ENCRYPTION_KEY: "testsecret", + REDIS_PASSWORD: "budibase", + MINIO_ACCESS_KEY: "budibase", + MINIO_SECRET_KEY: "budibase", + COUCH_DB_PASSWORD: "budibase", + COUCH_DB_USER: "budibase", + SELF_HOSTED: "1", + DISABLE_ACCOUNT_PORTAL: "1", + MULTI_TENANCY: "", + DISABLE_THREADING: "1", + SERVICE: "app-service", + DEPLOYMENT_ENVIRONMENT: "development", + BB_ADMIN_USER_EMAIL: "", + BB_ADMIN_USER_PASSWORD: "", + PLUGINS_DIR: "", + TENANT_FEATURE_FLAGS: "*:LICENSING,*:USER_GROUPS,*:ONBOARDING_TOUR", + HTTP_MIGRATIONS: "0", + HTTP_LOGGING: "0", + VERSION: "0.0.0+local", } + + config = { ...config, ...existingConfig } + + await updateDotEnv(config) } async function up() { diff --git a/packages/server/src/api/index.ts b/packages/server/src/api/index.ts index a01e3764f0..ad3d8307da 100644 --- a/packages/server/src/api/index.ts +++ b/packages/server/src/api/index.ts @@ -4,62 +4,75 @@ import currentApp from "../middleware/currentapp" import zlib from "zlib" import { mainRoutes, staticRoutes, publicRoutes } from "./routes" import { middleware as pro } from "@budibase/pro" +import { apiEnabled, automationsEnabled } from "../features" import migrations from "../middleware/appMigrations" +import { automationQueue } from "../automations" export { shutdown } from "./routes/public" const compress = require("koa-compress") export const router: Router = new Router() -router.get("/health", ctx => (ctx.status = 200)) +router.get("/health", async ctx => { + if (automationsEnabled()) { + if (!(await automationQueue.isReady())) { + ctx.status = 503 + return + } + } + ctx.status = 200 +}) router.get("/version", ctx => (ctx.body = envCore.VERSION)) router.use(middleware.errorHandling) -router - .use( - compress({ - threshold: 2048, - gzip: { - flush: zlib.constants.Z_SYNC_FLUSH, - }, - deflate: { - flush: zlib.constants.Z_SYNC_FLUSH, - }, - br: false, - }) - ) - // re-direct before any middlewares occur - .redirect("/", "/builder") - .use( - auth.buildAuthMiddleware([], { - publicAllowed: true, - }) - ) - // nothing in the server should allow query string tenants - // the server can be public anywhere, so nowhere should throw errors - // if the tenancy has not been set, it'll have to be discovered at application layer - .use( - auth.buildTenancyMiddleware([], [], { - noTenancyRequired: true, - }) - ) - .use(pro.licensing()) - // @ts-ignore - .use(currentApp) - .use(auth.auditLog) - // @ts-ignore - .use(migrations) +// only add the routes if they are enabled +if (apiEnabled()) { + router + .use( + compress({ + threshold: 2048, + gzip: { + flush: zlib.constants.Z_SYNC_FLUSH, + }, + deflate: { + flush: zlib.constants.Z_SYNC_FLUSH, + }, + br: false, + }) + ) + // re-direct before any middlewares occur + .redirect("/", "/builder") + .use( + auth.buildAuthMiddleware([], { + publicAllowed: true, + }) + ) + // nothing in the server should allow query string tenants + // the server can be public anywhere, so nowhere should throw errors + // if the tenancy has not been set, it'll have to be discovered at application layer + .use( + auth.buildTenancyMiddleware([], [], { + noTenancyRequired: true, + }) + ) + .use(pro.licensing()) + // @ts-ignore + .use(currentApp) + .use(auth.auditLog) + // @ts-ignore + .use(migrations) -// authenticated routes -for (let route of mainRoutes) { - router.use(route.routes()) - router.use(route.allowedMethods()) + // authenticated routes + for (let route of mainRoutes) { + router.use(route.routes()) + router.use(route.allowedMethods()) + } + + router.use(publicRoutes.routes()) + router.use(publicRoutes.allowedMethods()) + + // WARNING - static routes will catch everything else after them this must be last + router.use(staticRoutes.routes()) + router.use(staticRoutes.allowedMethods()) } - -router.use(publicRoutes.routes()) -router.use(publicRoutes.allowedMethods()) - -// WARNING - static routes will catch everything else after them this must be last -router.use(staticRoutes.routes()) -router.use(staticRoutes.allowedMethods()) diff --git a/packages/server/src/app.ts b/packages/server/src/app.ts index 4c0068be89..f6f1780030 100644 --- a/packages/server/src/app.ts +++ b/packages/server/src/app.ts @@ -9,7 +9,6 @@ import { ServiceType } from "@budibase/types" import { env as coreEnv } from "@budibase/backend-core" coreEnv._set("SERVICE_TYPE", ServiceType.APPS) -import { apiEnabled } from "./features" import createKoaApp from "./koa" import Koa from "koa" import { Server } from "http" @@ -18,12 +17,9 @@ import { startup } from "./startup" let app: Koa, server: Server async function start() { - // if API disabled, could run automations instead - if (apiEnabled()) { - const koa = createKoaApp() - app = koa.app - server = koa.server - } + const koa = createKoaApp() + app = koa.app + server = koa.server // startup includes automation runner - if enabled await startup(app, server) } diff --git a/packages/server/src/features.ts b/packages/server/src/features.ts index e12260ea32..f040cf82a2 100644 --- a/packages/server/src/features.ts +++ b/packages/server/src/features.ts @@ -22,3 +22,10 @@ export function automationsEnabled() { export function apiEnabled() { return featureList.includes(AppFeature.API) } + +export function printFeatures() { + if (!env.APP_FEATURES) { + return + } + console.log(`**** APP FEATURES SET: ${featureList.join(", ")} ****`) +} diff --git a/packages/server/src/startup.ts b/packages/server/src/startup.ts index 9144ff2b36..2db6e5ae6a 100644 --- a/packages/server/src/startup.ts +++ b/packages/server/src/startup.ts @@ -19,11 +19,14 @@ import * as pro from "@budibase/pro" import * as api from "./api" import sdk from "./sdk" import { initialise as initialiseWebsockets } from "./websockets" -import { automationsEnabled } from "./features" +import { automationsEnabled, printFeatures } from "./features" +import Koa from "koa" +import { Server } from "http" +import { AddressInfo } from "net" let STARTUP_RAN = false -async function initRoutes(app: any) { +async function initRoutes(app: Koa) { if (!env.isTest()) { const plugin = await bullboard.init() app.use(plugin) @@ -48,27 +51,31 @@ async function initPro() { }) } -function shutdown(server?: any) { +function shutdown(server?: Server) { if (server) { server.close() server.destroy() } } -export async function startup(app?: any, server?: any) { +export async function startup(app?: Koa, server?: Server) { if (STARTUP_RAN) { return } + printFeatures() STARTUP_RAN = true - if (server && !env.CLUSTER_MODE) { + if (app && server && !env.CLUSTER_MODE) { console.log(`Budibase running on ${JSON.stringify(server.address())}`) - env._set("PORT", server.address().port) + const address = server.address() as AddressInfo + env._set("PORT", address.port) } eventEmitter.emitPort(env.PORT) fileSystem.init() await redis.init() eventInit() - initialiseWebsockets(app, server) + if (app && server) { + initialiseWebsockets(app, server) + } // run migrations on startup if not done via http // not recommended in a clustered environment diff --git a/packages/server/src/tests/utilities/TestConfiguration.ts b/packages/server/src/tests/utilities/TestConfiguration.ts index 51b5fda3d4..b5810b9ba3 100644 --- a/packages/server/src/tests/utilities/TestConfiguration.ts +++ b/packages/server/src/tests/utilities/TestConfiguration.ts @@ -17,7 +17,6 @@ import { basicWebhook, } from "./structures" import { - auth, cache, constants, context, diff --git a/packages/worker/scripts/dev/manage.js b/packages/worker/scripts/dev/manage.js index 9e6a57d4bf..1b7c6f0ddd 100644 --- a/packages/worker/scripts/dev/manage.js +++ b/packages/worker/scripts/dev/manage.js @@ -1,44 +1,40 @@ #!/usr/bin/env node -const path = require("path") -const fs = require("fs") +const { parsed: existingConfig } = require("dotenv").config() +const updateDotEnv = require("update-dotenv") async function init() { - const envFilePath = path.join(process.cwd(), ".env") - if (!fs.existsSync(envFilePath)) { - const envFileJson = { - SELF_HOSTED: 1, - PORT: 4002, - CLUSTER_PORT: 10000, - JWT_SECRET: "testsecret", - INTERNAL_API_KEY: "budibase", - MINIO_ACCESS_KEY: "budibase", - MINIO_SECRET_KEY: "budibase", - REDIS_URL: "localhost:6379", - REDIS_PASSWORD: "budibase", - MINIO_URL: "http://localhost:4004", - COUCH_DB_URL: "http://budibase:budibase@localhost:4005", - COUCH_DB_USERNAME: "budibase", - COUCH_DB_PASSWORD: "budibase", - // empty string is false - MULTI_TENANCY: "", - DISABLE_ACCOUNT_PORTAL: 1, - ACCOUNT_PORTAL_URL: "http://localhost:10001", - ACCOUNT_PORTAL_API_KEY: "budibase", - PLATFORM_URL: "http://localhost:10000", - APPS_URL: "http://localhost:4001", - SERVICE: "worker-service", - DEPLOYMENT_ENVIRONMENT: "development", - TENANT_FEATURE_FLAGS: "*:LICENSING,*:USER_GROUPS,*:ONBOARDING_TOUR", - ENABLE_EMAIL_TEST_MODE: 1, - HTTP_LOGGING: 0, - VERSION: "0.0.0+local", - } - let envFile = "" - Object.keys(envFileJson).forEach(key => { - envFile += `${key}=${envFileJson[key]}\n` - }) - fs.writeFileSync(envFilePath, envFile) + let config = { + SELF_HOSTED: "1", + PORT: "4002", + CLUSTER_PORT: "10000", + JWT_SECRET: "testsecret", + INTERNAL_API_KEY: "budibase", + MINIO_ACCESS_KEY: "budibase", + MINIO_SECRET_KEY: "budibase", + REDIS_URL: "localhost:6379", + REDIS_PASSWORD: "budibase", + MINIO_URL: "http://localhost:4004", + COUCH_DB_URL: "http://budibase:budibase@localhost:4005", + COUCH_DB_USERNAME: "budibase", + COUCH_DB_PASSWORD: "budibase", + // empty string is false + MULTI_TENANCY: "", + DISABLE_ACCOUNT_PORTAL: "1", + ACCOUNT_PORTAL_URL: "http://localhost:10001", + ACCOUNT_PORTAL_API_KEY: "budibase", + PLATFORM_URL: "http://localhost:10000", + APPS_URL: "http://localhost:4001", + SERVICE: "worker-service", + DEPLOYMENT_ENVIRONMENT: "development", + TENANT_FEATURE_FLAGS: "*:LICENSING,*:USER_GROUPS,*:ONBOARDING_TOUR", + ENABLE_EMAIL_TEST_MODE: "1", + HTTP_LOGGING: "0", + VERSION: "0.0.0+local", } + + config = { ...config, ...existingConfig } + + await updateDotEnv(config) } // if more than init required use this to determine the command type diff --git a/packages/worker/src/api/controllers/system/environment.ts b/packages/worker/src/api/controllers/system/environment.ts index ade5f241e2..bf9270607f 100644 --- a/packages/worker/src/api/controllers/system/environment.ts +++ b/packages/worker/src/api/controllers/system/environment.ts @@ -1,6 +1,25 @@ import { Ctx } from "@budibase/types" import env from "../../../environment" import { env as coreEnv } from "@budibase/backend-core" +import nodeFetch from "node-fetch" + +let sqsAvailable: boolean +async function isSqsAvailable() { + if (sqsAvailable !== undefined) { + return sqsAvailable + } + + try { + await nodeFetch(coreEnv.COUCH_DB_SQL_URL, { + timeout: 1000, + }) + sqsAvailable = true + return true + } catch (e) { + sqsAvailable = false + return false + } +} export const fetch = async (ctx: Ctx) => { ctx.body = { @@ -12,4 +31,10 @@ export const fetch = async (ctx: Ctx) => { baseUrl: env.PLATFORM_URL, isDev: env.isDev() && !env.isTest(), } + + if (env.SELF_HOSTED) { + ctx.body.infrastructure = { + sqs: await isSqsAvailable(), + } + } } diff --git a/packages/worker/src/api/routes/system/tests/environment.spec.ts b/packages/worker/src/api/routes/system/tests/environment.spec.ts index 897cc970cc..2efbfa07c9 100644 --- a/packages/worker/src/api/routes/system/tests/environment.spec.ts +++ b/packages/worker/src/api/routes/system/tests/environment.spec.ts @@ -1,5 +1,7 @@ import { TestConfiguration } from "../../../../tests" +jest.unmock("node-fetch") + describe("/api/system/environment", () => { const config = new TestConfiguration() @@ -27,5 +29,22 @@ describe("/api/system/environment", () => { offlineMode: false, }) }) + + it("returns the expected environment for self hosters", async () => { + await config.withEnv({ SELF_HOSTED: true }, async () => { + const env = await config.api.environment.getEnvironment() + expect(env.body).toEqual({ + cloud: false, + disableAccountPortal: 0, + isDev: false, + multiTenancy: true, + baseUrl: "http://localhost:10000", + offlineMode: false, + infrastructure: { + sqs: false, + }, + }) + }) + }) }) }) diff --git a/packages/worker/src/tests/TestConfiguration.ts b/packages/worker/src/tests/TestConfiguration.ts index c43d1b9d13..8e163f0373 100644 --- a/packages/worker/src/tests/TestConfiguration.ts +++ b/packages/worker/src/tests/TestConfiguration.ts @@ -36,6 +36,7 @@ import { } from "@budibase/types" import API from "./api" import jwt, { Secret } from "jsonwebtoken" +import cloneDeep from "lodash/fp/cloneDeep" class TestConfiguration { server: any @@ -240,6 +241,34 @@ class TestConfiguration { return { message: "Admin user only endpoint.", status: 403 } } + async withEnv(newEnvVars: Partial, f: () => Promise) { + let cleanup = this.setEnv(newEnvVars) + try { + await f() + } finally { + cleanup() + } + } + + /* + * Sets the environment variables to the given values and returns a function + * that can be called to reset the environment variables to their original values. + */ + setEnv(newEnvVars: Partial): () => void { + const oldEnv = cloneDeep(env) + + let key: keyof typeof newEnvVars + for (key in newEnvVars) { + env._set(key, newEnvVars[key]) + } + + return () => { + for (const [key, value] of Object.entries(oldEnv)) { + env._set(key, value) + } + } + } + // USERS async createDefaultUser() {