From 6b491815a5d8b237c6b142de47c39cac6ab07de8 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Mon, 11 Dec 2023 15:22:02 +0000 Subject: [PATCH 01/25] Add isSqsAvailable key to the environment endpoint. --- packages/backend-core/src/environment.ts | 1 + .../src/api/controllers/system/environment.ts | 20 +++++++++++++++++++ 2 files changed, 21 insertions(+) diff --git a/packages/backend-core/src/environment.ts b/packages/backend-core/src/environment.ts index ed882fe96a..138dbbd9e0 100644 --- a/packages/backend-core/src/environment.ts +++ b/packages/backend-core/src/environment.ts @@ -107,6 +107,7 @@ const environment = { ENCRYPTION_KEY: process.env.ENCRYPTION_KEY, API_ENCRYPTION_KEY: getAPIEncryptionKey(), COUCH_DB_URL: process.env.COUCH_DB_URL || "http://localhost:4005", + COUCH_DB_SQL_URL: process.env.COUCH_DB_SQL_URL || "http://localhost:4984", COUCH_DB_USERNAME: process.env.COUCH_DB_USER, COUCH_DB_PASSWORD: process.env.COUCH_DB_PASSWORD, GOOGLE_CLIENT_ID: process.env.GOOGLE_CLIENT_ID, diff --git a/packages/worker/src/api/controllers/system/environment.ts b/packages/worker/src/api/controllers/system/environment.ts index ade5f241e2..c619d484f7 100644 --- a/packages/worker/src/api/controllers/system/environment.ts +++ b/packages/worker/src/api/controllers/system/environment.ts @@ -1,6 +1,25 @@ import { Ctx } from "@budibase/types" import env from "../../../environment" import { env as coreEnv } from "@budibase/backend-core" +import nodeFetch from "node-fetch" + +let sqsAvailable: boolean +async function isSqsAvailable() { + if (sqsAvailable !== undefined) { + return sqsAvailable + } + + try { + await nodeFetch(coreEnv.COUCH_DB_SQL_URL, { + timeout: 1000, + }) + } catch (e) { + sqsAvailable = false + return false + } + sqsAvailable = true + return true +} export const fetch = async (ctx: Ctx) => { ctx.body = { @@ -11,5 +30,6 @@ export const fetch = async (ctx: Ctx) => { disableAccountPortal: env.DISABLE_ACCOUNT_PORTAL, baseUrl: env.PLATFORM_URL, isDev: env.isDev() && !env.isTest(), + isSqsAvailable: await isSqsAvailable(), } } From 2c3b3d03e1734730d52bea73845c0f19521107cf Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Mon, 11 Dec 2023 16:49:44 +0000 Subject: [PATCH 02/25] Respond to PR comments. --- .../src/api/controllers/system/environment.ts | 7 +++- .../routes/system/tests/environment.spec.ts | 17 +++++++++ .../worker/src/tests/TestConfiguration.ts | 35 +++++++++++++++++++ 3 files changed, 58 insertions(+), 1 deletion(-) diff --git a/packages/worker/src/api/controllers/system/environment.ts b/packages/worker/src/api/controllers/system/environment.ts index c619d484f7..b62da949ad 100644 --- a/packages/worker/src/api/controllers/system/environment.ts +++ b/packages/worker/src/api/controllers/system/environment.ts @@ -30,6 +30,11 @@ export const fetch = async (ctx: Ctx) => { disableAccountPortal: env.DISABLE_ACCOUNT_PORTAL, baseUrl: env.PLATFORM_URL, isDev: env.isDev() && !env.isTest(), - isSqsAvailable: await isSqsAvailable(), + } + + if (env.SELF_HOSTED) { + ctx.body.infrastructure = { + sqs: await isSqsAvailable(), + } } } diff --git a/packages/worker/src/api/routes/system/tests/environment.spec.ts b/packages/worker/src/api/routes/system/tests/environment.spec.ts index 897cc970cc..5dc2e552e1 100644 --- a/packages/worker/src/api/routes/system/tests/environment.spec.ts +++ b/packages/worker/src/api/routes/system/tests/environment.spec.ts @@ -27,5 +27,22 @@ describe("/api/system/environment", () => { offlineMode: false, }) }) + + it("returns the expected environment for self hosters", async () => { + config.withEnv({ SELF_HOSTED: true }, async () => { + const env = await config.api.environment.getEnvironment() + expect(env.body).toEqual({ + cloud: true, + disableAccountPortal: 0, + isDev: false, + multiTenancy: true, + baseUrl: "http://localhost:10000", + offlineMode: false, + infrastructure: { + sqs: false, + }, + }) + }) + }) }) }) diff --git a/packages/worker/src/tests/TestConfiguration.ts b/packages/worker/src/tests/TestConfiguration.ts index c43d1b9d13..41a074ee89 100644 --- a/packages/worker/src/tests/TestConfiguration.ts +++ b/packages/worker/src/tests/TestConfiguration.ts @@ -36,6 +36,7 @@ import { } from "@budibase/types" import API from "./api" import jwt, { Secret } from "jsonwebtoken" +import { cloneDeep } from "lodash" class TestConfiguration { server: any @@ -240,6 +241,40 @@ class TestConfiguration { return { message: "Admin user only endpoint.", status: 403 } } + async withEnv(newEnvVars: Partial, f: () => Promise) { + let cleanup = this.setEnv(newEnvVars) + try { + await f() + } finally { + cleanup() + } + } + + /* + * Sets the environment variables to the given values and returns a function + * that can be called to reset the environment variables to their original values. + */ + setEnv(newEnvVars: Partial): () => void { + const oldEnv = cloneDeep(env) + const oldCoreEnv = cloneDeep(coreEnv) + + let key: keyof typeof newEnvVars + for (key in newEnvVars) { + env._set(key, newEnvVars[key]) + coreEnv._set(key, newEnvVars[key]) + } + + return () => { + for (const [key, value] of Object.entries(oldEnv)) { + env._set(key, value) + } + + for (const [key, value] of Object.entries(oldCoreEnv)) { + coreEnv._set(key, value) + } + } + } + // USERS async createDefaultUser() { From e0243b495ce5bf25f3b3297049b9a133fc940166 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Mon, 11 Dec 2023 17:20:00 +0000 Subject: [PATCH 03/25] Fix tests. --- packages/server/src/tests/utilities/TestConfiguration.ts | 8 +++++++- packages/worker/src/api/controllers/system/environment.ts | 4 ++-- .../src/api/routes/system/tests/environment.spec.ts | 6 ++++-- packages/worker/src/tests/TestConfiguration.ts | 7 +++++++ 4 files changed, 20 insertions(+), 5 deletions(-) diff --git a/packages/server/src/tests/utilities/TestConfiguration.ts b/packages/server/src/tests/utilities/TestConfiguration.ts index afaad64723..cbcb7888aa 100644 --- a/packages/server/src/tests/utilities/TestConfiguration.ts +++ b/packages/server/src/tests/utilities/TestConfiguration.ts @@ -17,7 +17,6 @@ import { basicWebhook, } from "./structures" import { - auth, cache, constants, context, @@ -227,6 +226,13 @@ class TestConfiguration { } return () => { + for (const key in newEnvVars) { + // @ts-ignore + delete env[key] + // @ts-ignore + delete coreEnv[key] + } + for (const [key, value] of Object.entries(oldEnv)) { env._set(key, value) } diff --git a/packages/worker/src/api/controllers/system/environment.ts b/packages/worker/src/api/controllers/system/environment.ts index b62da949ad..bf9270607f 100644 --- a/packages/worker/src/api/controllers/system/environment.ts +++ b/packages/worker/src/api/controllers/system/environment.ts @@ -13,12 +13,12 @@ async function isSqsAvailable() { await nodeFetch(coreEnv.COUCH_DB_SQL_URL, { timeout: 1000, }) + sqsAvailable = true + return true } catch (e) { sqsAvailable = false return false } - sqsAvailable = true - return true } export const fetch = async (ctx: Ctx) => { diff --git a/packages/worker/src/api/routes/system/tests/environment.spec.ts b/packages/worker/src/api/routes/system/tests/environment.spec.ts index 5dc2e552e1..2efbfa07c9 100644 --- a/packages/worker/src/api/routes/system/tests/environment.spec.ts +++ b/packages/worker/src/api/routes/system/tests/environment.spec.ts @@ -1,5 +1,7 @@ import { TestConfiguration } from "../../../../tests" +jest.unmock("node-fetch") + describe("/api/system/environment", () => { const config = new TestConfiguration() @@ -29,10 +31,10 @@ describe("/api/system/environment", () => { }) it("returns the expected environment for self hosters", async () => { - config.withEnv({ SELF_HOSTED: true }, async () => { + await config.withEnv({ SELF_HOSTED: true }, async () => { const env = await config.api.environment.getEnvironment() expect(env.body).toEqual({ - cloud: true, + cloud: false, disableAccountPortal: 0, isDev: false, multiTenancy: true, diff --git a/packages/worker/src/tests/TestConfiguration.ts b/packages/worker/src/tests/TestConfiguration.ts index 41a074ee89..918083cf56 100644 --- a/packages/worker/src/tests/TestConfiguration.ts +++ b/packages/worker/src/tests/TestConfiguration.ts @@ -265,6 +265,13 @@ class TestConfiguration { } return () => { + for (const key in newEnvVars) { + // @ts-ignore + delete env[key] + // @ts-ignore + delete coreEnv[key] + } + for (const [key, value] of Object.entries(oldEnv)) { env._set(key, value) } From 867d68d4298de105a5b361fb3b584eeaa5a28e3b Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Mon, 11 Dec 2023 17:25:38 +0000 Subject: [PATCH 04/25] Cleanup. --- packages/server/src/tests/utilities/TestConfiguration.ts | 7 ------- packages/worker/src/tests/TestConfiguration.ts | 7 ------- 2 files changed, 14 deletions(-) diff --git a/packages/server/src/tests/utilities/TestConfiguration.ts b/packages/server/src/tests/utilities/TestConfiguration.ts index cbcb7888aa..82704220f8 100644 --- a/packages/server/src/tests/utilities/TestConfiguration.ts +++ b/packages/server/src/tests/utilities/TestConfiguration.ts @@ -226,13 +226,6 @@ class TestConfiguration { } return () => { - for (const key in newEnvVars) { - // @ts-ignore - delete env[key] - // @ts-ignore - delete coreEnv[key] - } - for (const [key, value] of Object.entries(oldEnv)) { env._set(key, value) } diff --git a/packages/worker/src/tests/TestConfiguration.ts b/packages/worker/src/tests/TestConfiguration.ts index 918083cf56..41a074ee89 100644 --- a/packages/worker/src/tests/TestConfiguration.ts +++ b/packages/worker/src/tests/TestConfiguration.ts @@ -265,13 +265,6 @@ class TestConfiguration { } return () => { - for (const key in newEnvVars) { - // @ts-ignore - delete env[key] - // @ts-ignore - delete coreEnv[key] - } - for (const [key, value] of Object.entries(oldEnv)) { env._set(key, value) } From 779bd8f230a0a455f28cb8c7c6d61f02d3801be7 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Mon, 11 Dec 2023 17:28:33 +0000 Subject: [PATCH 05/25] Remove core env overriding in worker TestConfiguration. Should be split out in server, too. --- packages/worker/src/tests/TestConfiguration.ts | 6 ------ 1 file changed, 6 deletions(-) diff --git a/packages/worker/src/tests/TestConfiguration.ts b/packages/worker/src/tests/TestConfiguration.ts index 41a074ee89..747b520296 100644 --- a/packages/worker/src/tests/TestConfiguration.ts +++ b/packages/worker/src/tests/TestConfiguration.ts @@ -256,22 +256,16 @@ class TestConfiguration { */ setEnv(newEnvVars: Partial): () => void { const oldEnv = cloneDeep(env) - const oldCoreEnv = cloneDeep(coreEnv) let key: keyof typeof newEnvVars for (key in newEnvVars) { env._set(key, newEnvVars[key]) - coreEnv._set(key, newEnvVars[key]) } return () => { for (const [key, value] of Object.entries(oldEnv)) { env._set(key, value) } - - for (const [key, value] of Object.entries(oldCoreEnv)) { - coreEnv._set(key, value) - } } } From 3a7c30c7f3386f36222b1f68e0a539bc1901ac2d Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Mon, 11 Dec 2023 17:54:02 +0000 Subject: [PATCH 06/25] Seeing if CI is happy if I remove the unmock call. --- packages/worker/src/api/routes/system/tests/environment.spec.ts | 2 -- 1 file changed, 2 deletions(-) diff --git a/packages/worker/src/api/routes/system/tests/environment.spec.ts b/packages/worker/src/api/routes/system/tests/environment.spec.ts index 2efbfa07c9..9fe707d288 100644 --- a/packages/worker/src/api/routes/system/tests/environment.spec.ts +++ b/packages/worker/src/api/routes/system/tests/environment.spec.ts @@ -1,7 +1,5 @@ import { TestConfiguration } from "../../../../tests" -jest.unmock("node-fetch") - describe("/api/system/environment", () => { const config = new TestConfiguration() From b4f0423ca331c31ab95493be11460309b2cc4c8c Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Tue, 12 Dec 2023 14:43:53 +0000 Subject: [PATCH 07/25] Respond to Adri's feedback. --- packages/worker/src/tests/TestConfiguration.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/worker/src/tests/TestConfiguration.ts b/packages/worker/src/tests/TestConfiguration.ts index 747b520296..8e163f0373 100644 --- a/packages/worker/src/tests/TestConfiguration.ts +++ b/packages/worker/src/tests/TestConfiguration.ts @@ -36,7 +36,7 @@ import { } from "@budibase/types" import API from "./api" import jwt, { Secret } from "jsonwebtoken" -import { cloneDeep } from "lodash" +import cloneDeep from "lodash/fp/cloneDeep" class TestConfiguration { server: any From 0a7af05f01874723041930149d2b4e1fb3b2d371 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Tue, 12 Dec 2023 14:46:08 +0000 Subject: [PATCH 08/25] Unmock node-fetch again. --- packages/worker/src/api/routes/system/tests/environment.spec.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/packages/worker/src/api/routes/system/tests/environment.spec.ts b/packages/worker/src/api/routes/system/tests/environment.spec.ts index 9fe707d288..2efbfa07c9 100644 --- a/packages/worker/src/api/routes/system/tests/environment.spec.ts +++ b/packages/worker/src/api/routes/system/tests/environment.spec.ts @@ -1,5 +1,7 @@ import { TestConfiguration } from "../../../../tests" +jest.unmock("node-fetch") + describe("/api/system/environment", () => { const config = new TestConfiguration() From 6098a2d9e8b01a3f5214acb0b10b7c0e1bbac3a1 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Wed, 13 Dec 2023 10:47:16 +0000 Subject: [PATCH 09/25] Split out automation processing from the apps pods. --- charts/budibase/README.md | 10 + .../templates/app-service-deployment.yaml | 2 + .../automation-worker-service-deployment.yaml | 243 ++++++++++++++++++ .../automation-worker-service-hpa.yaml | 32 +++ .../automation-worker-service-service.yaml | 15 ++ charts/budibase/values.yaml | 66 +++++ packages/server/src/api/index.ts | 12 +- 7 files changed, 379 insertions(+), 1 deletion(-) create mode 100644 charts/budibase/templates/automation-worker-service-deployment.yaml create mode 100644 charts/budibase/templates/automation-worker-service-hpa.yaml create mode 100644 charts/budibase/templates/automation-worker-service-service.yaml diff --git a/charts/budibase/README.md b/charts/budibase/README.md index d8191026ce..5fc1b606ad 100644 --- a/charts/budibase/README.md +++ b/charts/budibase/README.md @@ -157,6 +157,16 @@ $ helm install --create-namespace --namespace budibase budibase . -f values.yaml | services.apps.replicaCount | int | `1` | The number of apps replicas to run. | | services.apps.resources | object | `{}` | The resources to use for apps pods. See for more information on how to set these. | | services.apps.startupProbe | object | HTTP health checks. | Startup probe configuration for apps pods. You shouldn't need to change this, but if you want to you can find more information here: | +| services.automationWorkers.autoscaling.enabled | bool | `false` | Whether to enable horizontal pod autoscaling for the apps service. | +| services.automationWorkers.autoscaling.maxReplicas | int | `10` | | +| services.automationWorkers.autoscaling.minReplicas | int | `1` | | +| services.automationWorkers.autoscaling.targetCPUUtilizationPercentage | int | `80` | Target CPU utilization percentage for the automation worker service. Note that for autoscaling to work, you will need to have metrics-server configured, and resources set for the automation worker pods. | +| services.automationWorkers.livenessProbe | object | HTTP health checks. | Liveness probe configuration for automation worker pods. You shouldn't need to change this, but if you want to you can find more information here: | +| services.automationWorkers.logLevel | string | `"info"` | The log level for the automation worker service. | +| services.automationWorkers.readinessProbe | object | HTTP health checks. | Readiness probe configuration for automation worker pods. You shouldn't need to change this, but if you want to you can find more information here: | +| services.automationWorkers.replicaCount | int | `1` | The number of automation worker replicas to run. | +| services.automationWorkers.resources | object | `{}` | The resources to use for automation worker pods. See for more information on how to set these. | +| services.automationWorkers.startupProbe | object | HTTP health checks. | Startup probe configuration for automation worker pods. You shouldn't need to change this, but if you want to you can find more information here: | | services.couchdb.backup.enabled | bool | `false` | Whether or not to enable periodic CouchDB backups. This works by replicating to another CouchDB instance. | | services.couchdb.backup.interval | string | `""` | Backup interval in seconds | | services.couchdb.backup.resources | object | `{}` | The resources to use for CouchDB backup pods. See for more information on how to set these. | diff --git a/charts/budibase/templates/app-service-deployment.yaml b/charts/budibase/templates/app-service-deployment.yaml index 7358e474ca..11651ee8bc 100644 --- a/charts/budibase/templates/app-service-deployment.yaml +++ b/charts/budibase/templates/app-service-deployment.yaml @@ -192,6 +192,8 @@ spec: - name: NODE_TLS_REJECT_UNAUTHORIZED value: {{ .Values.services.tlsRejectUnauthorized }} {{ end }} + - name: APP_FEATURES + value: "api" image: budibase/apps:{{ .Values.globals.appVersion | default .Chart.AppVersion }} imagePullPolicy: Always diff --git a/charts/budibase/templates/automation-worker-service-deployment.yaml b/charts/budibase/templates/automation-worker-service-deployment.yaml new file mode 100644 index 0000000000..440fcc4e2a --- /dev/null +++ b/charts/budibase/templates/automation-worker-service-deployment.yaml @@ -0,0 +1,243 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + annotations: +{{ if .Values.services.automationWorkers.deploymentAnnotations }} +{{- toYaml .Values.services.automationWorkers.deploymentAnnotations | indent 4 -}} +{{ end }} + labels: + io.kompose.service: automation-worker-service +{{ if .Values.services.automationWorkers.deploymentLabels }} +{{- toYaml .Values.services.automationWorkers.deploymentLabels | indent 4 -}} +{{ end }} + name: automation-worker-service +spec: + replicas: {{ .Values.services.automationWorkers.replicaCount }} + selector: + matchLabels: + io.kompose.service: automation-worker-service + strategy: + type: RollingUpdate + template: + metadata: + annotations: +{{ if .Values.services.automationWorkers.templateAnnotations }} +{{- toYaml .Values.services.automationWorkers.templateAnnotations | indent 8 -}} +{{ end }} + labels: + io.kompose.service: automation-worker-service +{{ if .Values.services.automationWorkers.templateLabels }} +{{- toYaml .Values.services.automationWorkers.templateLabels | indent 8 -}} +{{ end }} + spec: + containers: + - env: + - name: BUDIBASE_ENVIRONMENT + value: {{ .Values.globals.budibaseEnv }} + - name: DEPLOYMENT_ENVIRONMENT + value: "kubernetes" + - name: COUCH_DB_URL + {{ if .Values.services.couchdb.url }} + value: {{ .Values.services.couchdb.url }} + {{ else }} + value: http://{{ .Release.Name }}-svc-couchdb:{{ .Values.services.couchdb.port }} + {{ end }} + {{ if .Values.services.couchdb.enabled }} + - name: COUCH_DB_USER + valueFrom: + secretKeyRef: + name: {{ template "couchdb.fullname" . }} + key: adminUsername + - name: COUCH_DB_PASSWORD + valueFrom: + secretKeyRef: + name: {{ template "couchdb.fullname" . }} + key: adminPassword + {{ end }} + - name: ENABLE_ANALYTICS + value: {{ .Values.globals.enableAnalytics | quote }} + - name: API_ENCRYPTION_KEY + value: {{ .Values.globals.apiEncryptionKey | quote }} + - name: HTTP_LOGGING + value: {{ .Values.services.automationWorkers.httpLogging | quote }} + - name: INTERNAL_API_KEY + valueFrom: + secretKeyRef: + name: {{ template "budibase.fullname" . }} + key: internalApiKey + - name: INTERNAL_API_KEY_FALLBACK + value: {{ .Values.globals.internalApiKeyFallback | quote }} + - name: JWT_SECRET + valueFrom: + secretKeyRef: + name: {{ template "budibase.fullname" . }} + key: jwtSecret + - name: JWT_SECRET_FALLBACK + value: {{ .Values.globals.jwtSecretFallback | quote }} + {{ if .Values.services.objectStore.region }} + - name: AWS_REGION + value: {{ .Values.services.objectStore.region }} + {{ end }} + - name: MINIO_ENABLED + value: {{ .Values.services.objectStore.minio | quote }} + - name: MINIO_ACCESS_KEY + valueFrom: + secretKeyRef: + name: {{ template "budibase.fullname" . }} + key: objectStoreAccess + - name: MINIO_SECRET_KEY + valueFrom: + secretKeyRef: + name: {{ template "budibase.fullname" . }} + key: objectStoreSecret + - name: CLOUDFRONT_CDN + value: {{ .Values.services.objectStore.cloudfront.cdn | quote }} + - name: CLOUDFRONT_PUBLIC_KEY_ID + value: {{ .Values.services.objectStore.cloudfront.publicKeyId | quote }} + - name: CLOUDFRONT_PRIVATE_KEY_64 + value: {{ .Values.services.objectStore.cloudfront.privateKey64 | quote }} + - name: MINIO_URL + value: {{ .Values.services.objectStore.url }} + - name: PLUGIN_BUCKET_NAME + value: {{ .Values.services.objectStore.pluginBucketName | quote }} + - name: APPS_BUCKET_NAME + value: {{ .Values.services.objectStore.appsBucketName | quote }} + - name: GLOBAL_BUCKET_NAME + value: {{ .Values.services.objectStore.globalBucketName | quote }} + - name: BACKUPS_BUCKET_NAME + value: {{ .Values.services.objectStore.backupsBucketName | quote }} + - name: PORT + value: {{ .Values.services.automationWorkers.port | quote }} + {{ if .Values.services.worker.publicApiRateLimitPerSecond }} + - name: API_REQ_LIMIT_PER_SEC + value: {{ .Values.globals.automationWorkers.publicApiRateLimitPerSecond | quote }} + {{ end }} + - name: MULTI_TENANCY + value: {{ .Values.globals.multiTenancy | quote }} + - name: OFFLINE_MODE + value: {{ .Values.globals.offlineMode | quote }} + - name: LOG_LEVEL + value: {{ .Values.services.automationWorkers.logLevel | quote }} + - name: REDIS_PASSWORD + value: {{ .Values.services.redis.password }} + - name: REDIS_URL + {{ if .Values.services.redis.url }} + value: {{ .Values.services.redis.url }} + {{ else }} + value: redis-service:{{ .Values.services.redis.port }} + {{ end }} + - name: SELF_HOSTED + value: {{ .Values.globals.selfHosted | quote }} + - name: POSTHOG_TOKEN + value: {{ .Values.globals.posthogToken | quote }} + - name: WORKER_URL + value: http://worker-service:{{ .Values.services.worker.port }} + - name: PLATFORM_URL + value: {{ .Values.globals.platformUrl | quote }} + - name: ACCOUNT_PORTAL_URL + value: {{ .Values.globals.accountPortalUrl | quote }} + - name: ACCOUNT_PORTAL_API_KEY + value: {{ .Values.globals.accountPortalApiKey | quote }} + - name: COOKIE_DOMAIN + value: {{ .Values.globals.cookieDomain | quote }} + - name: HTTP_MIGRATIONS + value: {{ .Values.globals.httpMigrations | quote }} + - name: GOOGLE_CLIENT_ID + value: {{ .Values.globals.google.clientId | quote }} + - name: GOOGLE_CLIENT_SECRET + value: {{ .Values.globals.google.secret | quote }} + - name: AUTOMATION_MAX_ITERATIONS + value: {{ .Values.globals.automationMaxIterations | quote }} + - name: TENANT_FEATURE_FLAGS + value: {{ .Values.globals.tenantFeatureFlags | quote }} + - name: ENCRYPTION_KEY + value: {{ .Values.globals.bbEncryptionKey | quote }} + {{ if .Values.globals.bbAdminUserEmail }} + - name: BB_ADMIN_USER_EMAIL + value: {{ .Values.globals.bbAdminUserEmail | quote }} + {{ end }} + {{ if .Values.globals.bbAdminUserPassword }} + - name: BB_ADMIN_USER_PASSWORD + value: {{ .Values.globals.bbAdminUserPassword | quote }} + {{ end }} + {{ if .Values.globals.pluginsDir }} + - name: PLUGINS_DIR + value: {{ .Values.globals.pluginsDir | quote }} + {{ end }} + {{ if .Values.services.automationWorkers.nodeDebug }} + - name: NODE_DEBUG + value: {{ .Values.services.automationWorkers.nodeDebug | quote }} + {{ end }} + {{ if .Values.globals.datadogApmEnabled }} + - name: DD_LOGS_INJECTION + value: {{ .Values.globals.datadogApmEnabled | quote }} + - name: DD_APM_ENABLED + value: {{ .Values.globals.datadogApmEnabled | quote }} + - name: DD_APM_DD_URL + value: https://trace.agent.datadoghq.eu + {{ end }} + {{ if .Values.globals.globalAgentHttpProxy }} + - name: GLOBAL_AGENT_HTTP_PROXY + value: {{ .Values.globals.globalAgentHttpProxy | quote }} + {{ end }} + {{ if .Values.globals.globalAgentHttpsProxy }} + - name: GLOBAL_AGENT_HTTPS_PROXY + value: {{ .Values.globals.globalAgentHttpsProxy | quote }} + {{ end }} + {{ if .Values.globals.globalAgentNoProxy }} + - name: GLOBAL_AGENT_NO_PROXY + value: {{ .Values.globals.globalAgentNoProxy | quote }} + {{ end }} + {{ if .Values.services.tlsRejectUnauthorized }} + - name: NODE_TLS_REJECT_UNAUTHORIZED + value: {{ .Values.services.tlsRejectUnauthorized }} + {{ end }} + - name: APP_FEATURES + # We keep `api` enabled here for the /health endpoint + value: "automations,api" + + image: budibase/apps:{{ .Values.globals.appVersion | default .Chart.AppVersion }} + imagePullPolicy: Always + {{- if .Values.services.automationWorkers.startupProbe }} + {{- with .Values.services.automationWorkers.startupProbe }} + startupProbe: + {{- toYaml . | nindent 10 }} + {{- end }} + {{- end }} + {{- if .Values.services.automationWorkers.livenessProbe }} + {{- with .Values.services.automationWorkers.livenessProbe }} + livenessProbe: + {{- toYaml . | nindent 10 }} + {{- end }} + {{- end }} + {{- if .Values.services.automationWorkers.readinessProbe }} + {{- with .Values.services.automationWorkers.readinessProbe }} + readinessProbe: + {{- toYaml . | nindent 10 }} + {{- end }} + {{- end }} + name: bbapps + ports: + - containerPort: {{ .Values.services.automationWorkers.port }} + {{ with .Values.services.automationWorkers.resources }} + resources: + {{- toYaml . | nindent 10 }} + {{ end }} + {{- with .Values.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} + {{ if .Values.schedulerName }} + schedulerName: {{ .Values.schedulerName | quote }} + {{ end }} + {{ if .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml .Values.imagePullSecrets | nindent 6 }} + {{ end }} + restartPolicy: Always + serviceAccountName: "" +status: {} diff --git a/charts/budibase/templates/automation-worker-service-hpa.yaml b/charts/budibase/templates/automation-worker-service-hpa.yaml new file mode 100644 index 0000000000..f29223b61b --- /dev/null +++ b/charts/budibase/templates/automation-worker-service-hpa.yaml @@ -0,0 +1,32 @@ +{{- if .Values.services.automationWorkers.autoscaling.enabled }} +apiVersion: {{ ternary "autoscaling/v2" "autoscaling/v2beta2" (.Capabilities.APIVersions.Has "autoscaling/v2") }} +kind: HorizontalPodAutoscaler +metadata: + name: {{ include "budibase.fullname" . }}-apps + labels: + {{- include "budibase.labels" . | nindent 4 }} +spec: + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: automation-worker-service + minReplicas: {{ .Values.services.automationWorkers.autoscaling.minReplicas }} + maxReplicas: {{ .Values.services.automationWorkers.autoscaling.maxReplicas }} + metrics: + {{- if .Values.services.automationWorkers.autoscaling.targetCPUUtilizationPercentage }} + - type: Resource + resource: + name: cpu + target: + type: Utilization + averageUtilization: {{ .Values.services.automationWorkers.autoscaling.targetCPUUtilizationPercentage }} + {{- end }} + {{- if .Values.services.automationWorkers.autoscaling.targetMemoryUtilizationPercentage }} + - type: Resource + resource: + name: memory + target: + type: Utilization + averageUtilization: {{ .Values.services.automationWorkers.autoscaling.targetMemoryUtilizationPercentage }} + {{- end }} +{{- end }} diff --git a/charts/budibase/templates/automation-worker-service-service.yaml b/charts/budibase/templates/automation-worker-service-service.yaml new file mode 100644 index 0000000000..1b65227c60 --- /dev/null +++ b/charts/budibase/templates/automation-worker-service-service.yaml @@ -0,0 +1,15 @@ +apiVersion: v1 +kind: Service +metadata: + labels: + io.kompose.service: automation-worker-service + name: automation-worker-service +spec: + ports: + - name: {{ .Values.services.automationWorkers.port | quote }} + port: {{ .Values.services.automationWorkers.port }} + targetPort: {{ .Values.services.automationWorkers.port }} + selector: + io.kompose.service: automation-worker-service +status: + loadBalancer: {} diff --git a/charts/budibase/values.yaml b/charts/budibase/values.yaml index 13054e75fc..0fcc599d23 100644 --- a/charts/budibase/values.yaml +++ b/charts/budibase/values.yaml @@ -272,6 +272,72 @@ services: # and resources set for the apps pods. targetCPUUtilizationPercentage: 80 + automationWorkers: + # @ignore (you shouldn't need to change this) + port: 4002 + # -- The number of automation worker replicas to run. + replicaCount: 1 + # -- The log level for the automation worker service. + logLevel: info + # -- The resources to use for automation worker pods. See + # + # for more information on how to set these. + resources: {} + # -- Startup probe configuration for automation worker pods. You shouldn't + # need to change this, but if you want to you can find more information + # here: + # + # @default -- HTTP health checks. + startupProbe: + # @ignore + httpGet: + path: /health + port: 4002 + scheme: HTTP + # @ignore + failureThreshold: 30 + # @ignore + periodSeconds: 3 + # -- Readiness probe configuration for automation worker pods. You shouldn't + # need to change this, but if you want to you can find more information + # here: + # + # @default -- HTTP health checks. + readinessProbe: + # @ignore + httpGet: + path: /health + port: 4002 + scheme: HTTP + # @ignore + periodSeconds: 3 + # @ignore + failureThreshold: 1 + # -- Liveness probe configuration for automation worker pods. You shouldn't + # need to change this, but if you want to you can find more information + # here: + # + # @default -- HTTP health checks. + livenessProbe: + # @ignore + httpGet: + path: /health + port: 4002 + scheme: HTTP + # @ignore + failureThreshold: 3 + # @ignore + periodSeconds: 30 + autoscaling: + # -- Whether to enable horizontal pod autoscaling for the apps service. + enabled: false + minReplicas: 1 + maxReplicas: 10 + # -- Target CPU utilization percentage for the automation worker service. + # Note that for autoscaling to work, you will need to have metrics-server + # configured, and resources set for the automation worker pods. + targetCPUUtilizationPercentage: 80 + worker: # @ignore (you shouldn't need to change this) port: 4003 diff --git a/packages/server/src/api/index.ts b/packages/server/src/api/index.ts index a01e3764f0..444a346e72 100644 --- a/packages/server/src/api/index.ts +++ b/packages/server/src/api/index.ts @@ -5,13 +5,23 @@ import zlib from "zlib" import { mainRoutes, staticRoutes, publicRoutes } from "./routes" import { middleware as pro } from "@budibase/pro" import migrations from "../middleware/appMigrations" +import { automationsEnabled } from "src/features" +import { automationQueue } from "src/automations" export { shutdown } from "./routes/public" const compress = require("koa-compress") export const router: Router = new Router() -router.get("/health", ctx => (ctx.status = 200)) +router.get("/health", async ctx => { + if (automationsEnabled()) { + if (!(await automationQueue.isReady())) { + ctx.status = 503 + return + } + } + ctx.status = 200 +}) router.get("/version", ctx => (ctx.body = envCore.VERSION)) router.use(middleware.errorHandling) From ff04d0516ad9097ec335de965b2e9933e8f9f556 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Wed, 13 Dec 2023 10:59:35 +0000 Subject: [PATCH 10/25] Disable API completely in automations workers. --- .../templates/automation-worker-service-deployment.yaml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/charts/budibase/templates/automation-worker-service-deployment.yaml b/charts/budibase/templates/automation-worker-service-deployment.yaml index 440fcc4e2a..83071f17ac 100644 --- a/charts/budibase/templates/automation-worker-service-deployment.yaml +++ b/charts/budibase/templates/automation-worker-service-deployment.yaml @@ -193,8 +193,7 @@ spec: value: {{ .Values.services.tlsRejectUnauthorized }} {{ end }} - name: APP_FEATURES - # We keep `api` enabled here for the /health endpoint - value: "automations,api" + value: "automations" image: budibase/apps:{{ .Values.globals.appVersion | default .Chart.AppVersion }} imagePullPolicy: Always From dbaa2699ca163e2a3da96bb8d5e4f9437953ad29 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Wed, 13 Dec 2023 11:10:58 +0000 Subject: [PATCH 11/25] Fix broken imports. --- packages/server/src/api/index.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/server/src/api/index.ts b/packages/server/src/api/index.ts index 444a346e72..9eba33126e 100644 --- a/packages/server/src/api/index.ts +++ b/packages/server/src/api/index.ts @@ -5,8 +5,8 @@ import zlib from "zlib" import { mainRoutes, staticRoutes, publicRoutes } from "./routes" import { middleware as pro } from "@budibase/pro" import migrations from "../middleware/appMigrations" -import { automationsEnabled } from "src/features" -import { automationQueue } from "src/automations" +import { automationsEnabled } from "../features" +import { automationQueue } from "../automations" export { shutdown } from "./routes/public" const compress = require("koa-compress") From d9df2d34c88498067f86d1b89e7929bbdf738fbd Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Wed, 13 Dec 2023 11:23:46 +0000 Subject: [PATCH 12/25] Adding Koa API back in when automation feature only enabled, so that health check can still be provided, but rest of API is disabled. --- packages/server/src/api/index.ts | 94 +++++++++++++++++--------------- packages/server/src/app.ts | 10 +--- packages/server/src/features.ts | 7 +++ packages/server/src/startup.ts | 21 ++++--- 4 files changed, 73 insertions(+), 59 deletions(-) diff --git a/packages/server/src/api/index.ts b/packages/server/src/api/index.ts index a01e3764f0..56c61ce85a 100644 --- a/packages/server/src/api/index.ts +++ b/packages/server/src/api/index.ts @@ -4,6 +4,7 @@ import currentApp from "../middleware/currentapp" import zlib from "zlib" import { mainRoutes, staticRoutes, publicRoutes } from "./routes" import { middleware as pro } from "@budibase/pro" +import { apiEnabled } from "../features" import migrations from "../middleware/appMigrations" export { shutdown } from "./routes/public" @@ -16,50 +17,53 @@ router.get("/version", ctx => (ctx.body = envCore.VERSION)) router.use(middleware.errorHandling) -router - .use( - compress({ - threshold: 2048, - gzip: { - flush: zlib.constants.Z_SYNC_FLUSH, - }, - deflate: { - flush: zlib.constants.Z_SYNC_FLUSH, - }, - br: false, - }) - ) - // re-direct before any middlewares occur - .redirect("/", "/builder") - .use( - auth.buildAuthMiddleware([], { - publicAllowed: true, - }) - ) - // nothing in the server should allow query string tenants - // the server can be public anywhere, so nowhere should throw errors - // if the tenancy has not been set, it'll have to be discovered at application layer - .use( - auth.buildTenancyMiddleware([], [], { - noTenancyRequired: true, - }) - ) - .use(pro.licensing()) - // @ts-ignore - .use(currentApp) - .use(auth.auditLog) - // @ts-ignore - .use(migrations) +// only add the routes if they are enabled +if (apiEnabled()) { + router + .use( + compress({ + threshold: 2048, + gzip: { + flush: zlib.constants.Z_SYNC_FLUSH, + }, + deflate: { + flush: zlib.constants.Z_SYNC_FLUSH, + }, + br: false, + }) + ) + // re-direct before any middlewares occur + .redirect("/", "/builder") + .use( + auth.buildAuthMiddleware([], { + publicAllowed: true, + }) + ) + // nothing in the server should allow query string tenants + // the server can be public anywhere, so nowhere should throw errors + // if the tenancy has not been set, it'll have to be discovered at application layer + .use( + auth.buildTenancyMiddleware([], [], { + noTenancyRequired: true, + }) + ) + .use(pro.licensing()) + // @ts-ignore + .use(currentApp) + .use(auth.auditLog) + // @ts-ignore + .use(migrations) -// authenticated routes -for (let route of mainRoutes) { - router.use(route.routes()) - router.use(route.allowedMethods()) + // authenticated routes + for (let route of mainRoutes) { + router.use(route.routes()) + router.use(route.allowedMethods()) + } + + router.use(publicRoutes.routes()) + router.use(publicRoutes.allowedMethods()) + + // WARNING - static routes will catch everything else after them this must be last + router.use(staticRoutes.routes()) + router.use(staticRoutes.allowedMethods()) } - -router.use(publicRoutes.routes()) -router.use(publicRoutes.allowedMethods()) - -// WARNING - static routes will catch everything else after them this must be last -router.use(staticRoutes.routes()) -router.use(staticRoutes.allowedMethods()) diff --git a/packages/server/src/app.ts b/packages/server/src/app.ts index 4c0068be89..f6f1780030 100644 --- a/packages/server/src/app.ts +++ b/packages/server/src/app.ts @@ -9,7 +9,6 @@ import { ServiceType } from "@budibase/types" import { env as coreEnv } from "@budibase/backend-core" coreEnv._set("SERVICE_TYPE", ServiceType.APPS) -import { apiEnabled } from "./features" import createKoaApp from "./koa" import Koa from "koa" import { Server } from "http" @@ -18,12 +17,9 @@ import { startup } from "./startup" let app: Koa, server: Server async function start() { - // if API disabled, could run automations instead - if (apiEnabled()) { - const koa = createKoaApp() - app = koa.app - server = koa.server - } + const koa = createKoaApp() + app = koa.app + server = koa.server // startup includes automation runner - if enabled await startup(app, server) } diff --git a/packages/server/src/features.ts b/packages/server/src/features.ts index e12260ea32..f040cf82a2 100644 --- a/packages/server/src/features.ts +++ b/packages/server/src/features.ts @@ -22,3 +22,10 @@ export function automationsEnabled() { export function apiEnabled() { return featureList.includes(AppFeature.API) } + +export function printFeatures() { + if (!env.APP_FEATURES) { + return + } + console.log(`**** APP FEATURES SET: ${featureList.join(", ")} ****`) +} diff --git a/packages/server/src/startup.ts b/packages/server/src/startup.ts index 9144ff2b36..2db6e5ae6a 100644 --- a/packages/server/src/startup.ts +++ b/packages/server/src/startup.ts @@ -19,11 +19,14 @@ import * as pro from "@budibase/pro" import * as api from "./api" import sdk from "./sdk" import { initialise as initialiseWebsockets } from "./websockets" -import { automationsEnabled } from "./features" +import { automationsEnabled, printFeatures } from "./features" +import Koa from "koa" +import { Server } from "http" +import { AddressInfo } from "net" let STARTUP_RAN = false -async function initRoutes(app: any) { +async function initRoutes(app: Koa) { if (!env.isTest()) { const plugin = await bullboard.init() app.use(plugin) @@ -48,27 +51,31 @@ async function initPro() { }) } -function shutdown(server?: any) { +function shutdown(server?: Server) { if (server) { server.close() server.destroy() } } -export async function startup(app?: any, server?: any) { +export async function startup(app?: Koa, server?: Server) { if (STARTUP_RAN) { return } + printFeatures() STARTUP_RAN = true - if (server && !env.CLUSTER_MODE) { + if (app && server && !env.CLUSTER_MODE) { console.log(`Budibase running on ${JSON.stringify(server.address())}`) - env._set("PORT", server.address().port) + const address = server.address() as AddressInfo + env._set("PORT", address.port) } eventEmitter.emitPort(env.PORT) fileSystem.init() await redis.init() eventInit() - initialiseWebsockets(app, server) + if (app && server) { + initialiseWebsockets(app, server) + } // run migrations on startup if not done via http // not recommended in a clustered environment From 4934b4a4282034fe140915e1825e6516f9cf54b5 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Wed, 13 Dec 2023 11:58:56 +0000 Subject: [PATCH 13/25] Fix broken test. --- packages/backend-core/src/queue/inMemoryQueue.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/packages/backend-core/src/queue/inMemoryQueue.ts b/packages/backend-core/src/queue/inMemoryQueue.ts index a8add7ecb6..ac7cdf550b 100644 --- a/packages/backend-core/src/queue/inMemoryQueue.ts +++ b/packages/backend-core/src/queue/inMemoryQueue.ts @@ -68,6 +68,10 @@ class InMemoryQueue { }) } + async isReady() { + return true + } + // simply puts a message to the queue and emits to the queue for processing /** * Simple function to replicate the add message functionality of Bull, putting From 116b1ce9098b2f07aec1ea5502c8a12701b6ec0e Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Wed, 13 Dec 2023 13:14:13 +0100 Subject: [PATCH 14/25] Ensure .env is created properly --- packages/server/scripts/dev/manage.js | 75 +++++++++++++-------------- packages/worker/scripts/dev/manage.js | 70 ++++++++++++------------- 2 files changed, 69 insertions(+), 76 deletions(-) diff --git a/packages/server/scripts/dev/manage.js b/packages/server/scripts/dev/manage.js index b469c1ffc7..6dc0966f78 100644 --- a/packages/server/scripts/dev/manage.js +++ b/packages/server/scripts/dev/manage.js @@ -1,7 +1,8 @@ #!/usr/bin/env node const compose = require("docker-compose") const path = require("path") -const fs = require("fs") +const { parsed: existingConfig } = require("dotenv").config() +const updateDotEnv = require("update-dotenv") // This script wraps docker-compose allowing you to manage your dev infrastructure with simple commands. const CONFIG = { @@ -17,45 +18,41 @@ const Commands = { } async function init() { - const envFilePath = path.join(process.cwd(), ".env") - if (!fs.existsSync(envFilePath)) { - const envFileJson = { - PORT: 4001, - MINIO_URL: "http://localhost:4004", - COUCH_DB_URL: "http://budibase:budibase@localhost:4005", - REDIS_URL: "localhost:6379", - WORKER_URL: "http://localhost:4002", - INTERNAL_API_KEY: "budibase", - ACCOUNT_PORTAL_URL: "http://localhost:10001", - ACCOUNT_PORTAL_API_KEY: "budibase", - PLATFORM_URL: "http://localhost:10000", - JWT_SECRET: "testsecret", - ENCRYPTION_KEY: "testsecret", - REDIS_PASSWORD: "budibase", - MINIO_ACCESS_KEY: "budibase", - MINIO_SECRET_KEY: "budibase", - COUCH_DB_PASSWORD: "budibase", - COUCH_DB_USER: "budibase", - SELF_HOSTED: 1, - DISABLE_ACCOUNT_PORTAL: 1, - MULTI_TENANCY: "", - DISABLE_THREADING: 1, - SERVICE: "app-service", - DEPLOYMENT_ENVIRONMENT: "development", - BB_ADMIN_USER_EMAIL: "", - BB_ADMIN_USER_PASSWORD: "", - PLUGINS_DIR: "", - TENANT_FEATURE_FLAGS: "*:LICENSING,*:USER_GROUPS,*:ONBOARDING_TOUR", - HTTP_MIGRATIONS: "0", - HTTP_LOGGING: "0", - VERSION: "0.0.0+local", - } - let envFile = "" - Object.keys(envFileJson).forEach(key => { - envFile += `${key}=${envFileJson[key]}\n` - }) - fs.writeFileSync(envFilePath, envFile) + let config = { + PORT: "4001", + MINIO_URL: "http://localhost:4004", + COUCH_DB_URL: "http://budibase:budibase@localhost:4005", + REDIS_URL: "localhost:6379", + WORKER_URL: "http://localhost:4002", + INTERNAL_API_KEY: "budibase", + ACCOUNT_PORTAL_URL: "http://localhost:10001", + ACCOUNT_PORTAL_API_KEY: "budibase", + PLATFORM_URL: "http://localhost:10000", + JWT_SECRET: "testsecret", + ENCRYPTION_KEY: "testsecret", + REDIS_PASSWORD: "budibase", + MINIO_ACCESS_KEY: "budibase", + MINIO_SECRET_KEY: "budibase", + COUCH_DB_PASSWORD: "budibase", + COUCH_DB_USER: "budibase", + SELF_HOSTED: "1", + DISABLE_ACCOUNT_PORTAL: "1", + MULTI_TENANCY: "", + DISABLE_THREADING: "1", + SERVICE: "app-service", + DEPLOYMENT_ENVIRONMENT: "development", + BB_ADMIN_USER_EMAIL: "", + BB_ADMIN_USER_PASSWORD: "", + PLUGINS_DIR: "", + TENANT_FEATURE_FLAGS: "*:LICENSING,*:USER_GROUPS,*:ONBOARDING_TOUR", + HTTP_MIGRATIONS: "0", + HTTP_LOGGING: "0", + VERSION: "0.0.0+local", } + + config = { ...config, ...existingConfig } + + await updateDotEnv(config) } async function up() { diff --git a/packages/worker/scripts/dev/manage.js b/packages/worker/scripts/dev/manage.js index 9e6a57d4bf..1b7c6f0ddd 100644 --- a/packages/worker/scripts/dev/manage.js +++ b/packages/worker/scripts/dev/manage.js @@ -1,44 +1,40 @@ #!/usr/bin/env node -const path = require("path") -const fs = require("fs") +const { parsed: existingConfig } = require("dotenv").config() +const updateDotEnv = require("update-dotenv") async function init() { - const envFilePath = path.join(process.cwd(), ".env") - if (!fs.existsSync(envFilePath)) { - const envFileJson = { - SELF_HOSTED: 1, - PORT: 4002, - CLUSTER_PORT: 10000, - JWT_SECRET: "testsecret", - INTERNAL_API_KEY: "budibase", - MINIO_ACCESS_KEY: "budibase", - MINIO_SECRET_KEY: "budibase", - REDIS_URL: "localhost:6379", - REDIS_PASSWORD: "budibase", - MINIO_URL: "http://localhost:4004", - COUCH_DB_URL: "http://budibase:budibase@localhost:4005", - COUCH_DB_USERNAME: "budibase", - COUCH_DB_PASSWORD: "budibase", - // empty string is false - MULTI_TENANCY: "", - DISABLE_ACCOUNT_PORTAL: 1, - ACCOUNT_PORTAL_URL: "http://localhost:10001", - ACCOUNT_PORTAL_API_KEY: "budibase", - PLATFORM_URL: "http://localhost:10000", - APPS_URL: "http://localhost:4001", - SERVICE: "worker-service", - DEPLOYMENT_ENVIRONMENT: "development", - TENANT_FEATURE_FLAGS: "*:LICENSING,*:USER_GROUPS,*:ONBOARDING_TOUR", - ENABLE_EMAIL_TEST_MODE: 1, - HTTP_LOGGING: 0, - VERSION: "0.0.0+local", - } - let envFile = "" - Object.keys(envFileJson).forEach(key => { - envFile += `${key}=${envFileJson[key]}\n` - }) - fs.writeFileSync(envFilePath, envFile) + let config = { + SELF_HOSTED: "1", + PORT: "4002", + CLUSTER_PORT: "10000", + JWT_SECRET: "testsecret", + INTERNAL_API_KEY: "budibase", + MINIO_ACCESS_KEY: "budibase", + MINIO_SECRET_KEY: "budibase", + REDIS_URL: "localhost:6379", + REDIS_PASSWORD: "budibase", + MINIO_URL: "http://localhost:4004", + COUCH_DB_URL: "http://budibase:budibase@localhost:4005", + COUCH_DB_USERNAME: "budibase", + COUCH_DB_PASSWORD: "budibase", + // empty string is false + MULTI_TENANCY: "", + DISABLE_ACCOUNT_PORTAL: "1", + ACCOUNT_PORTAL_URL: "http://localhost:10001", + ACCOUNT_PORTAL_API_KEY: "budibase", + PLATFORM_URL: "http://localhost:10000", + APPS_URL: "http://localhost:4001", + SERVICE: "worker-service", + DEPLOYMENT_ENVIRONMENT: "development", + TENANT_FEATURE_FLAGS: "*:LICENSING,*:USER_GROUPS,*:ONBOARDING_TOUR", + ENABLE_EMAIL_TEST_MODE: "1", + HTTP_LOGGING: "0", + VERSION: "0.0.0+local", } + + config = { ...config, ...existingConfig } + + await updateDotEnv(config) } // if more than init required use this to determine the command type From 51c328996837a52fd04e311295208d9c0c334180 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Wed, 13 Dec 2023 12:30:11 +0000 Subject: [PATCH 15/25] Fix lint error. --- packages/server/src/api/index.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/packages/server/src/api/index.ts b/packages/server/src/api/index.ts index f4a9875855..ad3d8307da 100644 --- a/packages/server/src/api/index.ts +++ b/packages/server/src/api/index.ts @@ -4,9 +4,8 @@ import currentApp from "../middleware/currentapp" import zlib from "zlib" import { mainRoutes, staticRoutes, publicRoutes } from "./routes" import { middleware as pro } from "@budibase/pro" -import { apiEnabled } from "../features" +import { apiEnabled, automationsEnabled } from "../features" import migrations from "../middleware/appMigrations" -import { automationsEnabled } from "../features" import { automationQueue } from "../automations" export { shutdown } from "./routes/public" From d164a3788db9f4b8a0f389c944e9bcc84c9ac456 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Wed, 13 Dec 2023 12:36:24 +0000 Subject: [PATCH 16/25] Make it possible to disable automation workers. --- charts/budibase/README.md | 1 + .../templates/app-service-deployment.yaml | 3 +++ .../automation-worker-service-deployment.yaml | 2 ++ .../automation-worker-service-service.yaml | 15 --------------- charts/budibase/values.yaml | 3 +++ 5 files changed, 9 insertions(+), 15 deletions(-) delete mode 100644 charts/budibase/templates/automation-worker-service-service.yaml diff --git a/charts/budibase/README.md b/charts/budibase/README.md index 5fc1b606ad..342011bdb1 100644 --- a/charts/budibase/README.md +++ b/charts/budibase/README.md @@ -161,6 +161,7 @@ $ helm install --create-namespace --namespace budibase budibase . -f values.yaml | services.automationWorkers.autoscaling.maxReplicas | int | `10` | | | services.automationWorkers.autoscaling.minReplicas | int | `1` | | | services.automationWorkers.autoscaling.targetCPUUtilizationPercentage | int | `80` | Target CPU utilization percentage for the automation worker service. Note that for autoscaling to work, you will need to have metrics-server configured, and resources set for the automation worker pods. | +| services.automationWorkers.enabled | bool | `true` | Whether or not to enable the automation worker service. If you disable this, automations will be processed by the apps service. | | services.automationWorkers.livenessProbe | object | HTTP health checks. | Liveness probe configuration for automation worker pods. You shouldn't need to change this, but if you want to you can find more information here: | | services.automationWorkers.logLevel | string | `"info"` | The log level for the automation worker service. | | services.automationWorkers.readinessProbe | object | HTTP health checks. | Readiness probe configuration for automation worker pods. You shouldn't need to change this, but if you want to you can find more information here: | diff --git a/charts/budibase/templates/app-service-deployment.yaml b/charts/budibase/templates/app-service-deployment.yaml index 11651ee8bc..99d278ef97 100644 --- a/charts/budibase/templates/app-service-deployment.yaml +++ b/charts/budibase/templates/app-service-deployment.yaml @@ -192,8 +192,11 @@ spec: - name: NODE_TLS_REJECT_UNAUTHORIZED value: {{ .Values.services.tlsRejectUnauthorized }} {{ end }} + {{ }} + {{- if .Values.services.automationWorkers.enabled }} - name: APP_FEATURES value: "api" + {{- end }} image: budibase/apps:{{ .Values.globals.appVersion | default .Chart.AppVersion }} imagePullPolicy: Always diff --git a/charts/budibase/templates/automation-worker-service-deployment.yaml b/charts/budibase/templates/automation-worker-service-deployment.yaml index 83071f17ac..a83542607a 100644 --- a/charts/budibase/templates/automation-worker-service-deployment.yaml +++ b/charts/budibase/templates/automation-worker-service-deployment.yaml @@ -1,3 +1,4 @@ +{{- if .Values.services.automationWorkers.enabled }} apiVersion: apps/v1 kind: Deployment metadata: @@ -240,3 +241,4 @@ spec: restartPolicy: Always serviceAccountName: "" status: {} +{{- end }} \ No newline at end of file diff --git a/charts/budibase/templates/automation-worker-service-service.yaml b/charts/budibase/templates/automation-worker-service-service.yaml deleted file mode 100644 index 1b65227c60..0000000000 --- a/charts/budibase/templates/automation-worker-service-service.yaml +++ /dev/null @@ -1,15 +0,0 @@ -apiVersion: v1 -kind: Service -metadata: - labels: - io.kompose.service: automation-worker-service - name: automation-worker-service -spec: - ports: - - name: {{ .Values.services.automationWorkers.port | quote }} - port: {{ .Values.services.automationWorkers.port }} - targetPort: {{ .Values.services.automationWorkers.port }} - selector: - io.kompose.service: automation-worker-service -status: - loadBalancer: {} diff --git a/charts/budibase/values.yaml b/charts/budibase/values.yaml index 0fcc599d23..6dbcc20242 100644 --- a/charts/budibase/values.yaml +++ b/charts/budibase/values.yaml @@ -273,6 +273,9 @@ services: targetCPUUtilizationPercentage: 80 automationWorkers: + # -- Whether or not to enable the automation worker service. If you disable this, + # automations will be processed by the apps service. + enabled: true # @ignore (you shouldn't need to change this) port: 4002 # -- The number of automation worker replicas to run. From 33818daea9cce66a24d848416d9352ba2926ba94 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Wed, 13 Dec 2023 12:37:08 +0000 Subject: [PATCH 17/25] Fix whitespace. --- charts/budibase/templates/app-service-deployment.yaml | 2 -- 1 file changed, 2 deletions(-) diff --git a/charts/budibase/templates/app-service-deployment.yaml b/charts/budibase/templates/app-service-deployment.yaml index 99d278ef97..495d11aadc 100644 --- a/charts/budibase/templates/app-service-deployment.yaml +++ b/charts/budibase/templates/app-service-deployment.yaml @@ -192,12 +192,10 @@ spec: - name: NODE_TLS_REJECT_UNAUTHORIZED value: {{ .Values.services.tlsRejectUnauthorized }} {{ end }} - {{ }} {{- if .Values.services.automationWorkers.enabled }} - name: APP_FEATURES value: "api" {{- end }} - image: budibase/apps:{{ .Values.globals.appVersion | default .Chart.AppVersion }} imagePullPolicy: Always {{- if .Values.services.apps.startupProbe }} From fac6646ed8813edba5267d7875ad592710ae7876 Mon Sep 17 00:00:00 2001 From: Budibase Staging Release Bot <> Date: Wed, 13 Dec 2023 14:51:14 +0000 Subject: [PATCH 18/25] Bump version to 2.13.38 --- lerna.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lerna.json b/lerna.json index caae7fbaaa..3d309f0881 100644 --- a/lerna.json +++ b/lerna.json @@ -1,5 +1,5 @@ { - "version": "2.13.37", + "version": "2.13.38", "npmClient": "yarn", "packages": [ "packages/*", @@ -22,4 +22,4 @@ "loadEnvFiles": false } } -} +} \ No newline at end of file From c4af5214ef9f8857ae767ea7cc2f49a58f62aada Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Wed, 13 Dec 2023 15:08:23 +0000 Subject: [PATCH 19/25] Improving types around the writethrough cache, exposing the proper typing from the database and removing a log statement which is useless, errors are correctly propogated if they matter (and handled above this level with proper context) or in the 99% scenario it is not a real error (such as try once locks already being held) and a log is just spammy. --- .../src/cache/tests/writethrough.spec.ts | 13 +++++++------ packages/backend-core/src/cache/writethrough.ts | 10 +++++----- packages/backend-core/src/redis/redlockImpl.ts | 1 - packages/pro | 2 +- 4 files changed, 13 insertions(+), 13 deletions(-) diff --git a/packages/backend-core/src/cache/tests/writethrough.spec.ts b/packages/backend-core/src/cache/tests/writethrough.spec.ts index 97d3ece7a6..37887b4bd9 100644 --- a/packages/backend-core/src/cache/tests/writethrough.spec.ts +++ b/packages/backend-core/src/cache/tests/writethrough.spec.ts @@ -1,15 +1,16 @@ import { DBTestConfiguration } from "../../../tests/extra" -import { - structures, - expectFunctionWasCalledTimesWith, - mocks, -} from "../../../tests" +import { structures } from "../../../tests" import { Writethrough } from "../writethrough" import { getDB } from "../../db" +import { Document } from "@budibase/types" import tk from "timekeeper" tk.freeze(Date.now()) +interface ValueDoc extends Document { + value: any +} + const DELAY = 5000 describe("writethrough", () => { @@ -117,7 +118,7 @@ describe("writethrough", () => { describe("get", () => { it("should be able to retrieve", async () => { await config.doInTenant(async () => { - const response = await writethrough.get(docId) + const response = await writethrough.get(docId) expect(response.value).toBe(4) }) }) diff --git a/packages/backend-core/src/cache/writethrough.ts b/packages/backend-core/src/cache/writethrough.ts index c331d791a6..364cc35c67 100644 --- a/packages/backend-core/src/cache/writethrough.ts +++ b/packages/backend-core/src/cache/writethrough.ts @@ -84,16 +84,16 @@ async function put( return { ok: true, id: output._id, rev: output._rev } } -async function get(db: Database, id: string): Promise { +async function get(db: Database, id: string): Promise { const cache = await getCache() const cacheKey = makeCacheKey(db, id) let cacheItem: CacheItem = await cache.get(cacheKey) if (!cacheItem) { - const doc = await db.get(id) + const doc = await db.get(id) cacheItem = makeCacheItem(doc) await cache.store(cacheKey, cacheItem) } - return cacheItem.doc + return cacheItem.doc as T } async function remove(db: Database, docOrId: any, rev?: any): Promise { @@ -123,8 +123,8 @@ export class Writethrough { return put(this.db, doc, writeRateMs) } - async get(id: string) { - return get(this.db, id) + async get(id: string) { + return get(this.db, id) } async remove(docOrId: any, rev?: any) { diff --git a/packages/backend-core/src/redis/redlockImpl.ts b/packages/backend-core/src/redis/redlockImpl.ts index 4de2516ab2..e57a3721b5 100644 --- a/packages/backend-core/src/redis/redlockImpl.ts +++ b/packages/backend-core/src/redis/redlockImpl.ts @@ -137,7 +137,6 @@ export async function doWithLock( const result = await task() return { executed: true, result } } catch (e: any) { - logWarn(`lock type: ${opts.type} error`, e) // lock limit exceeded if (e.name === "LockError") { if (opts.type === LockType.TRY_ONCE) { diff --git a/packages/pro b/packages/pro index 056c2093db..3b89670d88 160000 --- a/packages/pro +++ b/packages/pro @@ -1 +1 @@ -Subproject commit 056c2093dbc93d9a10ea9f5050c84a84edd8100c +Subproject commit 3b89670d8817eee95a1ce53a35b95307ee3b8e76 From 48d70c45b96b9e149f2a36b8c56833c0b099e93a Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Wed, 13 Dec 2023 15:35:44 +0000 Subject: [PATCH 20/25] Updating pro. --- packages/pro | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/pro b/packages/pro index 3b89670d88..cae0b311d7 160000 --- a/packages/pro +++ b/packages/pro @@ -1 +1 @@ -Subproject commit 3b89670d8817eee95a1ce53a35b95307ee3b8e76 +Subproject commit cae0b311d7a304dbe9b41f610d1d543a2c9dd2d3 From c8128325c69fe1058a2aae722f020679f643d9f7 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Wed, 13 Dec 2023 15:39:04 +0000 Subject: [PATCH 21/25] Improving type handling. --- packages/backend-core/src/cache/writethrough.ts | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/packages/backend-core/src/cache/writethrough.ts b/packages/backend-core/src/cache/writethrough.ts index 364cc35c67..24e519dc7f 100644 --- a/packages/backend-core/src/cache/writethrough.ts +++ b/packages/backend-core/src/cache/writethrough.ts @@ -7,7 +7,7 @@ import * as locks from "../redis/redlockImpl" const DEFAULT_WRITE_RATE_MS = 10000 let CACHE: BaseCache | null = null -interface CacheItem { +interface CacheItem { doc: any lastWrite: number } @@ -24,7 +24,10 @@ function makeCacheKey(db: Database, key: string) { return db.name + key } -function makeCacheItem(doc: any, lastWrite: number | null = null): CacheItem { +function makeCacheItem( + doc: T, + lastWrite: number | null = null +): CacheItem { return { doc, lastWrite: lastWrite || Date.now() } } @@ -35,7 +38,7 @@ async function put( ) { const cache = await getCache() const key = doc._id - let cacheItem: CacheItem | undefined + let cacheItem: CacheItem | undefined if (key) { cacheItem = await cache.get(makeCacheKey(db, key)) } @@ -87,13 +90,13 @@ async function put( async function get(db: Database, id: string): Promise { const cache = await getCache() const cacheKey = makeCacheKey(db, id) - let cacheItem: CacheItem = await cache.get(cacheKey) + let cacheItem: CacheItem = await cache.get(cacheKey) if (!cacheItem) { const doc = await db.get(id) cacheItem = makeCacheItem(doc) await cache.store(cacheKey, cacheItem) } - return cacheItem.doc as T + return cacheItem.doc } async function remove(db: Database, docOrId: any, rev?: any): Promise { From 59581890cbbea5a319dd6101e5c015c9641cc841 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Wed, 13 Dec 2023 15:48:22 +0000 Subject: [PATCH 22/25] Rename automation worker container. --- .../templates/automation-worker-service-deployment.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/charts/budibase/templates/automation-worker-service-deployment.yaml b/charts/budibase/templates/automation-worker-service-deployment.yaml index a83542607a..4c41be3393 100644 --- a/charts/budibase/templates/automation-worker-service-deployment.yaml +++ b/charts/budibase/templates/automation-worker-service-deployment.yaml @@ -216,7 +216,7 @@ spec: {{- toYaml . | nindent 10 }} {{- end }} {{- end }} - name: bbapps + name: bbautomationworker ports: - containerPort: {{ .Values.services.automationWorkers.port }} {{ with .Values.services.automationWorkers.resources }} From 27b8cc7476a0c75d6555d6b032c116728b0faa64 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Wed, 13 Dec 2023 15:51:23 +0000 Subject: [PATCH 23/25] Updating pro. --- packages/pro | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/pro b/packages/pro index cae0b311d7..992486c100 160000 --- a/packages/pro +++ b/packages/pro @@ -1 +1 @@ -Subproject commit cae0b311d7a304dbe9b41f610d1d543a2c9dd2d3 +Subproject commit 992486c10044a7495496b97bdf5f454d4020bfba From 79fa567095c608b7e7a11a34624d9e60373450f8 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Wed, 13 Dec 2023 17:10:19 +0000 Subject: [PATCH 24/25] Allow the passing of extra envrionment variables to apps, worker, and automation worker pods. --- charts/budibase/templates/app-service-deployment.yaml | 4 ++++ .../templates/automation-worker-service-deployment.yaml | 4 ++++ charts/budibase/templates/worker-service-deployment.yaml | 4 ++++ charts/budibase/values.yaml | 9 +++++++++ 4 files changed, 21 insertions(+) diff --git a/charts/budibase/templates/app-service-deployment.yaml b/charts/budibase/templates/app-service-deployment.yaml index 495d11aadc..9fb435c2a3 100644 --- a/charts/budibase/templates/app-service-deployment.yaml +++ b/charts/budibase/templates/app-service-deployment.yaml @@ -196,6 +196,10 @@ spec: - name: APP_FEATURES value: "api" {{- end }} + {{- range .Values.services.apps.extraEnv }} + - name: {{ .name }} + value: {{ .value | quote }} + {{- end }} image: budibase/apps:{{ .Values.globals.appVersion | default .Chart.AppVersion }} imagePullPolicy: Always {{- if .Values.services.apps.startupProbe }} diff --git a/charts/budibase/templates/automation-worker-service-deployment.yaml b/charts/budibase/templates/automation-worker-service-deployment.yaml index 4c41be3393..46be6a4435 100644 --- a/charts/budibase/templates/automation-worker-service-deployment.yaml +++ b/charts/budibase/templates/automation-worker-service-deployment.yaml @@ -195,6 +195,10 @@ spec: {{ end }} - name: APP_FEATURES value: "automations" + {{- range .Values.services.automationWorkers.extraEnv }} + - name: {{ .name }} + value: {{ .value | quote }} + {{- end }} image: budibase/apps:{{ .Values.globals.appVersion | default .Chart.AppVersion }} imagePullPolicy: Always diff --git a/charts/budibase/templates/worker-service-deployment.yaml b/charts/budibase/templates/worker-service-deployment.yaml index 6427aa70e8..1d90aaf954 100644 --- a/charts/budibase/templates/worker-service-deployment.yaml +++ b/charts/budibase/templates/worker-service-deployment.yaml @@ -182,6 +182,10 @@ spec: - name: NODE_TLS_REJECT_UNAUTHORIZED value: {{ .Values.services.tlsRejectUnauthorized }} {{ end }} + {{- range .Values.services.worker.extraEnv }} + - name: {{ .name }} + value: {{ .value | quote }} + {{- end }} image: budibase/worker:{{ .Values.globals.appVersion | default .Chart.AppVersion }} imagePullPolicy: Always {{- if .Values.services.worker.startupProbe }} diff --git a/charts/budibase/values.yaml b/charts/budibase/values.yaml index 6dbcc20242..09262df463 100644 --- a/charts/budibase/values.yaml +++ b/charts/budibase/values.yaml @@ -220,6 +220,9 @@ services: # # for more information on how to set these. resources: {} + # -- Extra environment variables to set for apps pods. Takes a list of + # name=value pairs. + extraEnv: [] # -- Startup probe configuration for apps pods. You shouldn't need to # change this, but if you want to you can find more information here: # @@ -286,6 +289,9 @@ services: # # for more information on how to set these. resources: {} + # -- Extra environment variables to set for automation worker pods. Takes a list of + # name=value pairs. + extraEnv: [] # -- Startup probe configuration for automation worker pods. You shouldn't # need to change this, but if you want to you can find more information # here: @@ -354,6 +360,9 @@ services: # # for more information on how to set these. resources: {} + # -- Extra environment variables to set for worker pods. Takes a list of + # name=value pairs. + extraEnv: [] # -- Startup probe configuration for worker pods. You shouldn't need to # change this, but if you want to you can find more information here: # From 99eb6597fbc2562ebcca6ad14c5a90313049a9b7 Mon Sep 17 00:00:00 2001 From: Budibase Staging Release Bot <> Date: Wed, 13 Dec 2023 17:59:15 +0000 Subject: [PATCH 25/25] Bump version to 2.13.39 --- lerna.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lerna.json b/lerna.json index 3d309f0881..ed49a4267d 100644 --- a/lerna.json +++ b/lerna.json @@ -1,5 +1,5 @@ { - "version": "2.13.38", + "version": "2.13.39", "npmClient": "yarn", "packages": [ "packages/*",