From 0cde00842188bdb2400f728167e3ac8cbfa7aa72 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Fri, 21 Jun 2024 17:01:27 +0100 Subject: [PATCH 1/9] Update docker-compose.yaml for SQS. --- hosting/docker-compose.yaml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/hosting/docker-compose.yaml b/hosting/docker-compose.yaml index a72b36aef1..d9811935e4 100644 --- a/hosting/docker-compose.yaml +++ b/hosting/docker-compose.yaml @@ -27,6 +27,8 @@ services: BB_ADMIN_USER_PASSWORD: ${BB_ADMIN_USER_PASSWORD} PLUGINS_DIR: ${PLUGINS_DIR} OFFLINE_MODE: ${OFFLINE_MODE:-} + SQS_SEARCH_ENABLE: "true" + COUCH_DB_SQL_URL: "http://${COUCH_DB_USER}:${COUCH_DB_PASSWORD}@couchdb-service:4984" depends_on: - worker-service - redis-service @@ -54,6 +56,8 @@ services: REDIS_URL: redis-service:6379 REDIS_PASSWORD: ${REDIS_PASSWORD} OFFLINE_MODE: ${OFFLINE_MODE:-} + SQS_SEARCH_ENABLE: "true" + COUCH_DB_SQL_URL: "http://${COUCH_DB_USER}:${COUCH_DB_PASSWORD}@couchdb-service:4984" depends_on: - redis-service - minio-service @@ -97,7 +101,7 @@ services: couchdb-service: restart: unless-stopped - image: budibase/couchdb + image: budibase/couchdb:v3.3.3-sqs environment: - COUCHDB_PASSWORD=${COUCH_DB_PASSWORD} - COUCHDB_USER=${COUCH_DB_USER} From f64c48addff56882906d43a4978b3d80b81c99a7 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Fri, 21 Jun 2024 17:09:39 +0100 Subject: [PATCH 2/9] Add some jitter to the migration interval, and increase to a minimum of 5 seconds. --- packages/cli/src/hosting/utils.ts | 3 +- .../src/components/Updating.svelte | 34 ++++++++----------- 2 files changed, 16 insertions(+), 21 deletions(-) diff --git a/packages/cli/src/hosting/utils.ts b/packages/cli/src/hosting/utils.ts index 5c3ac33f44..05eb18dc1d 100644 --- a/packages/cli/src/hosting/utils.ts +++ b/packages/cli/src/hosting/utils.ts @@ -46,7 +46,8 @@ export function setServiceImage(service: string, image: string) { export async function downloadDockerCompose() { const filename = composeFilename() try { - await downloadFile(COMPOSE_URL, `./${filename}`) + fs.copyFileSync("../../hosting/docker-compose.yaml", `./${filename}`) + //await downloadFile(COMPOSE_URL, `./${filename}`) } catch (err) { console.error(error(`Failed to retrieve compose file - ${err}`)) } diff --git a/packages/frontend-core/src/components/Updating.svelte b/packages/frontend-core/src/components/Updating.svelte index 7d14e57aba..311a6b91c8 100644 --- a/packages/frontend-core/src/components/Updating.svelte +++ b/packages/frontend-core/src/components/Updating.svelte @@ -2,36 +2,30 @@ export let isMigrationDone export let onMigrationDone export let timeoutSeconds = 60 // 1 minute - export let minTimeSeconds = 3 - const loadTime = Date.now() - const intervalMs = 1000 let timedOut = false - let secondsWaited = 0 async function checkMigrationsFinished() { - setTimeout(async () => { + let totalWaitMs = 0 + while (true) { + const waitForMs = 5000 + Math.random() * 5000 + await new Promise(resolve => setTimeout(resolve, waitForMs)) + totalWaitMs += waitForMs + const isMigrated = await isMigrationDone() - - const timeoutMs = timeoutSeconds * 1000 - if (!isMigrated || secondsWaited <= minTimeSeconds) { - if (loadTime + timeoutMs > Date.now()) { - secondsWaited += 1 - return checkMigrationsFinished() - } - - return migrationTimeout() + if (isMigrated) { + onMigrationDone() + return } - onMigrationDone() - }, intervalMs) + if (totalWaitMs > timeoutSeconds * 1000) { + timedOut = true + return + } + } } checkMigrationsFinished() - - function migrationTimeout() { - timedOut = true - }
From bb0a0ce109df5b87a36127087b37865356dce82c Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Tue, 25 Jun 2024 11:01:29 +0100 Subject: [PATCH 3/9] Fix lint. --- packages/cli/src/hosting/utils.ts | 3 +-- packages/frontend-core/src/components/Updating.svelte | 1 + 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/cli/src/hosting/utils.ts b/packages/cli/src/hosting/utils.ts index 05eb18dc1d..5c3ac33f44 100644 --- a/packages/cli/src/hosting/utils.ts +++ b/packages/cli/src/hosting/utils.ts @@ -46,8 +46,7 @@ export function setServiceImage(service: string, image: string) { export async function downloadDockerCompose() { const filename = composeFilename() try { - fs.copyFileSync("../../hosting/docker-compose.yaml", `./${filename}`) - //await downloadFile(COMPOSE_URL, `./${filename}`) + await downloadFile(COMPOSE_URL, `./${filename}`) } catch (err) { console.error(error(`Failed to retrieve compose file - ${err}`)) } diff --git a/packages/frontend-core/src/components/Updating.svelte b/packages/frontend-core/src/components/Updating.svelte index 311a6b91c8..97e83e2322 100644 --- a/packages/frontend-core/src/components/Updating.svelte +++ b/packages/frontend-core/src/components/Updating.svelte @@ -7,6 +7,7 @@ async function checkMigrationsFinished() { let totalWaitMs = 0 + // eslint-disable-next-line no-constant-condition while (true) { const waitForMs = 5000 + Math.random() * 5000 await new Promise(resolve => setTimeout(resolve, waitForMs)) From b66591f52f6f9666560078a4d1e127072c218c2a Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Tue, 2 Jul 2024 15:40:50 +0100 Subject: [PATCH 4/9] Remove superfluous environment variables. --- hosting/docker-compose.yaml | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/hosting/docker-compose.yaml b/hosting/docker-compose.yaml index d9811935e4..d59acb8f9b 100644 --- a/hosting/docker-compose.yaml +++ b/hosting/docker-compose.yaml @@ -27,8 +27,6 @@ services: BB_ADMIN_USER_PASSWORD: ${BB_ADMIN_USER_PASSWORD} PLUGINS_DIR: ${PLUGINS_DIR} OFFLINE_MODE: ${OFFLINE_MODE:-} - SQS_SEARCH_ENABLE: "true" - COUCH_DB_SQL_URL: "http://${COUCH_DB_USER}:${COUCH_DB_PASSWORD}@couchdb-service:4984" depends_on: - worker-service - redis-service @@ -56,8 +54,6 @@ services: REDIS_URL: redis-service:6379 REDIS_PASSWORD: ${REDIS_PASSWORD} OFFLINE_MODE: ${OFFLINE_MODE:-} - SQS_SEARCH_ENABLE: "true" - COUCH_DB_SQL_URL: "http://${COUCH_DB_USER}:${COUCH_DB_PASSWORD}@couchdb-service:4984" depends_on: - redis-service - minio-service @@ -101,7 +97,7 @@ services: couchdb-service: restart: unless-stopped - image: budibase/couchdb:v3.3.3-sqs + image: budibase/couchdb:v3.3.3 environment: - COUCHDB_PASSWORD=${COUCH_DB_PASSWORD} - COUCHDB_USER=${COUCH_DB_USER} From 75c43e7c690d99766be6a2d072f30a369907af28 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Wed, 25 Sep 2024 17:01:28 +0100 Subject: [PATCH 5/9] Updating to specific SQS version. --- hosting/docker-compose.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hosting/docker-compose.yaml b/hosting/docker-compose.yaml index d59acb8f9b..6d14361d25 100644 --- a/hosting/docker-compose.yaml +++ b/hosting/docker-compose.yaml @@ -97,7 +97,7 @@ services: couchdb-service: restart: unless-stopped - image: budibase/couchdb:v3.3.3 + image: budibase/couchdb:v3.3.3-sqs-v2.1.1 environment: - COUCHDB_PASSWORD=${COUCH_DB_PASSWORD} - COUCHDB_USER=${COUCH_DB_USER} From c9d42a0e2964a3c5b1710421e5ec16eaa6881846 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Mon, 14 Oct 2024 14:51:17 +0100 Subject: [PATCH 6/9] Add SQS feature flags, remove scarf.sh URLs. --- hosting/docker-compose.yaml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/hosting/docker-compose.yaml b/hosting/docker-compose.yaml index 6d14361d25..1c4bf9a1f8 100644 --- a/hosting/docker-compose.yaml +++ b/hosting/docker-compose.yaml @@ -5,7 +5,7 @@ version: "3" services: app-service: restart: unless-stopped - image: budibase.docker.scarf.sh/budibase/apps + image: budibase/apps container_name: bbapps environment: SELF_HOSTED: 1 @@ -27,6 +27,7 @@ services: BB_ADMIN_USER_PASSWORD: ${BB_ADMIN_USER_PASSWORD} PLUGINS_DIR: ${PLUGINS_DIR} OFFLINE_MODE: ${OFFLINE_MODE:-} + TENANT_FEATURE_FLAGS: "*:SQS" depends_on: - worker-service - redis-service @@ -35,7 +36,7 @@ services: worker-service: restart: unless-stopped - image: budibase.docker.scarf.sh/budibase/worker + image: budibase/worker container_name: bbworker environment: SELF_HOSTED: 1 @@ -54,6 +55,7 @@ services: REDIS_URL: redis-service:6379 REDIS_PASSWORD: ${REDIS_PASSWORD} OFFLINE_MODE: ${OFFLINE_MODE:-} + TENANT_FEATURE_FLAGS: "*:SQS" depends_on: - redis-service - minio-service From 58b4a37fca1ac7df76d7eb6d0b14198582ef3dbb Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Mon, 14 Oct 2024 17:20:36 +0100 Subject: [PATCH 7/9] Enable SQS in code instead of in env vars. --- hosting/docker-compose.yaml | 2 -- packages/backend-core/src/features/features.ts | 2 +- packages/backend-core/src/features/tests/features.spec.ts | 6 ++++++ 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/hosting/docker-compose.yaml b/hosting/docker-compose.yaml index 1c4bf9a1f8..c7a22eb2b3 100644 --- a/hosting/docker-compose.yaml +++ b/hosting/docker-compose.yaml @@ -27,7 +27,6 @@ services: BB_ADMIN_USER_PASSWORD: ${BB_ADMIN_USER_PASSWORD} PLUGINS_DIR: ${PLUGINS_DIR} OFFLINE_MODE: ${OFFLINE_MODE:-} - TENANT_FEATURE_FLAGS: "*:SQS" depends_on: - worker-service - redis-service @@ -55,7 +54,6 @@ services: REDIS_URL: redis-service:6379 REDIS_PASSWORD: ${REDIS_PASSWORD} OFFLINE_MODE: ${OFFLINE_MODE:-} - TENANT_FEATURE_FLAGS: "*:SQS" depends_on: - redis-service - minio-service diff --git a/packages/backend-core/src/features/features.ts b/packages/backend-core/src/features/features.ts index 20b207bb02..e95472a784 100644 --- a/packages/backend-core/src/features/features.ts +++ b/packages/backend-core/src/features/features.ts @@ -269,7 +269,7 @@ export class FlagSet, T extends { [key: string]: V }> { export const flags = new FlagSet({ DEFAULT_VALUES: Flag.boolean(env.isDev()), AUTOMATION_BRANCHING: Flag.boolean(env.isDev()), - SQS: Flag.boolean(env.isDev()), + SQS: Flag.boolean(true), [FeatureFlag.AI_CUSTOM_CONFIGS]: Flag.boolean(env.isDev()), [FeatureFlag.ENRICHED_RELATIONSHIPS]: Flag.boolean(env.isDev()), [FeatureFlag.TABLES_DEFAULT_ADMIN]: Flag.boolean(env.isDev()), diff --git a/packages/backend-core/src/features/tests/features.spec.ts b/packages/backend-core/src/features/tests/features.spec.ts index 9af8a8f4bb..ced874f4af 100644 --- a/packages/backend-core/src/features/tests/features.spec.ts +++ b/packages/backend-core/src/features/tests/features.spec.ts @@ -10,6 +10,7 @@ const schema = { TEST_BOOLEAN: Flag.boolean(false), TEST_STRING: Flag.string("default value"), TEST_NUMBER: Flag.number(0), + TEST_BOOLEAN_DEFAULT_TRUE: Flag.boolean(true), } const flags = new FlagSet(schema) @@ -123,6 +124,11 @@ describe("feature flags", () => { }, expected: flags.defaults(), }, + { + it: "should be possible to override a default true flag to false", + environmentFlags: "default:!TEST_BOOLEAN_DEFAULT_TRUE", + expected: { TEST_BOOLEAN_DEFAULT_TRUE: false }, + }, ])( "$it", async ({ From 61558aff774f169545f75e640794aad2c87dd59a Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Mon, 14 Oct 2024 17:24:14 +0100 Subject: [PATCH 8/9] Update Helm chart for SQS. --- charts/budibase/templates/app-service-deployment.yaml | 6 ------ .../templates/automation-worker-service-deployment.yaml | 2 ++ charts/budibase/templates/worker-service-deployment.yaml | 6 ------ charts/budibase/values.yaml | 5 ----- 4 files changed, 2 insertions(+), 17 deletions(-) diff --git a/charts/budibase/templates/app-service-deployment.yaml b/charts/budibase/templates/app-service-deployment.yaml index 4d0560312f..5710749028 100644 --- a/charts/budibase/templates/app-service-deployment.yaml +++ b/charts/budibase/templates/app-service-deployment.yaml @@ -42,14 +42,8 @@ spec: {{ else }} value: http://{{ .Release.Name }}-svc-couchdb:{{ .Values.services.couchdb.port }} {{ end }} - {{ if .Values.globals.sqs.enabled }} - name: COUCH_DB_SQL_URL - {{ if .Values.globals.sqs.url }} - value: {{ .Values.globals.sqs.url }} - {{ else }} value: http://{{ .Release.Name }}-svc-couchdb:{{ .Values.globals.sqs.port }} - {{ end }} - {{ end }} {{ if .Values.services.couchdb.enabled }} - name: COUCH_DB_USER valueFrom: diff --git a/charts/budibase/templates/automation-worker-service-deployment.yaml b/charts/budibase/templates/automation-worker-service-deployment.yaml index 71089bd7ee..ee3cd6dbdb 100644 --- a/charts/budibase/templates/automation-worker-service-deployment.yaml +++ b/charts/budibase/templates/automation-worker-service-deployment.yaml @@ -43,6 +43,8 @@ spec: {{ else }} value: http://{{ .Release.Name }}-svc-couchdb:{{ .Values.services.couchdb.port }} {{ end }} + - name: COUCH_DB_SQL_URL + value: http://{{ .Release.Name }}-svc-couchdb:{{ .Values.globals.sqs.port }} {{ if .Values.services.couchdb.enabled }} - name: COUCH_DB_USER valueFrom: diff --git a/charts/budibase/templates/worker-service-deployment.yaml b/charts/budibase/templates/worker-service-deployment.yaml index dcab33fa58..a493980fd3 100644 --- a/charts/budibase/templates/worker-service-deployment.yaml +++ b/charts/budibase/templates/worker-service-deployment.yaml @@ -56,14 +56,8 @@ spec: {{ else }} value: http://{{ .Release.Name }}-svc-couchdb:{{ .Values.services.couchdb.port }} {{ end }} - {{ if .Values.globals.sqs.enabled }} - name: COUCH_DB_SQL_URL - {{ if .Values.globals.sqs.url }} - value: {{ .Values.globals.sqs.url }} - {{ else }} value: http://{{ .Release.Name }}-svc-couchdb:{{ .Values.globals.sqs.port }} - {{ end }} - {{ end }} - name: API_ENCRYPTION_KEY valueFrom: secretKeyRef: diff --git a/charts/budibase/values.yaml b/charts/budibase/values.yaml index 2c1525bd90..9a5cdcdc82 100644 --- a/charts/budibase/values.yaml +++ b/charts/budibase/values.yaml @@ -139,11 +139,6 @@ globals: password: "" sqs: - # -- Whether to use the CouchDB "structured query service" or not. This is disabled by - # default for now, but will become the default in a future release. - enabled: false - # @ignore - url: "" # @ignore port: "4984" From dd81e246bf4eff7c06fc192fef054abacda97ef8 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Mon, 14 Oct 2024 17:28:25 +0100 Subject: [PATCH 9/9] Allow customisation of SQS URL to match CouchDB URL. --- charts/budibase/templates/app-service-deployment.yaml | 4 ++++ .../templates/automation-worker-service-deployment.yaml | 4 ++++ charts/budibase/templates/worker-service-deployment.yaml | 4 ++++ charts/budibase/values.yaml | 2 ++ 4 files changed, 14 insertions(+) diff --git a/charts/budibase/templates/app-service-deployment.yaml b/charts/budibase/templates/app-service-deployment.yaml index 5710749028..278bd1767f 100644 --- a/charts/budibase/templates/app-service-deployment.yaml +++ b/charts/budibase/templates/app-service-deployment.yaml @@ -43,7 +43,11 @@ spec: value: http://{{ .Release.Name }}-svc-couchdb:{{ .Values.services.couchdb.port }} {{ end }} - name: COUCH_DB_SQL_URL + {{ if .Values.globals.sqs.url }} + value: {{ .Values.globals.sqs.url | quote }} + {{ else }} value: http://{{ .Release.Name }}-svc-couchdb:{{ .Values.globals.sqs.port }} + {{ end }} {{ if .Values.services.couchdb.enabled }} - name: COUCH_DB_USER valueFrom: diff --git a/charts/budibase/templates/automation-worker-service-deployment.yaml b/charts/budibase/templates/automation-worker-service-deployment.yaml index ee3cd6dbdb..e0d43d0ce6 100644 --- a/charts/budibase/templates/automation-worker-service-deployment.yaml +++ b/charts/budibase/templates/automation-worker-service-deployment.yaml @@ -44,7 +44,11 @@ spec: value: http://{{ .Release.Name }}-svc-couchdb:{{ .Values.services.couchdb.port }} {{ end }} - name: COUCH_DB_SQL_URL + {{ if .Values.globals.sqs.url }} + value: {{ .Values.globals.sqs.url | quote }} + {{ else }} value: http://{{ .Release.Name }}-svc-couchdb:{{ .Values.globals.sqs.port }} + {{ end }} {{ if .Values.services.couchdb.enabled }} - name: COUCH_DB_USER valueFrom: diff --git a/charts/budibase/templates/worker-service-deployment.yaml b/charts/budibase/templates/worker-service-deployment.yaml index a493980fd3..94fdd0b94e 100644 --- a/charts/budibase/templates/worker-service-deployment.yaml +++ b/charts/budibase/templates/worker-service-deployment.yaml @@ -57,7 +57,11 @@ spec: value: http://{{ .Release.Name }}-svc-couchdb:{{ .Values.services.couchdb.port }} {{ end }} - name: COUCH_DB_SQL_URL + {{ if .Values.globals.sqs.url }} + value: {{ .Values.globals.sqs.url | quote }} + {{ else }} value: http://{{ .Release.Name }}-svc-couchdb:{{ .Values.globals.sqs.port }} + {{ end }} - name: API_ENCRYPTION_KEY valueFrom: secretKeyRef: diff --git a/charts/budibase/values.yaml b/charts/budibase/values.yaml index 9a5cdcdc82..de2cdb9474 100644 --- a/charts/budibase/values.yaml +++ b/charts/budibase/values.yaml @@ -139,6 +139,8 @@ globals: password: "" sqs: + # @ignore + url: "" # @ignore port: "4984"