From 0cde00842188bdb2400f728167e3ac8cbfa7aa72 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Fri, 21 Jun 2024 17:01:27 +0100 Subject: [PATCH 01/26] Update docker-compose.yaml for SQS. --- hosting/docker-compose.yaml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/hosting/docker-compose.yaml b/hosting/docker-compose.yaml index a72b36aef1..d9811935e4 100644 --- a/hosting/docker-compose.yaml +++ b/hosting/docker-compose.yaml @@ -27,6 +27,8 @@ services: BB_ADMIN_USER_PASSWORD: ${BB_ADMIN_USER_PASSWORD} PLUGINS_DIR: ${PLUGINS_DIR} OFFLINE_MODE: ${OFFLINE_MODE:-} + SQS_SEARCH_ENABLE: "true" + COUCH_DB_SQL_URL: "http://${COUCH_DB_USER}:${COUCH_DB_PASSWORD}@couchdb-service:4984" depends_on: - worker-service - redis-service @@ -54,6 +56,8 @@ services: REDIS_URL: redis-service:6379 REDIS_PASSWORD: ${REDIS_PASSWORD} OFFLINE_MODE: ${OFFLINE_MODE:-} + SQS_SEARCH_ENABLE: "true" + COUCH_DB_SQL_URL: "http://${COUCH_DB_USER}:${COUCH_DB_PASSWORD}@couchdb-service:4984" depends_on: - redis-service - minio-service @@ -97,7 +101,7 @@ services: couchdb-service: restart: unless-stopped - image: budibase/couchdb + image: budibase/couchdb:v3.3.3-sqs environment: - COUCHDB_PASSWORD=${COUCH_DB_PASSWORD} - COUCHDB_USER=${COUCH_DB_USER} From f64c48addff56882906d43a4978b3d80b81c99a7 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Fri, 21 Jun 2024 17:09:39 +0100 Subject: [PATCH 02/26] Add some jitter to the migration interval, and increase to a minimum of 5 seconds. --- packages/cli/src/hosting/utils.ts | 3 +- .../src/components/Updating.svelte | 34 ++++++++----------- 2 files changed, 16 insertions(+), 21 deletions(-) diff --git a/packages/cli/src/hosting/utils.ts b/packages/cli/src/hosting/utils.ts index 5c3ac33f44..05eb18dc1d 100644 --- a/packages/cli/src/hosting/utils.ts +++ b/packages/cli/src/hosting/utils.ts @@ -46,7 +46,8 @@ export function setServiceImage(service: string, image: string) { export async function downloadDockerCompose() { const filename = composeFilename() try { - await downloadFile(COMPOSE_URL, `./${filename}`) + fs.copyFileSync("../../hosting/docker-compose.yaml", `./${filename}`) + //await downloadFile(COMPOSE_URL, `./${filename}`) } catch (err) { console.error(error(`Failed to retrieve compose file - ${err}`)) } diff --git a/packages/frontend-core/src/components/Updating.svelte b/packages/frontend-core/src/components/Updating.svelte index 7d14e57aba..311a6b91c8 100644 --- a/packages/frontend-core/src/components/Updating.svelte +++ b/packages/frontend-core/src/components/Updating.svelte @@ -2,36 +2,30 @@ export let isMigrationDone export let onMigrationDone export let timeoutSeconds = 60 // 1 minute - export let minTimeSeconds = 3 - const loadTime = Date.now() - const intervalMs = 1000 let timedOut = false - let secondsWaited = 0 async function checkMigrationsFinished() { - setTimeout(async () => { + let totalWaitMs = 0 + while (true) { + const waitForMs = 5000 + Math.random() * 5000 + await new Promise(resolve => setTimeout(resolve, waitForMs)) + totalWaitMs += waitForMs + const isMigrated = await isMigrationDone() - - const timeoutMs = timeoutSeconds * 1000 - if (!isMigrated || secondsWaited <= minTimeSeconds) { - if (loadTime + timeoutMs > Date.now()) { - secondsWaited += 1 - return checkMigrationsFinished() - } - - return migrationTimeout() + if (isMigrated) { + onMigrationDone() + return } - onMigrationDone() - }, intervalMs) + if (totalWaitMs > timeoutSeconds * 1000) { + timedOut = true + return + } + } } checkMigrationsFinished() - - function migrationTimeout() { - timedOut = true - }
From bb0a0ce109df5b87a36127087b37865356dce82c Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Tue, 25 Jun 2024 11:01:29 +0100 Subject: [PATCH 03/26] Fix lint. --- packages/cli/src/hosting/utils.ts | 3 +-- packages/frontend-core/src/components/Updating.svelte | 1 + 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/cli/src/hosting/utils.ts b/packages/cli/src/hosting/utils.ts index 05eb18dc1d..5c3ac33f44 100644 --- a/packages/cli/src/hosting/utils.ts +++ b/packages/cli/src/hosting/utils.ts @@ -46,8 +46,7 @@ export function setServiceImage(service: string, image: string) { export async function downloadDockerCompose() { const filename = composeFilename() try { - fs.copyFileSync("../../hosting/docker-compose.yaml", `./${filename}`) - //await downloadFile(COMPOSE_URL, `./${filename}`) + await downloadFile(COMPOSE_URL, `./${filename}`) } catch (err) { console.error(error(`Failed to retrieve compose file - ${err}`)) } diff --git a/packages/frontend-core/src/components/Updating.svelte b/packages/frontend-core/src/components/Updating.svelte index 311a6b91c8..97e83e2322 100644 --- a/packages/frontend-core/src/components/Updating.svelte +++ b/packages/frontend-core/src/components/Updating.svelte @@ -7,6 +7,7 @@ async function checkMigrationsFinished() { let totalWaitMs = 0 + // eslint-disable-next-line no-constant-condition while (true) { const waitForMs = 5000 + Math.random() * 5000 await new Promise(resolve => setTimeout(resolve, waitForMs)) From b66591f52f6f9666560078a4d1e127072c218c2a Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Tue, 2 Jul 2024 15:40:50 +0100 Subject: [PATCH 04/26] Remove superfluous environment variables. --- hosting/docker-compose.yaml | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/hosting/docker-compose.yaml b/hosting/docker-compose.yaml index d9811935e4..d59acb8f9b 100644 --- a/hosting/docker-compose.yaml +++ b/hosting/docker-compose.yaml @@ -27,8 +27,6 @@ services: BB_ADMIN_USER_PASSWORD: ${BB_ADMIN_USER_PASSWORD} PLUGINS_DIR: ${PLUGINS_DIR} OFFLINE_MODE: ${OFFLINE_MODE:-} - SQS_SEARCH_ENABLE: "true" - COUCH_DB_SQL_URL: "http://${COUCH_DB_USER}:${COUCH_DB_PASSWORD}@couchdb-service:4984" depends_on: - worker-service - redis-service @@ -56,8 +54,6 @@ services: REDIS_URL: redis-service:6379 REDIS_PASSWORD: ${REDIS_PASSWORD} OFFLINE_MODE: ${OFFLINE_MODE:-} - SQS_SEARCH_ENABLE: "true" - COUCH_DB_SQL_URL: "http://${COUCH_DB_USER}:${COUCH_DB_PASSWORD}@couchdb-service:4984" depends_on: - redis-service - minio-service @@ -101,7 +97,7 @@ services: couchdb-service: restart: unless-stopped - image: budibase/couchdb:v3.3.3-sqs + image: budibase/couchdb:v3.3.3 environment: - COUCHDB_PASSWORD=${COUCH_DB_PASSWORD} - COUCHDB_USER=${COUCH_DB_USER} From 75c43e7c690d99766be6a2d072f30a369907af28 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Wed, 25 Sep 2024 17:01:28 +0100 Subject: [PATCH 05/26] Updating to specific SQS version. --- hosting/docker-compose.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hosting/docker-compose.yaml b/hosting/docker-compose.yaml index d59acb8f9b..6d14361d25 100644 --- a/hosting/docker-compose.yaml +++ b/hosting/docker-compose.yaml @@ -97,7 +97,7 @@ services: couchdb-service: restart: unless-stopped - image: budibase/couchdb:v3.3.3 + image: budibase/couchdb:v3.3.3-sqs-v2.1.1 environment: - COUCHDB_PASSWORD=${COUCH_DB_PASSWORD} - COUCHDB_USER=${COUCH_DB_USER} From cb78e0bc13326d5ee1c3f721e123732b5028068d Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Fri, 11 Oct 2024 12:20:04 +0200 Subject: [PATCH 06/26] Add extra tests --- .../src/api/routes/tests/search.spec.ts | 41 +++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/packages/server/src/api/routes/tests/search.spec.ts b/packages/server/src/api/routes/tests/search.spec.ts index 1ccc9bfdc9..3b3492e74c 100644 --- a/packages/server/src/api/routes/tests/search.spec.ts +++ b/packages/server/src/api/routes/tests/search.spec.ts @@ -2329,6 +2329,47 @@ describe.each([ equal: { ["name"]: "baz" }, }).toContainExactly([{ name: "baz", productCat: undefined }]) }) + + describe("logical filters", () => { + it("should allow nested ands with single conditions", async () => { + await expectQuery({ + $and: { + conditions: [ + { + $and: { + conditions: [ + { + equal: { ["productCat.name"]: "foo" }, + }, + ], + }, + }, + ], + }, + }).toContainExactly([ + { name: "foo", productCat: [{ _id: productCatRows[0]._id }] }, + ]) + }) + + it("should allow nested ands with exclusive conditions", async () => { + await expectQuery({ + $and: { + conditions: [ + { + $and: { + conditions: [ + { + equal: { ["productCat.name"]: "foo" }, + notEqual: { ["productCat.name"]: "foo" }, + }, + ], + }, + }, + ], + }, + }).toContainExactly([]) + }) + }) }) isSql && From 37450823bba30ee0c715e83fe2592c26cffd73f2 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Fri, 11 Oct 2024 12:25:10 +0200 Subject: [PATCH 07/26] More tests --- .../src/api/routes/tests/search.spec.ts | 39 +++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/packages/server/src/api/routes/tests/search.spec.ts b/packages/server/src/api/routes/tests/search.spec.ts index 3b3492e74c..1ed2f66676 100644 --- a/packages/server/src/api/routes/tests/search.spec.ts +++ b/packages/server/src/api/routes/tests/search.spec.ts @@ -2369,6 +2369,45 @@ describe.each([ }, }).toContainExactly([]) }) + + it("should allow nested ands with multiple conditions", async () => { + await expectQuery({ + $and: { + conditions: [ + { + $and: { + conditions: [ + { + equal: { ["productCat.name"]: "foo" }, + }, + ], + }, + notEqual: { ["productCat.name"]: "foo" }, + }, + ], + }, + }).toContainExactly([]) + }) + + it("should allow nesting or under and with single conditions", async () => { + await expectQuery({ + $and: { + conditions: [ + { + $or: { + conditions: [ + { + equal: { ["productCat.name"]: "foo" }, + }, + ], + }, + }, + ], + }, + }).toContainExactly([ + { name: "foo", productCat: [{ _id: productCatRows[0]._id }] }, + ]) + }) }) }) From f192a30da0a00680b73e0bfc8987675259a86338 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Fri, 11 Oct 2024 12:29:34 +0200 Subject: [PATCH 08/26] More tests --- .../src/api/routes/tests/search.spec.ts | 186 +++++++++++------- 1 file changed, 118 insertions(+), 68 deletions(-) diff --git a/packages/server/src/api/routes/tests/search.spec.ts b/packages/server/src/api/routes/tests/search.spec.ts index 1ed2f66676..a3f8f1577d 100644 --- a/packages/server/src/api/routes/tests/search.spec.ts +++ b/packages/server/src/api/routes/tests/search.spec.ts @@ -2331,82 +2331,132 @@ describe.each([ }) describe("logical filters", () => { - it("should allow nested ands with single conditions", async () => { - await expectQuery({ - $and: { - conditions: [ - { - $and: { - conditions: [ - { - equal: { ["productCat.name"]: "foo" }, - }, - ], + describe("just $ands", () => { + it("should allow nested ands with single conditions", async () => { + await expectQuery({ + $and: { + conditions: [ + { + $and: { + conditions: [ + { + equal: { ["productCat.name"]: "foo" }, + }, + ], + }, }, - }, - ], - }, - }).toContainExactly([ - { name: "foo", productCat: [{ _id: productCatRows[0]._id }] }, - ]) + ], + }, + }).toContainExactly([ + { name: "foo", productCat: [{ _id: productCatRows[0]._id }] }, + ]) + }) + + it("should allow nested ands with exclusive conditions", async () => { + await expectQuery({ + $and: { + conditions: [ + { + $and: { + conditions: [ + { + equal: { ["productCat.name"]: "foo" }, + notEqual: { ["productCat.name"]: "foo" }, + }, + ], + }, + }, + ], + }, + }).toContainExactly([]) + }) + + it("should allow nested ands with multiple conditions", async () => { + await expectQuery({ + $and: { + conditions: [ + { + $and: { + conditions: [ + { + equal: { ["productCat.name"]: "foo" }, + }, + ], + }, + notEqual: { ["productCat.name"]: "foo" }, + }, + ], + }, + }).toContainExactly([]) + }) }) - it("should allow nested ands with exclusive conditions", async () => { - await expectQuery({ - $and: { - conditions: [ - { - $and: { - conditions: [ - { - equal: { ["productCat.name"]: "foo" }, - notEqual: { ["productCat.name"]: "foo" }, - }, - ], + describe("just $ors", () => { + it("should allow nested ands with single conditions", async () => { + await expectQuery({ + $or: { + conditions: [ + { + $or: { + conditions: [ + { + equal: { ["productCat.name"]: "foo" }, + }, + ], + }, }, - }, - ], - }, - }).toContainExactly([]) - }) + ], + }, + }).toContainExactly([ + { name: "foo", productCat: [{ _id: productCatRows[0]._id }] }, + ]) + }) - it("should allow nested ands with multiple conditions", async () => { - await expectQuery({ - $and: { - conditions: [ - { - $and: { - conditions: [ - { - equal: { ["productCat.name"]: "foo" }, - }, - ], + it("should allow nested ands with exclusive conditions", async () => { + await expectQuery({ + $or: { + conditions: [ + { + $or: { + conditions: [ + { + equal: { ["productCat.name"]: "foo" }, + notEqual: { ["productCat.name"]: "foo" }, + }, + ], + }, }, - notEqual: { ["productCat.name"]: "foo" }, - }, - ], - }, - }).toContainExactly([]) - }) + ], + }, + }).toContainExactly([ + { name: "foo", productCat: [{ _id: productCatRows[0]._id }] }, + { name: "bar", productCat: [{ _id: productCatRows[1]._id }] }, + { name: "baz", productCat: undefined }, + ]) + }) - it("should allow nesting or under and with single conditions", async () => { - await expectQuery({ - $and: { - conditions: [ - { - $or: { - conditions: [ - { - equal: { ["productCat.name"]: "foo" }, - }, - ], + it("should allow nested ands with multiple conditions", async () => { + await expectQuery({ + $or: { + conditions: [ + { + $or: { + conditions: [ + { + equal: { ["productCat.name"]: "foo" }, + }, + ], + }, + notEqual: { ["productCat.name"]: "foo" }, }, - }, - ], - }, - }).toContainExactly([ - { name: "foo", productCat: [{ _id: productCatRows[0]._id }] }, - ]) + ], + }, + }).toContainExactly([ + { name: "foo", productCat: [{ _id: productCatRows[0]._id }] }, + { name: "bar", productCat: [{ _id: productCatRows[1]._id }] }, + { name: "baz", productCat: undefined }, + ]) + }) }) }) }) From 2311f8aa5079fc6e2aef61db7255c5532fe446b2 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Fri, 11 Oct 2024 12:55:23 +0200 Subject: [PATCH 09/26] Don't break or conditions on nested joins --- packages/backend-core/src/sql/sql.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/packages/backend-core/src/sql/sql.ts b/packages/backend-core/src/sql/sql.ts index 382eca3f76..f77e76023e 100644 --- a/packages/backend-core/src/sql/sql.ts +++ b/packages/backend-core/src/sql/sql.ts @@ -521,8 +521,11 @@ class InternalBuilder { const [filterTableName, ...otherProperties] = key.split(".") const property = otherProperties.join(".") const alias = getTableAlias(filterTableName) - return fn(q, alias ? `${alias}.${property}` : property, value) + return q.andWhere(subquery => + fn(subquery, alias ? `${alias}.${property}` : property, value) + ) } + for (const key in structure) { const value = structure[key] const updatedKey = dbCore.removeKeyNumbering(key) From cf089eff26af1598f85a81de9a6726a8d4b85773 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Fri, 11 Oct 2024 12:59:33 +0200 Subject: [PATCH 10/26] Fix ors --- packages/backend-core/src/sql/sql.ts | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/packages/backend-core/src/sql/sql.ts b/packages/backend-core/src/sql/sql.ts index f77e76023e..2b697d42ae 100644 --- a/packages/backend-core/src/sql/sql.ts +++ b/packages/backend-core/src/sql/sql.ts @@ -406,6 +406,7 @@ class InternalBuilder { addRelationshipForFilter( query: Knex.QueryBuilder, filterKey: string, + isOr: boolean, whereCb: (query: Knex.QueryBuilder) => Knex.QueryBuilder ): Knex.QueryBuilder { const mainKnex = this.knex @@ -470,7 +471,12 @@ class InternalBuilder { ) ) } - query = query.whereExists(whereCb(subQuery)) + + if (isOr) { + query = query.orWhereExists(whereCb(subQuery)) + } else { + query = query.whereExists(whereCb(subQuery)) + } break } } @@ -555,9 +561,14 @@ class InternalBuilder { value ) } else if (shouldProcessRelationship) { - query = builder.addRelationshipForFilter(query, updatedKey, q => { - return handleRelationship(q, updatedKey, value) - }) + query = builder.addRelationshipForFilter( + query, + updatedKey, + !!allOr, + q => { + return handleRelationship(q, updatedKey, value) + } + ) } } } From bfdead820c4b4ff13d7558fc487e42da37f688b6 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Fri, 11 Oct 2024 13:00:05 +0200 Subject: [PATCH 11/26] Cleanup tests --- .../src/api/routes/tests/search.spec.ts | 68 +++++++++++++++++-- 1 file changed, 63 insertions(+), 5 deletions(-) diff --git a/packages/server/src/api/routes/tests/search.spec.ts b/packages/server/src/api/routes/tests/search.spec.ts index a3f8f1577d..364b6aa77a 100644 --- a/packages/server/src/api/routes/tests/search.spec.ts +++ b/packages/server/src/api/routes/tests/search.spec.ts @@ -2332,6 +2332,33 @@ describe.each([ describe("logical filters", () => { describe("just $ands", () => { + it("should allow single conditions", async () => { + await expectQuery({ + $and: { + conditions: [ + { + equal: { ["productCat.name"]: "foo" }, + }, + ], + }, + }).toContainExactly([ + { name: "foo", productCat: [{ _id: productCatRows[0]._id }] }, + ]) + }) + + it("should allow exclusive conditions", async () => { + await expectQuery({ + $and: { + conditions: [ + { + equal: { ["productCat.name"]: "foo" }, + notEqual: { ["productCat.name"]: "foo" }, + }, + ], + }, + }).toContainExactly([]) + }) + it("should allow nested ands with single conditions", async () => { await expectQuery({ $and: { @@ -2392,7 +2419,38 @@ describe.each([ }) describe("just $ors", () => { - it("should allow nested ands with single conditions", async () => { + it("should allow single conditions", async () => { + await expectQuery({ + $or: { + conditions: [ + { + equal: { ["productCat.name"]: "foo" }, + }, + ], + }, + }).toContainExactly([ + { name: "foo", productCat: [{ _id: productCatRows[0]._id }] }, + ]) + }) + + it("should allow exclusive conditions", async () => { + await expectQuery({ + $or: { + conditions: [ + { + equal: { ["productCat.name"]: "foo" }, + notEqual: { ["productCat.name"]: "foo" }, + }, + ], + }, + }).toContainExactly([ + { name: "foo", productCat: [{ _id: productCatRows[0]._id }] }, + { name: "bar", productCat: [{ _id: productCatRows[1]._id }] }, + // { name: "baz", productCat: undefined }, // TODO + ]) + }) + + it("should allow nested ors with single conditions", async () => { await expectQuery({ $or: { conditions: [ @@ -2412,7 +2470,7 @@ describe.each([ ]) }) - it("should allow nested ands with exclusive conditions", async () => { + it("should allow nested ors with exclusive conditions", async () => { await expectQuery({ $or: { conditions: [ @@ -2431,11 +2489,11 @@ describe.each([ }).toContainExactly([ { name: "foo", productCat: [{ _id: productCatRows[0]._id }] }, { name: "bar", productCat: [{ _id: productCatRows[1]._id }] }, - { name: "baz", productCat: undefined }, + // { name: "baz", productCat: undefined }, // TODO ]) }) - it("should allow nested ands with multiple conditions", async () => { + it("should allow nested ors with multiple conditions", async () => { await expectQuery({ $or: { conditions: [ @@ -2454,7 +2512,7 @@ describe.each([ }).toContainExactly([ { name: "foo", productCat: [{ _id: productCatRows[0]._id }] }, { name: "bar", productCat: [{ _id: productCatRows[1]._id }] }, - { name: "baz", productCat: undefined }, + // { name: "baz", productCat: undefined }, // TODO ]) }) }) From f73b7d4824fbf12862351f5e5a6fdff140017948 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Fri, 11 Oct 2024 13:06:37 +0200 Subject: [PATCH 12/26] More tests --- .../src/api/routes/tests/search.spec.ts | 249 ++++++++++-------- 1 file changed, 135 insertions(+), 114 deletions(-) diff --git a/packages/server/src/api/routes/tests/search.spec.ts b/packages/server/src/api/routes/tests/search.spec.ts index 364b6aa77a..6b63b94a43 100644 --- a/packages/server/src/api/routes/tests/search.spec.ts +++ b/packages/server/src/api/routes/tests/search.spec.ts @@ -23,6 +23,7 @@ import { EmptyFilterOption, FieldType, JsonFieldSubType, + LogicalOperator, RelationshipType, Row, RowSearchParams, @@ -2331,7 +2332,9 @@ describe.each([ }) describe("logical filters", () => { - describe("just $ands", () => { + const logicalOperators = [LogicalOperator.AND, LogicalOperator.OR] + + describe("$and", () => { it("should allow single conditions", async () => { await expectQuery({ $and: { @@ -2359,66 +2362,75 @@ describe.each([ }).toContainExactly([]) }) - it("should allow nested ands with single conditions", async () => { - await expectQuery({ - $and: { - conditions: [ - { - $and: { - conditions: [ - { - equal: { ["productCat.name"]: "foo" }, - }, - ], + it.each([logicalOperators])( + "should allow nested ands with single conditions (with %s as root)", + async rootOperator => { + await expectQuery({ + [rootOperator]: { + conditions: [ + { + $and: { + conditions: [ + { + equal: { ["productCat.name"]: "foo" }, + }, + ], + }, }, - }, - ], - }, - }).toContainExactly([ - { name: "foo", productCat: [{ _id: productCatRows[0]._id }] }, - ]) - }) + ], + }, + }).toContainExactly([ + { name: "foo", productCat: [{ _id: productCatRows[0]._id }] }, + ]) + } + ) - it("should allow nested ands with exclusive conditions", async () => { - await expectQuery({ - $and: { - conditions: [ - { - $and: { - conditions: [ - { - equal: { ["productCat.name"]: "foo" }, - notEqual: { ["productCat.name"]: "foo" }, - }, - ], + it.each([logicalOperators])( + "should allow nested ands with exclusive conditions (with %s as root)", + async rootOperator => { + await expectQuery({ + [rootOperator]: { + conditions: [ + { + $and: { + conditions: [ + { + equal: { ["productCat.name"]: "foo" }, + notEqual: { ["productCat.name"]: "foo" }, + }, + ], + }, }, - }, - ], - }, - }).toContainExactly([]) - }) + ], + }, + }).toContainExactly([]) + } + ) - it("should allow nested ands with multiple conditions", async () => { - await expectQuery({ - $and: { - conditions: [ - { - $and: { - conditions: [ - { - equal: { ["productCat.name"]: "foo" }, - }, - ], + it.each([logicalOperators])( + "should allow nested ands with multiple conditions (with %s as root)", + async rootOperator => { + await expectQuery({ + [rootOperator]: { + conditions: [ + { + $and: { + conditions: [ + { + equal: { ["productCat.name"]: "foo" }, + }, + ], + }, + notEqual: { ["productCat.name"]: "foo" }, }, - notEqual: { ["productCat.name"]: "foo" }, - }, - ], - }, - }).toContainExactly([]) - }) + ], + }, + }).toContainExactly([]) + } + ) }) - describe("just $ors", () => { + describe("$ors", () => { it("should allow single conditions", async () => { await expectQuery({ $or: { @@ -2450,71 +2462,80 @@ describe.each([ ]) }) - it("should allow nested ors with single conditions", async () => { - await expectQuery({ - $or: { - conditions: [ - { - $or: { - conditions: [ - { - equal: { ["productCat.name"]: "foo" }, - }, - ], + it.each([logicalOperators])( + "should allow nested ors with single conditions (with %s as root)", + async rootOperator => { + await expectQuery({ + [rootOperator]: { + conditions: [ + { + $or: { + conditions: [ + { + equal: { ["productCat.name"]: "foo" }, + }, + ], + }, }, - }, - ], - }, - }).toContainExactly([ - { name: "foo", productCat: [{ _id: productCatRows[0]._id }] }, - ]) - }) + ], + }, + }).toContainExactly([ + { name: "foo", productCat: [{ _id: productCatRows[0]._id }] }, + ]) + } + ) - it("should allow nested ors with exclusive conditions", async () => { - await expectQuery({ - $or: { - conditions: [ - { - $or: { - conditions: [ - { - equal: { ["productCat.name"]: "foo" }, - notEqual: { ["productCat.name"]: "foo" }, - }, - ], + it.each([logicalOperators])( + "should allow nested ors with exclusive conditions (with %s as root)", + async rootOperator => { + await expectQuery({ + [rootOperator]: { + conditions: [ + { + $or: { + conditions: [ + { + equal: { ["productCat.name"]: "foo" }, + notEqual: { ["productCat.name"]: "foo" }, + }, + ], + }, }, - }, - ], - }, - }).toContainExactly([ - { name: "foo", productCat: [{ _id: productCatRows[0]._id }] }, - { name: "bar", productCat: [{ _id: productCatRows[1]._id }] }, - // { name: "baz", productCat: undefined }, // TODO - ]) - }) + ], + }, + }).toContainExactly([ + { name: "foo", productCat: [{ _id: productCatRows[0]._id }] }, + { name: "bar", productCat: [{ _id: productCatRows[1]._id }] }, + // { name: "baz", productCat: undefined }, // TODO + ]) + } + ) - it("should allow nested ors with multiple conditions", async () => { - await expectQuery({ - $or: { - conditions: [ - { - $or: { - conditions: [ - { - equal: { ["productCat.name"]: "foo" }, - }, - ], + it.each([logicalOperators])( + "should allow nested ors with multiple conditions (with %s as root)", + async rootOperator => { + await expectQuery({ + [rootOperator]: { + conditions: [ + { + $or: { + conditions: [ + { + equal: { ["productCat.name"]: "foo" }, + }, + ], + }, + notEqual: { ["productCat.name"]: "foo" }, }, - notEqual: { ["productCat.name"]: "foo" }, - }, - ], - }, - }).toContainExactly([ - { name: "foo", productCat: [{ _id: productCatRows[0]._id }] }, - { name: "bar", productCat: [{ _id: productCatRows[1]._id }] }, - // { name: "baz", productCat: undefined }, // TODO - ]) - }) + ], + }, + }).toContainExactly([ + { name: "foo", productCat: [{ _id: productCatRows[0]._id }] }, + { name: "bar", productCat: [{ _id: productCatRows[1]._id }] }, + // { name: "baz", productCat: undefined }, // TODO + ]) + } + ) }) }) }) From ca7a7bcef9f55d71baf6e0d551ee71da42fe8fed Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Fri, 11 Oct 2024 13:13:34 +0200 Subject: [PATCH 13/26] Fix tests --- .../src/api/routes/tests/search.spec.ts | 45 +++++++++---------- 1 file changed, 21 insertions(+), 24 deletions(-) diff --git a/packages/server/src/api/routes/tests/search.spec.ts b/packages/server/src/api/routes/tests/search.spec.ts index 6b63b94a43..3ab35c9294 100644 --- a/packages/server/src/api/routes/tests/search.spec.ts +++ b/packages/server/src/api/routes/tests/search.spec.ts @@ -2511,31 +2511,28 @@ describe.each([ } ) - it.each([logicalOperators])( - "should allow nested ors with multiple conditions (with %s as root)", - async rootOperator => { - await expectQuery({ - [rootOperator]: { - conditions: [ - { - $or: { - conditions: [ - { - equal: { ["productCat.name"]: "foo" }, - }, - ], - }, - notEqual: { ["productCat.name"]: "foo" }, + it("should allow nested ors with multiple conditions", async () => { + await expectQuery({ + $or: { + conditions: [ + { + $or: { + conditions: [ + { + equal: { ["productCat.name"]: "foo" }, + }, + ], }, - ], - }, - }).toContainExactly([ - { name: "foo", productCat: [{ _id: productCatRows[0]._id }] }, - { name: "bar", productCat: [{ _id: productCatRows[1]._id }] }, - // { name: "baz", productCat: undefined }, // TODO - ]) - } - ) + notEqual: { ["productCat.name"]: "foo" }, + }, + ], + }, + }).toContainExactly([ + { name: "foo", productCat: [{ _id: productCatRows[0]._id }] }, + { name: "bar", productCat: [{ _id: productCatRows[1]._id }] }, + // { name: "baz", productCat: undefined }, // TODO + ]) + }) }) }) }) From a6cb1d072a49df425e94bd684473c8536b9233a9 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Fri, 11 Oct 2024 17:39:01 +0200 Subject: [PATCH 14/26] Fix sql alias test --- packages/server/src/integrations/tests/sqlAlias.spec.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/server/src/integrations/tests/sqlAlias.spec.ts b/packages/server/src/integrations/tests/sqlAlias.spec.ts index fc5af4238c..890c8c4663 100644 --- a/packages/server/src/integrations/tests/sqlAlias.spec.ts +++ b/packages/server/src/integrations/tests/sqlAlias.spec.ts @@ -79,7 +79,7 @@ describe("Captures of real examples", () => { sql: expect.stringContaining( multiline( `where exists (select 1 from "tasks" as "b" inner join "products_tasks" as "c" on "b"."taskid" = "c"."taskid" where "c"."productid" = "a"."productid" - and COALESCE("b"."taskname" = $1, FALSE)` + and (COALESCE("b"."taskname" = $1, FALSE))` ) ), }) @@ -144,7 +144,7 @@ describe("Captures of real examples", () => { ], sql: expect.stringContaining( multiline( - `where exists (select 1 from "persons" as "c" where "c"."personid" = "a"."executorid" and "c"."year" between $1 and $2)` + `where exists (select 1 from "persons" as "c" where "c"."personid" = "a"."executorid" and ("c"."year" between $1 and $2))` ) ), }) From 329d4fc01b4bf65592fa20fa2325ccbf24a92017 Mon Sep 17 00:00:00 2001 From: Christos Alexiou Date: Mon, 14 Oct 2024 11:47:15 +0300 Subject: [PATCH 15/26] Move conditionals to env section --- .github/workflows/deploy-featurebranch.yml | 44 ++++++++-------------- 1 file changed, 16 insertions(+), 28 deletions(-) diff --git a/.github/workflows/deploy-featurebranch.yml b/.github/workflows/deploy-featurebranch.yml index 0e19f0649f..872faa98fa 100644 --- a/.github/workflows/deploy-featurebranch.yml +++ b/.github/workflows/deploy-featurebranch.yml @@ -2,13 +2,11 @@ name: deploy-featurebranch on: pull_request: - types: [ - labeled, - # default types below (https://docs.github.com/en/actions/using-workflows/events-that-trigger-workflows#pull_request) - opened, - synchronize, - reopened, - ] + types: + - labeled + - opened + - synchronize + - reopened jobs: release: @@ -22,31 +20,21 @@ jobs: contains(github.event.pull_request.labels.*.name, 'feature-branch-enterprise') ) runs-on: ubuntu-latest + env: + PAYLOAD_BRANCH: ${{ github.head_ref }} + PAYLOAD_PR_NUMBER: ${{ github.event.pull_request.number }} + PAYLOAD_LICENSE_TYPE: | + ${{ + contains(github.event.pull_request.labels.*.name, 'feature-branch') && 'free' || + contains(github.event.pull_request.labels.*.name, 'feature-branch-pro') && 'pro' || + contains(github.event.pull_request.labels.*.name, 'feature-branch-team') && 'team' || + contains(github.event.pull_request.labels.*.name, 'feature-branch-business') && 'business' || + contains(github.event.pull_request.labels.*.name, 'feature-branch-enterprise') && 'enterprise' || 'free' + }} steps: - uses: actions/checkout@v4 - - name: Set PAYLOAD_LICENSE_TYPE - id: set_license_type - run: | - if [[ "${{ contains(github.event.pull_request.labels.*.name, 'feature-branch') }}" == "true" ]]; then - echo "PAYLOAD_LICENSE_TYPE=free" >> $GITHUB_ENV - elif [[ "${{ contains(github.event.pull_request.labels.*.name, 'feature-branch-pro') }}" == "true" ]]; then - echo "PAYLOAD_LICENSE_TYPE=pro" >> $GITHUB_ENV - elif [[ "${{ contains(github.event.pull_request.labels.*.name, 'feature-branch-team') }}" == "true" ]]; then - echo "PAYLOAD_LICENSE_TYPE=team" >> $GITHUB_ENV - elif [[ "${{ contains(github.event.pull_request.labels.*.name, 'feature-branch-business') }}" == "true" ]]; then - echo "PAYLOAD_LICENSE_TYPE=business" >> $GITHUB_ENV - elif [[ "${{ contains(github.event.pull_request.labels.*.name, 'feature-branch-enterprise') }}" == "true" ]]; then - echo "PAYLOAD_LICENSE_TYPE=enterprise" >> $GITHUB_ENV - else - echo "PAYLOAD_LICENSE_TYPE=free" >> $GITHUB_ENV - fi - - uses: passeidireto/trigger-external-workflow-action@main - env: - PAYLOAD_BRANCH: ${{ github.head_ref }} - PAYLOAD_PR_NUMBER: ${{ github.event.pull_request.number }} - PAYLOAD_LICENSE_TYPE: ${{ env.PAYLOAD_LICENSE_TYPE }} with: repository: budibase/budibase-deploys event: featurebranch-qa-deploy From 1052f3608421dcc43de702883b70d86f4b322e2f Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Mon, 14 Oct 2024 13:00:26 +0200 Subject: [PATCH 16/26] Fix invalid schemas while fetching --- packages/server/src/sdk/app/tables/getters.ts | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/packages/server/src/sdk/app/tables/getters.ts b/packages/server/src/sdk/app/tables/getters.ts index 5ff000fe12..280fb5378a 100644 --- a/packages/server/src/sdk/app/tables/getters.ts +++ b/packages/server/src/sdk/app/tables/getters.ts @@ -20,7 +20,13 @@ export async function processTable(table: Table): Promise { if (!table) { return table } + + table = { ...table } if (table._id && isExternalTableID(table._id)) { + // Old created external tables via Budibase might have a missing field name breaking some UI such as filters + if (table.schema["id"] && !table.schema["id"].name) { + table.schema["id"].name = "id" + } return { ...table, type: "table", From 61967b6b4b460c8c45a43efa725653da97446db2 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Mon, 14 Oct 2024 13:01:20 +0200 Subject: [PATCH 17/26] Fix invalid schema --- .../_components/panels/Tables/CreateExternalTableModal.svelte | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/builder/src/pages/builder/app/[application]/data/datasource/[datasourceId]/_components/panels/Tables/CreateExternalTableModal.svelte b/packages/builder/src/pages/builder/app/[application]/data/datasource/[datasourceId]/_components/panels/Tables/CreateExternalTableModal.svelte index 129b975309..b5a8dfc8ac 100644 --- a/packages/builder/src/pages/builder/app/[application]/data/datasource/[datasourceId]/_components/panels/Tables/CreateExternalTableModal.svelte +++ b/packages/builder/src/pages/builder/app/[application]/data/datasource/[datasourceId]/_components/panels/Tables/CreateExternalTableModal.svelte @@ -23,6 +23,7 @@ sourceType: DB_TYPE_EXTERNAL, schema: { id: { + name: "id", autocolumn: true, type: "number", }, From e9ad3e58bec11d4f8cd105318e51b824b2943de7 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Mon, 14 Oct 2024 12:14:41 +0100 Subject: [PATCH 18/26] Support default values on multi-user columns. --- .../server/src/api/routes/tests/row.spec.ts | 33 +++++++++++++++++++ .../src/utilities/rowProcessor/index.ts | 15 ++++++--- packages/shared-core/src/table.ts | 2 +- .../types/src/documents/app/table/schema.ts | 1 + 4 files changed, 45 insertions(+), 6 deletions(-) diff --git a/packages/server/src/api/routes/tests/row.spec.ts b/packages/server/src/api/routes/tests/row.spec.ts index 6490b4770a..2717a1b7d5 100644 --- a/packages/server/src/api/routes/tests/row.spec.ts +++ b/packages/server/src/api/routes/tests/row.spec.ts @@ -789,6 +789,39 @@ describe.each([ }) }) + describe("multi-user column", () => { + beforeAll(async () => { + table = await config.api.table.save( + saveTableRequest({ + schema: { + users: { + name: "users", + type: FieldType.BB_REFERENCE, + subtype: BBReferenceFieldSubType.USER, + default: ["{{ [Current User]._id }}"], + }, + }, + }) + ) + }) + + it("creates a new row with a default value successfully", async () => { + const row = await config.api.row.save(table._id!, {}) + expect(row.users).toHaveLength(1) + expect(row.users[0]._id).toEqual(config.getUser()._id) + }) + + it("does not use default value if value specified", async () => { + const id = `us_${utils.newid()}` + await config.createUser({ _id: id }) + const row = await config.api.row.save(table._id!, { + users: [id], + }) + expect(row.users).toHaveLength(1) + expect(row.users[0]._id).toEqual(id) + }) + }) + describe("bindings", () => { describe("string column", () => { beforeAll(async () => { diff --git a/packages/server/src/utilities/rowProcessor/index.ts b/packages/server/src/utilities/rowProcessor/index.ts index e63750bff9..717ac01969 100644 --- a/packages/server/src/utilities/rowProcessor/index.ts +++ b/packages/server/src/utilities/rowProcessor/index.ts @@ -33,7 +33,7 @@ import { PROTECTED_EXTERNAL_COLUMNS, PROTECTED_INTERNAL_COLUMNS, } from "@budibase/shared-core" -import { processString } from "@budibase/string-templates" +import { processString, processStringSync } from "@budibase/string-templates" import { getTableFromSource, isUserMetadataTable, @@ -134,10 +134,15 @@ async function processDefaultValues(table: Table, row: Row) { for (const [key, schema] of Object.entries(table.schema)) { if ("default" in schema && schema.default != null && row[key] == null) { - const processed = - typeof schema.default === "string" - ? await processString(schema.default, ctx) - : schema.default + let processed: string | string[] + if (Array.isArray(schema.default)) { + processed = schema.default.map(val => processStringSync(val, ctx)) + } else if (typeof schema.default === "string") { + processed = processStringSync(schema.default, ctx) + } else { + processed = schema.default + } + try { row[key] = coerce(processed, schema.type) } catch (err: any) { diff --git a/packages/shared-core/src/table.ts b/packages/shared-core/src/table.ts index 677b1e2357..e81e8266cc 100644 --- a/packages/shared-core/src/table.ts +++ b/packages/shared-core/src/table.ts @@ -66,7 +66,7 @@ const allowDefaultColumnByType: Record = { [FieldType.ATTACHMENT_SINGLE]: false, [FieldType.SIGNATURE_SINGLE]: false, [FieldType.LINK]: false, - [FieldType.BB_REFERENCE]: false, + [FieldType.BB_REFERENCE]: true, [FieldType.BB_REFERENCE_SINGLE]: true, } diff --git a/packages/types/src/documents/app/table/schema.ts b/packages/types/src/documents/app/table/schema.ts index f5bb081fd5..b98a0a3d4a 100644 --- a/packages/types/src/documents/app/table/schema.ts +++ b/packages/types/src/documents/app/table/schema.ts @@ -121,6 +121,7 @@ export interface BBReferenceFieldMetadata type: FieldType.BB_REFERENCE subtype: BBReferenceFieldSubType relationshipType?: RelationshipType + default?: string[] } export interface BBReferenceSingleFieldMetadata extends Omit { From 7c9b61181f0e70a1f9ee515a9bd02afec727f8ef Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Mon, 14 Oct 2024 12:15:45 +0100 Subject: [PATCH 19/26] Fix lint. --- packages/server/src/utilities/rowProcessor/index.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/server/src/utilities/rowProcessor/index.ts b/packages/server/src/utilities/rowProcessor/index.ts index 717ac01969..dd17bc9ba6 100644 --- a/packages/server/src/utilities/rowProcessor/index.ts +++ b/packages/server/src/utilities/rowProcessor/index.ts @@ -33,7 +33,7 @@ import { PROTECTED_EXTERNAL_COLUMNS, PROTECTED_INTERNAL_COLUMNS, } from "@budibase/shared-core" -import { processString, processStringSync } from "@budibase/string-templates" +import { processStringSync } from "@budibase/string-templates" import { getTableFromSource, isUserMetadataTable, From 874df3f54dfc658b9a501ba4a7df24e0201125b2 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Mon, 14 Oct 2024 13:20:34 +0200 Subject: [PATCH 20/26] Fix wrong schema while fetching from datasource --- .../server/src/api/controllers/table/index.ts | 21 ++++++++++--------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/packages/server/src/api/controllers/table/index.ts b/packages/server/src/api/controllers/table/index.ts index efe1a88e4a..2f2f93bffe 100644 --- a/packages/server/src/api/controllers/table/index.ts +++ b/packages/server/src/api/controllers/table/index.ts @@ -71,19 +71,20 @@ export async function fetch(ctx: UserCtx) { const datasources = await sdk.datasources.getExternalDatasources() - const external = datasources.flatMap(datasource => { + const external: Table[] = [] + for (const datasource of datasources) { let entities = datasource.entities if (entities) { - return Object.values(entities).map
((entity: Table) => ({ - ...entity, - sourceType: TableSourceType.EXTERNAL, - sourceId: datasource._id!, - sql: isSQL(datasource), - })) - } else { - return [] + for (const entity of Object.values(entities)) { + external.push({ + ...(await processTable(entity)), + sourceType: TableSourceType.EXTERNAL, + sourceId: datasource._id!, + sql: isSQL(datasource), + }) + } } - }) + } const result: FetchTablesResponse = [] for (const table of [...internal, ...external]) { From c9d42a0e2964a3c5b1710421e5ec16eaa6881846 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Mon, 14 Oct 2024 14:51:17 +0100 Subject: [PATCH 21/26] Add SQS feature flags, remove scarf.sh URLs. --- hosting/docker-compose.yaml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/hosting/docker-compose.yaml b/hosting/docker-compose.yaml index 6d14361d25..1c4bf9a1f8 100644 --- a/hosting/docker-compose.yaml +++ b/hosting/docker-compose.yaml @@ -5,7 +5,7 @@ version: "3" services: app-service: restart: unless-stopped - image: budibase.docker.scarf.sh/budibase/apps + image: budibase/apps container_name: bbapps environment: SELF_HOSTED: 1 @@ -27,6 +27,7 @@ services: BB_ADMIN_USER_PASSWORD: ${BB_ADMIN_USER_PASSWORD} PLUGINS_DIR: ${PLUGINS_DIR} OFFLINE_MODE: ${OFFLINE_MODE:-} + TENANT_FEATURE_FLAGS: "*:SQS" depends_on: - worker-service - redis-service @@ -35,7 +36,7 @@ services: worker-service: restart: unless-stopped - image: budibase.docker.scarf.sh/budibase/worker + image: budibase/worker container_name: bbworker environment: SELF_HOSTED: 1 @@ -54,6 +55,7 @@ services: REDIS_URL: redis-service:6379 REDIS_PASSWORD: ${REDIS_PASSWORD} OFFLINE_MODE: ${OFFLINE_MODE:-} + TENANT_FEATURE_FLAGS: "*:SQS" depends_on: - redis-service - minio-service From 7ea2c187a7a6572da98587c0dd8e55dd3a8c854e Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Mon, 14 Oct 2024 16:17:24 +0200 Subject: [PATCH 22/26] Simplify --- packages/backend-core/src/sql/sql.ts | 22 +++++++--------------- 1 file changed, 7 insertions(+), 15 deletions(-) diff --git a/packages/backend-core/src/sql/sql.ts b/packages/backend-core/src/sql/sql.ts index 2b697d42ae..b415a6f1b7 100644 --- a/packages/backend-core/src/sql/sql.ts +++ b/packages/backend-core/src/sql/sql.ts @@ -406,7 +406,6 @@ class InternalBuilder { addRelationshipForFilter( query: Knex.QueryBuilder, filterKey: string, - isOr: boolean, whereCb: (query: Knex.QueryBuilder) => Knex.QueryBuilder ): Knex.QueryBuilder { const mainKnex = this.knex @@ -471,12 +470,7 @@ class InternalBuilder { ) ) } - - if (isOr) { - query = query.orWhereExists(whereCb(subQuery)) - } else { - query = query.whereExists(whereCb(subQuery)) - } + query = query.whereExists(whereCb(subQuery)) break } } @@ -561,14 +555,12 @@ class InternalBuilder { value ) } else if (shouldProcessRelationship) { - query = builder.addRelationshipForFilter( - query, - updatedKey, - !!allOr, - q => { - return handleRelationship(q, updatedKey, value) - } - ) + if (allOr) { + query = query.or + } + query = builder.addRelationshipForFilter(query, updatedKey, q => { + return handleRelationship(q, updatedKey, value) + }) } } } From 58b4a37fca1ac7df76d7eb6d0b14198582ef3dbb Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Mon, 14 Oct 2024 17:20:36 +0100 Subject: [PATCH 23/26] Enable SQS in code instead of in env vars. --- hosting/docker-compose.yaml | 2 -- packages/backend-core/src/features/features.ts | 2 +- packages/backend-core/src/features/tests/features.spec.ts | 6 ++++++ 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/hosting/docker-compose.yaml b/hosting/docker-compose.yaml index 1c4bf9a1f8..c7a22eb2b3 100644 --- a/hosting/docker-compose.yaml +++ b/hosting/docker-compose.yaml @@ -27,7 +27,6 @@ services: BB_ADMIN_USER_PASSWORD: ${BB_ADMIN_USER_PASSWORD} PLUGINS_DIR: ${PLUGINS_DIR} OFFLINE_MODE: ${OFFLINE_MODE:-} - TENANT_FEATURE_FLAGS: "*:SQS" depends_on: - worker-service - redis-service @@ -55,7 +54,6 @@ services: REDIS_URL: redis-service:6379 REDIS_PASSWORD: ${REDIS_PASSWORD} OFFLINE_MODE: ${OFFLINE_MODE:-} - TENANT_FEATURE_FLAGS: "*:SQS" depends_on: - redis-service - minio-service diff --git a/packages/backend-core/src/features/features.ts b/packages/backend-core/src/features/features.ts index 20b207bb02..e95472a784 100644 --- a/packages/backend-core/src/features/features.ts +++ b/packages/backend-core/src/features/features.ts @@ -269,7 +269,7 @@ export class FlagSet, T extends { [key: string]: V }> { export const flags = new FlagSet({ DEFAULT_VALUES: Flag.boolean(env.isDev()), AUTOMATION_BRANCHING: Flag.boolean(env.isDev()), - SQS: Flag.boolean(env.isDev()), + SQS: Flag.boolean(true), [FeatureFlag.AI_CUSTOM_CONFIGS]: Flag.boolean(env.isDev()), [FeatureFlag.ENRICHED_RELATIONSHIPS]: Flag.boolean(env.isDev()), [FeatureFlag.TABLES_DEFAULT_ADMIN]: Flag.boolean(env.isDev()), diff --git a/packages/backend-core/src/features/tests/features.spec.ts b/packages/backend-core/src/features/tests/features.spec.ts index 9af8a8f4bb..ced874f4af 100644 --- a/packages/backend-core/src/features/tests/features.spec.ts +++ b/packages/backend-core/src/features/tests/features.spec.ts @@ -10,6 +10,7 @@ const schema = { TEST_BOOLEAN: Flag.boolean(false), TEST_STRING: Flag.string("default value"), TEST_NUMBER: Flag.number(0), + TEST_BOOLEAN_DEFAULT_TRUE: Flag.boolean(true), } const flags = new FlagSet(schema) @@ -123,6 +124,11 @@ describe("feature flags", () => { }, expected: flags.defaults(), }, + { + it: "should be possible to override a default true flag to false", + environmentFlags: "default:!TEST_BOOLEAN_DEFAULT_TRUE", + expected: { TEST_BOOLEAN_DEFAULT_TRUE: false }, + }, ])( "$it", async ({ From 61558aff774f169545f75e640794aad2c87dd59a Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Mon, 14 Oct 2024 17:24:14 +0100 Subject: [PATCH 24/26] Update Helm chart for SQS. --- charts/budibase/templates/app-service-deployment.yaml | 6 ------ .../templates/automation-worker-service-deployment.yaml | 2 ++ charts/budibase/templates/worker-service-deployment.yaml | 6 ------ charts/budibase/values.yaml | 5 ----- 4 files changed, 2 insertions(+), 17 deletions(-) diff --git a/charts/budibase/templates/app-service-deployment.yaml b/charts/budibase/templates/app-service-deployment.yaml index 4d0560312f..5710749028 100644 --- a/charts/budibase/templates/app-service-deployment.yaml +++ b/charts/budibase/templates/app-service-deployment.yaml @@ -42,14 +42,8 @@ spec: {{ else }} value: http://{{ .Release.Name }}-svc-couchdb:{{ .Values.services.couchdb.port }} {{ end }} - {{ if .Values.globals.sqs.enabled }} - name: COUCH_DB_SQL_URL - {{ if .Values.globals.sqs.url }} - value: {{ .Values.globals.sqs.url }} - {{ else }} value: http://{{ .Release.Name }}-svc-couchdb:{{ .Values.globals.sqs.port }} - {{ end }} - {{ end }} {{ if .Values.services.couchdb.enabled }} - name: COUCH_DB_USER valueFrom: diff --git a/charts/budibase/templates/automation-worker-service-deployment.yaml b/charts/budibase/templates/automation-worker-service-deployment.yaml index 71089bd7ee..ee3cd6dbdb 100644 --- a/charts/budibase/templates/automation-worker-service-deployment.yaml +++ b/charts/budibase/templates/automation-worker-service-deployment.yaml @@ -43,6 +43,8 @@ spec: {{ else }} value: http://{{ .Release.Name }}-svc-couchdb:{{ .Values.services.couchdb.port }} {{ end }} + - name: COUCH_DB_SQL_URL + value: http://{{ .Release.Name }}-svc-couchdb:{{ .Values.globals.sqs.port }} {{ if .Values.services.couchdb.enabled }} - name: COUCH_DB_USER valueFrom: diff --git a/charts/budibase/templates/worker-service-deployment.yaml b/charts/budibase/templates/worker-service-deployment.yaml index dcab33fa58..a493980fd3 100644 --- a/charts/budibase/templates/worker-service-deployment.yaml +++ b/charts/budibase/templates/worker-service-deployment.yaml @@ -56,14 +56,8 @@ spec: {{ else }} value: http://{{ .Release.Name }}-svc-couchdb:{{ .Values.services.couchdb.port }} {{ end }} - {{ if .Values.globals.sqs.enabled }} - name: COUCH_DB_SQL_URL - {{ if .Values.globals.sqs.url }} - value: {{ .Values.globals.sqs.url }} - {{ else }} value: http://{{ .Release.Name }}-svc-couchdb:{{ .Values.globals.sqs.port }} - {{ end }} - {{ end }} - name: API_ENCRYPTION_KEY valueFrom: secretKeyRef: diff --git a/charts/budibase/values.yaml b/charts/budibase/values.yaml index 2c1525bd90..9a5cdcdc82 100644 --- a/charts/budibase/values.yaml +++ b/charts/budibase/values.yaml @@ -139,11 +139,6 @@ globals: password: "" sqs: - # -- Whether to use the CouchDB "structured query service" or not. This is disabled by - # default for now, but will become the default in a future release. - enabled: false - # @ignore - url: "" # @ignore port: "4984" From dd81e246bf4eff7c06fc192fef054abacda97ef8 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Mon, 14 Oct 2024 17:28:25 +0100 Subject: [PATCH 25/26] Allow customisation of SQS URL to match CouchDB URL. --- charts/budibase/templates/app-service-deployment.yaml | 4 ++++ .../templates/automation-worker-service-deployment.yaml | 4 ++++ charts/budibase/templates/worker-service-deployment.yaml | 4 ++++ charts/budibase/values.yaml | 2 ++ 4 files changed, 14 insertions(+) diff --git a/charts/budibase/templates/app-service-deployment.yaml b/charts/budibase/templates/app-service-deployment.yaml index 5710749028..278bd1767f 100644 --- a/charts/budibase/templates/app-service-deployment.yaml +++ b/charts/budibase/templates/app-service-deployment.yaml @@ -43,7 +43,11 @@ spec: value: http://{{ .Release.Name }}-svc-couchdb:{{ .Values.services.couchdb.port }} {{ end }} - name: COUCH_DB_SQL_URL + {{ if .Values.globals.sqs.url }} + value: {{ .Values.globals.sqs.url | quote }} + {{ else }} value: http://{{ .Release.Name }}-svc-couchdb:{{ .Values.globals.sqs.port }} + {{ end }} {{ if .Values.services.couchdb.enabled }} - name: COUCH_DB_USER valueFrom: diff --git a/charts/budibase/templates/automation-worker-service-deployment.yaml b/charts/budibase/templates/automation-worker-service-deployment.yaml index ee3cd6dbdb..e0d43d0ce6 100644 --- a/charts/budibase/templates/automation-worker-service-deployment.yaml +++ b/charts/budibase/templates/automation-worker-service-deployment.yaml @@ -44,7 +44,11 @@ spec: value: http://{{ .Release.Name }}-svc-couchdb:{{ .Values.services.couchdb.port }} {{ end }} - name: COUCH_DB_SQL_URL + {{ if .Values.globals.sqs.url }} + value: {{ .Values.globals.sqs.url | quote }} + {{ else }} value: http://{{ .Release.Name }}-svc-couchdb:{{ .Values.globals.sqs.port }} + {{ end }} {{ if .Values.services.couchdb.enabled }} - name: COUCH_DB_USER valueFrom: diff --git a/charts/budibase/templates/worker-service-deployment.yaml b/charts/budibase/templates/worker-service-deployment.yaml index a493980fd3..94fdd0b94e 100644 --- a/charts/budibase/templates/worker-service-deployment.yaml +++ b/charts/budibase/templates/worker-service-deployment.yaml @@ -57,7 +57,11 @@ spec: value: http://{{ .Release.Name }}-svc-couchdb:{{ .Values.services.couchdb.port }} {{ end }} - name: COUCH_DB_SQL_URL + {{ if .Values.globals.sqs.url }} + value: {{ .Values.globals.sqs.url | quote }} + {{ else }} value: http://{{ .Release.Name }}-svc-couchdb:{{ .Values.globals.sqs.port }} + {{ end }} - name: API_ENCRYPTION_KEY valueFrom: secretKeyRef: diff --git a/charts/budibase/values.yaml b/charts/budibase/values.yaml index 9a5cdcdc82..de2cdb9474 100644 --- a/charts/budibase/values.yaml +++ b/charts/budibase/values.yaml @@ -139,6 +139,8 @@ globals: password: "" sqs: + # @ignore + url: "" # @ignore port: "4984" From 68498a0c54eff40d8c2d2702bab0f702908c715b Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Tue, 15 Oct 2024 12:46:27 +0100 Subject: [PATCH 26/26] Write Redis data to the persistent data dir in single image. --- hosting/single/Dockerfile | 3 +++ hosting/single/redis.conf | 7 +++++++ hosting/single/runner.sh | 8 ++++++-- 3 files changed, 16 insertions(+), 2 deletions(-) create mode 100644 hosting/single/redis.conf diff --git a/hosting/single/Dockerfile b/hosting/single/Dockerfile index ded0bc17dc..a1230f3c37 100644 --- a/hosting/single/Dockerfile +++ b/hosting/single/Dockerfile @@ -69,6 +69,9 @@ WORKDIR /minio COPY scripts/install-minio.sh ./install.sh RUN chmod +x install.sh && ./install.sh +# setup redis +COPY hosting/single/redis.conf /etc/redis/redis.conf + # setup runner file WORKDIR / COPY hosting/single/runner.sh . diff --git a/hosting/single/redis.conf b/hosting/single/redis.conf new file mode 100644 index 0000000000..00740ffece --- /dev/null +++ b/hosting/single/redis.conf @@ -0,0 +1,7 @@ +dir "DATA_DIR/redis" + +appendonly yes +appendfsync everysec + +auto-aof-rewrite-percentage 100 +auto-aof-rewrite-min-size 64mb \ No newline at end of file diff --git a/hosting/single/runner.sh b/hosting/single/runner.sh index 95464dd031..d9b8719f0f 100644 --- a/hosting/single/runner.sh +++ b/hosting/single/runner.sh @@ -75,13 +75,17 @@ fi for LINE in $(cat ${DATA_DIR}/.env); do export $LINE; done ln -s ${DATA_DIR}/.env /app/.env ln -s ${DATA_DIR}/.env /worker/.env + # make these directories in runner, incase of mount mkdir -p ${DATA_DIR}/minio +mkdir -p ${DATA_DIR}/redis chown -R couchdb:couchdb ${DATA_DIR}/couch + +sed -i "s#DATA_DIR#${DATA_DIR}#g" /etc/redis/redis.conf if [[ -n "${REDIS_PASSWORD}" ]]; then - redis-server --requirepass $REDIS_PASSWORD > /dev/stdout 2>&1 & + redis-server /etc/redis/redis.conf --requirepass $REDIS_PASSWORD > /dev/stdout 2>&1 & else - redis-server > /dev/stdout 2>&1 & + redis-server /etc/redis/redis.conf > /dev/stdout 2>&1 & fi /bbcouch-runner.sh &