From 0cde00842188bdb2400f728167e3ac8cbfa7aa72 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Fri, 21 Jun 2024 17:01:27 +0100 Subject: [PATCH 01/19] Update docker-compose.yaml for SQS. --- hosting/docker-compose.yaml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/hosting/docker-compose.yaml b/hosting/docker-compose.yaml index a72b36aef1..d9811935e4 100644 --- a/hosting/docker-compose.yaml +++ b/hosting/docker-compose.yaml @@ -27,6 +27,8 @@ services: BB_ADMIN_USER_PASSWORD: ${BB_ADMIN_USER_PASSWORD} PLUGINS_DIR: ${PLUGINS_DIR} OFFLINE_MODE: ${OFFLINE_MODE:-} + SQS_SEARCH_ENABLE: "true" + COUCH_DB_SQL_URL: "http://${COUCH_DB_USER}:${COUCH_DB_PASSWORD}@couchdb-service:4984" depends_on: - worker-service - redis-service @@ -54,6 +56,8 @@ services: REDIS_URL: redis-service:6379 REDIS_PASSWORD: ${REDIS_PASSWORD} OFFLINE_MODE: ${OFFLINE_MODE:-} + SQS_SEARCH_ENABLE: "true" + COUCH_DB_SQL_URL: "http://${COUCH_DB_USER}:${COUCH_DB_PASSWORD}@couchdb-service:4984" depends_on: - redis-service - minio-service @@ -97,7 +101,7 @@ services: couchdb-service: restart: unless-stopped - image: budibase/couchdb + image: budibase/couchdb:v3.3.3-sqs environment: - COUCHDB_PASSWORD=${COUCH_DB_PASSWORD} - COUCHDB_USER=${COUCH_DB_USER} From f64c48addff56882906d43a4978b3d80b81c99a7 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Fri, 21 Jun 2024 17:09:39 +0100 Subject: [PATCH 02/19] Add some jitter to the migration interval, and increase to a minimum of 5 seconds. --- packages/cli/src/hosting/utils.ts | 3 +- .../src/components/Updating.svelte | 34 ++++++++----------- 2 files changed, 16 insertions(+), 21 deletions(-) diff --git a/packages/cli/src/hosting/utils.ts b/packages/cli/src/hosting/utils.ts index 5c3ac33f44..05eb18dc1d 100644 --- a/packages/cli/src/hosting/utils.ts +++ b/packages/cli/src/hosting/utils.ts @@ -46,7 +46,8 @@ export function setServiceImage(service: string, image: string) { export async function downloadDockerCompose() { const filename = composeFilename() try { - await downloadFile(COMPOSE_URL, `./${filename}`) + fs.copyFileSync("../../hosting/docker-compose.yaml", `./${filename}`) + //await downloadFile(COMPOSE_URL, `./${filename}`) } catch (err) { console.error(error(`Failed to retrieve compose file - ${err}`)) } diff --git a/packages/frontend-core/src/components/Updating.svelte b/packages/frontend-core/src/components/Updating.svelte index 7d14e57aba..311a6b91c8 100644 --- a/packages/frontend-core/src/components/Updating.svelte +++ b/packages/frontend-core/src/components/Updating.svelte @@ -2,36 +2,30 @@ export let isMigrationDone export let onMigrationDone export let timeoutSeconds = 60 // 1 minute - export let minTimeSeconds = 3 - const loadTime = Date.now() - const intervalMs = 1000 let timedOut = false - let secondsWaited = 0 async function checkMigrationsFinished() { - setTimeout(async () => { + let totalWaitMs = 0 + while (true) { + const waitForMs = 5000 + Math.random() * 5000 + await new Promise(resolve => setTimeout(resolve, waitForMs)) + totalWaitMs += waitForMs + const isMigrated = await isMigrationDone() - - const timeoutMs = timeoutSeconds * 1000 - if (!isMigrated || secondsWaited <= minTimeSeconds) { - if (loadTime + timeoutMs > Date.now()) { - secondsWaited += 1 - return checkMigrationsFinished() - } - - return migrationTimeout() + if (isMigrated) { + onMigrationDone() + return } - onMigrationDone() - }, intervalMs) + if (totalWaitMs > timeoutSeconds * 1000) { + timedOut = true + return + } + } } checkMigrationsFinished() - - function migrationTimeout() { - timedOut = true - }
From bb0a0ce109df5b87a36127087b37865356dce82c Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Tue, 25 Jun 2024 11:01:29 +0100 Subject: [PATCH 03/19] Fix lint. --- packages/cli/src/hosting/utils.ts | 3 +-- packages/frontend-core/src/components/Updating.svelte | 1 + 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/cli/src/hosting/utils.ts b/packages/cli/src/hosting/utils.ts index 05eb18dc1d..5c3ac33f44 100644 --- a/packages/cli/src/hosting/utils.ts +++ b/packages/cli/src/hosting/utils.ts @@ -46,8 +46,7 @@ export function setServiceImage(service: string, image: string) { export async function downloadDockerCompose() { const filename = composeFilename() try { - fs.copyFileSync("../../hosting/docker-compose.yaml", `./${filename}`) - //await downloadFile(COMPOSE_URL, `./${filename}`) + await downloadFile(COMPOSE_URL, `./${filename}`) } catch (err) { console.error(error(`Failed to retrieve compose file - ${err}`)) } diff --git a/packages/frontend-core/src/components/Updating.svelte b/packages/frontend-core/src/components/Updating.svelte index 311a6b91c8..97e83e2322 100644 --- a/packages/frontend-core/src/components/Updating.svelte +++ b/packages/frontend-core/src/components/Updating.svelte @@ -7,6 +7,7 @@ async function checkMigrationsFinished() { let totalWaitMs = 0 + // eslint-disable-next-line no-constant-condition while (true) { const waitForMs = 5000 + Math.random() * 5000 await new Promise(resolve => setTimeout(resolve, waitForMs)) From b66591f52f6f9666560078a4d1e127072c218c2a Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Tue, 2 Jul 2024 15:40:50 +0100 Subject: [PATCH 04/19] Remove superfluous environment variables. --- hosting/docker-compose.yaml | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/hosting/docker-compose.yaml b/hosting/docker-compose.yaml index d9811935e4..d59acb8f9b 100644 --- a/hosting/docker-compose.yaml +++ b/hosting/docker-compose.yaml @@ -27,8 +27,6 @@ services: BB_ADMIN_USER_PASSWORD: ${BB_ADMIN_USER_PASSWORD} PLUGINS_DIR: ${PLUGINS_DIR} OFFLINE_MODE: ${OFFLINE_MODE:-} - SQS_SEARCH_ENABLE: "true" - COUCH_DB_SQL_URL: "http://${COUCH_DB_USER}:${COUCH_DB_PASSWORD}@couchdb-service:4984" depends_on: - worker-service - redis-service @@ -56,8 +54,6 @@ services: REDIS_URL: redis-service:6379 REDIS_PASSWORD: ${REDIS_PASSWORD} OFFLINE_MODE: ${OFFLINE_MODE:-} - SQS_SEARCH_ENABLE: "true" - COUCH_DB_SQL_URL: "http://${COUCH_DB_USER}:${COUCH_DB_PASSWORD}@couchdb-service:4984" depends_on: - redis-service - minio-service @@ -101,7 +97,7 @@ services: couchdb-service: restart: unless-stopped - image: budibase/couchdb:v3.3.3-sqs + image: budibase/couchdb:v3.3.3 environment: - COUCHDB_PASSWORD=${COUCH_DB_PASSWORD} - COUCHDB_USER=${COUCH_DB_USER} From 75c43e7c690d99766be6a2d072f30a369907af28 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Wed, 25 Sep 2024 17:01:28 +0100 Subject: [PATCH 05/19] Updating to specific SQS version. --- hosting/docker-compose.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hosting/docker-compose.yaml b/hosting/docker-compose.yaml index d59acb8f9b..6d14361d25 100644 --- a/hosting/docker-compose.yaml +++ b/hosting/docker-compose.yaml @@ -97,7 +97,7 @@ services: couchdb-service: restart: unless-stopped - image: budibase/couchdb:v3.3.3 + image: budibase/couchdb:v3.3.3-sqs-v2.1.1 environment: - COUCHDB_PASSWORD=${COUCH_DB_PASSWORD} - COUCHDB_USER=${COUCH_DB_USER} From cb78e0bc13326d5ee1c3f721e123732b5028068d Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Fri, 11 Oct 2024 12:20:04 +0200 Subject: [PATCH 06/19] Add extra tests --- .../src/api/routes/tests/search.spec.ts | 41 +++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/packages/server/src/api/routes/tests/search.spec.ts b/packages/server/src/api/routes/tests/search.spec.ts index 1ccc9bfdc9..3b3492e74c 100644 --- a/packages/server/src/api/routes/tests/search.spec.ts +++ b/packages/server/src/api/routes/tests/search.spec.ts @@ -2329,6 +2329,47 @@ describe.each([ equal: { ["name"]: "baz" }, }).toContainExactly([{ name: "baz", productCat: undefined }]) }) + + describe("logical filters", () => { + it("should allow nested ands with single conditions", async () => { + await expectQuery({ + $and: { + conditions: [ + { + $and: { + conditions: [ + { + equal: { ["productCat.name"]: "foo" }, + }, + ], + }, + }, + ], + }, + }).toContainExactly([ + { name: "foo", productCat: [{ _id: productCatRows[0]._id }] }, + ]) + }) + + it("should allow nested ands with exclusive conditions", async () => { + await expectQuery({ + $and: { + conditions: [ + { + $and: { + conditions: [ + { + equal: { ["productCat.name"]: "foo" }, + notEqual: { ["productCat.name"]: "foo" }, + }, + ], + }, + }, + ], + }, + }).toContainExactly([]) + }) + }) }) isSql && From 37450823bba30ee0c715e83fe2592c26cffd73f2 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Fri, 11 Oct 2024 12:25:10 +0200 Subject: [PATCH 07/19] More tests --- .../src/api/routes/tests/search.spec.ts | 39 +++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/packages/server/src/api/routes/tests/search.spec.ts b/packages/server/src/api/routes/tests/search.spec.ts index 3b3492e74c..1ed2f66676 100644 --- a/packages/server/src/api/routes/tests/search.spec.ts +++ b/packages/server/src/api/routes/tests/search.spec.ts @@ -2369,6 +2369,45 @@ describe.each([ }, }).toContainExactly([]) }) + + it("should allow nested ands with multiple conditions", async () => { + await expectQuery({ + $and: { + conditions: [ + { + $and: { + conditions: [ + { + equal: { ["productCat.name"]: "foo" }, + }, + ], + }, + notEqual: { ["productCat.name"]: "foo" }, + }, + ], + }, + }).toContainExactly([]) + }) + + it("should allow nesting or under and with single conditions", async () => { + await expectQuery({ + $and: { + conditions: [ + { + $or: { + conditions: [ + { + equal: { ["productCat.name"]: "foo" }, + }, + ], + }, + }, + ], + }, + }).toContainExactly([ + { name: "foo", productCat: [{ _id: productCatRows[0]._id }] }, + ]) + }) }) }) From f192a30da0a00680b73e0bfc8987675259a86338 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Fri, 11 Oct 2024 12:29:34 +0200 Subject: [PATCH 08/19] More tests --- .../src/api/routes/tests/search.spec.ts | 186 +++++++++++------- 1 file changed, 118 insertions(+), 68 deletions(-) diff --git a/packages/server/src/api/routes/tests/search.spec.ts b/packages/server/src/api/routes/tests/search.spec.ts index 1ed2f66676..a3f8f1577d 100644 --- a/packages/server/src/api/routes/tests/search.spec.ts +++ b/packages/server/src/api/routes/tests/search.spec.ts @@ -2331,82 +2331,132 @@ describe.each([ }) describe("logical filters", () => { - it("should allow nested ands with single conditions", async () => { - await expectQuery({ - $and: { - conditions: [ - { - $and: { - conditions: [ - { - equal: { ["productCat.name"]: "foo" }, - }, - ], + describe("just $ands", () => { + it("should allow nested ands with single conditions", async () => { + await expectQuery({ + $and: { + conditions: [ + { + $and: { + conditions: [ + { + equal: { ["productCat.name"]: "foo" }, + }, + ], + }, }, - }, - ], - }, - }).toContainExactly([ - { name: "foo", productCat: [{ _id: productCatRows[0]._id }] }, - ]) + ], + }, + }).toContainExactly([ + { name: "foo", productCat: [{ _id: productCatRows[0]._id }] }, + ]) + }) + + it("should allow nested ands with exclusive conditions", async () => { + await expectQuery({ + $and: { + conditions: [ + { + $and: { + conditions: [ + { + equal: { ["productCat.name"]: "foo" }, + notEqual: { ["productCat.name"]: "foo" }, + }, + ], + }, + }, + ], + }, + }).toContainExactly([]) + }) + + it("should allow nested ands with multiple conditions", async () => { + await expectQuery({ + $and: { + conditions: [ + { + $and: { + conditions: [ + { + equal: { ["productCat.name"]: "foo" }, + }, + ], + }, + notEqual: { ["productCat.name"]: "foo" }, + }, + ], + }, + }).toContainExactly([]) + }) }) - it("should allow nested ands with exclusive conditions", async () => { - await expectQuery({ - $and: { - conditions: [ - { - $and: { - conditions: [ - { - equal: { ["productCat.name"]: "foo" }, - notEqual: { ["productCat.name"]: "foo" }, - }, - ], + describe("just $ors", () => { + it("should allow nested ands with single conditions", async () => { + await expectQuery({ + $or: { + conditions: [ + { + $or: { + conditions: [ + { + equal: { ["productCat.name"]: "foo" }, + }, + ], + }, }, - }, - ], - }, - }).toContainExactly([]) - }) + ], + }, + }).toContainExactly([ + { name: "foo", productCat: [{ _id: productCatRows[0]._id }] }, + ]) + }) - it("should allow nested ands with multiple conditions", async () => { - await expectQuery({ - $and: { - conditions: [ - { - $and: { - conditions: [ - { - equal: { ["productCat.name"]: "foo" }, - }, - ], + it("should allow nested ands with exclusive conditions", async () => { + await expectQuery({ + $or: { + conditions: [ + { + $or: { + conditions: [ + { + equal: { ["productCat.name"]: "foo" }, + notEqual: { ["productCat.name"]: "foo" }, + }, + ], + }, }, - notEqual: { ["productCat.name"]: "foo" }, - }, - ], - }, - }).toContainExactly([]) - }) + ], + }, + }).toContainExactly([ + { name: "foo", productCat: [{ _id: productCatRows[0]._id }] }, + { name: "bar", productCat: [{ _id: productCatRows[1]._id }] }, + { name: "baz", productCat: undefined }, + ]) + }) - it("should allow nesting or under and with single conditions", async () => { - await expectQuery({ - $and: { - conditions: [ - { - $or: { - conditions: [ - { - equal: { ["productCat.name"]: "foo" }, - }, - ], + it("should allow nested ands with multiple conditions", async () => { + await expectQuery({ + $or: { + conditions: [ + { + $or: { + conditions: [ + { + equal: { ["productCat.name"]: "foo" }, + }, + ], + }, + notEqual: { ["productCat.name"]: "foo" }, }, - }, - ], - }, - }).toContainExactly([ - { name: "foo", productCat: [{ _id: productCatRows[0]._id }] }, - ]) + ], + }, + }).toContainExactly([ + { name: "foo", productCat: [{ _id: productCatRows[0]._id }] }, + { name: "bar", productCat: [{ _id: productCatRows[1]._id }] }, + { name: "baz", productCat: undefined }, + ]) + }) }) }) }) From 2311f8aa5079fc6e2aef61db7255c5532fe446b2 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Fri, 11 Oct 2024 12:55:23 +0200 Subject: [PATCH 09/19] Don't break or conditions on nested joins --- packages/backend-core/src/sql/sql.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/packages/backend-core/src/sql/sql.ts b/packages/backend-core/src/sql/sql.ts index 382eca3f76..f77e76023e 100644 --- a/packages/backend-core/src/sql/sql.ts +++ b/packages/backend-core/src/sql/sql.ts @@ -521,8 +521,11 @@ class InternalBuilder { const [filterTableName, ...otherProperties] = key.split(".") const property = otherProperties.join(".") const alias = getTableAlias(filterTableName) - return fn(q, alias ? `${alias}.${property}` : property, value) + return q.andWhere(subquery => + fn(subquery, alias ? `${alias}.${property}` : property, value) + ) } + for (const key in structure) { const value = structure[key] const updatedKey = dbCore.removeKeyNumbering(key) From cf089eff26af1598f85a81de9a6726a8d4b85773 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Fri, 11 Oct 2024 12:59:33 +0200 Subject: [PATCH 10/19] Fix ors --- packages/backend-core/src/sql/sql.ts | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/packages/backend-core/src/sql/sql.ts b/packages/backend-core/src/sql/sql.ts index f77e76023e..2b697d42ae 100644 --- a/packages/backend-core/src/sql/sql.ts +++ b/packages/backend-core/src/sql/sql.ts @@ -406,6 +406,7 @@ class InternalBuilder { addRelationshipForFilter( query: Knex.QueryBuilder, filterKey: string, + isOr: boolean, whereCb: (query: Knex.QueryBuilder) => Knex.QueryBuilder ): Knex.QueryBuilder { const mainKnex = this.knex @@ -470,7 +471,12 @@ class InternalBuilder { ) ) } - query = query.whereExists(whereCb(subQuery)) + + if (isOr) { + query = query.orWhereExists(whereCb(subQuery)) + } else { + query = query.whereExists(whereCb(subQuery)) + } break } } @@ -555,9 +561,14 @@ class InternalBuilder { value ) } else if (shouldProcessRelationship) { - query = builder.addRelationshipForFilter(query, updatedKey, q => { - return handleRelationship(q, updatedKey, value) - }) + query = builder.addRelationshipForFilter( + query, + updatedKey, + !!allOr, + q => { + return handleRelationship(q, updatedKey, value) + } + ) } } } From bfdead820c4b4ff13d7558fc487e42da37f688b6 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Fri, 11 Oct 2024 13:00:05 +0200 Subject: [PATCH 11/19] Cleanup tests --- .../src/api/routes/tests/search.spec.ts | 68 +++++++++++++++++-- 1 file changed, 63 insertions(+), 5 deletions(-) diff --git a/packages/server/src/api/routes/tests/search.spec.ts b/packages/server/src/api/routes/tests/search.spec.ts index a3f8f1577d..364b6aa77a 100644 --- a/packages/server/src/api/routes/tests/search.spec.ts +++ b/packages/server/src/api/routes/tests/search.spec.ts @@ -2332,6 +2332,33 @@ describe.each([ describe("logical filters", () => { describe("just $ands", () => { + it("should allow single conditions", async () => { + await expectQuery({ + $and: { + conditions: [ + { + equal: { ["productCat.name"]: "foo" }, + }, + ], + }, + }).toContainExactly([ + { name: "foo", productCat: [{ _id: productCatRows[0]._id }] }, + ]) + }) + + it("should allow exclusive conditions", async () => { + await expectQuery({ + $and: { + conditions: [ + { + equal: { ["productCat.name"]: "foo" }, + notEqual: { ["productCat.name"]: "foo" }, + }, + ], + }, + }).toContainExactly([]) + }) + it("should allow nested ands with single conditions", async () => { await expectQuery({ $and: { @@ -2392,7 +2419,38 @@ describe.each([ }) describe("just $ors", () => { - it("should allow nested ands with single conditions", async () => { + it("should allow single conditions", async () => { + await expectQuery({ + $or: { + conditions: [ + { + equal: { ["productCat.name"]: "foo" }, + }, + ], + }, + }).toContainExactly([ + { name: "foo", productCat: [{ _id: productCatRows[0]._id }] }, + ]) + }) + + it("should allow exclusive conditions", async () => { + await expectQuery({ + $or: { + conditions: [ + { + equal: { ["productCat.name"]: "foo" }, + notEqual: { ["productCat.name"]: "foo" }, + }, + ], + }, + }).toContainExactly([ + { name: "foo", productCat: [{ _id: productCatRows[0]._id }] }, + { name: "bar", productCat: [{ _id: productCatRows[1]._id }] }, + // { name: "baz", productCat: undefined }, // TODO + ]) + }) + + it("should allow nested ors with single conditions", async () => { await expectQuery({ $or: { conditions: [ @@ -2412,7 +2470,7 @@ describe.each([ ]) }) - it("should allow nested ands with exclusive conditions", async () => { + it("should allow nested ors with exclusive conditions", async () => { await expectQuery({ $or: { conditions: [ @@ -2431,11 +2489,11 @@ describe.each([ }).toContainExactly([ { name: "foo", productCat: [{ _id: productCatRows[0]._id }] }, { name: "bar", productCat: [{ _id: productCatRows[1]._id }] }, - { name: "baz", productCat: undefined }, + // { name: "baz", productCat: undefined }, // TODO ]) }) - it("should allow nested ands with multiple conditions", async () => { + it("should allow nested ors with multiple conditions", async () => { await expectQuery({ $or: { conditions: [ @@ -2454,7 +2512,7 @@ describe.each([ }).toContainExactly([ { name: "foo", productCat: [{ _id: productCatRows[0]._id }] }, { name: "bar", productCat: [{ _id: productCatRows[1]._id }] }, - { name: "baz", productCat: undefined }, + // { name: "baz", productCat: undefined }, // TODO ]) }) }) From f73b7d4824fbf12862351f5e5a6fdff140017948 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Fri, 11 Oct 2024 13:06:37 +0200 Subject: [PATCH 12/19] More tests --- .../src/api/routes/tests/search.spec.ts | 249 ++++++++++-------- 1 file changed, 135 insertions(+), 114 deletions(-) diff --git a/packages/server/src/api/routes/tests/search.spec.ts b/packages/server/src/api/routes/tests/search.spec.ts index 364b6aa77a..6b63b94a43 100644 --- a/packages/server/src/api/routes/tests/search.spec.ts +++ b/packages/server/src/api/routes/tests/search.spec.ts @@ -23,6 +23,7 @@ import { EmptyFilterOption, FieldType, JsonFieldSubType, + LogicalOperator, RelationshipType, Row, RowSearchParams, @@ -2331,7 +2332,9 @@ describe.each([ }) describe("logical filters", () => { - describe("just $ands", () => { + const logicalOperators = [LogicalOperator.AND, LogicalOperator.OR] + + describe("$and", () => { it("should allow single conditions", async () => { await expectQuery({ $and: { @@ -2359,66 +2362,75 @@ describe.each([ }).toContainExactly([]) }) - it("should allow nested ands with single conditions", async () => { - await expectQuery({ - $and: { - conditions: [ - { - $and: { - conditions: [ - { - equal: { ["productCat.name"]: "foo" }, - }, - ], + it.each([logicalOperators])( + "should allow nested ands with single conditions (with %s as root)", + async rootOperator => { + await expectQuery({ + [rootOperator]: { + conditions: [ + { + $and: { + conditions: [ + { + equal: { ["productCat.name"]: "foo" }, + }, + ], + }, }, - }, - ], - }, - }).toContainExactly([ - { name: "foo", productCat: [{ _id: productCatRows[0]._id }] }, - ]) - }) + ], + }, + }).toContainExactly([ + { name: "foo", productCat: [{ _id: productCatRows[0]._id }] }, + ]) + } + ) - it("should allow nested ands with exclusive conditions", async () => { - await expectQuery({ - $and: { - conditions: [ - { - $and: { - conditions: [ - { - equal: { ["productCat.name"]: "foo" }, - notEqual: { ["productCat.name"]: "foo" }, - }, - ], + it.each([logicalOperators])( + "should allow nested ands with exclusive conditions (with %s as root)", + async rootOperator => { + await expectQuery({ + [rootOperator]: { + conditions: [ + { + $and: { + conditions: [ + { + equal: { ["productCat.name"]: "foo" }, + notEqual: { ["productCat.name"]: "foo" }, + }, + ], + }, }, - }, - ], - }, - }).toContainExactly([]) - }) + ], + }, + }).toContainExactly([]) + } + ) - it("should allow nested ands with multiple conditions", async () => { - await expectQuery({ - $and: { - conditions: [ - { - $and: { - conditions: [ - { - equal: { ["productCat.name"]: "foo" }, - }, - ], + it.each([logicalOperators])( + "should allow nested ands with multiple conditions (with %s as root)", + async rootOperator => { + await expectQuery({ + [rootOperator]: { + conditions: [ + { + $and: { + conditions: [ + { + equal: { ["productCat.name"]: "foo" }, + }, + ], + }, + notEqual: { ["productCat.name"]: "foo" }, }, - notEqual: { ["productCat.name"]: "foo" }, - }, - ], - }, - }).toContainExactly([]) - }) + ], + }, + }).toContainExactly([]) + } + ) }) - describe("just $ors", () => { + describe("$ors", () => { it("should allow single conditions", async () => { await expectQuery({ $or: { @@ -2450,71 +2462,80 @@ describe.each([ ]) }) - it("should allow nested ors with single conditions", async () => { - await expectQuery({ - $or: { - conditions: [ - { - $or: { - conditions: [ - { - equal: { ["productCat.name"]: "foo" }, - }, - ], + it.each([logicalOperators])( + "should allow nested ors with single conditions (with %s as root)", + async rootOperator => { + await expectQuery({ + [rootOperator]: { + conditions: [ + { + $or: { + conditions: [ + { + equal: { ["productCat.name"]: "foo" }, + }, + ], + }, }, - }, - ], - }, - }).toContainExactly([ - { name: "foo", productCat: [{ _id: productCatRows[0]._id }] }, - ]) - }) + ], + }, + }).toContainExactly([ + { name: "foo", productCat: [{ _id: productCatRows[0]._id }] }, + ]) + } + ) - it("should allow nested ors with exclusive conditions", async () => { - await expectQuery({ - $or: { - conditions: [ - { - $or: { - conditions: [ - { - equal: { ["productCat.name"]: "foo" }, - notEqual: { ["productCat.name"]: "foo" }, - }, - ], + it.each([logicalOperators])( + "should allow nested ors with exclusive conditions (with %s as root)", + async rootOperator => { + await expectQuery({ + [rootOperator]: { + conditions: [ + { + $or: { + conditions: [ + { + equal: { ["productCat.name"]: "foo" }, + notEqual: { ["productCat.name"]: "foo" }, + }, + ], + }, }, - }, - ], - }, - }).toContainExactly([ - { name: "foo", productCat: [{ _id: productCatRows[0]._id }] }, - { name: "bar", productCat: [{ _id: productCatRows[1]._id }] }, - // { name: "baz", productCat: undefined }, // TODO - ]) - }) + ], + }, + }).toContainExactly([ + { name: "foo", productCat: [{ _id: productCatRows[0]._id }] }, + { name: "bar", productCat: [{ _id: productCatRows[1]._id }] }, + // { name: "baz", productCat: undefined }, // TODO + ]) + } + ) - it("should allow nested ors with multiple conditions", async () => { - await expectQuery({ - $or: { - conditions: [ - { - $or: { - conditions: [ - { - equal: { ["productCat.name"]: "foo" }, - }, - ], + it.each([logicalOperators])( + "should allow nested ors with multiple conditions (with %s as root)", + async rootOperator => { + await expectQuery({ + [rootOperator]: { + conditions: [ + { + $or: { + conditions: [ + { + equal: { ["productCat.name"]: "foo" }, + }, + ], + }, + notEqual: { ["productCat.name"]: "foo" }, }, - notEqual: { ["productCat.name"]: "foo" }, - }, - ], - }, - }).toContainExactly([ - { name: "foo", productCat: [{ _id: productCatRows[0]._id }] }, - { name: "bar", productCat: [{ _id: productCatRows[1]._id }] }, - // { name: "baz", productCat: undefined }, // TODO - ]) - }) + ], + }, + }).toContainExactly([ + { name: "foo", productCat: [{ _id: productCatRows[0]._id }] }, + { name: "bar", productCat: [{ _id: productCatRows[1]._id }] }, + // { name: "baz", productCat: undefined }, // TODO + ]) + } + ) }) }) }) From ca7a7bcef9f55d71baf6e0d551ee71da42fe8fed Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Fri, 11 Oct 2024 13:13:34 +0200 Subject: [PATCH 13/19] Fix tests --- .../src/api/routes/tests/search.spec.ts | 45 +++++++++---------- 1 file changed, 21 insertions(+), 24 deletions(-) diff --git a/packages/server/src/api/routes/tests/search.spec.ts b/packages/server/src/api/routes/tests/search.spec.ts index 6b63b94a43..3ab35c9294 100644 --- a/packages/server/src/api/routes/tests/search.spec.ts +++ b/packages/server/src/api/routes/tests/search.spec.ts @@ -2511,31 +2511,28 @@ describe.each([ } ) - it.each([logicalOperators])( - "should allow nested ors with multiple conditions (with %s as root)", - async rootOperator => { - await expectQuery({ - [rootOperator]: { - conditions: [ - { - $or: { - conditions: [ - { - equal: { ["productCat.name"]: "foo" }, - }, - ], - }, - notEqual: { ["productCat.name"]: "foo" }, + it("should allow nested ors with multiple conditions", async () => { + await expectQuery({ + $or: { + conditions: [ + { + $or: { + conditions: [ + { + equal: { ["productCat.name"]: "foo" }, + }, + ], }, - ], - }, - }).toContainExactly([ - { name: "foo", productCat: [{ _id: productCatRows[0]._id }] }, - { name: "bar", productCat: [{ _id: productCatRows[1]._id }] }, - // { name: "baz", productCat: undefined }, // TODO - ]) - } - ) + notEqual: { ["productCat.name"]: "foo" }, + }, + ], + }, + }).toContainExactly([ + { name: "foo", productCat: [{ _id: productCatRows[0]._id }] }, + { name: "bar", productCat: [{ _id: productCatRows[1]._id }] }, + // { name: "baz", productCat: undefined }, // TODO + ]) + }) }) }) }) From a6cb1d072a49df425e94bd684473c8536b9233a9 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Fri, 11 Oct 2024 17:39:01 +0200 Subject: [PATCH 14/19] Fix sql alias test --- packages/server/src/integrations/tests/sqlAlias.spec.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/server/src/integrations/tests/sqlAlias.spec.ts b/packages/server/src/integrations/tests/sqlAlias.spec.ts index fc5af4238c..890c8c4663 100644 --- a/packages/server/src/integrations/tests/sqlAlias.spec.ts +++ b/packages/server/src/integrations/tests/sqlAlias.spec.ts @@ -79,7 +79,7 @@ describe("Captures of real examples", () => { sql: expect.stringContaining( multiline( `where exists (select 1 from "tasks" as "b" inner join "products_tasks" as "c" on "b"."taskid" = "c"."taskid" where "c"."productid" = "a"."productid" - and COALESCE("b"."taskname" = $1, FALSE)` + and (COALESCE("b"."taskname" = $1, FALSE))` ) ), }) @@ -144,7 +144,7 @@ describe("Captures of real examples", () => { ], sql: expect.stringContaining( multiline( - `where exists (select 1 from "persons" as "c" where "c"."personid" = "a"."executorid" and "c"."year" between $1 and $2)` + `where exists (select 1 from "persons" as "c" where "c"."personid" = "a"."executorid" and ("c"."year" between $1 and $2))` ) ), }) From c9d42a0e2964a3c5b1710421e5ec16eaa6881846 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Mon, 14 Oct 2024 14:51:17 +0100 Subject: [PATCH 15/19] Add SQS feature flags, remove scarf.sh URLs. --- hosting/docker-compose.yaml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/hosting/docker-compose.yaml b/hosting/docker-compose.yaml index 6d14361d25..1c4bf9a1f8 100644 --- a/hosting/docker-compose.yaml +++ b/hosting/docker-compose.yaml @@ -5,7 +5,7 @@ version: "3" services: app-service: restart: unless-stopped - image: budibase.docker.scarf.sh/budibase/apps + image: budibase/apps container_name: bbapps environment: SELF_HOSTED: 1 @@ -27,6 +27,7 @@ services: BB_ADMIN_USER_PASSWORD: ${BB_ADMIN_USER_PASSWORD} PLUGINS_DIR: ${PLUGINS_DIR} OFFLINE_MODE: ${OFFLINE_MODE:-} + TENANT_FEATURE_FLAGS: "*:SQS" depends_on: - worker-service - redis-service @@ -35,7 +36,7 @@ services: worker-service: restart: unless-stopped - image: budibase.docker.scarf.sh/budibase/worker + image: budibase/worker container_name: bbworker environment: SELF_HOSTED: 1 @@ -54,6 +55,7 @@ services: REDIS_URL: redis-service:6379 REDIS_PASSWORD: ${REDIS_PASSWORD} OFFLINE_MODE: ${OFFLINE_MODE:-} + TENANT_FEATURE_FLAGS: "*:SQS" depends_on: - redis-service - minio-service From 7ea2c187a7a6572da98587c0dd8e55dd3a8c854e Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Mon, 14 Oct 2024 16:17:24 +0200 Subject: [PATCH 16/19] Simplify --- packages/backend-core/src/sql/sql.ts | 22 +++++++--------------- 1 file changed, 7 insertions(+), 15 deletions(-) diff --git a/packages/backend-core/src/sql/sql.ts b/packages/backend-core/src/sql/sql.ts index 2b697d42ae..b415a6f1b7 100644 --- a/packages/backend-core/src/sql/sql.ts +++ b/packages/backend-core/src/sql/sql.ts @@ -406,7 +406,6 @@ class InternalBuilder { addRelationshipForFilter( query: Knex.QueryBuilder, filterKey: string, - isOr: boolean, whereCb: (query: Knex.QueryBuilder) => Knex.QueryBuilder ): Knex.QueryBuilder { const mainKnex = this.knex @@ -471,12 +470,7 @@ class InternalBuilder { ) ) } - - if (isOr) { - query = query.orWhereExists(whereCb(subQuery)) - } else { - query = query.whereExists(whereCb(subQuery)) - } + query = query.whereExists(whereCb(subQuery)) break } } @@ -561,14 +555,12 @@ class InternalBuilder { value ) } else if (shouldProcessRelationship) { - query = builder.addRelationshipForFilter( - query, - updatedKey, - !!allOr, - q => { - return handleRelationship(q, updatedKey, value) - } - ) + if (allOr) { + query = query.or + } + query = builder.addRelationshipForFilter(query, updatedKey, q => { + return handleRelationship(q, updatedKey, value) + }) } } } From 58b4a37fca1ac7df76d7eb6d0b14198582ef3dbb Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Mon, 14 Oct 2024 17:20:36 +0100 Subject: [PATCH 17/19] Enable SQS in code instead of in env vars. --- hosting/docker-compose.yaml | 2 -- packages/backend-core/src/features/features.ts | 2 +- packages/backend-core/src/features/tests/features.spec.ts | 6 ++++++ 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/hosting/docker-compose.yaml b/hosting/docker-compose.yaml index 1c4bf9a1f8..c7a22eb2b3 100644 --- a/hosting/docker-compose.yaml +++ b/hosting/docker-compose.yaml @@ -27,7 +27,6 @@ services: BB_ADMIN_USER_PASSWORD: ${BB_ADMIN_USER_PASSWORD} PLUGINS_DIR: ${PLUGINS_DIR} OFFLINE_MODE: ${OFFLINE_MODE:-} - TENANT_FEATURE_FLAGS: "*:SQS" depends_on: - worker-service - redis-service @@ -55,7 +54,6 @@ services: REDIS_URL: redis-service:6379 REDIS_PASSWORD: ${REDIS_PASSWORD} OFFLINE_MODE: ${OFFLINE_MODE:-} - TENANT_FEATURE_FLAGS: "*:SQS" depends_on: - redis-service - minio-service diff --git a/packages/backend-core/src/features/features.ts b/packages/backend-core/src/features/features.ts index 20b207bb02..e95472a784 100644 --- a/packages/backend-core/src/features/features.ts +++ b/packages/backend-core/src/features/features.ts @@ -269,7 +269,7 @@ export class FlagSet, T extends { [key: string]: V }> { export const flags = new FlagSet({ DEFAULT_VALUES: Flag.boolean(env.isDev()), AUTOMATION_BRANCHING: Flag.boolean(env.isDev()), - SQS: Flag.boolean(env.isDev()), + SQS: Flag.boolean(true), [FeatureFlag.AI_CUSTOM_CONFIGS]: Flag.boolean(env.isDev()), [FeatureFlag.ENRICHED_RELATIONSHIPS]: Flag.boolean(env.isDev()), [FeatureFlag.TABLES_DEFAULT_ADMIN]: Flag.boolean(env.isDev()), diff --git a/packages/backend-core/src/features/tests/features.spec.ts b/packages/backend-core/src/features/tests/features.spec.ts index 9af8a8f4bb..ced874f4af 100644 --- a/packages/backend-core/src/features/tests/features.spec.ts +++ b/packages/backend-core/src/features/tests/features.spec.ts @@ -10,6 +10,7 @@ const schema = { TEST_BOOLEAN: Flag.boolean(false), TEST_STRING: Flag.string("default value"), TEST_NUMBER: Flag.number(0), + TEST_BOOLEAN_DEFAULT_TRUE: Flag.boolean(true), } const flags = new FlagSet(schema) @@ -123,6 +124,11 @@ describe("feature flags", () => { }, expected: flags.defaults(), }, + { + it: "should be possible to override a default true flag to false", + environmentFlags: "default:!TEST_BOOLEAN_DEFAULT_TRUE", + expected: { TEST_BOOLEAN_DEFAULT_TRUE: false }, + }, ])( "$it", async ({ From 61558aff774f169545f75e640794aad2c87dd59a Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Mon, 14 Oct 2024 17:24:14 +0100 Subject: [PATCH 18/19] Update Helm chart for SQS. --- charts/budibase/templates/app-service-deployment.yaml | 6 ------ .../templates/automation-worker-service-deployment.yaml | 2 ++ charts/budibase/templates/worker-service-deployment.yaml | 6 ------ charts/budibase/values.yaml | 5 ----- 4 files changed, 2 insertions(+), 17 deletions(-) diff --git a/charts/budibase/templates/app-service-deployment.yaml b/charts/budibase/templates/app-service-deployment.yaml index 4d0560312f..5710749028 100644 --- a/charts/budibase/templates/app-service-deployment.yaml +++ b/charts/budibase/templates/app-service-deployment.yaml @@ -42,14 +42,8 @@ spec: {{ else }} value: http://{{ .Release.Name }}-svc-couchdb:{{ .Values.services.couchdb.port }} {{ end }} - {{ if .Values.globals.sqs.enabled }} - name: COUCH_DB_SQL_URL - {{ if .Values.globals.sqs.url }} - value: {{ .Values.globals.sqs.url }} - {{ else }} value: http://{{ .Release.Name }}-svc-couchdb:{{ .Values.globals.sqs.port }} - {{ end }} - {{ end }} {{ if .Values.services.couchdb.enabled }} - name: COUCH_DB_USER valueFrom: diff --git a/charts/budibase/templates/automation-worker-service-deployment.yaml b/charts/budibase/templates/automation-worker-service-deployment.yaml index 71089bd7ee..ee3cd6dbdb 100644 --- a/charts/budibase/templates/automation-worker-service-deployment.yaml +++ b/charts/budibase/templates/automation-worker-service-deployment.yaml @@ -43,6 +43,8 @@ spec: {{ else }} value: http://{{ .Release.Name }}-svc-couchdb:{{ .Values.services.couchdb.port }} {{ end }} + - name: COUCH_DB_SQL_URL + value: http://{{ .Release.Name }}-svc-couchdb:{{ .Values.globals.sqs.port }} {{ if .Values.services.couchdb.enabled }} - name: COUCH_DB_USER valueFrom: diff --git a/charts/budibase/templates/worker-service-deployment.yaml b/charts/budibase/templates/worker-service-deployment.yaml index dcab33fa58..a493980fd3 100644 --- a/charts/budibase/templates/worker-service-deployment.yaml +++ b/charts/budibase/templates/worker-service-deployment.yaml @@ -56,14 +56,8 @@ spec: {{ else }} value: http://{{ .Release.Name }}-svc-couchdb:{{ .Values.services.couchdb.port }} {{ end }} - {{ if .Values.globals.sqs.enabled }} - name: COUCH_DB_SQL_URL - {{ if .Values.globals.sqs.url }} - value: {{ .Values.globals.sqs.url }} - {{ else }} value: http://{{ .Release.Name }}-svc-couchdb:{{ .Values.globals.sqs.port }} - {{ end }} - {{ end }} - name: API_ENCRYPTION_KEY valueFrom: secretKeyRef: diff --git a/charts/budibase/values.yaml b/charts/budibase/values.yaml index 2c1525bd90..9a5cdcdc82 100644 --- a/charts/budibase/values.yaml +++ b/charts/budibase/values.yaml @@ -139,11 +139,6 @@ globals: password: "" sqs: - # -- Whether to use the CouchDB "structured query service" or not. This is disabled by - # default for now, but will become the default in a future release. - enabled: false - # @ignore - url: "" # @ignore port: "4984" From dd81e246bf4eff7c06fc192fef054abacda97ef8 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Mon, 14 Oct 2024 17:28:25 +0100 Subject: [PATCH 19/19] Allow customisation of SQS URL to match CouchDB URL. --- charts/budibase/templates/app-service-deployment.yaml | 4 ++++ .../templates/automation-worker-service-deployment.yaml | 4 ++++ charts/budibase/templates/worker-service-deployment.yaml | 4 ++++ charts/budibase/values.yaml | 2 ++ 4 files changed, 14 insertions(+) diff --git a/charts/budibase/templates/app-service-deployment.yaml b/charts/budibase/templates/app-service-deployment.yaml index 5710749028..278bd1767f 100644 --- a/charts/budibase/templates/app-service-deployment.yaml +++ b/charts/budibase/templates/app-service-deployment.yaml @@ -43,7 +43,11 @@ spec: value: http://{{ .Release.Name }}-svc-couchdb:{{ .Values.services.couchdb.port }} {{ end }} - name: COUCH_DB_SQL_URL + {{ if .Values.globals.sqs.url }} + value: {{ .Values.globals.sqs.url | quote }} + {{ else }} value: http://{{ .Release.Name }}-svc-couchdb:{{ .Values.globals.sqs.port }} + {{ end }} {{ if .Values.services.couchdb.enabled }} - name: COUCH_DB_USER valueFrom: diff --git a/charts/budibase/templates/automation-worker-service-deployment.yaml b/charts/budibase/templates/automation-worker-service-deployment.yaml index ee3cd6dbdb..e0d43d0ce6 100644 --- a/charts/budibase/templates/automation-worker-service-deployment.yaml +++ b/charts/budibase/templates/automation-worker-service-deployment.yaml @@ -44,7 +44,11 @@ spec: value: http://{{ .Release.Name }}-svc-couchdb:{{ .Values.services.couchdb.port }} {{ end }} - name: COUCH_DB_SQL_URL + {{ if .Values.globals.sqs.url }} + value: {{ .Values.globals.sqs.url | quote }} + {{ else }} value: http://{{ .Release.Name }}-svc-couchdb:{{ .Values.globals.sqs.port }} + {{ end }} {{ if .Values.services.couchdb.enabled }} - name: COUCH_DB_USER valueFrom: diff --git a/charts/budibase/templates/worker-service-deployment.yaml b/charts/budibase/templates/worker-service-deployment.yaml index a493980fd3..94fdd0b94e 100644 --- a/charts/budibase/templates/worker-service-deployment.yaml +++ b/charts/budibase/templates/worker-service-deployment.yaml @@ -57,7 +57,11 @@ spec: value: http://{{ .Release.Name }}-svc-couchdb:{{ .Values.services.couchdb.port }} {{ end }} - name: COUCH_DB_SQL_URL + {{ if .Values.globals.sqs.url }} + value: {{ .Values.globals.sqs.url | quote }} + {{ else }} value: http://{{ .Release.Name }}-svc-couchdb:{{ .Values.globals.sqs.port }} + {{ end }} - name: API_ENCRYPTION_KEY valueFrom: secretKeyRef: diff --git a/charts/budibase/values.yaml b/charts/budibase/values.yaml index 9a5cdcdc82..de2cdb9474 100644 --- a/charts/budibase/values.yaml +++ b/charts/budibase/values.yaml @@ -139,6 +139,8 @@ globals: password: "" sqs: + # @ignore + url: "" # @ignore port: "4984"