Merge branch 'master' into v3-ui

charts/budibase/templates/app-service-deployment.yaml

@@ -42,14 +42,12 @@ spec:
           {{ else }}
           value: http://{{ .Release.Name }}-svc-couchdb:{{ .Values.services.couchdb.port }}
           {{ end }}
-        {{ if .Values.globals.sqs.enabled }}
         - name: COUCH_DB_SQL_URL
-        {{ if .Values.globals.sqs.url }}
+          {{ if .Values.globals.sqs.url }}
-          value: {{ .Values.globals.sqs.url }}
+          value: {{ .Values.globals.sqs.url | quote }}
-        {{ else }}
+          {{ else }}
           value: http://{{ .Release.Name }}-svc-couchdb:{{ .Values.globals.sqs.port }}
-        {{ end }}
+          {{ end }}
-        {{ end }}
         {{ if .Values.services.couchdb.enabled }}
         - name: COUCH_DB_USER
           valueFrom:

charts/budibase/templates/automation-worker-service-deployment.yaml

@@ -43,6 +43,12 @@ spec:
           {{ else }}
           value: http://{{ .Release.Name }}-svc-couchdb:{{ .Values.services.couchdb.port }}
           {{ end }}
+        - name: COUCH_DB_SQL_URL
+          {{ if .Values.globals.sqs.url }}
+          value: {{ .Values.globals.sqs.url | quote }}
+          {{ else }}
+          value: http://{{ .Release.Name }}-svc-couchdb:{{ .Values.globals.sqs.port }}
+          {{ end }}
         {{ if .Values.services.couchdb.enabled }}
         - name: COUCH_DB_USER
           valueFrom:

charts/budibase/templates/worker-service-deployment.yaml

@@ -56,14 +56,12 @@ spec:
           {{ else }}
           value: http://{{ .Release.Name }}-svc-couchdb:{{ .Values.services.couchdb.port }}
           {{ end }}
-        {{ if .Values.globals.sqs.enabled }}
         - name: COUCH_DB_SQL_URL
-        {{ if .Values.globals.sqs.url }}
+          {{ if .Values.globals.sqs.url }}
-          value: {{ .Values.globals.sqs.url }}
+          value: {{ .Values.globals.sqs.url | quote }}
-        {{ else }}
+          {{ else }}
           value: http://{{ .Release.Name }}-svc-couchdb:{{ .Values.globals.sqs.port }}
-        {{ end }}
+          {{ end }}
-        {{ end }}
         - name: API_ENCRYPTION_KEY
           valueFrom:
             secretKeyRef:

charts/budibase/values.yaml

@@ -139,9 +139,6 @@ globals:
     password: ""
 
   sqs:
-    # -- Whether to use the CouchDB "structured query service" or not. This is disabled by
-    # default for now, but will become the default in a future release.
-    enabled: false
     # @ignore
     url: ""
     # @ignore

hosting/docker-compose.yaml

@@ -5,7 +5,7 @@ version: "3"
 services:
   app-service:
     restart: unless-stopped
-    image: budibase.docker.scarf.sh/budibase/apps
+    image: budibase/apps
     container_name: bbapps
     environment:
       SELF_HOSTED: 1
@@ -35,7 +35,7 @@ services:
 
   worker-service:
     restart: unless-stopped
-    image: budibase.docker.scarf.sh/budibase/worker
+    image: budibase/worker
     container_name: bbworker
     environment:
       SELF_HOSTED: 1
@@ -97,7 +97,7 @@ services:
 
   couchdb-service:
     restart: unless-stopped
-    image: budibase/couchdb
+    image: budibase/couchdb:v3.3.3-sqs-v2.1.1
     environment:
       - COUCHDB_PASSWORD=${COUCH_DB_PASSWORD}
       - COUCHDB_USER=${COUCH_DB_USER}

packages/backend-core/src/features/features.ts

@@ -269,7 +269,7 @@ export class FlagSet<V extends Flag<any>, T extends { [key: string]: V }> {
 export const flags = new FlagSet({
   DEFAULT_VALUES: Flag.boolean(env.isDev()),
   AUTOMATION_BRANCHING: Flag.boolean(env.isDev()),
-  SQS: Flag.boolean(env.isDev()),
+  SQS: Flag.boolean(true),
   [FeatureFlag.AI_CUSTOM_CONFIGS]: Flag.boolean(env.isDev()),
   [FeatureFlag.ENRICHED_RELATIONSHIPS]: Flag.boolean(env.isDev()),
   [FeatureFlag.TABLES_DEFAULT_ADMIN]: Flag.boolean(env.isDev()),

packages/backend-core/src/features/tests/features.spec.ts

@@ -10,6 +10,7 @@ const schema = {
   TEST_BOOLEAN: Flag.boolean(false),
   TEST_STRING: Flag.string("default value"),
   TEST_NUMBER: Flag.number(0),
+  TEST_BOOLEAN_DEFAULT_TRUE: Flag.boolean(true),
 }
 const flags = new FlagSet(schema)
 
@@ -123,6 +124,11 @@ describe("feature flags", () => {
       },
       expected: flags.defaults(),
     },
+    {
+      it: "should be possible to override a default true flag to false",
+      environmentFlags: "default:!TEST_BOOLEAN_DEFAULT_TRUE",
+      expected: { TEST_BOOLEAN_DEFAULT_TRUE: false },
+    },
   ])(
     "$it",
     async ({

packages/backend-core/src/sql/sql.ts

@@ -521,8 +521,11 @@ class InternalBuilder {
         const [filterTableName, ...otherProperties] = key.split(".")
         const property = otherProperties.join(".")
         const alias = getTableAlias(filterTableName)
-        return fn(q, alias ? `${alias}.${property}` : property, value)
+        return q.andWhere(subquery =>
+          fn(subquery, alias ? `${alias}.${property}` : property, value)
+        )
       }
+
       for (const key in structure) {
         const value = structure[key]
         const updatedKey = dbCore.removeKeyNumbering(key)
@@ -552,6 +555,9 @@ class InternalBuilder {
             value
           )
         } else if (shouldProcessRelationship) {
+          if (allOr) {
+            query = query.or
+          }
           query = builder.addRelationshipForFilter(query, updatedKey, q => {
             return handleRelationship(q, updatedKey, value)
           })

packages/frontend-core/src/components/Updating.svelte

@@ -2,36 +2,31 @@
   export let isMigrationDone
   export let onMigrationDone
   export let timeoutSeconds = 60 // 1 minute
-  export let minTimeSeconds = 3
 
-  const loadTime = Date.now()
-  const intervalMs = 1000
   let timedOut = false
-  let secondsWaited = 0
 
   async function checkMigrationsFinished() {
-    setTimeout(async () => {
+    let totalWaitMs = 0
+    // eslint-disable-next-line no-constant-condition
+    while (true) {
+      const waitForMs = 5000 + Math.random() * 5000
+      await new Promise(resolve => setTimeout(resolve, waitForMs))
+      totalWaitMs += waitForMs
+
       const isMigrated = await isMigrationDone()
-
+      if (isMigrated) {
-      const timeoutMs = timeoutSeconds * 1000
+        onMigrationDone()
-      if (!isMigrated || secondsWaited <= minTimeSeconds) {
+        return
-        if (loadTime + timeoutMs > Date.now()) {
-          secondsWaited += 1
-          return checkMigrationsFinished()
-        }
-
-        return migrationTimeout()
       }
 
-      onMigrationDone()
+      if (totalWaitMs > timeoutSeconds * 1000) {
-    }, intervalMs)
+        timedOut = true
+        return
+      }
+    }
   }
 
   checkMigrationsFinished()
-
-  function migrationTimeout() {
-    timedOut = true
-  }
 </script>
 
 <div class="loading" class:timeout={timedOut}>

packages/server/src/api/routes/tests/search.spec.ts

@@ -24,6 +24,7 @@ import {
   EmptyFilterOption,
   FieldType,
   JsonFieldSubType,
+  LogicalOperator,
   RelationshipType,
   Row,
   RowSearchParams,
@@ -2415,6 +2416,211 @@ describe.each([
             equal: { ["name"]: "baz" },
           }).toContainExactly([{ name: "baz", productCat: undefined }])
         })
+
+        describe("logical filters", () => {
+          const logicalOperators = [LogicalOperator.AND, LogicalOperator.OR]
+
+          describe("$and", () => {
+            it("should allow single conditions", async () => {
+              await expectQuery({
+                $and: {
+                  conditions: [
+                    {
+                      equal: { ["productCat.name"]: "foo" },
+                    },
+                  ],
+                },
+              }).toContainExactly([
+                { name: "foo", productCat: [{ _id: productCatRows[0]._id }] },
+              ])
+            })
+
+            it("should allow exclusive conditions", async () => {
+              await expectQuery({
+                $and: {
+                  conditions: [
+                    {
+                      equal: { ["productCat.name"]: "foo" },
+                      notEqual: { ["productCat.name"]: "foo" },
+                    },
+                  ],
+                },
+              }).toContainExactly([])
+            })
+
+            it.each([logicalOperators])(
+              "should allow nested ands with single conditions (with %s as root)",
+              async rootOperator => {
+                await expectQuery({
+                  [rootOperator]: {
+                    conditions: [
+                      {
+                        $and: {
+                          conditions: [
+                            {
+                              equal: { ["productCat.name"]: "foo" },
+                            },
+                          ],
+                        },
+                      },
+                    ],
+                  },
+                }).toContainExactly([
+                  { name: "foo", productCat: [{ _id: productCatRows[0]._id }] },
+                ])
+              }
+            )
+
+            it.each([logicalOperators])(
+              "should allow nested ands with exclusive conditions (with %s as root)",
+              async rootOperator => {
+                await expectQuery({
+                  [rootOperator]: {
+                    conditions: [
+                      {
+                        $and: {
+                          conditions: [
+                            {
+                              equal: { ["productCat.name"]: "foo" },
+                              notEqual: { ["productCat.name"]: "foo" },
+                            },
+                          ],
+                        },
+                      },
+                    ],
+                  },
+                }).toContainExactly([])
+              }
+            )
+
+            it.each([logicalOperators])(
+              "should allow nested ands with multiple conditions (with %s as root)",
+              async rootOperator => {
+                await expectQuery({
+                  [rootOperator]: {
+                    conditions: [
+                      {
+                        $and: {
+                          conditions: [
+                            {
+                              equal: { ["productCat.name"]: "foo" },
+                            },
+                          ],
+                        },
+                        notEqual: { ["productCat.name"]: "foo" },
+                      },
+                    ],
+                  },
+                }).toContainExactly([])
+              }
+            )
+          })
+
+          describe("$ors", () => {
+            it("should allow single conditions", async () => {
+              await expectQuery({
+                $or: {
+                  conditions: [
+                    {
+                      equal: { ["productCat.name"]: "foo" },
+                    },
+                  ],
+                },
+              }).toContainExactly([
+                { name: "foo", productCat: [{ _id: productCatRows[0]._id }] },
+              ])
+            })
+
+            it("should allow exclusive conditions", async () => {
+              await expectQuery({
+                $or: {
+                  conditions: [
+                    {
+                      equal: { ["productCat.name"]: "foo" },
+                      notEqual: { ["productCat.name"]: "foo" },
+                    },
+                  ],
+                },
+              }).toContainExactly([
+                { name: "foo", productCat: [{ _id: productCatRows[0]._id }] },
+                { name: "bar", productCat: [{ _id: productCatRows[1]._id }] },
+                // { name: "baz", productCat: undefined }, // TODO
+              ])
+            })
+
+            it.each([logicalOperators])(
+              "should allow nested ors with single conditions (with %s as root)",
+              async rootOperator => {
+                await expectQuery({
+                  [rootOperator]: {
+                    conditions: [
+                      {
+                        $or: {
+                          conditions: [
+                            {
+                              equal: { ["productCat.name"]: "foo" },
+                            },
+                          ],
+                        },
+                      },
+                    ],
+                  },
+                }).toContainExactly([
+                  { name: "foo", productCat: [{ _id: productCatRows[0]._id }] },
+                ])
+              }
+            )
+
+            it.each([logicalOperators])(
+              "should allow nested ors with exclusive conditions (with %s as root)",
+              async rootOperator => {
+                await expectQuery({
+                  [rootOperator]: {
+                    conditions: [
+                      {
+                        $or: {
+                          conditions: [
+                            {
+                              equal: { ["productCat.name"]: "foo" },
+                              notEqual: { ["productCat.name"]: "foo" },
+                            },
+                          ],
+                        },
+                      },
+                    ],
+                  },
+                }).toContainExactly([
+                  { name: "foo", productCat: [{ _id: productCatRows[0]._id }] },
+                  { name: "bar", productCat: [{ _id: productCatRows[1]._id }] },
+                  // { name: "baz", productCat: undefined }, // TODO
+                ])
+              }
+            )
+
+            it("should allow nested ors with multiple conditions", async () => {
+              await expectQuery({
+                $or: {
+                  conditions: [
+                    {
+                      $or: {
+                        conditions: [
+                          {
+                            equal: { ["productCat.name"]: "foo" },
+                          },
+                        ],
+                      },
+                      notEqual: { ["productCat.name"]: "foo" },
+                    },
+                  ],
+                },
+              }).toContainExactly([
+                { name: "foo", productCat: [{ _id: productCatRows[0]._id }] },
+                { name: "bar", productCat: [{ _id: productCatRows[1]._id }] },
+                // { name: "baz", productCat: undefined }, // TODO
+              ])
+            })
+          })
+        })
       })
 
     isSql &&

packages/server/src/integrations/tests/sqlAlias.spec.ts

@@ -79,7 +79,7 @@ describe("Captures of real examples", () => {
         sql: expect.stringContaining(
           multiline(
             `where exists (select 1 from "tasks" as "b" inner join "products_tasks" as "c" on "b"."taskid" = "c"."taskid" where "c"."productid" = "a"."productid" 
-                 and COALESCE("b"."taskname" = $1, FALSE)`
+                 and (COALESCE("b"."taskname" = $1, FALSE))`
           )
         ),
       })
@@ -144,7 +144,7 @@ describe("Captures of real examples", () => {
         ],
         sql: expect.stringContaining(
           multiline(
-            `where exists (select 1 from "persons" as "c" where "c"."personid" = "a"."executorid" and "c"."year" between $1 and $2)`
+            `where exists (select 1 from "persons" as "c" where "c"."personid" = "a"."executorid" and ("c"."year" between $1 and $2))`
           )
         ),
       })