From 0cde00842188bdb2400f728167e3ac8cbfa7aa72 Mon Sep 17 00:00:00 2001
From: Sam Rose <hello@samwho.dev>
Date: Fri, 21 Jun 2024 17:01:27 +0100
Subject: [PATCH 01/27] Update docker-compose.yaml for SQS.

---
 hosting/docker-compose.yaml | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/hosting/docker-compose.yaml b/hosting/docker-compose.yaml
index a72b36aef1..d9811935e4 100644
--- a/hosting/docker-compose.yaml
+++ b/hosting/docker-compose.yaml
@@ -27,6 +27,8 @@ services:
       BB_ADMIN_USER_PASSWORD: ${BB_ADMIN_USER_PASSWORD}
       PLUGINS_DIR: ${PLUGINS_DIR}
       OFFLINE_MODE: ${OFFLINE_MODE:-}
+      SQS_SEARCH_ENABLE: "true"
+      COUCH_DB_SQL_URL: "http://${COUCH_DB_USER}:${COUCH_DB_PASSWORD}@couchdb-service:4984"
     depends_on:
       - worker-service
       - redis-service
@@ -54,6 +56,8 @@ services:
       REDIS_URL: redis-service:6379
       REDIS_PASSWORD: ${REDIS_PASSWORD}
       OFFLINE_MODE: ${OFFLINE_MODE:-}
+      SQS_SEARCH_ENABLE: "true"
+      COUCH_DB_SQL_URL: "http://${COUCH_DB_USER}:${COUCH_DB_PASSWORD}@couchdb-service:4984"
     depends_on:
       - redis-service
       - minio-service
@@ -97,7 +101,7 @@ services:
 
   couchdb-service:
     restart: unless-stopped
-    image: budibase/couchdb
+    image: budibase/couchdb:v3.3.3-sqs
     environment:
       - COUCHDB_PASSWORD=${COUCH_DB_PASSWORD}
       - COUCHDB_USER=${COUCH_DB_USER}

From f64c48addff56882906d43a4978b3d80b81c99a7 Mon Sep 17 00:00:00 2001
From: Sam Rose <hello@samwho.dev>
Date: Fri, 21 Jun 2024 17:09:39 +0100
Subject: [PATCH 02/27] Add some jitter to the migration interval, and increase
 to a minimum of 5 seconds.

---
 packages/cli/src/hosting/utils.ts             |  3 +-
 .../src/components/Updating.svelte            | 34 ++++++++-----------
 2 files changed, 16 insertions(+), 21 deletions(-)

diff --git a/packages/cli/src/hosting/utils.ts b/packages/cli/src/hosting/utils.ts
index 5c3ac33f44..05eb18dc1d 100644
--- a/packages/cli/src/hosting/utils.ts
+++ b/packages/cli/src/hosting/utils.ts
@@ -46,7 +46,8 @@ export function setServiceImage(service: string, image: string) {
 export async function downloadDockerCompose() {
   const filename = composeFilename()
   try {
-    await downloadFile(COMPOSE_URL, `./${filename}`)
+    fs.copyFileSync("../../hosting/docker-compose.yaml", `./${filename}`)
+    //await downloadFile(COMPOSE_URL, `./${filename}`)
   } catch (err) {
     console.error(error(`Failed to retrieve compose file - ${err}`))
   }
diff --git a/packages/frontend-core/src/components/Updating.svelte b/packages/frontend-core/src/components/Updating.svelte
index 7d14e57aba..311a6b91c8 100644
--- a/packages/frontend-core/src/components/Updating.svelte
+++ b/packages/frontend-core/src/components/Updating.svelte
@@ -2,36 +2,30 @@
   export let isMigrationDone
   export let onMigrationDone
   export let timeoutSeconds = 60 // 1 minute
-  export let minTimeSeconds = 3
 
-  const loadTime = Date.now()
-  const intervalMs = 1000
   let timedOut = false
-  let secondsWaited = 0
 
   async function checkMigrationsFinished() {
-    setTimeout(async () => {
+    let totalWaitMs = 0
+    while (true) {
+      const waitForMs = 5000 + Math.random() * 5000
+      await new Promise(resolve => setTimeout(resolve, waitForMs))
+      totalWaitMs += waitForMs
+
       const isMigrated = await isMigrationDone()
-
-      const timeoutMs = timeoutSeconds * 1000
-      if (!isMigrated || secondsWaited <= minTimeSeconds) {
-        if (loadTime + timeoutMs > Date.now()) {
-          secondsWaited += 1
-          return checkMigrationsFinished()
-        }
-
-        return migrationTimeout()
+      if (isMigrated) {
+        onMigrationDone()
+        return
       }
 
-      onMigrationDone()
-    }, intervalMs)
+      if (totalWaitMs > timeoutSeconds * 1000) {
+        timedOut = true
+        return
+      }
+    }
   }
 
   checkMigrationsFinished()
-
-  function migrationTimeout() {
-    timedOut = true
-  }
 </script>
 
 <div class="loading" class:timeout={timedOut}>

From bb0a0ce109df5b87a36127087b37865356dce82c Mon Sep 17 00:00:00 2001
From: Sam Rose <hello@samwho.dev>
Date: Tue, 25 Jun 2024 11:01:29 +0100
Subject: [PATCH 03/27] Fix lint.

---
 packages/cli/src/hosting/utils.ts                     | 3 +--
 packages/frontend-core/src/components/Updating.svelte | 1 +
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/packages/cli/src/hosting/utils.ts b/packages/cli/src/hosting/utils.ts
index 05eb18dc1d..5c3ac33f44 100644
--- a/packages/cli/src/hosting/utils.ts
+++ b/packages/cli/src/hosting/utils.ts
@@ -46,8 +46,7 @@ export function setServiceImage(service: string, image: string) {
 export async function downloadDockerCompose() {
   const filename = composeFilename()
   try {
-    fs.copyFileSync("../../hosting/docker-compose.yaml", `./${filename}`)
-    //await downloadFile(COMPOSE_URL, `./${filename}`)
+    await downloadFile(COMPOSE_URL, `./${filename}`)
   } catch (err) {
     console.error(error(`Failed to retrieve compose file - ${err}`))
   }
diff --git a/packages/frontend-core/src/components/Updating.svelte b/packages/frontend-core/src/components/Updating.svelte
index 311a6b91c8..97e83e2322 100644
--- a/packages/frontend-core/src/components/Updating.svelte
+++ b/packages/frontend-core/src/components/Updating.svelte
@@ -7,6 +7,7 @@
 
   async function checkMigrationsFinished() {
     let totalWaitMs = 0
+    // eslint-disable-next-line no-constant-condition
     while (true) {
       const waitForMs = 5000 + Math.random() * 5000
       await new Promise(resolve => setTimeout(resolve, waitForMs))

From b66591f52f6f9666560078a4d1e127072c218c2a Mon Sep 17 00:00:00 2001
From: Sam Rose <hello@samwho.dev>
Date: Tue, 2 Jul 2024 15:40:50 +0100
Subject: [PATCH 04/27] Remove superfluous environment variables.

---
 hosting/docker-compose.yaml | 6 +-----
 1 file changed, 1 insertion(+), 5 deletions(-)

diff --git a/hosting/docker-compose.yaml b/hosting/docker-compose.yaml
index d9811935e4..d59acb8f9b 100644
--- a/hosting/docker-compose.yaml
+++ b/hosting/docker-compose.yaml
@@ -27,8 +27,6 @@ services:
       BB_ADMIN_USER_PASSWORD: ${BB_ADMIN_USER_PASSWORD}
       PLUGINS_DIR: ${PLUGINS_DIR}
       OFFLINE_MODE: ${OFFLINE_MODE:-}
-      SQS_SEARCH_ENABLE: "true"
-      COUCH_DB_SQL_URL: "http://${COUCH_DB_USER}:${COUCH_DB_PASSWORD}@couchdb-service:4984"
     depends_on:
       - worker-service
       - redis-service
@@ -56,8 +54,6 @@ services:
       REDIS_URL: redis-service:6379
       REDIS_PASSWORD: ${REDIS_PASSWORD}
       OFFLINE_MODE: ${OFFLINE_MODE:-}
-      SQS_SEARCH_ENABLE: "true"
-      COUCH_DB_SQL_URL: "http://${COUCH_DB_USER}:${COUCH_DB_PASSWORD}@couchdb-service:4984"
     depends_on:
       - redis-service
       - minio-service
@@ -101,7 +97,7 @@ services:
 
   couchdb-service:
     restart: unless-stopped
-    image: budibase/couchdb:v3.3.3-sqs
+    image: budibase/couchdb:v3.3.3
     environment:
       - COUCHDB_PASSWORD=${COUCH_DB_PASSWORD}
       - COUCHDB_USER=${COUCH_DB_USER}

From 75c43e7c690d99766be6a2d072f30a369907af28 Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Wed, 25 Sep 2024 17:01:28 +0100
Subject: [PATCH 05/27] Updating to specific SQS version.

---
 hosting/docker-compose.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/hosting/docker-compose.yaml b/hosting/docker-compose.yaml
index d59acb8f9b..6d14361d25 100644
--- a/hosting/docker-compose.yaml
+++ b/hosting/docker-compose.yaml
@@ -97,7 +97,7 @@ services:
 
   couchdb-service:
     restart: unless-stopped
-    image: budibase/couchdb:v3.3.3
+    image: budibase/couchdb:v3.3.3-sqs-v2.1.1
     environment:
       - COUCHDB_PASSWORD=${COUCH_DB_PASSWORD}
       - COUCHDB_USER=${COUCH_DB_USER}

From 5f91c7d8da4960b7fd45d8999000e4f8bc836749 Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Thu, 10 Oct 2024 16:11:03 +0100
Subject: [PATCH 06/27] new test case.

---
 .../server/src/api/routes/tests/role.spec.ts  | 33 +++++++++++++++++++
 1 file changed, 33 insertions(+)

diff --git a/packages/server/src/api/routes/tests/role.spec.ts b/packages/server/src/api/routes/tests/role.spec.ts
index 127be789b9..6531461e43 100644
--- a/packages/server/src/api/routes/tests/role.spec.ts
+++ b/packages/server/src/api/routes/tests/role.spec.ts
@@ -161,4 +161,37 @@ describe("/roles", () => {
       expect(res[2]).toBe("PUBLIC")
     })
   })
+
+  describe("accessible - multi-inheritance", () => {
+    it("should list access correctly for multi-inheritance role", async () => {
+      const role1 = "custom_role_1",
+        role2 = "custom_role_2",
+        role3 = "custom_role_3"
+      const { _id: roleId1 } = await config.api.roles.save({
+        name: role1,
+        inherits: roles.BUILTIN_ROLE_IDS.BASIC,
+        permissionId: permissions.BuiltinPermissionID.WRITE,
+        version: "name",
+      })
+      const { _id: roleId2 } = await config.api.roles.save({
+        name: role2,
+        inherits: roles.BUILTIN_ROLE_IDS.POWER,
+        permissionId: permissions.BuiltinPermissionID.POWER,
+        version: "name",
+      })
+      await config.api.roles.save({
+        name: role3,
+        inherits: role1,
+        permissionId: permissions.BuiltinPermissionID.READ_ONLY,
+        version: "name",
+      })
+      const headers = await config.roleHeaders({
+        roleId: role3,
+      })
+      const res = await config.api.roles.accessible(headers, {
+        status: 200,
+      })
+      expect(res.length).toBe(4)
+    })
+  })
 })

From 324616be59e4832b6026edbeae73fff0d9b9d853 Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Thu, 10 Oct 2024 18:15:23 +0100
Subject: [PATCH 07/27] Finishing multi-inheritance test case and getting
 accessibility to be detected correctly.

---
 packages/backend-core/src/security/roles.ts   | 121 ++++++++++++------
 .../server/src/api/routes/tests/role.spec.ts  |   9 +-
 packages/shared-core/src/helpers/index.ts     |   1 +
 3 files changed, 86 insertions(+), 45 deletions(-)

diff --git a/packages/backend-core/src/security/roles.ts b/packages/backend-core/src/security/roles.ts
index 108bc0414c..9d23463fa3 100644
--- a/packages/backend-core/src/security/roles.ts
+++ b/packages/backend-core/src/security/roles.ts
@@ -9,7 +9,7 @@ import {
 import { getAppDB } from "../context"
 import { Screen, Role as RoleDoc, RoleUIMetadata } from "@budibase/types"
 import cloneDeep from "lodash/fp/cloneDeep"
-import { RoleColor } from "@budibase/shared-core"
+import { RoleColor, helpers } from "@budibase/shared-core"
 
 export const BUILTIN_ROLE_IDS = {
   ADMIN: "ADMIN",
@@ -204,6 +204,36 @@ export function lowerBuiltinRoleID(roleId1?: string, roleId2?: string): string {
     : roleId1
 }
 
+/**
+ * Given a list of roles, this will pick the role out, accounting for built ins.
+ */
+export function findRole(
+  roleId: string,
+  roles: RoleDoc[],
+  opts?: { defaultPublic?: boolean }
+): RoleDoc {
+  // built in roles mostly come from the in-code implementation,
+  // but can be extended by a doc stored about them (e.g. permissions)
+  let role: RoleDoc | undefined = getBuiltinRole(roleId)
+  if (!role) {
+    // make sure has the prefix (if it has it then it won't be added)
+    roleId = prefixRoleID(roleId)
+  }
+  const dbRole = roles.find(
+    role => role._id && role._id === getExternalRoleID(roleId, role.version)
+  )
+  if (!dbRole && !isBuiltin(roleId) && opts?.defaultPublic) {
+    return cloneDeep(BUILTIN_ROLES.PUBLIC)
+  }
+  if (!dbRole && (!role || Object.keys(role).length === 0)) {
+    throw new Error("Role could not be found")
+  }
+  role = Object.assign(role || {}, dbRole)
+  // finalise the ID
+  role._id = getExternalRoleID(role._id!, role.version)
+  return role
+}
+
 /**
  * Gets the role object, this is mainly useful for two purposes, to check if the level exists and
  * to check if the role inherits any others.
@@ -215,29 +245,15 @@ export async function getRole(
   roleId: string,
   opts?: { defaultPublic?: boolean }
 ): Promise<RoleDoc> {
-  // built in roles mostly come from the in-code implementation,
-  // but can be extended by a doc stored about them (e.g. permissions)
-  let role: RoleDoc | undefined = getBuiltinRole(roleId)
-  if (!role) {
-    // make sure has the prefix (if it has it then it won't be added)
-    roleId = prefixRoleID(roleId)
-  }
-  try {
-    const db = getAppDB()
-    const dbRole = await db.get<RoleDoc>(getDBRoleID(roleId))
-    role = Object.assign(role || {}, dbRole)
-    // finalise the ID
-    role._id = getExternalRoleID(role._id!, role.version)
-  } catch (err) {
-    if (!isBuiltin(roleId) && opts?.defaultPublic) {
-      return cloneDeep(BUILTIN_ROLES.PUBLIC)
-    }
-    // only throw an error if there is no role at all
-    if (!role || Object.keys(role).length === 0) {
-      throw err
+  const db = getAppDB()
+  const roleList = []
+  if (!isBuiltin(roleId)) {
+    const role = await db.tryGet<RoleDoc>(getDBRoleID(roleId))
+    if (role) {
+      roleList.push(role)
     }
   }
-  return role
+  return findRole(roleId, roleList, opts)
 }
 
 /**
@@ -247,13 +263,14 @@ async function getAllUserRoles(
   userRoleId: string,
   opts?: { defaultPublic?: boolean }
 ): Promise<RoleDoc[]> {
+  const allRoles = await getAllRoles()
+  if (helpers.roles.checkForRoleInheritanceLoops(allRoles)) {
+    throw new Error("Loop detected in roles - cannot list roles")
+  }
   // admins have access to all roles
   if (userRoleId === BUILTIN_IDS.ADMIN) {
-    return getAllRoles()
+    return allRoles
   }
-  let currentRole = await getRole(userRoleId, opts)
-  let roles = currentRole ? [currentRole] : []
-  let roleIds = [userRoleId]
   const rolesFound = (ids: string | string[]) => {
     if (Array.isArray(ids)) {
       return ids.filter(id => roleIds.includes(id)).length === ids.length
@@ -261,23 +278,49 @@ async function getAllUserRoles(
       return roleIds.includes(ids)
     }
   }
-  // get all the inherited roles
-  while (
-    currentRole &&
-    currentRole.inherits &&
-    !rolesFound(currentRole.inherits)
-  ) {
-    if (Array.isArray(currentRole.inherits)) {
-      // TODO: role inheritance
+
+  const roleIds = [userRoleId]
+  const roles: RoleDoc[] = []
+  const iterateInherited = (role: RoleDoc) => {
+    if (!role || !role._id) {
+      return
+    }
+    roleIds.push(role._id)
+    roles.push(role)
+    if (Array.isArray(role.inherits)) {
+      role.inherits.forEach(roleId => {
+        const foundRole = findRole(roleId, allRoles, opts)
+        if (foundRole) {
+          iterateInherited(foundRole)
+        }
+      })
     } else {
-      roleIds.push(currentRole.inherits)
-      currentRole = await getRole(currentRole.inherits)
-      if (currentRole) {
-        roles.push(currentRole)
+      while (role && role.inherits && !rolesFound(role.inherits)) {
+        if (Array.isArray(role.inherits)) {
+          iterateInherited(role)
+          break
+        } else {
+          roleIds.push(role.inherits)
+          role = findRole(role.inherits, allRoles, opts)
+          if (role) {
+            roles.push(role)
+          }
+        }
       }
     }
   }
-  return roles
+
+  // get all the inherited roles
+  iterateInherited(findRole(userRoleId, allRoles, opts))
+  const foundRoleIds: string[] = []
+  return roles.filter(role => {
+    if (role._id && !foundRoleIds.includes(role._id)) {
+      foundRoleIds.push(role._id)
+      return true
+    } else {
+      return false
+    }
+  })
 }
 
 export async function getUserRoleIdHierarchy(
diff --git a/packages/server/src/api/routes/tests/role.spec.ts b/packages/server/src/api/routes/tests/role.spec.ts
index f9eb1c3651..682ebf2f7a 100644
--- a/packages/server/src/api/routes/tests/role.spec.ts
+++ b/packages/server/src/api/routes/tests/role.spec.ts
@@ -155,10 +155,7 @@ describe("/roles", () => {
           status: 200,
         }
       )
-      expect(res.length).toBe(3)
-      expect(res[0]).toBe(customRoleName)
-      expect(res[1]).toBe("BASIC")
-      expect(res[2]).toBe("PUBLIC")
+      expect(res).toEqual([customRoleName, "BASIC", "PUBLIC"])
     })
   })
 
@@ -181,7 +178,7 @@ describe("/roles", () => {
       })
       await config.api.roles.save({
         name: role3,
-        inherits: role1,
+        inherits: [roleId1!, roleId2!],
         permissionId: permissions.BuiltinPermissionID.READ_ONLY,
         version: "name",
       })
@@ -191,7 +188,7 @@ describe("/roles", () => {
       const res = await config.api.roles.accessible(headers, {
         status: 200,
       })
-      expect(res.length).toBe(4)
+      expect(res).toEqual([role3, role1, "BASIC", "PUBLIC", role2, "POWER"])
     })
   })
 })
diff --git a/packages/shared-core/src/helpers/index.ts b/packages/shared-core/src/helpers/index.ts
index 503f71e4eb..7603a9b88b 100644
--- a/packages/shared-core/src/helpers/index.ts
+++ b/packages/shared-core/src/helpers/index.ts
@@ -3,3 +3,4 @@ export * from "./integrations"
 export * as cron from "./cron"
 export * as schema from "./schema"
 export * as views from "./views"
+export * as roles from "./roles"

From cb78e0bc13326d5ee1c3f721e123732b5028068d Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Fri, 11 Oct 2024 12:20:04 +0200
Subject: [PATCH 08/27] Add extra tests

---
 .../src/api/routes/tests/search.spec.ts       | 41 +++++++++++++++++++
 1 file changed, 41 insertions(+)

diff --git a/packages/server/src/api/routes/tests/search.spec.ts b/packages/server/src/api/routes/tests/search.spec.ts
index 1ccc9bfdc9..3b3492e74c 100644
--- a/packages/server/src/api/routes/tests/search.spec.ts
+++ b/packages/server/src/api/routes/tests/search.spec.ts
@@ -2329,6 +2329,47 @@ describe.each([
             equal: { ["name"]: "baz" },
           }).toContainExactly([{ name: "baz", productCat: undefined }])
         })
+
+        describe("logical filters", () => {
+          it("should allow nested ands with single conditions", async () => {
+            await expectQuery({
+              $and: {
+                conditions: [
+                  {
+                    $and: {
+                      conditions: [
+                        {
+                          equal: { ["productCat.name"]: "foo" },
+                        },
+                      ],
+                    },
+                  },
+                ],
+              },
+            }).toContainExactly([
+              { name: "foo", productCat: [{ _id: productCatRows[0]._id }] },
+            ])
+          })
+
+          it("should allow nested ands with exclusive conditions", async () => {
+            await expectQuery({
+              $and: {
+                conditions: [
+                  {
+                    $and: {
+                      conditions: [
+                        {
+                          equal: { ["productCat.name"]: "foo" },
+                          notEqual: { ["productCat.name"]: "foo" },
+                        },
+                      ],
+                    },
+                  },
+                ],
+              },
+            }).toContainExactly([])
+          })
+        })
       })
 
     isSql &&

From 37450823bba30ee0c715e83fe2592c26cffd73f2 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Fri, 11 Oct 2024 12:25:10 +0200
Subject: [PATCH 09/27] More tests

---
 .../src/api/routes/tests/search.spec.ts       | 39 +++++++++++++++++++
 1 file changed, 39 insertions(+)

diff --git a/packages/server/src/api/routes/tests/search.spec.ts b/packages/server/src/api/routes/tests/search.spec.ts
index 3b3492e74c..1ed2f66676 100644
--- a/packages/server/src/api/routes/tests/search.spec.ts
+++ b/packages/server/src/api/routes/tests/search.spec.ts
@@ -2369,6 +2369,45 @@ describe.each([
               },
             }).toContainExactly([])
           })
+
+          it("should allow nested ands with multiple conditions", async () => {
+            await expectQuery({
+              $and: {
+                conditions: [
+                  {
+                    $and: {
+                      conditions: [
+                        {
+                          equal: { ["productCat.name"]: "foo" },
+                        },
+                      ],
+                    },
+                    notEqual: { ["productCat.name"]: "foo" },
+                  },
+                ],
+              },
+            }).toContainExactly([])
+          })
+
+          it("should allow nesting or under and with single conditions", async () => {
+            await expectQuery({
+              $and: {
+                conditions: [
+                  {
+                    $or: {
+                      conditions: [
+                        {
+                          equal: { ["productCat.name"]: "foo" },
+                        },
+                      ],
+                    },
+                  },
+                ],
+              },
+            }).toContainExactly([
+              { name: "foo", productCat: [{ _id: productCatRows[0]._id }] },
+            ])
+          })
         })
       })
 

From f192a30da0a00680b73e0bfc8987675259a86338 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Fri, 11 Oct 2024 12:29:34 +0200
Subject: [PATCH 10/27] More tests

---
 .../src/api/routes/tests/search.spec.ts       | 186 +++++++++++-------
 1 file changed, 118 insertions(+), 68 deletions(-)

diff --git a/packages/server/src/api/routes/tests/search.spec.ts b/packages/server/src/api/routes/tests/search.spec.ts
index 1ed2f66676..a3f8f1577d 100644
--- a/packages/server/src/api/routes/tests/search.spec.ts
+++ b/packages/server/src/api/routes/tests/search.spec.ts
@@ -2331,82 +2331,132 @@ describe.each([
         })
 
         describe("logical filters", () => {
-          it("should allow nested ands with single conditions", async () => {
-            await expectQuery({
-              $and: {
-                conditions: [
-                  {
-                    $and: {
-                      conditions: [
-                        {
-                          equal: { ["productCat.name"]: "foo" },
-                        },
-                      ],
+          describe("just $ands", () => {
+            it("should allow nested ands with single conditions", async () => {
+              await expectQuery({
+                $and: {
+                  conditions: [
+                    {
+                      $and: {
+                        conditions: [
+                          {
+                            equal: { ["productCat.name"]: "foo" },
+                          },
+                        ],
+                      },
                     },
-                  },
-                ],
-              },
-            }).toContainExactly([
-              { name: "foo", productCat: [{ _id: productCatRows[0]._id }] },
-            ])
+                  ],
+                },
+              }).toContainExactly([
+                { name: "foo", productCat: [{ _id: productCatRows[0]._id }] },
+              ])
+            })
+
+            it("should allow nested ands with exclusive conditions", async () => {
+              await expectQuery({
+                $and: {
+                  conditions: [
+                    {
+                      $and: {
+                        conditions: [
+                          {
+                            equal: { ["productCat.name"]: "foo" },
+                            notEqual: { ["productCat.name"]: "foo" },
+                          },
+                        ],
+                      },
+                    },
+                  ],
+                },
+              }).toContainExactly([])
+            })
+
+            it("should allow nested ands with multiple conditions", async () => {
+              await expectQuery({
+                $and: {
+                  conditions: [
+                    {
+                      $and: {
+                        conditions: [
+                          {
+                            equal: { ["productCat.name"]: "foo" },
+                          },
+                        ],
+                      },
+                      notEqual: { ["productCat.name"]: "foo" },
+                    },
+                  ],
+                },
+              }).toContainExactly([])
+            })
           })
 
-          it("should allow nested ands with exclusive conditions", async () => {
-            await expectQuery({
-              $and: {
-                conditions: [
-                  {
-                    $and: {
-                      conditions: [
-                        {
-                          equal: { ["productCat.name"]: "foo" },
-                          notEqual: { ["productCat.name"]: "foo" },
-                        },
-                      ],
+          describe("just $ors", () => {
+            it("should allow nested ands with single conditions", async () => {
+              await expectQuery({
+                $or: {
+                  conditions: [
+                    {
+                      $or: {
+                        conditions: [
+                          {
+                            equal: { ["productCat.name"]: "foo" },
+                          },
+                        ],
+                      },
                     },
-                  },
-                ],
-              },
-            }).toContainExactly([])
-          })
+                  ],
+                },
+              }).toContainExactly([
+                { name: "foo", productCat: [{ _id: productCatRows[0]._id }] },
+              ])
+            })
 
-          it("should allow nested ands with multiple conditions", async () => {
-            await expectQuery({
-              $and: {
-                conditions: [
-                  {
-                    $and: {
-                      conditions: [
-                        {
-                          equal: { ["productCat.name"]: "foo" },
-                        },
-                      ],
+            it("should allow nested ands with exclusive conditions", async () => {
+              await expectQuery({
+                $or: {
+                  conditions: [
+                    {
+                      $or: {
+                        conditions: [
+                          {
+                            equal: { ["productCat.name"]: "foo" },
+                            notEqual: { ["productCat.name"]: "foo" },
+                          },
+                        ],
+                      },
                     },
-                    notEqual: { ["productCat.name"]: "foo" },
-                  },
-                ],
-              },
-            }).toContainExactly([])
-          })
+                  ],
+                },
+              }).toContainExactly([
+                { name: "foo", productCat: [{ _id: productCatRows[0]._id }] },
+                { name: "bar", productCat: [{ _id: productCatRows[1]._id }] },
+                { name: "baz", productCat: undefined },
+              ])
+            })
 
-          it("should allow nesting or under and with single conditions", async () => {
-            await expectQuery({
-              $and: {
-                conditions: [
-                  {
-                    $or: {
-                      conditions: [
-                        {
-                          equal: { ["productCat.name"]: "foo" },
-                        },
-                      ],
+            it("should allow nested ands with multiple conditions", async () => {
+              await expectQuery({
+                $or: {
+                  conditions: [
+                    {
+                      $or: {
+                        conditions: [
+                          {
+                            equal: { ["productCat.name"]: "foo" },
+                          },
+                        ],
+                      },
+                      notEqual: { ["productCat.name"]: "foo" },
                     },
-                  },
-                ],
-              },
-            }).toContainExactly([
-              { name: "foo", productCat: [{ _id: productCatRows[0]._id }] },
-            ])
+                  ],
+                },
+              }).toContainExactly([
+                { name: "foo", productCat: [{ _id: productCatRows[0]._id }] },
+                { name: "bar", productCat: [{ _id: productCatRows[1]._id }] },
+                { name: "baz", productCat: undefined },
+              ])
+            })
           })
         })
       })

From 2311f8aa5079fc6e2aef61db7255c5532fe446b2 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Fri, 11 Oct 2024 12:55:23 +0200
Subject: [PATCH 11/27] Don't break or conditions on nested joins

---
 packages/backend-core/src/sql/sql.ts | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/packages/backend-core/src/sql/sql.ts b/packages/backend-core/src/sql/sql.ts
index 382eca3f76..f77e76023e 100644
--- a/packages/backend-core/src/sql/sql.ts
+++ b/packages/backend-core/src/sql/sql.ts
@@ -521,8 +521,11 @@ class InternalBuilder {
         const [filterTableName, ...otherProperties] = key.split(".")
         const property = otherProperties.join(".")
         const alias = getTableAlias(filterTableName)
-        return fn(q, alias ? `${alias}.${property}` : property, value)
+        return q.andWhere(subquery =>
+          fn(subquery, alias ? `${alias}.${property}` : property, value)
+        )
       }
+
       for (const key in structure) {
         const value = structure[key]
         const updatedKey = dbCore.removeKeyNumbering(key)

From cf089eff26af1598f85a81de9a6726a8d4b85773 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Fri, 11 Oct 2024 12:59:33 +0200
Subject: [PATCH 12/27] Fix ors

---
 packages/backend-core/src/sql/sql.ts | 19 +++++++++++++++----
 1 file changed, 15 insertions(+), 4 deletions(-)

diff --git a/packages/backend-core/src/sql/sql.ts b/packages/backend-core/src/sql/sql.ts
index f77e76023e..2b697d42ae 100644
--- a/packages/backend-core/src/sql/sql.ts
+++ b/packages/backend-core/src/sql/sql.ts
@@ -406,6 +406,7 @@ class InternalBuilder {
   addRelationshipForFilter(
     query: Knex.QueryBuilder,
     filterKey: string,
+    isOr: boolean,
     whereCb: (query: Knex.QueryBuilder) => Knex.QueryBuilder
   ): Knex.QueryBuilder {
     const mainKnex = this.knex
@@ -470,7 +471,12 @@ class InternalBuilder {
             )
           )
         }
-        query = query.whereExists(whereCb(subQuery))
+
+        if (isOr) {
+          query = query.orWhereExists(whereCb(subQuery))
+        } else {
+          query = query.whereExists(whereCb(subQuery))
+        }
         break
       }
     }
@@ -555,9 +561,14 @@ class InternalBuilder {
             value
           )
         } else if (shouldProcessRelationship) {
-          query = builder.addRelationshipForFilter(query, updatedKey, q => {
-            return handleRelationship(q, updatedKey, value)
-          })
+          query = builder.addRelationshipForFilter(
+            query,
+            updatedKey,
+            !!allOr,
+            q => {
+              return handleRelationship(q, updatedKey, value)
+            }
+          )
         }
       }
     }

From bfdead820c4b4ff13d7558fc487e42da37f688b6 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Fri, 11 Oct 2024 13:00:05 +0200
Subject: [PATCH 13/27] Cleanup tests

---
 .../src/api/routes/tests/search.spec.ts       | 68 +++++++++++++++++--
 1 file changed, 63 insertions(+), 5 deletions(-)

diff --git a/packages/server/src/api/routes/tests/search.spec.ts b/packages/server/src/api/routes/tests/search.spec.ts
index a3f8f1577d..364b6aa77a 100644
--- a/packages/server/src/api/routes/tests/search.spec.ts
+++ b/packages/server/src/api/routes/tests/search.spec.ts
@@ -2332,6 +2332,33 @@ describe.each([
 
         describe("logical filters", () => {
           describe("just $ands", () => {
+            it("should allow single conditions", async () => {
+              await expectQuery({
+                $and: {
+                  conditions: [
+                    {
+                      equal: { ["productCat.name"]: "foo" },
+                    },
+                  ],
+                },
+              }).toContainExactly([
+                { name: "foo", productCat: [{ _id: productCatRows[0]._id }] },
+              ])
+            })
+
+            it("should allow exclusive conditions", async () => {
+              await expectQuery({
+                $and: {
+                  conditions: [
+                    {
+                      equal: { ["productCat.name"]: "foo" },
+                      notEqual: { ["productCat.name"]: "foo" },
+                    },
+                  ],
+                },
+              }).toContainExactly([])
+            })
+
             it("should allow nested ands with single conditions", async () => {
               await expectQuery({
                 $and: {
@@ -2392,7 +2419,38 @@ describe.each([
           })
 
           describe("just $ors", () => {
-            it("should allow nested ands with single conditions", async () => {
+            it("should allow single conditions", async () => {
+              await expectQuery({
+                $or: {
+                  conditions: [
+                    {
+                      equal: { ["productCat.name"]: "foo" },
+                    },
+                  ],
+                },
+              }).toContainExactly([
+                { name: "foo", productCat: [{ _id: productCatRows[0]._id }] },
+              ])
+            })
+
+            it("should allow exclusive conditions", async () => {
+              await expectQuery({
+                $or: {
+                  conditions: [
+                    {
+                      equal: { ["productCat.name"]: "foo" },
+                      notEqual: { ["productCat.name"]: "foo" },
+                    },
+                  ],
+                },
+              }).toContainExactly([
+                { name: "foo", productCat: [{ _id: productCatRows[0]._id }] },
+                { name: "bar", productCat: [{ _id: productCatRows[1]._id }] },
+                // { name: "baz", productCat: undefined }, // TODO
+              ])
+            })
+
+            it("should allow nested ors with single conditions", async () => {
               await expectQuery({
                 $or: {
                   conditions: [
@@ -2412,7 +2470,7 @@ describe.each([
               ])
             })
 
-            it("should allow nested ands with exclusive conditions", async () => {
+            it("should allow nested ors with exclusive conditions", async () => {
               await expectQuery({
                 $or: {
                   conditions: [
@@ -2431,11 +2489,11 @@ describe.each([
               }).toContainExactly([
                 { name: "foo", productCat: [{ _id: productCatRows[0]._id }] },
                 { name: "bar", productCat: [{ _id: productCatRows[1]._id }] },
-                { name: "baz", productCat: undefined },
+                // { name: "baz", productCat: undefined }, // TODO
               ])
             })
 
-            it("should allow nested ands with multiple conditions", async () => {
+            it("should allow nested ors with multiple conditions", async () => {
               await expectQuery({
                 $or: {
                   conditions: [
@@ -2454,7 +2512,7 @@ describe.each([
               }).toContainExactly([
                 { name: "foo", productCat: [{ _id: productCatRows[0]._id }] },
                 { name: "bar", productCat: [{ _id: productCatRows[1]._id }] },
-                { name: "baz", productCat: undefined },
+                // { name: "baz", productCat: undefined }, // TODO
               ])
             })
           })

From f73b7d4824fbf12862351f5e5a6fdff140017948 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Fri, 11 Oct 2024 13:06:37 +0200
Subject: [PATCH 14/27] More tests

---
 .../src/api/routes/tests/search.spec.ts       | 249 ++++++++++--------
 1 file changed, 135 insertions(+), 114 deletions(-)

diff --git a/packages/server/src/api/routes/tests/search.spec.ts b/packages/server/src/api/routes/tests/search.spec.ts
index 364b6aa77a..6b63b94a43 100644
--- a/packages/server/src/api/routes/tests/search.spec.ts
+++ b/packages/server/src/api/routes/tests/search.spec.ts
@@ -23,6 +23,7 @@ import {
   EmptyFilterOption,
   FieldType,
   JsonFieldSubType,
+  LogicalOperator,
   RelationshipType,
   Row,
   RowSearchParams,
@@ -2331,7 +2332,9 @@ describe.each([
         })
 
         describe("logical filters", () => {
-          describe("just $ands", () => {
+          const logicalOperators = [LogicalOperator.AND, LogicalOperator.OR]
+
+          describe("$and", () => {
             it("should allow single conditions", async () => {
               await expectQuery({
                 $and: {
@@ -2359,66 +2362,75 @@ describe.each([
               }).toContainExactly([])
             })
 
-            it("should allow nested ands with single conditions", async () => {
-              await expectQuery({
-                $and: {
-                  conditions: [
-                    {
-                      $and: {
-                        conditions: [
-                          {
-                            equal: { ["productCat.name"]: "foo" },
-                          },
-                        ],
+            it.each([logicalOperators])(
+              "should allow nested ands with single conditions (with %s as root)",
+              async rootOperator => {
+                await expectQuery({
+                  [rootOperator]: {
+                    conditions: [
+                      {
+                        $and: {
+                          conditions: [
+                            {
+                              equal: { ["productCat.name"]: "foo" },
+                            },
+                          ],
+                        },
                       },
-                    },
-                  ],
-                },
-              }).toContainExactly([
-                { name: "foo", productCat: [{ _id: productCatRows[0]._id }] },
-              ])
-            })
+                    ],
+                  },
+                }).toContainExactly([
+                  { name: "foo", productCat: [{ _id: productCatRows[0]._id }] },
+                ])
+              }
+            )
 
-            it("should allow nested ands with exclusive conditions", async () => {
-              await expectQuery({
-                $and: {
-                  conditions: [
-                    {
-                      $and: {
-                        conditions: [
-                          {
-                            equal: { ["productCat.name"]: "foo" },
-                            notEqual: { ["productCat.name"]: "foo" },
-                          },
-                        ],
+            it.each([logicalOperators])(
+              "should allow nested ands with exclusive conditions (with %s as root)",
+              async rootOperator => {
+                await expectQuery({
+                  [rootOperator]: {
+                    conditions: [
+                      {
+                        $and: {
+                          conditions: [
+                            {
+                              equal: { ["productCat.name"]: "foo" },
+                              notEqual: { ["productCat.name"]: "foo" },
+                            },
+                          ],
+                        },
                       },
-                    },
-                  ],
-                },
-              }).toContainExactly([])
-            })
+                    ],
+                  },
+                }).toContainExactly([])
+              }
+            )
 
-            it("should allow nested ands with multiple conditions", async () => {
-              await expectQuery({
-                $and: {
-                  conditions: [
-                    {
-                      $and: {
-                        conditions: [
-                          {
-                            equal: { ["productCat.name"]: "foo" },
-                          },
-                        ],
+            it.each([logicalOperators])(
+              "should allow nested ands with multiple conditions (with %s as root)",
+              async rootOperator => {
+                await expectQuery({
+                  [rootOperator]: {
+                    conditions: [
+                      {
+                        $and: {
+                          conditions: [
+                            {
+                              equal: { ["productCat.name"]: "foo" },
+                            },
+                          ],
+                        },
+                        notEqual: { ["productCat.name"]: "foo" },
                       },
-                      notEqual: { ["productCat.name"]: "foo" },
-                    },
-                  ],
-                },
-              }).toContainExactly([])
-            })
+                    ],
+                  },
+                }).toContainExactly([])
+              }
+            )
           })
 
-          describe("just $ors", () => {
+          describe("$ors", () => {
             it("should allow single conditions", async () => {
               await expectQuery({
                 $or: {
@@ -2450,71 +2462,80 @@ describe.each([
               ])
             })
 
-            it("should allow nested ors with single conditions", async () => {
-              await expectQuery({
-                $or: {
-                  conditions: [
-                    {
-                      $or: {
-                        conditions: [
-                          {
-                            equal: { ["productCat.name"]: "foo" },
-                          },
-                        ],
+            it.each([logicalOperators])(
+              "should allow nested ors with single conditions (with %s as root)",
+              async rootOperator => {
+                await expectQuery({
+                  [rootOperator]: {
+                    conditions: [
+                      {
+                        $or: {
+                          conditions: [
+                            {
+                              equal: { ["productCat.name"]: "foo" },
+                            },
+                          ],
+                        },
                       },
-                    },
-                  ],
-                },
-              }).toContainExactly([
-                { name: "foo", productCat: [{ _id: productCatRows[0]._id }] },
-              ])
-            })
+                    ],
+                  },
+                }).toContainExactly([
+                  { name: "foo", productCat: [{ _id: productCatRows[0]._id }] },
+                ])
+              }
+            )
 
-            it("should allow nested ors with exclusive conditions", async () => {
-              await expectQuery({
-                $or: {
-                  conditions: [
-                    {
-                      $or: {
-                        conditions: [
-                          {
-                            equal: { ["productCat.name"]: "foo" },
-                            notEqual: { ["productCat.name"]: "foo" },
-                          },
-                        ],
+            it.each([logicalOperators])(
+              "should allow nested ors with exclusive conditions (with %s as root)",
+              async rootOperator => {
+                await expectQuery({
+                  [rootOperator]: {
+                    conditions: [
+                      {
+                        $or: {
+                          conditions: [
+                            {
+                              equal: { ["productCat.name"]: "foo" },
+                              notEqual: { ["productCat.name"]: "foo" },
+                            },
+                          ],
+                        },
                       },
-                    },
-                  ],
-                },
-              }).toContainExactly([
-                { name: "foo", productCat: [{ _id: productCatRows[0]._id }] },
-                { name: "bar", productCat: [{ _id: productCatRows[1]._id }] },
-                // { name: "baz", productCat: undefined }, // TODO
-              ])
-            })
+                    ],
+                  },
+                }).toContainExactly([
+                  { name: "foo", productCat: [{ _id: productCatRows[0]._id }] },
+                  { name: "bar", productCat: [{ _id: productCatRows[1]._id }] },
+                  // { name: "baz", productCat: undefined }, // TODO
+                ])
+              }
+            )
 
-            it("should allow nested ors with multiple conditions", async () => {
-              await expectQuery({
-                $or: {
-                  conditions: [
-                    {
-                      $or: {
-                        conditions: [
-                          {
-                            equal: { ["productCat.name"]: "foo" },
-                          },
-                        ],
+            it.each([logicalOperators])(
+              "should allow nested ors with multiple conditions (with %s as root)",
+              async rootOperator => {
+                await expectQuery({
+                  [rootOperator]: {
+                    conditions: [
+                      {
+                        $or: {
+                          conditions: [
+                            {
+                              equal: { ["productCat.name"]: "foo" },
+                            },
+                          ],
+                        },
+                        notEqual: { ["productCat.name"]: "foo" },
                       },
-                      notEqual: { ["productCat.name"]: "foo" },
-                    },
-                  ],
-                },
-              }).toContainExactly([
-                { name: "foo", productCat: [{ _id: productCatRows[0]._id }] },
-                { name: "bar", productCat: [{ _id: productCatRows[1]._id }] },
-                // { name: "baz", productCat: undefined }, // TODO
-              ])
-            })
+                    ],
+                  },
+                }).toContainExactly([
+                  { name: "foo", productCat: [{ _id: productCatRows[0]._id }] },
+                  { name: "bar", productCat: [{ _id: productCatRows[1]._id }] },
+                  // { name: "baz", productCat: undefined }, // TODO
+                ])
+              }
+            )
           })
         })
       })

From ca7a7bcef9f55d71baf6e0d551ee71da42fe8fed Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Fri, 11 Oct 2024 13:13:34 +0200
Subject: [PATCH 15/27] Fix tests

---
 .../src/api/routes/tests/search.spec.ts       | 45 +++++++++----------
 1 file changed, 21 insertions(+), 24 deletions(-)

diff --git a/packages/server/src/api/routes/tests/search.spec.ts b/packages/server/src/api/routes/tests/search.spec.ts
index 6b63b94a43..3ab35c9294 100644
--- a/packages/server/src/api/routes/tests/search.spec.ts
+++ b/packages/server/src/api/routes/tests/search.spec.ts
@@ -2511,31 +2511,28 @@ describe.each([
               }
             )
 
-            it.each([logicalOperators])(
-              "should allow nested ors with multiple conditions (with %s as root)",
-              async rootOperator => {
-                await expectQuery({
-                  [rootOperator]: {
-                    conditions: [
-                      {
-                        $or: {
-                          conditions: [
-                            {
-                              equal: { ["productCat.name"]: "foo" },
-                            },
-                          ],
-                        },
-                        notEqual: { ["productCat.name"]: "foo" },
+            it("should allow nested ors with multiple conditions", async () => {
+              await expectQuery({
+                $or: {
+                  conditions: [
+                    {
+                      $or: {
+                        conditions: [
+                          {
+                            equal: { ["productCat.name"]: "foo" },
+                          },
+                        ],
                       },
-                    ],
-                  },
-                }).toContainExactly([
-                  { name: "foo", productCat: [{ _id: productCatRows[0]._id }] },
-                  { name: "bar", productCat: [{ _id: productCatRows[1]._id }] },
-                  // { name: "baz", productCat: undefined }, // TODO
-                ])
-              }
-            )
+                      notEqual: { ["productCat.name"]: "foo" },
+                    },
+                  ],
+                },
+              }).toContainExactly([
+                { name: "foo", productCat: [{ _id: productCatRows[0]._id }] },
+                { name: "bar", productCat: [{ _id: productCatRows[1]._id }] },
+                // { name: "baz", productCat: undefined }, // TODO
+              ])
+            })
           })
         })
       })

From a6cb1d072a49df425e94bd684473c8536b9233a9 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Fri, 11 Oct 2024 17:39:01 +0200
Subject: [PATCH 16/27] Fix sql alias test

---
 packages/server/src/integrations/tests/sqlAlias.spec.ts | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/packages/server/src/integrations/tests/sqlAlias.spec.ts b/packages/server/src/integrations/tests/sqlAlias.spec.ts
index fc5af4238c..890c8c4663 100644
--- a/packages/server/src/integrations/tests/sqlAlias.spec.ts
+++ b/packages/server/src/integrations/tests/sqlAlias.spec.ts
@@ -79,7 +79,7 @@ describe("Captures of real examples", () => {
         sql: expect.stringContaining(
           multiline(
             `where exists (select 1 from "tasks" as "b" inner join "products_tasks" as "c" on "b"."taskid" = "c"."taskid" where "c"."productid" = "a"."productid" 
-                 and COALESCE("b"."taskname" = $1, FALSE)`
+                 and (COALESCE("b"."taskname" = $1, FALSE))`
           )
         ),
       })
@@ -144,7 +144,7 @@ describe("Captures of real examples", () => {
         ],
         sql: expect.stringContaining(
           multiline(
-            `where exists (select 1 from "persons" as "c" where "c"."personid" = "a"."executorid" and "c"."year" between $1 and $2)`
+            `where exists (select 1 from "persons" as "c" where "c"."personid" = "a"."executorid" and ("c"."year" between $1 and $2))`
           )
         ),
       })

From 329d4fc01b4bf65592fa20fa2325ccbf24a92017 Mon Sep 17 00:00:00 2001
From: Christos Alexiou <christos@budibase.com>
Date: Mon, 14 Oct 2024 11:47:15 +0300
Subject: [PATCH 17/27] Move conditionals to env section

---
 .github/workflows/deploy-featurebranch.yml | 44 ++++++++--------------
 1 file changed, 16 insertions(+), 28 deletions(-)

diff --git a/.github/workflows/deploy-featurebranch.yml b/.github/workflows/deploy-featurebranch.yml
index 0e19f0649f..872faa98fa 100644
--- a/.github/workflows/deploy-featurebranch.yml
+++ b/.github/workflows/deploy-featurebranch.yml
@@ -2,13 +2,11 @@ name: deploy-featurebranch
 
 on:
   pull_request:
-    types: [
-      labeled,
-      # default types below (https://docs.github.com/en/actions/using-workflows/events-that-trigger-workflows#pull_request)
-      opened,
-      synchronize,
-      reopened,
-    ]
+    types: 
+      - labeled
+      - opened
+      - synchronize
+      - reopened
 
 jobs:
   release:
@@ -22,31 +20,21 @@ jobs:
         contains(github.event.pull_request.labels.*.name, 'feature-branch-enterprise')
       )
     runs-on: ubuntu-latest
+    env:
+      PAYLOAD_BRANCH: ${{ github.head_ref }}
+      PAYLOAD_PR_NUMBER: ${{ github.event.pull_request.number }}
+      PAYLOAD_LICENSE_TYPE: |
+        ${{ 
+          contains(github.event.pull_request.labels.*.name, 'feature-branch') && 'free' ||
+          contains(github.event.pull_request.labels.*.name, 'feature-branch-pro') && 'pro' ||
+          contains(github.event.pull_request.labels.*.name, 'feature-branch-team') && 'team' ||
+          contains(github.event.pull_request.labels.*.name, 'feature-branch-business') && 'business' ||
+          contains(github.event.pull_request.labels.*.name, 'feature-branch-enterprise') && 'enterprise' || 'free'
+        }}
     steps:
       - uses: actions/checkout@v4
 
-      - name: Set PAYLOAD_LICENSE_TYPE
-        id: set_license_type
-        run: |
-          if [[ "${{ contains(github.event.pull_request.labels.*.name, 'feature-branch') }}" == "true" ]]; then
-            echo "PAYLOAD_LICENSE_TYPE=free" >> $GITHUB_ENV
-          elif [[ "${{ contains(github.event.pull_request.labels.*.name, 'feature-branch-pro') }}" == "true" ]]; then
-            echo "PAYLOAD_LICENSE_TYPE=pro" >> $GITHUB_ENV
-          elif [[ "${{ contains(github.event.pull_request.labels.*.name, 'feature-branch-team') }}" == "true" ]]; then
-            echo "PAYLOAD_LICENSE_TYPE=team" >> $GITHUB_ENV
-          elif [[ "${{ contains(github.event.pull_request.labels.*.name, 'feature-branch-business') }}" == "true" ]]; then
-            echo "PAYLOAD_LICENSE_TYPE=business" >> $GITHUB_ENV
-          elif [[ "${{ contains(github.event.pull_request.labels.*.name, 'feature-branch-enterprise') }}" == "true" ]]; then
-            echo "PAYLOAD_LICENSE_TYPE=enterprise" >> $GITHUB_ENV
-          else
-            echo "PAYLOAD_LICENSE_TYPE=free" >> $GITHUB_ENV
-          fi
-
       - uses: passeidireto/trigger-external-workflow-action@main
-        env:
-          PAYLOAD_BRANCH: ${{ github.head_ref }}
-          PAYLOAD_PR_NUMBER: ${{ github.event.pull_request.number }}
-          PAYLOAD_LICENSE_TYPE: ${{ env.PAYLOAD_LICENSE_TYPE }}
         with:
           repository: budibase/budibase-deploys
           event: featurebranch-qa-deploy

From c9d42a0e2964a3c5b1710421e5ec16eaa6881846 Mon Sep 17 00:00:00 2001
From: Sam Rose <hello@samwho.dev>
Date: Mon, 14 Oct 2024 14:51:17 +0100
Subject: [PATCH 18/27] Add SQS feature flags, remove scarf.sh URLs.

---
 hosting/docker-compose.yaml | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/hosting/docker-compose.yaml b/hosting/docker-compose.yaml
index 6d14361d25..1c4bf9a1f8 100644
--- a/hosting/docker-compose.yaml
+++ b/hosting/docker-compose.yaml
@@ -5,7 +5,7 @@ version: "3"
 services:
   app-service:
     restart: unless-stopped
-    image: budibase.docker.scarf.sh/budibase/apps
+    image: budibase/apps
     container_name: bbapps
     environment:
       SELF_HOSTED: 1
@@ -27,6 +27,7 @@ services:
       BB_ADMIN_USER_PASSWORD: ${BB_ADMIN_USER_PASSWORD}
       PLUGINS_DIR: ${PLUGINS_DIR}
       OFFLINE_MODE: ${OFFLINE_MODE:-}
+      TENANT_FEATURE_FLAGS: "*:SQS"
     depends_on:
       - worker-service
       - redis-service
@@ -35,7 +36,7 @@ services:
 
   worker-service:
     restart: unless-stopped
-    image: budibase.docker.scarf.sh/budibase/worker
+    image: budibase/worker
     container_name: bbworker
     environment:
       SELF_HOSTED: 1
@@ -54,6 +55,7 @@ services:
       REDIS_URL: redis-service:6379
       REDIS_PASSWORD: ${REDIS_PASSWORD}
       OFFLINE_MODE: ${OFFLINE_MODE:-}
+      TENANT_FEATURE_FLAGS: "*:SQS"
     depends_on:
       - redis-service
       - minio-service

From 7ea2c187a7a6572da98587c0dd8e55dd3a8c854e Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Mon, 14 Oct 2024 16:17:24 +0200
Subject: [PATCH 19/27] Simplify

---
 packages/backend-core/src/sql/sql.ts | 22 +++++++---------------
 1 file changed, 7 insertions(+), 15 deletions(-)

diff --git a/packages/backend-core/src/sql/sql.ts b/packages/backend-core/src/sql/sql.ts
index 2b697d42ae..b415a6f1b7 100644
--- a/packages/backend-core/src/sql/sql.ts
+++ b/packages/backend-core/src/sql/sql.ts
@@ -406,7 +406,6 @@ class InternalBuilder {
   addRelationshipForFilter(
     query: Knex.QueryBuilder,
     filterKey: string,
-    isOr: boolean,
     whereCb: (query: Knex.QueryBuilder) => Knex.QueryBuilder
   ): Knex.QueryBuilder {
     const mainKnex = this.knex
@@ -471,12 +470,7 @@ class InternalBuilder {
             )
           )
         }
-
-        if (isOr) {
-          query = query.orWhereExists(whereCb(subQuery))
-        } else {
-          query = query.whereExists(whereCb(subQuery))
-        }
+        query = query.whereExists(whereCb(subQuery))
         break
       }
     }
@@ -561,14 +555,12 @@ class InternalBuilder {
             value
           )
         } else if (shouldProcessRelationship) {
-          query = builder.addRelationshipForFilter(
-            query,
-            updatedKey,
-            !!allOr,
-            q => {
-              return handleRelationship(q, updatedKey, value)
-            }
-          )
+          if (allOr) {
+            query = query.or
+          }
+          query = builder.addRelationshipForFilter(query, updatedKey, q => {
+            return handleRelationship(q, updatedKey, value)
+          })
         }
       }
     }

From 58b4a37fca1ac7df76d7eb6d0b14198582ef3dbb Mon Sep 17 00:00:00 2001
From: Sam Rose <hello@samwho.dev>
Date: Mon, 14 Oct 2024 17:20:36 +0100
Subject: [PATCH 20/27] Enable SQS in code instead of in env vars.

---
 hosting/docker-compose.yaml                               | 2 --
 packages/backend-core/src/features/features.ts            | 2 +-
 packages/backend-core/src/features/tests/features.spec.ts | 6 ++++++
 3 files changed, 7 insertions(+), 3 deletions(-)

diff --git a/hosting/docker-compose.yaml b/hosting/docker-compose.yaml
index 1c4bf9a1f8..c7a22eb2b3 100644
--- a/hosting/docker-compose.yaml
+++ b/hosting/docker-compose.yaml
@@ -27,7 +27,6 @@ services:
       BB_ADMIN_USER_PASSWORD: ${BB_ADMIN_USER_PASSWORD}
       PLUGINS_DIR: ${PLUGINS_DIR}
       OFFLINE_MODE: ${OFFLINE_MODE:-}
-      TENANT_FEATURE_FLAGS: "*:SQS"
     depends_on:
       - worker-service
       - redis-service
@@ -55,7 +54,6 @@ services:
       REDIS_URL: redis-service:6379
       REDIS_PASSWORD: ${REDIS_PASSWORD}
       OFFLINE_MODE: ${OFFLINE_MODE:-}
-      TENANT_FEATURE_FLAGS: "*:SQS"
     depends_on:
       - redis-service
       - minio-service
diff --git a/packages/backend-core/src/features/features.ts b/packages/backend-core/src/features/features.ts
index 20b207bb02..e95472a784 100644
--- a/packages/backend-core/src/features/features.ts
+++ b/packages/backend-core/src/features/features.ts
@@ -269,7 +269,7 @@ export class FlagSet<V extends Flag<any>, T extends { [key: string]: V }> {
 export const flags = new FlagSet({
   DEFAULT_VALUES: Flag.boolean(env.isDev()),
   AUTOMATION_BRANCHING: Flag.boolean(env.isDev()),
-  SQS: Flag.boolean(env.isDev()),
+  SQS: Flag.boolean(true),
   [FeatureFlag.AI_CUSTOM_CONFIGS]: Flag.boolean(env.isDev()),
   [FeatureFlag.ENRICHED_RELATIONSHIPS]: Flag.boolean(env.isDev()),
   [FeatureFlag.TABLES_DEFAULT_ADMIN]: Flag.boolean(env.isDev()),
diff --git a/packages/backend-core/src/features/tests/features.spec.ts b/packages/backend-core/src/features/tests/features.spec.ts
index 9af8a8f4bb..ced874f4af 100644
--- a/packages/backend-core/src/features/tests/features.spec.ts
+++ b/packages/backend-core/src/features/tests/features.spec.ts
@@ -10,6 +10,7 @@ const schema = {
   TEST_BOOLEAN: Flag.boolean(false),
   TEST_STRING: Flag.string("default value"),
   TEST_NUMBER: Flag.number(0),
+  TEST_BOOLEAN_DEFAULT_TRUE: Flag.boolean(true),
 }
 const flags = new FlagSet(schema)
 
@@ -123,6 +124,11 @@ describe("feature flags", () => {
       },
       expected: flags.defaults(),
     },
+    {
+      it: "should be possible to override a default true flag to false",
+      environmentFlags: "default:!TEST_BOOLEAN_DEFAULT_TRUE",
+      expected: { TEST_BOOLEAN_DEFAULT_TRUE: false },
+    },
   ])(
     "$it",
     async ({

From 61558aff774f169545f75e640794aad2c87dd59a Mon Sep 17 00:00:00 2001
From: Sam Rose <hello@samwho.dev>
Date: Mon, 14 Oct 2024 17:24:14 +0100
Subject: [PATCH 21/27] Update Helm chart for SQS.

---
 charts/budibase/templates/app-service-deployment.yaml       | 6 ------
 .../templates/automation-worker-service-deployment.yaml     | 2 ++
 charts/budibase/templates/worker-service-deployment.yaml    | 6 ------
 charts/budibase/values.yaml                                 | 5 -----
 4 files changed, 2 insertions(+), 17 deletions(-)

diff --git a/charts/budibase/templates/app-service-deployment.yaml b/charts/budibase/templates/app-service-deployment.yaml
index 4d0560312f..5710749028 100644
--- a/charts/budibase/templates/app-service-deployment.yaml
+++ b/charts/budibase/templates/app-service-deployment.yaml
@@ -42,14 +42,8 @@ spec:
           {{ else }}
           value: http://{{ .Release.Name }}-svc-couchdb:{{ .Values.services.couchdb.port }}
           {{ end }}
-        {{ if .Values.globals.sqs.enabled }}
         - name: COUCH_DB_SQL_URL
-        {{ if .Values.globals.sqs.url }}
-          value: {{ .Values.globals.sqs.url }}
-        {{ else }}
           value: http://{{ .Release.Name }}-svc-couchdb:{{ .Values.globals.sqs.port }}
-        {{ end }}
-        {{ end }}
         {{ if .Values.services.couchdb.enabled }}
         - name: COUCH_DB_USER
           valueFrom:
diff --git a/charts/budibase/templates/automation-worker-service-deployment.yaml b/charts/budibase/templates/automation-worker-service-deployment.yaml
index 71089bd7ee..ee3cd6dbdb 100644
--- a/charts/budibase/templates/automation-worker-service-deployment.yaml
+++ b/charts/budibase/templates/automation-worker-service-deployment.yaml
@@ -43,6 +43,8 @@ spec:
           {{ else }}
           value: http://{{ .Release.Name }}-svc-couchdb:{{ .Values.services.couchdb.port }}
           {{ end }}
+        - name: COUCH_DB_SQL_URL
+          value: http://{{ .Release.Name }}-svc-couchdb:{{ .Values.globals.sqs.port }}
         {{ if .Values.services.couchdb.enabled }}
         - name: COUCH_DB_USER
           valueFrom:
diff --git a/charts/budibase/templates/worker-service-deployment.yaml b/charts/budibase/templates/worker-service-deployment.yaml
index dcab33fa58..a493980fd3 100644
--- a/charts/budibase/templates/worker-service-deployment.yaml
+++ b/charts/budibase/templates/worker-service-deployment.yaml
@@ -56,14 +56,8 @@ spec:
           {{ else }}
           value: http://{{ .Release.Name }}-svc-couchdb:{{ .Values.services.couchdb.port }}
           {{ end }}
-        {{ if .Values.globals.sqs.enabled }}
         - name: COUCH_DB_SQL_URL
-        {{ if .Values.globals.sqs.url }}
-          value: {{ .Values.globals.sqs.url }}
-        {{ else }}
           value: http://{{ .Release.Name }}-svc-couchdb:{{ .Values.globals.sqs.port }}
-        {{ end }}
-        {{ end }}
         - name: API_ENCRYPTION_KEY
           valueFrom:
             secretKeyRef:
diff --git a/charts/budibase/values.yaml b/charts/budibase/values.yaml
index 2c1525bd90..9a5cdcdc82 100644
--- a/charts/budibase/values.yaml
+++ b/charts/budibase/values.yaml
@@ -139,11 +139,6 @@ globals:
     password: ""
 
   sqs:
-    # -- Whether to use the CouchDB "structured query service" or not. This is disabled by
-    # default for now, but will become the default in a future release.
-    enabled: false
-    # @ignore
-    url: ""
     # @ignore
     port: "4984"
 

From dd81e246bf4eff7c06fc192fef054abacda97ef8 Mon Sep 17 00:00:00 2001
From: Sam Rose <hello@samwho.dev>
Date: Mon, 14 Oct 2024 17:28:25 +0100
Subject: [PATCH 22/27] Allow customisation of SQS URL to match CouchDB URL.

---
 charts/budibase/templates/app-service-deployment.yaml         | 4 ++++
 .../templates/automation-worker-service-deployment.yaml       | 4 ++++
 charts/budibase/templates/worker-service-deployment.yaml      | 4 ++++
 charts/budibase/values.yaml                                   | 2 ++
 4 files changed, 14 insertions(+)

diff --git a/charts/budibase/templates/app-service-deployment.yaml b/charts/budibase/templates/app-service-deployment.yaml
index 5710749028..278bd1767f 100644
--- a/charts/budibase/templates/app-service-deployment.yaml
+++ b/charts/budibase/templates/app-service-deployment.yaml
@@ -43,7 +43,11 @@ spec:
           value: http://{{ .Release.Name }}-svc-couchdb:{{ .Values.services.couchdb.port }}
           {{ end }}
         - name: COUCH_DB_SQL_URL
+          {{ if .Values.globals.sqs.url }}
+          value: {{ .Values.globals.sqs.url | quote }}
+          {{ else }}
           value: http://{{ .Release.Name }}-svc-couchdb:{{ .Values.globals.sqs.port }}
+          {{ end }}
         {{ if .Values.services.couchdb.enabled }}
         - name: COUCH_DB_USER
           valueFrom:
diff --git a/charts/budibase/templates/automation-worker-service-deployment.yaml b/charts/budibase/templates/automation-worker-service-deployment.yaml
index ee3cd6dbdb..e0d43d0ce6 100644
--- a/charts/budibase/templates/automation-worker-service-deployment.yaml
+++ b/charts/budibase/templates/automation-worker-service-deployment.yaml
@@ -44,7 +44,11 @@ spec:
           value: http://{{ .Release.Name }}-svc-couchdb:{{ .Values.services.couchdb.port }}
           {{ end }}
         - name: COUCH_DB_SQL_URL
+          {{ if .Values.globals.sqs.url }}
+          value: {{ .Values.globals.sqs.url | quote }}
+          {{ else }}
           value: http://{{ .Release.Name }}-svc-couchdb:{{ .Values.globals.sqs.port }}
+          {{ end }}
         {{ if .Values.services.couchdb.enabled }}
         - name: COUCH_DB_USER
           valueFrom:
diff --git a/charts/budibase/templates/worker-service-deployment.yaml b/charts/budibase/templates/worker-service-deployment.yaml
index a493980fd3..94fdd0b94e 100644
--- a/charts/budibase/templates/worker-service-deployment.yaml
+++ b/charts/budibase/templates/worker-service-deployment.yaml
@@ -57,7 +57,11 @@ spec:
           value: http://{{ .Release.Name }}-svc-couchdb:{{ .Values.services.couchdb.port }}
           {{ end }}
         - name: COUCH_DB_SQL_URL
+          {{ if .Values.globals.sqs.url }}
+          value: {{ .Values.globals.sqs.url | quote }}
+          {{ else }}
           value: http://{{ .Release.Name }}-svc-couchdb:{{ .Values.globals.sqs.port }}
+          {{ end }}
         - name: API_ENCRYPTION_KEY
           valueFrom:
             secretKeyRef:
diff --git a/charts/budibase/values.yaml b/charts/budibase/values.yaml
index 9a5cdcdc82..de2cdb9474 100644
--- a/charts/budibase/values.yaml
+++ b/charts/budibase/values.yaml
@@ -139,6 +139,8 @@ globals:
     password: ""
 
   sqs:
+    # @ignore
+    url: ""
     # @ignore
     port: "4984"
 

From 676cb3f92e9dcc49b7f2a5de1177d34217bf06f2 Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Mon, 14 Oct 2024 18:00:41 +0100
Subject: [PATCH 23/27] Handling role numbering.

---
 packages/backend-core/src/security/roles.ts | 29 +++++++++++++++++----
 1 file changed, 24 insertions(+), 5 deletions(-)

diff --git a/packages/backend-core/src/security/roles.ts b/packages/backend-core/src/security/roles.ts
index 9d23463fa3..3c36a91c48 100644
--- a/packages/backend-core/src/security/roles.ts
+++ b/packages/backend-core/src/security/roles.ts
@@ -157,7 +157,7 @@ export function builtinRoleToNumber(id: string) {
       break
     }
     if (Array.isArray(role.inherits)) {
-      // TODO: role inheritance
+      throw new Error("Built-in roles don't support multi-inheritance")
     } else {
       role = builtins[role.inherits!]
     }
@@ -176,17 +176,36 @@ export async function roleToNumber(id: string) {
   const hierarchy = (await getUserRoleHierarchy(id, {
     defaultPublic: true,
   })) as RoleDoc[]
-  for (let role of hierarchy) {
+  const findNumber = (role: RoleDoc): number => {
     if (!role.inherits) {
-      continue
+      return 0
     }
     if (Array.isArray(role.inherits)) {
-      // TODO: role inheritance
+      // find the built-in roles, get their number, sort it, then get the last one
+      const highestBuiltin: number | undefined = role.inherits
+        .map(roleId => {
+          const foundRole = hierarchy.find(role => role._id === roleId)
+          if (foundRole) {
+            return findNumber(foundRole) + 1
+          }
+        })
+        .filter(number => !!number)
+        .sort()
+        .pop()
+      if (highestBuiltin != undefined) {
+        return highestBuiltin
+      }
     } else if (isBuiltin(role.inherits)) {
       return builtinRoleToNumber(role.inherits) + 1
     }
+    return 0
   }
-  return 0
+  let highest = 0
+  for (let role of hierarchy) {
+    const roleNumber = findNumber(role)
+    highest = Math.max(roleNumber, highest)
+  }
+  return highest
 }
 
 /**

From 26ee50b10b54534022c63262b948fbcbafca57f1 Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Mon, 14 Oct 2024 18:57:46 +0100
Subject: [PATCH 24/27] Adding test case for multi-inheritance

---
 .../src/api/routes/tests/permissions.spec.ts  | 82 ++++++++++++++++++-
 .../src/tests/utilities/TestConfiguration.ts  | 28 +++++++
 2 files changed, 109 insertions(+), 1 deletion(-)

diff --git a/packages/server/src/api/routes/tests/permissions.spec.ts b/packages/server/src/api/routes/tests/permissions.spec.ts
index a479adb4cf..46ce656459 100644
--- a/packages/server/src/api/routes/tests/permissions.spec.ts
+++ b/packages/server/src/api/routes/tests/permissions.spec.ts
@@ -1,5 +1,5 @@
 import { roles } from "@budibase/backend-core"
-import { Document, PermissionLevel, Row } from "@budibase/types"
+import { Document, PermissionLevel, Role, Row, Table } from "@budibase/types"
 import * as setup from "./utilities"
 import { generator, mocks } from "@budibase/backend-core/tests"
 
@@ -288,6 +288,86 @@ describe("/permission", () => {
     })
   })
 
+  describe("multi-inheritance permissions", () => {
+    let table1: Table, table2: Table, role1: Role, role2: Role
+    beforeEach(async () => {
+      table1 = await config.createTable()
+      table2 = await config.createTable()
+      await config.api.row.save(table1._id!, {
+        name: "a",
+      })
+      await config.api.row.save(table2._id!, {
+        name: "b",
+      })
+      role1 = await config.api.roles.save(
+        {
+          name: "role1",
+          permissionId: PermissionLevel.WRITE,
+          inherits: BUILTIN_ROLE_IDS.BASIC,
+        },
+        { status: 200 }
+      )
+      role2 = await config.api.roles.save(
+        {
+          name: "role2",
+          permissionId: PermissionLevel.WRITE,
+          inherits: BUILTIN_ROLE_IDS.BASIC,
+        },
+        { status: 200 }
+      )
+      await config.api.permission.add({
+        roleId: role1._id!,
+        level: PermissionLevel.READ,
+        resourceId: table1._id!,
+      })
+      await config.api.permission.add({
+        roleId: role2._id!,
+        level: PermissionLevel.READ,
+        resourceId: table2._id!,
+      })
+    })
+
+    it("should be unable to search for table 2 using role 1", async () => {
+      await config.setRole(role1._id!, async () => {
+        const response2 = await config.api.row.search(
+          table2._id!,
+          {
+            query: {},
+          },
+          { status: 403 }
+        )
+        expect(response2.rows).toBeUndefined()
+      })
+    })
+
+    it("should be able to fetch two tables, with different roles, using multi-inheritance", async () => {
+      const role3 = await config.api.roles.save({
+        name: "role3",
+        permissionId: PermissionLevel.WRITE,
+        inherits: [role1._id!, role2._id!],
+      })
+
+      await config.setRole(role3._id!, async () => {
+        const response1 = await config.api.row.search(
+          table1._id!,
+          {
+            query: {},
+          },
+          { status: 200 }
+        )
+        const response2 = await config.api.row.search(
+          table2._id!,
+          {
+            query: {},
+          },
+          { status: 200 }
+        )
+        expect(response1.rows[0].name).toEqual("a")
+        expect(response2.rows[0].name).toEqual("b")
+      })
+    })
+  })
+
   describe("fetch builtins", () => {
     it("should be able to fetch builtin definitions", async () => {
       const res = await request
diff --git a/packages/server/src/tests/utilities/TestConfiguration.ts b/packages/server/src/tests/utilities/TestConfiguration.ts
index f320df2ff8..a3f2fb7adc 100644
--- a/packages/server/src/tests/utilities/TestConfiguration.ts
+++ b/packages/server/src/tests/utilities/TestConfiguration.ts
@@ -428,6 +428,34 @@ export default class TestConfiguration {
 
   // HEADERS
 
+  // sets the role for the headers, for the period of a callback
+  async setRole(roleId: string, cb: () => Promise<unknown>) {
+    const roleUser = await this.createUser({
+      roles: {
+        [this.prodAppId!]: roleId,
+      },
+      builder: { global: false },
+      admin: { global: false },
+    })
+    await this.login({
+      roleId,
+      userId: roleUser._id!,
+      builder: false,
+      prodApp: true,
+    })
+    const temp = this.user
+    this.user = roleUser
+    await cb()
+    if (temp) {
+      this.user = temp
+      await this.login({
+        userId: temp._id!,
+        builder: true,
+        prodApp: false,
+      })
+    }
+  }
+
   defaultHeaders(extras = {}, prodApp = false) {
     const tenantId = this.getTenantId()
     const user = this.getUser()

From a56a22804240e973d8c46f93b7a2e327b83b7136 Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Mon, 14 Oct 2024 18:57:54 +0100
Subject: [PATCH 25/27] Fixes based on test case.

---
 packages/backend-core/src/security/roles.ts     | 2 +-
 packages/server/src/tests/utilities/api/role.ts | 4 ++++
 2 files changed, 5 insertions(+), 1 deletion(-)

diff --git a/packages/backend-core/src/security/roles.ts b/packages/backend-core/src/security/roles.ts
index 3c36a91c48..cd55e2f728 100644
--- a/packages/backend-core/src/security/roles.ts
+++ b/packages/backend-core/src/security/roles.ts
@@ -516,7 +516,7 @@ export function getDBRoleID(roleName: string) {
 export function getExternalRoleID(roleId: string, version?: string) {
   // for built-in roles we want to remove the DB role ID element (role_)
   if (
-    roleId.startsWith(DocumentType.ROLE) &&
+    roleId.startsWith(`${DocumentType.ROLE}${SEPARATOR}`) &&
     (isBuiltin(roleId) || version === RoleIDVersion.NAME)
   ) {
     const parts = roleId.split(SEPARATOR)
diff --git a/packages/server/src/tests/utilities/api/role.ts b/packages/server/src/tests/utilities/api/role.ts
index 31bffc6f85..05165cd38e 100644
--- a/packages/server/src/tests/utilities/api/role.ts
+++ b/packages/server/src/tests/utilities/api/role.ts
@@ -22,6 +22,10 @@ export class RoleAPI extends TestAPI {
   }
 
   save = async (body: SaveRoleRequest, expectations?: Expectations) => {
+    // the tests should always be creating the "new" version of roles
+    if (body.version === undefined) {
+      body.version = "name"
+    }
     return await this._post<SaveRoleResponse>(`/api/roles`, {
       body,
       expectations,

From bb43049d55e5a43d9019ca0d6dae54d6552ed1a6 Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Tue, 15 Oct 2024 12:07:31 +0100
Subject: [PATCH 26/27] Adding loop protection.

---
 packages/server/src/api/controllers/role.ts | 17 +++++++++++++++--
 1 file changed, 15 insertions(+), 2 deletions(-)

diff --git a/packages/server/src/api/controllers/role.ts b/packages/server/src/api/controllers/role.ts
index f3fd5d7b46..76638eea23 100644
--- a/packages/server/src/api/controllers/role.ts
+++ b/packages/server/src/api/controllers/role.ts
@@ -19,7 +19,7 @@ import {
   UserMetadata,
   DocumentType,
 } from "@budibase/types"
-import { RoleColor, sdk as sharedSdk } from "@budibase/shared-core"
+import { RoleColor, sdk as sharedSdk, helpers } from "@budibase/shared-core"
 import sdk from "../../sdk"
 
 const UpdateRolesOptions = {
@@ -81,9 +81,10 @@ export async function save(ctx: UserCtx<SaveRoleRequest, SaveRoleResponse>) {
     _id = dbCore.prefixRoleID(_id)
   }
 
+  const allRoles = await roles.getAllRoles()
   let dbRole: Role | undefined
   if (!isCreate && _id?.startsWith(DocumentType.ROLE)) {
-    dbRole = await db.get<Role>(_id)
+    dbRole = allRoles.find(role => role._id === _id)
   }
   if (dbRole && dbRole.name !== name && isNewVersion) {
     ctx.throw(400, "Cannot change custom role name")
@@ -97,6 +98,18 @@ export async function save(ctx: UserCtx<SaveRoleRequest, SaveRoleResponse>) {
   if (dbRole?.permissions && !role.permissions) {
     role.permissions = dbRole.permissions
   }
+
+  // add the new role to the list and check for loops
+  const index = allRoles.findIndex(r => r._id === role._id)
+  if (index === -1) {
+    allRoles.push(role)
+  } else {
+    allRoles[index] = role
+  }
+  if (helpers.roles.checkForRoleInheritanceLoops(allRoles)) {
+    ctx.throw(400, "Role inheritance contains a loop, this is not supported")
+  }
+
   const foundRev = ctx.request.body._rev || dbRole?._rev
   if (foundRev) {
     role._rev = foundRev

From 68498a0c54eff40d8c2d2702bab0f702908c715b Mon Sep 17 00:00:00 2001
From: Sam Rose <hello@samwho.dev>
Date: Tue, 15 Oct 2024 12:46:27 +0100
Subject: [PATCH 27/27] Write Redis data to the persistent data dir in single
 image.

---
 hosting/single/Dockerfile | 3 +++
 hosting/single/redis.conf | 7 +++++++
 hosting/single/runner.sh  | 8 ++++++--
 3 files changed, 16 insertions(+), 2 deletions(-)
 create mode 100644 hosting/single/redis.conf

diff --git a/hosting/single/Dockerfile b/hosting/single/Dockerfile
index ded0bc17dc..a1230f3c37 100644
--- a/hosting/single/Dockerfile
+++ b/hosting/single/Dockerfile
@@ -69,6 +69,9 @@ WORKDIR /minio
 COPY scripts/install-minio.sh ./install.sh
 RUN chmod +x install.sh && ./install.sh
 
+# setup redis
+COPY hosting/single/redis.conf /etc/redis/redis.conf
+
 # setup runner file
 WORKDIR /
 COPY hosting/single/runner.sh .
diff --git a/hosting/single/redis.conf b/hosting/single/redis.conf
new file mode 100644
index 0000000000..00740ffece
--- /dev/null
+++ b/hosting/single/redis.conf
@@ -0,0 +1,7 @@
+dir "DATA_DIR/redis"
+
+appendonly yes
+appendfsync everysec
+
+auto-aof-rewrite-percentage 100
+auto-aof-rewrite-min-size 64mb
\ No newline at end of file
diff --git a/hosting/single/runner.sh b/hosting/single/runner.sh
index 95464dd031..d9b8719f0f 100644
--- a/hosting/single/runner.sh
+++ b/hosting/single/runner.sh
@@ -75,13 +75,17 @@ fi
 for LINE in $(cat ${DATA_DIR}/.env); do export $LINE; done
 ln -s ${DATA_DIR}/.env /app/.env
 ln -s ${DATA_DIR}/.env /worker/.env
+
 # make these directories in runner, incase of mount
 mkdir -p ${DATA_DIR}/minio
+mkdir -p ${DATA_DIR}/redis
 chown -R couchdb:couchdb ${DATA_DIR}/couch
+
+sed -i "s#DATA_DIR#${DATA_DIR}#g" /etc/redis/redis.conf
 if [[ -n "${REDIS_PASSWORD}" ]]; then
-  redis-server --requirepass $REDIS_PASSWORD > /dev/stdout 2>&1 &
+  redis-server /etc/redis/redis.conf --requirepass $REDIS_PASSWORD > /dev/stdout 2>&1 &
 else
-  redis-server > /dev/stdout 2>&1 &
+  redis-server /etc/redis/redis.conf > /dev/stdout 2>&1 &
 fi
 /bbcouch-runner.sh &