Merge branch 'feature/role-multi-inheritance' of github.com:Budibase/budibase into new-rbac-ui

.github/workflows/deploy-featurebranch.yml

@@ -2,13 +2,11 @@ name: deploy-featurebranch
 
 on:
   pull_request:
-    types: [
+    types: 
-      labeled,
+      - labeled
-      # default types below (https://docs.github.com/en/actions/using-workflows/events-that-trigger-workflows#pull_request)
+      - opened
-      opened,
+      - synchronize
-      synchronize,
+      - reopened
-      reopened,
-    ]
 
 jobs:
   release:
@@ -22,31 +20,21 @@ jobs:
         contains(github.event.pull_request.labels.*.name, 'feature-branch-enterprise')
       )
     runs-on: ubuntu-latest
+    env:
+      PAYLOAD_BRANCH: ${{ github.head_ref }}
+      PAYLOAD_PR_NUMBER: ${{ github.event.pull_request.number }}
+      PAYLOAD_LICENSE_TYPE: |
+        ${{ 
+          contains(github.event.pull_request.labels.*.name, 'feature-branch') && 'free' ||
+          contains(github.event.pull_request.labels.*.name, 'feature-branch-pro') && 'pro' ||
+          contains(github.event.pull_request.labels.*.name, 'feature-branch-team') && 'team' ||
+          contains(github.event.pull_request.labels.*.name, 'feature-branch-business') && 'business' ||
+          contains(github.event.pull_request.labels.*.name, 'feature-branch-enterprise') && 'enterprise' || 'free'
+        }}
     steps:
       - uses: actions/checkout@v4
 
-      - name: Set PAYLOAD_LICENSE_TYPE
-        id: set_license_type
-        run: |
-          if [[ "${{ contains(github.event.pull_request.labels.*.name, 'feature-branch') }}" == "true" ]]; then
-            echo "PAYLOAD_LICENSE_TYPE=free" >> $GITHUB_ENV
-          elif [[ "${{ contains(github.event.pull_request.labels.*.name, 'feature-branch-pro') }}" == "true" ]]; then
-            echo "PAYLOAD_LICENSE_TYPE=pro" >> $GITHUB_ENV
-          elif [[ "${{ contains(github.event.pull_request.labels.*.name, 'feature-branch-team') }}" == "true" ]]; then
-            echo "PAYLOAD_LICENSE_TYPE=team" >> $GITHUB_ENV
-          elif [[ "${{ contains(github.event.pull_request.labels.*.name, 'feature-branch-business') }}" == "true" ]]; then
-            echo "PAYLOAD_LICENSE_TYPE=business" >> $GITHUB_ENV
-          elif [[ "${{ contains(github.event.pull_request.labels.*.name, 'feature-branch-enterprise') }}" == "true" ]]; then
-            echo "PAYLOAD_LICENSE_TYPE=enterprise" >> $GITHUB_ENV
-          else
-            echo "PAYLOAD_LICENSE_TYPE=free" >> $GITHUB_ENV
-          fi
-
       - uses: passeidireto/trigger-external-workflow-action@main
-        env:
-          PAYLOAD_BRANCH: ${{ github.head_ref }}
-          PAYLOAD_PR_NUMBER: ${{ github.event.pull_request.number }}
-          PAYLOAD_LICENSE_TYPE: ${{ env.PAYLOAD_LICENSE_TYPE }}
         with:
           repository: budibase/budibase-deploys
           event: featurebranch-qa-deploy

charts/budibase/templates/app-service-deployment.yaml

@@ -42,14 +42,12 @@ spec:
           {{ else }}
           value: http://{{ .Release.Name }}-svc-couchdb:{{ .Values.services.couchdb.port }}
           {{ end }}
-        {{ if .Values.globals.sqs.enabled }}
         - name: COUCH_DB_SQL_URL
-        {{ if .Values.globals.sqs.url }}
+          {{ if .Values.globals.sqs.url }}
-          value: {{ .Values.globals.sqs.url }}
+          value: {{ .Values.globals.sqs.url | quote }}
-        {{ else }}
+          {{ else }}
           value: http://{{ .Release.Name }}-svc-couchdb:{{ .Values.globals.sqs.port }}
-        {{ end }}
+          {{ end }}
-        {{ end }}
         {{ if .Values.services.couchdb.enabled }}
         - name: COUCH_DB_USER
           valueFrom:

charts/budibase/templates/automation-worker-service-deployment.yaml

@@ -43,6 +43,12 @@ spec:
           {{ else }}
           value: http://{{ .Release.Name }}-svc-couchdb:{{ .Values.services.couchdb.port }}
           {{ end }}
+        - name: COUCH_DB_SQL_URL
+          {{ if .Values.globals.sqs.url }}
+          value: {{ .Values.globals.sqs.url | quote }}
+          {{ else }}
+          value: http://{{ .Release.Name }}-svc-couchdb:{{ .Values.globals.sqs.port }}
+          {{ end }}
         {{ if .Values.services.couchdb.enabled }}
         - name: COUCH_DB_USER
           valueFrom:

charts/budibase/templates/worker-service-deployment.yaml

@@ -56,14 +56,12 @@ spec:
           {{ else }}
           value: http://{{ .Release.Name }}-svc-couchdb:{{ .Values.services.couchdb.port }}
           {{ end }}
-        {{ if .Values.globals.sqs.enabled }}
         - name: COUCH_DB_SQL_URL
-        {{ if .Values.globals.sqs.url }}
+          {{ if .Values.globals.sqs.url }}
-          value: {{ .Values.globals.sqs.url }}
+          value: {{ .Values.globals.sqs.url | quote }}
-        {{ else }}
+          {{ else }}
           value: http://{{ .Release.Name }}-svc-couchdb:{{ .Values.globals.sqs.port }}
-        {{ end }}
+          {{ end }}
-        {{ end }}
         - name: API_ENCRYPTION_KEY
           valueFrom:
             secretKeyRef:

charts/budibase/values.yaml

@@ -139,9 +139,6 @@ globals:
     password: ""
 
   sqs:
-    # -- Whether to use the CouchDB "structured query service" or not. This is disabled by
-    # default for now, but will become the default in a future release.
-    enabled: false
     # @ignore
     url: ""
     # @ignore

hosting/docker-compose.yaml

@@ -5,7 +5,7 @@ version: "3"
 services:
   app-service:
     restart: unless-stopped
-    image: budibase.docker.scarf.sh/budibase/apps
+    image: budibase/apps
     container_name: bbapps
     environment:
       SELF_HOSTED: 1
@@ -35,7 +35,7 @@ services:
 
   worker-service:
     restart: unless-stopped
-    image: budibase.docker.scarf.sh/budibase/worker
+    image: budibase/worker
     container_name: bbworker
     environment:
       SELF_HOSTED: 1
@@ -97,7 +97,7 @@ services:
 
   couchdb-service:
     restart: unless-stopped
-    image: budibase/couchdb
+    image: budibase/couchdb:v3.3.3-sqs-v2.1.1
     environment:
       - COUCHDB_PASSWORD=${COUCH_DB_PASSWORD}
       - COUCHDB_USER=${COUCH_DB_USER}

hosting/single/Dockerfile

@@ -69,6 +69,9 @@ WORKDIR /minio
 COPY scripts/install-minio.sh ./install.sh
 RUN chmod +x install.sh && ./install.sh
 
+# setup redis
+COPY hosting/single/redis.conf /etc/redis/redis.conf
+
 # setup runner file
 WORKDIR /
 COPY hosting/single/runner.sh .

hosting/single/redis.conf

@@ -0,0 +1,7 @@
+dir "DATA_DIR/redis"
+
+appendonly yes
+appendfsync everysec
+
+auto-aof-rewrite-percentage 100
+auto-aof-rewrite-min-size 64mb

hosting/single/runner.sh

@@ -75,13 +75,17 @@ fi
 for LINE in $(cat ${DATA_DIR}/.env); do export $LINE; done
 ln -s ${DATA_DIR}/.env /app/.env
 ln -s ${DATA_DIR}/.env /worker/.env
+
 # make these directories in runner, incase of mount
 mkdir -p ${DATA_DIR}/minio
+mkdir -p ${DATA_DIR}/redis
 chown -R couchdb:couchdb ${DATA_DIR}/couch
+
+sed -i "s#DATA_DIR#${DATA_DIR}#g" /etc/redis/redis.conf
 if [[ -n "${REDIS_PASSWORD}" ]]; then
-  redis-server --requirepass $REDIS_PASSWORD > /dev/stdout 2>&1 &
+  redis-server /etc/redis/redis.conf --requirepass $REDIS_PASSWORD > /dev/stdout 2>&1 &
 else
-  redis-server > /dev/stdout 2>&1 &
+  redis-server /etc/redis/redis.conf > /dev/stdout 2>&1 &
 fi
 /bbcouch-runner.sh &
 

packages/backend-core/src/features/features.ts

@@ -269,7 +269,7 @@ export class FlagSet<V extends Flag<any>, T extends { [key: string]: V }> {
 export const flags = new FlagSet({
   DEFAULT_VALUES: Flag.boolean(env.isDev()),
   AUTOMATION_BRANCHING: Flag.boolean(env.isDev()),
-  SQS: Flag.boolean(env.isDev()),
+  SQS: Flag.boolean(true),
   [FeatureFlag.AI_CUSTOM_CONFIGS]: Flag.boolean(env.isDev()),
   [FeatureFlag.ENRICHED_RELATIONSHIPS]: Flag.boolean(env.isDev()),
   [FeatureFlag.TABLES_DEFAULT_ADMIN]: Flag.boolean(env.isDev()),

packages/backend-core/src/features/tests/features.spec.ts

@@ -10,6 +10,7 @@ const schema = {
   TEST_BOOLEAN: Flag.boolean(false),
   TEST_STRING: Flag.string("default value"),
   TEST_NUMBER: Flag.number(0),
+  TEST_BOOLEAN_DEFAULT_TRUE: Flag.boolean(true),
 }
 const flags = new FlagSet(schema)
 
@@ -123,6 +124,11 @@ describe("feature flags", () => {
       },
       expected: flags.defaults(),
     },
+    {
+      it: "should be possible to override a default true flag to false",
+      environmentFlags: "default:!TEST_BOOLEAN_DEFAULT_TRUE",
+      expected: { TEST_BOOLEAN_DEFAULT_TRUE: false },
+    },
   ])(
     "$it",
     async ({

packages/backend-core/src/security/roles.ts

@@ -9,7 +9,7 @@ import {
 import { getAppDB } from "../context"
 import { Screen, Role as RoleDoc, RoleUIMetadata } from "@budibase/types"
 import cloneDeep from "lodash/fp/cloneDeep"
-import { RoleColor } from "@budibase/shared-core"
+import { RoleColor, helpers } from "@budibase/shared-core"
 
 export const BUILTIN_ROLE_IDS = {
   ADMIN: "ADMIN",
@@ -157,7 +157,7 @@ export function builtinRoleToNumber(id: string) {
       break
     }
     if (Array.isArray(role.inherits)) {
-      // TODO: role inheritance
+      throw new Error("Built-in roles don't support multi-inheritance")
     } else {
       role = builtins[role.inherits!]
     }
@@ -176,17 +176,36 @@ export async function roleToNumber(id: string) {
   const hierarchy = (await getUserRoleHierarchy(id, {
     defaultPublic: true,
   })) as RoleDoc[]
-  for (let role of hierarchy) {
+  const findNumber = (role: RoleDoc): number => {
     if (!role.inherits) {
-      continue
+      return 0
     }
     if (Array.isArray(role.inherits)) {
-      // TODO: role inheritance
+      // find the built-in roles, get their number, sort it, then get the last one
+      const highestBuiltin: number | undefined = role.inherits
+        .map(roleId => {
+          const foundRole = hierarchy.find(role => role._id === roleId)
+          if (foundRole) {
+            return findNumber(foundRole) + 1
+          }
+        })
+        .filter(number => !!number)
+        .sort()
+        .pop()
+      if (highestBuiltin != undefined) {
+        return highestBuiltin
+      }
     } else if (isBuiltin(role.inherits)) {
       return builtinRoleToNumber(role.inherits) + 1
     }
+    return 0
   }
-  return 0
+  let highest = 0
+  for (let role of hierarchy) {
+    const roleNumber = findNumber(role)
+    highest = Math.max(roleNumber, highest)
+  }
+  return highest
 }
 
 /**
@@ -204,6 +223,36 @@ export function lowerBuiltinRoleID(roleId1?: string, roleId2?: string): string {
     : roleId1
 }
 
+/**
+ * Given a list of roles, this will pick the role out, accounting for built ins.
+ */
+export function findRole(
+  roleId: string,
+  roles: RoleDoc[],
+  opts?: { defaultPublic?: boolean }
+): RoleDoc {
+  // built in roles mostly come from the in-code implementation,
+  // but can be extended by a doc stored about them (e.g. permissions)
+  let role: RoleDoc | undefined = getBuiltinRole(roleId)
+  if (!role) {
+    // make sure has the prefix (if it has it then it won't be added)
+    roleId = prefixRoleID(roleId)
+  }
+  const dbRole = roles.find(
+    role => role._id && role._id === getExternalRoleID(roleId, role.version)
+  )
+  if (!dbRole && !isBuiltin(roleId) && opts?.defaultPublic) {
+    return cloneDeep(BUILTIN_ROLES.PUBLIC)
+  }
+  if (!dbRole && (!role || Object.keys(role).length === 0)) {
+    throw new Error("Role could not be found")
+  }
+  role = Object.assign(role || {}, dbRole)
+  // finalise the ID
+  role._id = getExternalRoleID(role._id!, role.version)
+  return role
+}
+
 /**
  * Gets the role object, this is mainly useful for two purposes, to check if the level exists and
  * to check if the role inherits any others.
@@ -215,29 +264,15 @@ export async function getRole(
   roleId: string,
   opts?: { defaultPublic?: boolean }
 ): Promise<RoleDoc> {
-  // built in roles mostly come from the in-code implementation,
+  const db = getAppDB()
-  // but can be extended by a doc stored about them (e.g. permissions)
+  const roleList = []
-  let role: RoleDoc | undefined = getBuiltinRole(roleId)
+  if (!isBuiltin(roleId)) {
-  if (!role) {
+    const role = await db.tryGet<RoleDoc>(getDBRoleID(roleId))
-    // make sure has the prefix (if it has it then it won't be added)
+    if (role) {
-    roleId = prefixRoleID(roleId)
+      roleList.push(role)
-  }
-  try {
-    const db = getAppDB()
-    const dbRole = await db.get<RoleDoc>(getDBRoleID(roleId))
-    role = Object.assign(role || {}, dbRole)
-    // finalise the ID
-    role._id = getExternalRoleID(role._id!, role.version)
-  } catch (err) {
-    if (!isBuiltin(roleId) && opts?.defaultPublic) {
-      return cloneDeep(BUILTIN_ROLES.PUBLIC)
-    }
-    // only throw an error if there is no role at all
-    if (!role || Object.keys(role).length === 0) {
-      throw err
     }
   }
-  return role
+  return findRole(roleId, roleList, opts)
 }
 
 /**
@@ -247,13 +282,14 @@ async function getAllUserRoles(
   userRoleId: string,
   opts?: { defaultPublic?: boolean }
 ): Promise<RoleDoc[]> {
+  const allRoles = await getAllRoles()
+  if (helpers.roles.checkForRoleInheritanceLoops(allRoles)) {
+    throw new Error("Loop detected in roles - cannot list roles")
+  }
   // admins have access to all roles
   if (userRoleId === BUILTIN_IDS.ADMIN) {
-    return getAllRoles()
+    return allRoles
   }
-  let currentRole = await getRole(userRoleId, opts)
-  let roles = currentRole ? [currentRole] : []
-  let roleIds = [userRoleId]
   const rolesFound = (ids: string | string[]) => {
     if (Array.isArray(ids)) {
       return ids.filter(id => roleIds.includes(id)).length === ids.length
@@ -261,23 +297,49 @@ async function getAllUserRoles(
       return roleIds.includes(ids)
     }
   }
-  // get all the inherited roles
+
-  while (
+  const roleIds = [userRoleId]
-    currentRole &&
+  const roles: RoleDoc[] = []
-    currentRole.inherits &&
+  const iterateInherited = (role: RoleDoc) => {
-    !rolesFound(currentRole.inherits)
+    if (!role || !role._id) {
-  ) {
+      return
-    if (Array.isArray(currentRole.inherits)) {
+    }
-      // TODO: role inheritance
+    roleIds.push(role._id)
+    roles.push(role)
+    if (Array.isArray(role.inherits)) {
+      role.inherits.forEach(roleId => {
+        const foundRole = findRole(roleId, allRoles, opts)
+        if (foundRole) {
+          iterateInherited(foundRole)
+        }
+      })
     } else {
-      roleIds.push(currentRole.inherits)
+      while (role && role.inherits && !rolesFound(role.inherits)) {
-      currentRole = await getRole(currentRole.inherits)
+        if (Array.isArray(role.inherits)) {
-      if (currentRole) {
+          iterateInherited(role)
-        roles.push(currentRole)
+          break
+        } else {
+          roleIds.push(role.inherits)
+          role = findRole(role.inherits, allRoles, opts)
+          if (role) {
+            roles.push(role)
+          }
+        }
       }
     }
   }
-  return roles
+
+  // get all the inherited roles
+  iterateInherited(findRole(userRoleId, allRoles, opts))
+  const foundRoleIds: string[] = []
+  return roles.filter(role => {
+    if (role._id && !foundRoleIds.includes(role._id)) {
+      foundRoleIds.push(role._id)
+      return true
+    } else {
+      return false
+    }
+  })
 }
 
 export async function getUserRoleIdHierarchy(
@@ -454,7 +516,7 @@ export function getDBRoleID(roleName: string) {
 export function getExternalRoleID(roleId: string, version?: string) {
   // for built-in roles we want to remove the DB role ID element (role_)
   if (
-    roleId.startsWith(DocumentType.ROLE) &&
+    roleId.startsWith(`${DocumentType.ROLE}${SEPARATOR}`) &&
     (isBuiltin(roleId) || version === RoleIDVersion.NAME)
   ) {
     const parts = roleId.split(SEPARATOR)

packages/backend-core/src/sql/sql.ts

@@ -521,8 +521,11 @@ class InternalBuilder {
         const [filterTableName, ...otherProperties] = key.split(".")
         const property = otherProperties.join(".")
         const alias = getTableAlias(filterTableName)
-        return fn(q, alias ? `${alias}.${property}` : property, value)
+        return q.andWhere(subquery =>
+          fn(subquery, alias ? `${alias}.${property}` : property, value)
+        )
       }
+
       for (const key in structure) {
         const value = structure[key]
         const updatedKey = dbCore.removeKeyNumbering(key)
@@ -552,6 +555,9 @@ class InternalBuilder {
             value
           )
         } else if (shouldProcessRelationship) {
+          if (allOr) {
+            query = query.or
+          }
           query = builder.addRelationshipForFilter(query, updatedKey, q => {
             return handleRelationship(q, updatedKey, value)
           })

packages/frontend-core/src/components/Updating.svelte

@@ -2,36 +2,31 @@
   export let isMigrationDone
   export let onMigrationDone
   export let timeoutSeconds = 60 // 1 minute
-  export let minTimeSeconds = 3
 
-  const loadTime = Date.now()
-  const intervalMs = 1000
   let timedOut = false
-  let secondsWaited = 0
 
   async function checkMigrationsFinished() {
-    setTimeout(async () => {
+    let totalWaitMs = 0
+    // eslint-disable-next-line no-constant-condition
+    while (true) {
+      const waitForMs = 5000 + Math.random() * 5000
+      await new Promise(resolve => setTimeout(resolve, waitForMs))
+      totalWaitMs += waitForMs
+
       const isMigrated = await isMigrationDone()
-
+      if (isMigrated) {
-      const timeoutMs = timeoutSeconds * 1000
+        onMigrationDone()
-      if (!isMigrated || secondsWaited <= minTimeSeconds) {
+        return
-        if (loadTime + timeoutMs > Date.now()) {
-          secondsWaited += 1
-          return checkMigrationsFinished()
-        }
-
-        return migrationTimeout()
       }
 
-      onMigrationDone()
+      if (totalWaitMs > timeoutSeconds * 1000) {
-    }, intervalMs)
+        timedOut = true
+        return
+      }
+    }
   }
 
   checkMigrationsFinished()
-
-  function migrationTimeout() {
-    timedOut = true
-  }
 </script>
 
 <div class="loading" class:timeout={timedOut}>

packages/server/src/api/controllers/role.ts

@@ -19,7 +19,7 @@ import {
   UserMetadata,
   DocumentType,
 } from "@budibase/types"
-import { RoleColor, sdk as sharedSdk } from "@budibase/shared-core"
+import { RoleColor, sdk as sharedSdk, helpers } from "@budibase/shared-core"
 import sdk from "../../sdk"
 import { builderSocket } from "../../websockets"
 
@@ -82,9 +82,10 @@ export async function save(ctx: UserCtx<SaveRoleRequest, SaveRoleResponse>) {
     _id = dbCore.prefixRoleID(_id)
   }
 
+  const allRoles = await roles.getAllRoles()
   let dbRole: Role | undefined
   if (!isCreate && _id?.startsWith(DocumentType.ROLE)) {
-    dbRole = await db.get<Role>(_id)
+    dbRole = allRoles.find(role => role._id === _id)
   }
   if (dbRole && dbRole.name !== name && isNewVersion) {
     ctx.throw(400, "Cannot change custom role name")
@@ -98,6 +99,18 @@ export async function save(ctx: UserCtx<SaveRoleRequest, SaveRoleResponse>) {
   if (dbRole?.permissions && !role.permissions) {
     role.permissions = dbRole.permissions
   }
+
+  // add the new role to the list and check for loops
+  const index = allRoles.findIndex(r => r._id === role._id)
+  if (index === -1) {
+    allRoles.push(role)
+  } else {
+    allRoles[index] = role
+  }
+  if (helpers.roles.checkForRoleInheritanceLoops(allRoles)) {
+    ctx.throw(400, "Role inheritance contains a loop, this is not supported")
+  }
+
   const foundRev = ctx.request.body._rev || dbRole?._rev
   if (foundRev) {
     role._rev = foundRev

packages/server/src/api/routes/tests/permissions.spec.ts

@@ -1,5 +1,5 @@
 import { roles } from "@budibase/backend-core"
-import { Document, PermissionLevel, Row } from "@budibase/types"
+import { Document, PermissionLevel, Role, Row, Table } from "@budibase/types"
 import * as setup from "./utilities"
 import { generator, mocks } from "@budibase/backend-core/tests"
 
@@ -288,6 +288,86 @@ describe("/permission", () => {
     })
   })
 
+  describe("multi-inheritance permissions", () => {
+    let table1: Table, table2: Table, role1: Role, role2: Role
+    beforeEach(async () => {
+      table1 = await config.createTable()
+      table2 = await config.createTable()
+      await config.api.row.save(table1._id!, {
+        name: "a",
+      })
+      await config.api.row.save(table2._id!, {
+        name: "b",
+      })
+      role1 = await config.api.roles.save(
+        {
+          name: "role1",
+          permissionId: PermissionLevel.WRITE,
+          inherits: BUILTIN_ROLE_IDS.BASIC,
+        },
+        { status: 200 }
+      )
+      role2 = await config.api.roles.save(
+        {
+          name: "role2",
+          permissionId: PermissionLevel.WRITE,
+          inherits: BUILTIN_ROLE_IDS.BASIC,
+        },
+        { status: 200 }
+      )
+      await config.api.permission.add({
+        roleId: role1._id!,
+        level: PermissionLevel.READ,
+        resourceId: table1._id!,
+      })
+      await config.api.permission.add({
+        roleId: role2._id!,
+        level: PermissionLevel.READ,
+        resourceId: table2._id!,
+      })
+    })
+
+    it("should be unable to search for table 2 using role 1", async () => {
+      await config.setRole(role1._id!, async () => {
+        const response2 = await config.api.row.search(
+          table2._id!,
+          {
+            query: {},
+          },
+          { status: 403 }
+        )
+        expect(response2.rows).toBeUndefined()
+      })
+    })
+
+    it("should be able to fetch two tables, with different roles, using multi-inheritance", async () => {
+      const role3 = await config.api.roles.save({
+        name: "role3",
+        permissionId: PermissionLevel.WRITE,
+        inherits: [role1._id!, role2._id!],
+      })
+
+      await config.setRole(role3._id!, async () => {
+        const response1 = await config.api.row.search(
+          table1._id!,
+          {
+            query: {},
+          },
+          { status: 200 }
+        )
+        const response2 = await config.api.row.search(
+          table2._id!,
+          {
+            query: {},
+          },
+          { status: 200 }
+        )
+        expect(response1.rows[0].name).toEqual("a")
+        expect(response2.rows[0].name).toEqual("b")
+      })
+    })
+  })
+
   describe("fetch builtins", () => {
     it("should be able to fetch builtin definitions", async () => {
       const res = await request

packages/server/src/api/routes/tests/role.spec.js

@@ -1,182 +0,0 @@
-const { roles, events, permissions } = require("@budibase/backend-core")
-const setup = require("./utilities")
-const { PermissionLevel } = require("@budibase/types")
-const { basicRole } = setup.structures
-const { BUILTIN_ROLE_IDS } = roles
-const { BuiltinPermissionID } = permissions
-
-describe("/roles", () => {
-  let request = setup.getRequest()
-  let config = setup.getConfig()
-
-  afterAll(setup.afterAll)
-
-  beforeAll(async () => {
-    await config.init()
-  })
-
-  const createRole = async role => {
-    if (!role) {
-      role = basicRole()
-    }
-
-    return request
-      .post(`/api/roles`)
-      .send(role)
-      .set(config.defaultHeaders())
-      .expect("Content-Type", /json/)
-      .expect(200)
-  }
-
-  describe("create", () => {
-    it("returns a success message when role is successfully created", async () => {
-      const role = basicRole()
-      const res = await createRole(role)
-
-      expect(res.body._id).toBeDefined()
-      expect(res.body._rev).toBeDefined()
-      expect(events.role.updated).not.toBeCalled()
-      expect(events.role.created).toBeCalledTimes(1)
-      expect(events.role.created).toBeCalledWith(res.body)
-    })
-  })
-
-  describe("update", () => {
-    it("updates a role", async () => {
-      const role = basicRole()
-      let res = await createRole(role)
-      jest.clearAllMocks()
-      res = await createRole(res.body)
-
-      expect(res.body._id).toBeDefined()
-      expect(res.body._rev).toBeDefined()
-      expect(events.role.created).not.toBeCalled()
-      expect(events.role.updated).toBeCalledTimes(1)
-      expect(events.role.updated).toBeCalledWith(res.body)
-    })
-  })
-
-  describe("fetch", () => {
-    beforeAll(async () => {
-      // Recreate the app
-      await config.init()
-    })
-
-    it("should list custom roles, plus 2 default roles", async () => {
-      const customRole = await config.createRole()
-
-      const res = await request
-        .get(`/api/roles`)
-        .set(config.defaultHeaders())
-        .expect("Content-Type", /json/)
-        .expect(200)
-
-      expect(res.body.length).toBe(5)
-
-      const adminRole = res.body.find(r => r._id === BUILTIN_ROLE_IDS.ADMIN)
-      expect(adminRole).toBeDefined()
-      expect(adminRole.inherits).toEqual(BUILTIN_ROLE_IDS.POWER)
-      expect(adminRole.permissionId).toEqual(BuiltinPermissionID.ADMIN)
-
-      const powerUserRole = res.body.find(r => r._id === BUILTIN_ROLE_IDS.POWER)
-      expect(powerUserRole).toBeDefined()
-      expect(powerUserRole.inherits).toEqual(BUILTIN_ROLE_IDS.BASIC)
-      expect(powerUserRole.permissionId).toEqual(BuiltinPermissionID.POWER)
-
-      const customRoleFetched = res.body.find(r => r._id === customRole.name)
-      expect(customRoleFetched).toBeDefined()
-      expect(customRoleFetched.inherits).toEqual(BUILTIN_ROLE_IDS.BASIC)
-      expect(customRoleFetched.permissionId).toEqual(
-        BuiltinPermissionID.READ_ONLY
-      )
-    })
-
-    it("should be able to get the role with a permission added", async () => {
-      const table = await config.createTable()
-      await config.api.permission.add({
-        roleId: BUILTIN_ROLE_IDS.POWER,
-        resourceId: table._id,
-        level: PermissionLevel.READ,
-      })
-      const res = await request
-        .get(`/api/roles`)
-        .set(config.defaultHeaders())
-        .expect("Content-Type", /json/)
-        .expect(200)
-      expect(res.body.length).toBeGreaterThan(0)
-      const power = res.body.find(role => role._id === BUILTIN_ROLE_IDS.POWER)
-      expect(power.permissions[table._id]).toEqual(["read"])
-    })
-  })
-
-  describe("destroy", () => {
-    it("should delete custom roles", async () => {
-      const customRole = await config.createRole({
-        name: "user",
-        permissionId: BuiltinPermissionID.READ_ONLY,
-        inherits: BUILTIN_ROLE_IDS.BASIC,
-      })
-      delete customRole._rev_tree
-      await request
-        .delete(`/api/roles/${customRole._id}/${customRole._rev}`)
-        .set(config.defaultHeaders())
-        .expect(200)
-      await request
-        .get(`/api/roles/${customRole._id}`)
-        .set(config.defaultHeaders())
-        .expect(404)
-      expect(events.role.deleted).toBeCalledTimes(1)
-      expect(events.role.deleted).toBeCalledWith(customRole)
-    })
-  })
-
-  describe("accessible", () => {
-    it("should be able to fetch accessible roles (with builder)", async () => {
-      const res = await request
-        .get("/api/roles/accessible")
-        .set(config.defaultHeaders())
-        .expect(200)
-      expect(res.body.length).toBe(5)
-      expect(typeof res.body[0]).toBe("string")
-    })
-
-    it("should be able to fetch accessible roles (basic user)", async () => {
-      const res = await request
-        .get("/api/roles/accessible")
-        .set(await config.basicRoleHeaders())
-        .expect(200)
-      expect(res.body.length).toBe(2)
-      expect(res.body[0]).toBe("BASIC")
-      expect(res.body[1]).toBe("PUBLIC")
-    })
-
-    it("should be able to fetch accessible roles (no user)", async () => {
-      const res = await request
-        .get("/api/roles/accessible")
-        .set(config.publicHeaders())
-        .expect(200)
-      expect(res.body.length).toBe(1)
-      expect(res.body[0]).toBe("PUBLIC")
-    })
-
-    it("should not fetch higher level accessible roles when a custom role header is provided", async () => {
-      await createRole({
-        name: `custom_role_1`,
-        inherits: roles.BUILTIN_ROLE_IDS.BASIC,
-        permissionId: permissions.BuiltinPermissionID.READ_ONLY,
-        version: "name",
-      })
-      const res = await request
-        .get("/api/roles/accessible")
-        .set({
-          ...config.defaultHeaders(),
-          "x-budibase-role": "custom_role_1",
-        })
-        .expect(200)
-      expect(res.body.length).toBe(3)
-      expect(res.body[0]).toBe("custom_role_1")
-      expect(res.body[1]).toBe("BASIC")
-      expect(res.body[2]).toBe("PUBLIC")
-    })
-  })
-})

packages/server/src/api/routes/tests/role.spec.ts

@@ -142,7 +142,7 @@ describe("/roles", () => {
     })
 
     it("should not fetch higher level accessible roles when a custom role header is provided", async () => {
-      const customRoleName = "CUSTOM_ROLE"
+      const customRoleName = "custom_role_1"
       await config.api.roles.save({
         name: customRoleName,
         inherits: roles.BUILTIN_ROLE_IDS.BASIC,
@@ -155,10 +155,40 @@ describe("/roles", () => {
           status: 200,
         }
       )
-      expect(res.length).toBe(3)
+      expect(res).toEqual([customRoleName, "BASIC", "PUBLIC"])
-      expect(res[0]).toBe(customRoleName)
+    })
-      expect(res[1]).toBe("BASIC")
+  })
-      expect(res[2]).toBe("PUBLIC")
+
+  describe("accessible - multi-inheritance", () => {
+    it("should list access correctly for multi-inheritance role", async () => {
+      const role1 = "multi_role_1",
+        role2 = "multi_role_2",
+        role3 = "multi_role_3"
+      const { _id: roleId1 } = await config.api.roles.save({
+        name: role1,
+        inherits: roles.BUILTIN_ROLE_IDS.BASIC,
+        permissionId: permissions.BuiltinPermissionID.WRITE,
+        version: "name",
+      })
+      const { _id: roleId2 } = await config.api.roles.save({
+        name: role2,
+        inherits: roles.BUILTIN_ROLE_IDS.POWER,
+        permissionId: permissions.BuiltinPermissionID.POWER,
+        version: "name",
+      })
+      await config.api.roles.save({
+        name: role3,
+        inherits: [roleId1!, roleId2!],
+        permissionId: permissions.BuiltinPermissionID.READ_ONLY,
+        version: "name",
+      })
+      const headers = await config.roleHeaders({
+        roleId: role3,
+      })
+      const res = await config.api.roles.accessible(headers, {
+        status: 200,
+      })
+      expect(res).toEqual([role3, role1, "BASIC", "PUBLIC", role2, "POWER"])
     })
   })
 })

packages/server/src/api/routes/tests/search.spec.ts

@@ -24,6 +24,7 @@ import {
   EmptyFilterOption,
   FieldType,
   JsonFieldSubType,
+  LogicalOperator,
   RelationshipType,
   Row,
   RowSearchParams,
@@ -2415,6 +2416,211 @@ describe.each([
             equal: { ["name"]: "baz" },
           }).toContainExactly([{ name: "baz", productCat: undefined }])
         })
+
+        describe("logical filters", () => {
+          const logicalOperators = [LogicalOperator.AND, LogicalOperator.OR]
+
+          describe("$and", () => {
+            it("should allow single conditions", async () => {
+              await expectQuery({
+                $and: {
+                  conditions: [
+                    {
+                      equal: { ["productCat.name"]: "foo" },
+                    },
+                  ],
+                },
+              }).toContainExactly([
+                { name: "foo", productCat: [{ _id: productCatRows[0]._id }] },
+              ])
+            })
+
+            it("should allow exclusive conditions", async () => {
+              await expectQuery({
+                $and: {
+                  conditions: [
+                    {
+                      equal: { ["productCat.name"]: "foo" },
+                      notEqual: { ["productCat.name"]: "foo" },
+                    },
+                  ],
+                },
+              }).toContainExactly([])
+            })
+
+            it.each([logicalOperators])(
+              "should allow nested ands with single conditions (with %s as root)",
+              async rootOperator => {
+                await expectQuery({
+                  [rootOperator]: {
+                    conditions: [
+                      {
+                        $and: {
+                          conditions: [
+                            {
+                              equal: { ["productCat.name"]: "foo" },
+                            },
+                          ],
+                        },
+                      },
+                    ],
+                  },
+                }).toContainExactly([
+                  { name: "foo", productCat: [{ _id: productCatRows[0]._id }] },
+                ])
+              }
+            )
+
+            it.each([logicalOperators])(
+              "should allow nested ands with exclusive conditions (with %s as root)",
+              async rootOperator => {
+                await expectQuery({
+                  [rootOperator]: {
+                    conditions: [
+                      {
+                        $and: {
+                          conditions: [
+                            {
+                              equal: { ["productCat.name"]: "foo" },
+                              notEqual: { ["productCat.name"]: "foo" },
+                            },
+                          ],
+                        },
+                      },
+                    ],
+                  },
+                }).toContainExactly([])
+              }
+            )
+
+            it.each([logicalOperators])(
+              "should allow nested ands with multiple conditions (with %s as root)",
+              async rootOperator => {
+                await expectQuery({
+                  [rootOperator]: {
+                    conditions: [
+                      {
+                        $and: {
+                          conditions: [
+                            {
+                              equal: { ["productCat.name"]: "foo" },
+                            },
+                          ],
+                        },
+                        notEqual: { ["productCat.name"]: "foo" },
+                      },
+                    ],
+                  },
+                }).toContainExactly([])
+              }
+            )
+          })
+
+          describe("$ors", () => {
+            it("should allow single conditions", async () => {
+              await expectQuery({
+                $or: {
+                  conditions: [
+                    {
+                      equal: { ["productCat.name"]: "foo" },
+                    },
+                  ],
+                },
+              }).toContainExactly([
+                { name: "foo", productCat: [{ _id: productCatRows[0]._id }] },
+              ])
+            })
+
+            it("should allow exclusive conditions", async () => {
+              await expectQuery({
+                $or: {
+                  conditions: [
+                    {
+                      equal: { ["productCat.name"]: "foo" },
+                      notEqual: { ["productCat.name"]: "foo" },
+                    },
+                  ],
+                },
+              }).toContainExactly([
+                { name: "foo", productCat: [{ _id: productCatRows[0]._id }] },
+                { name: "bar", productCat: [{ _id: productCatRows[1]._id }] },
+                // { name: "baz", productCat: undefined }, // TODO
+              ])
+            })
+
+            it.each([logicalOperators])(
+              "should allow nested ors with single conditions (with %s as root)",
+              async rootOperator => {
+                await expectQuery({
+                  [rootOperator]: {
+                    conditions: [
+                      {
+                        $or: {
+                          conditions: [
+                            {
+                              equal: { ["productCat.name"]: "foo" },
+                            },
+                          ],
+                        },
+                      },
+                    ],
+                  },
+                }).toContainExactly([
+                  { name: "foo", productCat: [{ _id: productCatRows[0]._id }] },
+                ])
+              }
+            )
+
+            it.each([logicalOperators])(
+              "should allow nested ors with exclusive conditions (with %s as root)",
+              async rootOperator => {
+                await expectQuery({
+                  [rootOperator]: {
+                    conditions: [
+                      {
+                        $or: {
+                          conditions: [
+                            {
+                              equal: { ["productCat.name"]: "foo" },
+                              notEqual: { ["productCat.name"]: "foo" },
+                            },
+                          ],
+                        },
+                      },
+                    ],
+                  },
+                }).toContainExactly([
+                  { name: "foo", productCat: [{ _id: productCatRows[0]._id }] },
+                  { name: "bar", productCat: [{ _id: productCatRows[1]._id }] },
+                  // { name: "baz", productCat: undefined }, // TODO
+                ])
+              }
+            )
+
+            it("should allow nested ors with multiple conditions", async () => {
+              await expectQuery({
+                $or: {
+                  conditions: [
+                    {
+                      $or: {
+                        conditions: [
+                          {
+                            equal: { ["productCat.name"]: "foo" },
+                          },
+                        ],
+                      },
+                      notEqual: { ["productCat.name"]: "foo" },
+                    },
+                  ],
+                },
+              }).toContainExactly([
+                { name: "foo", productCat: [{ _id: productCatRows[0]._id }] },
+                { name: "bar", productCat: [{ _id: productCatRows[1]._id }] },
+                // { name: "baz", productCat: undefined }, // TODO
+              ])
+            })
+          })
+        })
       })
 
     isSql &&

packages/server/src/integrations/tests/sqlAlias.spec.ts

@@ -79,7 +79,7 @@ describe("Captures of real examples", () => {
         sql: expect.stringContaining(
           multiline(
             `where exists (select 1 from "tasks" as "b" inner join "products_tasks" as "c" on "b"."taskid" = "c"."taskid" where "c"."productid" = "a"."productid" 
-                 and COALESCE("b"."taskname" = $1, FALSE)`
+                 and (COALESCE("b"."taskname" = $1, FALSE))`
           )
         ),
       })
@@ -144,7 +144,7 @@ describe("Captures of real examples", () => {
         ],
         sql: expect.stringContaining(
           multiline(
-            `where exists (select 1 from "persons" as "c" where "c"."personid" = "a"."executorid" and "c"."year" between $1 and $2)`
+            `where exists (select 1 from "persons" as "c" where "c"."personid" = "a"."executorid" and ("c"."year" between $1 and $2))`
           )
         ),
       })

packages/server/src/tests/utilities/TestConfiguration.ts

@@ -428,6 +428,34 @@ export default class TestConfiguration {
 
   // HEADERS
 
+  // sets the role for the headers, for the period of a callback
+  async setRole(roleId: string, cb: () => Promise<unknown>) {
+    const roleUser = await this.createUser({
+      roles: {
+        [this.prodAppId!]: roleId,
+      },
+      builder: { global: false },
+      admin: { global: false },
+    })
+    await this.login({
+      roleId,
+      userId: roleUser._id!,
+      builder: false,
+      prodApp: true,
+    })
+    const temp = this.user
+    this.user = roleUser
+    await cb()
+    if (temp) {
+      this.user = temp
+      await this.login({
+        userId: temp._id!,
+        builder: true,
+        prodApp: false,
+      })
+    }
+  }
+
   defaultHeaders(extras = {}, prodApp = false) {
     const tenantId = this.getTenantId()
     const user = this.getUser()

packages/server/src/tests/utilities/api/role.ts

@@ -22,6 +22,10 @@ export class RoleAPI extends TestAPI {
   }
 
   save = async (body: SaveRoleRequest, expectations?: Expectations) => {
+    // the tests should always be creating the "new" version of roles
+    if (body.version === undefined) {
+      body.version = "name"
+    }
     return await this._post<SaveRoleResponse>(`/api/roles`, {
       body,
       expectations,

packages/shared-core/src/helpers/index.ts

@@ -3,3 +3,4 @@ export * from "./integrations"
 export * as cron from "./cron"
 export * as schema from "./schema"
 export * as views from "./views"
+export * as roles from "./roles"