Merge pull request #15665 from Budibase/fix/group-perf-improvements

Group performance improvements

packages/backend-core/src/db/couch/DatabaseImpl.ts

@@ -222,9 +222,12 @@ export class DatabaseImpl implements Database {
   }
 
   async getMultiple<T extends Document>(
-    ids: string[],
+    ids?: string[],
     opts?: { allowMissing?: boolean; excludeDocs?: boolean }
   ): Promise<T[]> {
+    if (!ids || ids.length === 0) {
+      return []
+    }
     // get unique
     ids = [...new Set(ids)]
     const includeDocs = !opts?.excludeDocs
@@ -249,7 +252,7 @@ export class DatabaseImpl implements Database {
     if (!opts?.allowMissing && someMissing) {
       const missing = response.rows.filter(row => rowUnavailable(row))
       const missingIds = missing.map(row => row.key).join(", ")
-      throw new Error(`Unable to get documents: ${missingIds}`)
+      throw new Error(`Unable to get bulk documents: ${missingIds}`)
     }
     return rows.map(row => (includeDocs ? row.doc! : row.value))
   }

packages/backend-core/src/db/instrumentation.ts

@@ -52,13 +52,13 @@ export class DDInstrumentedDatabase implements Database {
   }
 
   getMultiple<T extends Document>(
-    ids: string[],
+    ids?: string[],
     opts?: { allowMissing?: boolean | undefined } | undefined
   ): Promise<T[]> {
     return tracer.trace("db.getMultiple", async span => {
       span.addTags({
         db_name: this.name,
-        num_docs: ids.length,
+        num_docs: ids?.length || 0,
         allow_missing: opts?.allowMissing,
       })
       const docs = await this.db.getMultiple<T>(ids, opts)

packages/backend-core/src/users/db.ts

@@ -26,8 +26,9 @@ import {
 import {
   getAccountHolderFromUsers,
   isAdmin,
-  isCreator,
+  creatorsInList,
   validateUniqueUser,
+  isCreatorAsync,
 } from "./utils"
 import {
   getFirstPlatformUser,
@@ -261,8 +262,16 @@ export class UserDB {
     }
 
     const change = dbUser ? 0 : 1 // no change if there is existing user
-    const creatorsChange =
-      (await isCreator(dbUser)) !== (await isCreator(user)) ? 1 : 0
+
+    let creatorsChange = 0
+    if (dbUser) {
+      const [isDbUserCreator, isUserCreator] = await creatorsInList([
+        dbUser,
+        user,
+      ])
+      creatorsChange = isDbUserCreator !== isUserCreator ? 1 : 0
+    }
+
     return UserDB.quotas.addUsers(change, creatorsChange, async () => {
       if (!opts.isAccountHolder) {
         await validateUniqueUser(email, tenantId)
@@ -353,7 +362,7 @@ export class UserDB {
       }
       newUser.userGroups = groups || []
       newUsers.push(newUser)
-      if (await isCreator(newUser)) {
+      if (await isCreatorAsync(newUser)) {
         newCreators.push(newUser)
       }
     }
@@ -453,10 +462,8 @@ export class UserDB {
     }))
     const dbResponse = await usersCore.bulkUpdateGlobalUsers(toDelete)
 
-    const creatorsEval = await Promise.all(usersToDelete.map(isCreator))
-    const creatorsToDeleteCount = creatorsEval.filter(
-      creator => !!creator
-    ).length
+    const creatorsEval = await creatorsInList(usersToDelete)
+    const creatorsToDeleteCount = creatorsEval.filter(creator => creator).length
 
     const ssoUsersToDelete: AnyDocument[] = []
     for (let user of usersToDelete) {
@@ -533,7 +540,7 @@ export class UserDB {
 
     await db.remove(userId, dbUser._rev!)
 
-    const creatorsToDelete = (await isCreator(dbUser)) ? 1 : 0
+    const creatorsToDelete = (await isCreatorAsync(dbUser)) ? 1 : 0
     await UserDB.quotas.removeUsers(1, creatorsToDelete)
     await eventHelpers.handleDeleteEvents(dbUser)
     await cache.user.invalidateUser(userId)

packages/backend-core/src/users/test/utils.spec.ts

@@ -2,39 +2,39 @@ import { User, UserGroup } from "@budibase/types"
 import { generator, structures } from "../../../tests"
 import { DBTestConfiguration } from "../../../tests/extra"
 import { getGlobalDB } from "../../context"
-import { isCreator } from "../utils"
+import { isCreatorSync, creatorsInList } from "../utils"
 
 const config = new DBTestConfiguration()
 
 describe("Users", () => {
-  it("User is a creator if it is configured as a global builder", async () => {
+  it("User is a creator if it is configured as a global builder", () => {
     const user: User = structures.users.user({ builder: { global: true } })
-    expect(await isCreator(user)).toBe(true)
+    expect(isCreatorSync(user, [])).toBe(true)
   })
 
-  it("User is a creator if it is configured as a global admin", async () => {
+  it("User is a creator if it is configured as a global admin", () => {
     const user: User = structures.users.user({ admin: { global: true } })
-    expect(await isCreator(user)).toBe(true)
+    expect(isCreatorSync(user, [])).toBe(true)
   })
 
-  it("User is a creator if it is configured with creator permission", async () => {
+  it("User is a creator if it is configured with creator permission", () => {
     const user: User = structures.users.user({ builder: { creator: true } })
-    expect(await isCreator(user)).toBe(true)
+    expect(isCreatorSync(user, [])).toBe(true)
   })
 
-  it("User is a creator if it is a builder in some application", async () => {
+  it("User is a creator if it is a builder in some application", () => {
     const user: User = structures.users.user({ builder: { apps: ["app1"] } })
-    expect(await isCreator(user)).toBe(true)
+    expect(isCreatorSync(user, [])).toBe(true)
   })
 
-  it("User is a creator if it has CREATOR permission in some application", async () => {
+  it("User is a creator if it has CREATOR permission in some application", () => {
     const user: User = structures.users.user({ roles: { app1: "CREATOR" } })
-    expect(await isCreator(user)).toBe(true)
+    expect(isCreatorSync(user, [])).toBe(true)
   })
 
-  it("User is a creator if it has ADMIN permission in some application", async () => {
+  it("User is a creator if it has ADMIN permission in some application", () => {
     const user: User = structures.users.user({ roles: { app1: "ADMIN" } })
-    expect(await isCreator(user)).toBe(true)
+    expect(isCreatorSync(user, [])).toBe(true)
   })
 
   it("User is a creator if it remains to a group with ADMIN permissions", async () => {
@@ -59,7 +59,7 @@ describe("Users", () => {
       await db.put(group)
       for (let user of users) {
         await db.put(user)
-        const creator = await isCreator(user)
+        const creator = (await creatorsInList([user]))[0]
         expect(creator).toBe(true)
       }
     })

packages/backend-core/src/users/users.ts

@@ -22,7 +22,7 @@ import {
 } from "@budibase/types"
 import * as context from "../context"
 import { getGlobalDB } from "../context"
-import { isCreator } from "./utils"
+import { creatorsInList } from "./utils"
 import { UserDB } from "./db"
 import { dataFilters } from "@budibase/shared-core"
 
@@ -305,8 +305,8 @@ export async function getCreatorCount() {
   let creators = 0
   async function iterate(startPage?: string) {
     const page = await paginatedUsers({ bookmark: startPage })
-    const creatorsEval = await Promise.all(page.data.map(isCreator))
-    creators += creatorsEval.filter(creator => !!creator).length
+    const creatorsEval = await creatorsInList(page.data)
+    creators += creatorsEval.filter(creator => creator).length
     if (page.hasNextPage) {
       await iterate(page.nextPage)
     }

packages/backend-core/src/users/utils.ts

@@ -16,30 +16,47 @@ export const hasAdminPermissions = sdk.users.hasAdminPermissions
 export const hasBuilderPermissions = sdk.users.hasBuilderPermissions
 export const hasAppBuilderPermissions = sdk.users.hasAppBuilderPermissions
 
-export async function isCreator(user?: User | ContextUser) {
+export async function creatorsInList(
+  users: (User | ContextUser)[],
+  groups?: UserGroup[]
+) {
+  const groupIds = [
+    ...new Set(
+      users.filter(user => user.userGroups).flatMap(user => user.userGroups!)
+    ),
+  ]
+  const db = context.getGlobalDB()
+  groups = await db.getMultiple<UserGroup>(groupIds, { allowMissing: true })
+  return users.map(user => isCreatorSync(user, groups))
+}
+
+// fetches groups if no provided, but is async and shouldn't be looped with
+export async function isCreatorAsync(user: User | ContextUser) {
+  let groups: UserGroup[] = []
+  if (user.userGroups) {
+    const db = context.getGlobalDB()
+    groups = await db.getMultiple<UserGroup>(user.userGroups)
+  }
+  return isCreatorSync(user, groups)
+}
+
+export function isCreatorSync(user: User | ContextUser, groups?: UserGroup[]) {
   const isCreatorByUserDefinition = sdk.users.isCreator(user)
   if (!isCreatorByUserDefinition && user) {
-    return await isCreatorByGroupMembership(user)
+    return isCreatorByGroupMembership(user, groups)
   }
   return isCreatorByUserDefinition
 }
 
-async function isCreatorByGroupMembership(user?: User | ContextUser) {
-  const userGroups = user?.userGroups || []
-  if (userGroups.length > 0) {
-    const db = context.getGlobalDB()
-    const groups: UserGroup[] = []
-    for (let groupId of userGroups) {
-      try {
-        const group = await db.get<UserGroup>(groupId)
-        groups.push(group)
-      } catch (e: any) {
-        if (e.error !== "not_found") {
-          throw e
-        }
-      }
-    }
-    return groups.some(group =>
+function isCreatorByGroupMembership(
+  user: User | ContextUser,
+  groups?: UserGroup[]
+) {
+  const userGroups = groups?.filter(
+    group => user.userGroups?.indexOf(group._id!) !== -1
+  )
+  if (userGroups && userGroups.length > 0) {
+    return userGroups.some(group =>
       Object.values(group.roles || {}).includes(BUILTIN_ROLE_IDS.ADMIN)
     )
   }

packages/backend-core/tests/core/users/users.spec.ts

@@ -1,52 +0,0 @@
-import { range } from "lodash/fp"
-import { structures } from "../.."
-
-jest.mock("../../../src/context")
-jest.mock("../../../src/db")
-
-import * as context from "../../../src/context"
-import * as db from "../../../src/db"
-
-import { getCreatorCount } from "../../../src/users/users"
-
-describe("Users", () => {
-  let getGlobalDBMock: jest.SpyInstance
-  let paginationMock: jest.SpyInstance
-
-  beforeEach(() => {
-    jest.resetAllMocks()
-
-    getGlobalDBMock = jest.spyOn(context, "getGlobalDB")
-    paginationMock = jest.spyOn(db, "pagination")
-
-    jest.spyOn(db, "getGlobalUserParams")
-  })
-
-  it("retrieves the number of creators", async () => {
-    const getUsers = (offset: number, limit: number, creators = false) => {
-      const opts = creators ? { builder: { global: true } } : undefined
-      return range(offset, limit).map(() => structures.users.user(opts))
-    }
-    const page1Data = getUsers(0, 8)
-    const page2Data = getUsers(8, 12, true)
-    getGlobalDBMock.mockImplementation(() => ({
-      name: "fake-db",
-      allDocs: () => ({
-        rows: [...page1Data, ...page2Data],
-      }),
-    }))
-    paginationMock.mockImplementationOnce(() => ({
-      data: page1Data,
-      hasNextPage: true,
-      nextPage: "1",
-    }))
-    paginationMock.mockImplementation(() => ({
-      data: page2Data,
-      hasNextPage: false,
-      nextPage: undefined,
-    }))
-    const creatorsCount = await getCreatorCount()
-    expect(creatorsCount).toBe(4)
-    expect(paginationMock).toHaveBeenCalledTimes(2)
-  })
-})

packages/pro

@@ -1 +1 @@
-Subproject commit b28dbd549284cf450be7f25ad85aadf614d08f0b
+Subproject commit 2dd06c2fcb3cf10d5f16f5d8fe6cd344c8e905a5

packages/server/src/middleware/authorized.ts

@@ -34,7 +34,7 @@ const checkAuthorized = async (
   const isCreatorApi = permType === PermissionType.CREATOR
   const isBuilderApi = permType === PermissionType.BUILDER
   const isGlobalBuilder = users.isGlobalBuilder(ctx.user)
-  const isCreator = await users.isCreator(ctx.user)
+  const isCreator = await users.isCreatorAsync(ctx.user)
   const isBuilder = appId
     ? users.isBuilder(ctx.user, appId)
     : users.hasBuilderPermissions(ctx.user)

packages/types/src/sdk/db.ts

@@ -136,7 +136,7 @@ export interface Database {
   get<T extends Document>(id?: string): Promise<T>
   tryGet<T extends Document>(id?: string): Promise<T | undefined>
   getMultiple<T extends Document>(
-    ids: string[],
+    ids?: string[],
     opts?: { allowMissing?: boolean; excludeDocs?: boolean }
   ): Promise<T[]>
   remove(idOrDoc: Document): Promise<Nano.DocumentDestroyResponse>