From f75cfc1db6832a250eaf7e340654ffb7c8b2ad62 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Fri, 28 Feb 2025 17:11:42 +0000 Subject: [PATCH] Wide variety of improvements to performance, making some group operations synchronous to avoid re-fetching groups repeatedly. --- packages/backend-core/src/users/db.ts | 25 +++++--- .../backend-core/src/users/test/utils.spec.ts | 28 ++++----- packages/backend-core/src/users/users.ts | 6 +- packages/backend-core/src/users/utils.ts | 59 +++++++++++++------ packages/pro | 2 +- packages/server/src/middleware/authorized.ts | 2 +- 6 files changed, 77 insertions(+), 45 deletions(-) diff --git a/packages/backend-core/src/users/db.ts b/packages/backend-core/src/users/db.ts index 2b15338925..677feed678 100644 --- a/packages/backend-core/src/users/db.ts +++ b/packages/backend-core/src/users/db.ts @@ -26,8 +26,9 @@ import { import { getAccountHolderFromUsers, isAdmin, - isCreator, + creatorsInList, validateUniqueUser, + isCreatorAsync, } from "./utils" import { getFirstPlatformUser, @@ -261,8 +262,16 @@ export class UserDB { } const change = dbUser ? 0 : 1 // no change if there is existing user - const creatorsChange = - (await isCreator(dbUser)) !== (await isCreator(user)) ? 1 : 0 + + let creatorsChange = 0 + if (dbUser) { + const [isDbUserCreator, isUserCreator] = await creatorsInList([ + dbUser, + user, + ]) + creatorsChange = isDbUserCreator !== isUserCreator ? 1 : 0 + } + return UserDB.quotas.addUsers(change, creatorsChange, async () => { if (!opts.isAccountHolder) { await validateUniqueUser(email, tenantId) @@ -353,7 +362,7 @@ export class UserDB { } newUser.userGroups = groups || [] newUsers.push(newUser) - if (await isCreator(newUser)) { + if (await isCreatorAsync(newUser)) { newCreators.push(newUser) } } @@ -453,10 +462,8 @@ export class UserDB { })) const dbResponse = await usersCore.bulkUpdateGlobalUsers(toDelete) - const creatorsEval = await Promise.all(usersToDelete.map(isCreator)) - const creatorsToDeleteCount = creatorsEval.filter( - creator => !!creator - ).length + const creatorsEval = await creatorsInList(usersToDelete) + const creatorsToDeleteCount = creatorsEval.filter(creator => creator).length const ssoUsersToDelete: AnyDocument[] = [] for (let user of usersToDelete) { @@ -533,7 +540,7 @@ export class UserDB { await db.remove(userId, dbUser._rev!) - const creatorsToDelete = (await isCreator(dbUser)) ? 1 : 0 + const creatorsToDelete = (await isCreatorAsync(dbUser)) ? 1 : 0 await UserDB.quotas.removeUsers(1, creatorsToDelete) await eventHelpers.handleDeleteEvents(dbUser) await cache.user.invalidateUser(userId) diff --git a/packages/backend-core/src/users/test/utils.spec.ts b/packages/backend-core/src/users/test/utils.spec.ts index cb98b8972b..b52397c979 100644 --- a/packages/backend-core/src/users/test/utils.spec.ts +++ b/packages/backend-core/src/users/test/utils.spec.ts @@ -2,39 +2,39 @@ import { User, UserGroup } from "@budibase/types" import { generator, structures } from "../../../tests" import { DBTestConfiguration } from "../../../tests/extra" import { getGlobalDB } from "../../context" -import { isCreator } from "../utils" +import { isCreatorSync, creatorsInList } from "../utils" const config = new DBTestConfiguration() describe("Users", () => { - it("User is a creator if it is configured as a global builder", async () => { + it("User is a creator if it is configured as a global builder", () => { const user: User = structures.users.user({ builder: { global: true } }) - expect(await isCreator(user)).toBe(true) + expect(isCreatorSync(user, [])).toBe(true) }) - it("User is a creator if it is configured as a global admin", async () => { + it("User is a creator if it is configured as a global admin", () => { const user: User = structures.users.user({ admin: { global: true } }) - expect(await isCreator(user)).toBe(true) + expect(isCreatorSync(user, [])).toBe(true) }) - it("User is a creator if it is configured with creator permission", async () => { + it("User is a creator if it is configured with creator permission", () => { const user: User = structures.users.user({ builder: { creator: true } }) - expect(await isCreator(user)).toBe(true) + expect(isCreatorSync(user, [])).toBe(true) }) - it("User is a creator if it is a builder in some application", async () => { + it("User is a creator if it is a builder in some application", () => { const user: User = structures.users.user({ builder: { apps: ["app1"] } }) - expect(await isCreator(user)).toBe(true) + expect(isCreatorSync(user, [])).toBe(true) }) - it("User is a creator if it has CREATOR permission in some application", async () => { + it("User is a creator if it has CREATOR permission in some application", () => { const user: User = structures.users.user({ roles: { app1: "CREATOR" } }) - expect(await isCreator(user)).toBe(true) + expect(isCreatorSync(user, [])).toBe(true) }) - it("User is a creator if it has ADMIN permission in some application", async () => { + it("User is a creator if it has ADMIN permission in some application", () => { const user: User = structures.users.user({ roles: { app1: "ADMIN" } }) - expect(await isCreator(user)).toBe(true) + expect(isCreatorSync(user, [])).toBe(true) }) it("User is a creator if it remains to a group with ADMIN permissions", async () => { @@ -59,7 +59,7 @@ describe("Users", () => { await db.put(group) for (let user of users) { await db.put(user) - const creator = await isCreator(user) + const creator = (await creatorsInList([user]))[0] expect(creator).toBe(true) } }) diff --git a/packages/backend-core/src/users/users.ts b/packages/backend-core/src/users/users.ts index 0bff428fa9..36abfcfb2d 100644 --- a/packages/backend-core/src/users/users.ts +++ b/packages/backend-core/src/users/users.ts @@ -22,7 +22,7 @@ import { } from "@budibase/types" import * as context from "../context" import { getGlobalDB } from "../context" -import { isCreator } from "./utils" +import { creatorsInList } from "./utils" import { UserDB } from "./db" import { dataFilters } from "@budibase/shared-core" @@ -305,8 +305,8 @@ export async function getCreatorCount() { let creators = 0 async function iterate(startPage?: string) { const page = await paginatedUsers({ bookmark: startPage }) - const creatorsEval = await Promise.all(page.data.map(isCreator)) - creators += creatorsEval.filter(creator => !!creator).length + const creatorsEval = await creatorsInList(page.data) + creators += creatorsEval.filter(creator => creator).length if (page.hasNextPage) { await iterate(page.nextPage) } diff --git a/packages/backend-core/src/users/utils.ts b/packages/backend-core/src/users/utils.ts index 91b667ce17..b9d8dc5823 100644 --- a/packages/backend-core/src/users/utils.ts +++ b/packages/backend-core/src/users/utils.ts @@ -16,30 +16,55 @@ export const hasAdminPermissions = sdk.users.hasAdminPermissions export const hasBuilderPermissions = sdk.users.hasBuilderPermissions export const hasAppBuilderPermissions = sdk.users.hasAppBuilderPermissions -export async function isCreator(user?: User | ContextUser) { +async function getGroups(groupIds: string[]) { + if (groupIds.length) { + const db = context.getGlobalDB() + return await db.getMultiple(groupIds) + } + return [] +} + +export async function creatorsInList( + users: (User | ContextUser)[], + groups?: UserGroup[] +) { + if (!groups) { + const groupIds = users.flatMap(user => user.userGroups || []) + if (groupIds.length) { + groups = await getGroups(groupIds) + } + } + return users.map(user => isCreatorSync(user, groups || [])) +} + +// fetches groups if no provided, but is async and shouldn't be looped with +export async function isCreatorAsync( + user: User | ContextUser, + groups?: UserGroup[] +) { + if (!groups) { + groups = await getGroups(user.userGroups || []) + } + return isCreatorSync(user, groups) +} + +export function isCreatorSync(user: User | ContextUser, groups: UserGroup[]) { const isCreatorByUserDefinition = sdk.users.isCreator(user) if (!isCreatorByUserDefinition && user) { - return await isCreatorByGroupMembership(user) + return isCreatorByGroupMembership(user, groups) } return isCreatorByUserDefinition } -async function isCreatorByGroupMembership(user?: User | ContextUser) { - const userGroups = user?.userGroups || [] +function isCreatorByGroupMembership( + user: User | ContextUser, + groups: UserGroup[] +) { + const userGroups = groups.filter( + group => user.userGroups?.indexOf(group._id!) !== -1 + ) if (userGroups.length > 0) { - const db = context.getGlobalDB() - const groups: UserGroup[] = [] - for (let groupId of userGroups) { - try { - const group = await db.get(groupId) - groups.push(group) - } catch (e: any) { - if (e.error !== "not_found") { - throw e - } - } - } - return groups.some(group => + return userGroups.some(group => Object.values(group.roles || {}).includes(BUILTIN_ROLE_IDS.ADMIN) ) } diff --git a/packages/pro b/packages/pro index e3843dd4ea..a1903c9b8c 160000 --- a/packages/pro +++ b/packages/pro @@ -1 +1 @@ -Subproject commit e3843dd4eaced68ae063355b77df200dbc789c98 +Subproject commit a1903c9b8c5985f72ddb33b40b4a37521aaf8fa0 diff --git a/packages/server/src/middleware/authorized.ts b/packages/server/src/middleware/authorized.ts index 3bead7f80d..0c4943cf5e 100644 --- a/packages/server/src/middleware/authorized.ts +++ b/packages/server/src/middleware/authorized.ts @@ -34,7 +34,7 @@ const checkAuthorized = async ( const isCreatorApi = permType === PermissionType.CREATOR const isBuilderApi = permType === PermissionType.BUILDER const isGlobalBuilder = users.isGlobalBuilder(ctx.user) - const isCreator = await users.isCreator(ctx.user) + const isCreator = await users.isCreatorAsync(ctx.user) const isBuilder = appId ? users.isBuilder(ctx.user, appId) : users.hasBuilderPermissions(ctx.user)