From e699f4684a7423f1809a221be6192c434c624d6d Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Tue, 4 Apr 2023 18:03:56 +0100 Subject: [PATCH] Updating the global user sync to be more accurate and also remove old user metadata from apps that users don't have access to anymore. --- packages/server/src/api/controllers/user.ts | 106 ++----------- packages/server/src/api/routes/user.ts | 5 - .../server/src/sdk/app/applications/sync.ts | 143 +++++++++++++++--- packages/server/src/sdk/users/utils.ts | 23 ++- packages/server/src/utilities/global.ts | 11 +- 5 files changed, 151 insertions(+), 137 deletions(-) diff --git a/packages/server/src/api/controllers/user.ts b/packages/server/src/api/controllers/user.ts index 1ae1a68824..b66f11bc1c 100644 --- a/packages/server/src/api/controllers/user.ts +++ b/packages/server/src/api/controllers/user.ts @@ -1,98 +1,12 @@ import { generateUserMetadataID, generateUserFlagID } from "../../db/utils" import { InternalTables } from "../../db/utils" -import { getGlobalUsers, getRawGlobalUser } from "../../utilities/global" +import { getGlobalUsers } from "../../utilities/global" import { getFullUser } from "../../utilities/users" -import { - context, - roles as rolesCore, - db as dbCore, -} from "@budibase/backend-core" -import { BBContext, Ctx, SyncUserRequest, User } from "@budibase/types" +import { context } from "@budibase/backend-core" +import { UserCtx } from "@budibase/types" import sdk from "../../sdk" -export async function syncUser(ctx: Ctx) { - let deleting = false, - user: User | any - const userId = ctx.params.id - - const previousUser = ctx.request.body?.previousUser - - try { - user = (await getRawGlobalUser(userId)) as User - } catch (err: any) { - if (err && err.status === 404) { - user = {} - deleting = true - } else { - throw err - } - } - - let previousApps = previousUser - ? Object.keys(previousUser.roles).map(appId => appId) - : [] - - const roles = deleting ? {} : user.roles - // remove props which aren't useful to metadata - delete user.password - delete user.forceResetPassword - delete user.roles - // run through all production appIDs in the users roles - let prodAppIds - // if they are a builder then get all production app IDs - if ((user.builder && user.builder.global) || deleting) { - prodAppIds = await dbCore.getProdAppIDs() - } else { - prodAppIds = Object.entries(roles) - .filter(entry => entry[1] !== rolesCore.BUILTIN_ROLE_IDS.PUBLIC) - .map(([appId]) => appId) - } - for (let prodAppId of new Set([...prodAppIds, ...previousApps])) { - const roleId = roles[prodAppId] - const deleteFromApp = !roleId - const devAppId = dbCore.getDevelopmentAppID(prodAppId) - for (let appId of [prodAppId, devAppId]) { - if (!(await dbCore.dbExists(appId))) { - continue - } - await context.doInAppContext(appId, async () => { - const db = context.getAppDB() - const metadataId = generateUserMetadataID(userId) - let metadata - try { - metadata = await db.get(metadataId) - } catch (err) { - if (deleteFromApp) { - return - } - metadata = { - tableId: InternalTables.USER_METADATA, - } - } - - if (deleteFromApp) { - await db.remove(metadata) - return - } - - // assign the roleId for the metadata doc - if (roleId) { - metadata.roleId = roleId - } - let combined = sdk.users.combineMetadataAndUser(user, metadata) - // if its null then there was no updates required - if (combined) { - await db.put(combined) - } - }) - } - } - ctx.body = { - message: "User synced.", - } -} - -export async function fetchMetadata(ctx: BBContext) { +export async function fetchMetadata(ctx: UserCtx) { const global = await getGlobalUsers() const metadata = await sdk.users.rawUserMetadata() const users = [] @@ -111,7 +25,7 @@ export async function fetchMetadata(ctx: BBContext) { ctx.body = users } -export async function updateSelfMetadata(ctx: BBContext) { +export async function updateSelfMetadata(ctx: UserCtx) { // overwrite the ID with current users ctx.request.body._id = ctx.user?._id // make sure no stale rev @@ -121,7 +35,7 @@ export async function updateSelfMetadata(ctx: BBContext) { await updateMetadata(ctx) } -export async function updateMetadata(ctx: BBContext) { +export async function updateMetadata(ctx: UserCtx) { const db = context.getAppDB() const user = ctx.request.body // this isn't applicable to the user @@ -133,7 +47,7 @@ export async function updateMetadata(ctx: BBContext) { ctx.body = await db.put(metadata) } -export async function destroyMetadata(ctx: BBContext) { +export async function destroyMetadata(ctx: UserCtx) { const db = context.getAppDB() try { const dbUser = await db.get(ctx.params.id) @@ -146,11 +60,11 @@ export async function destroyMetadata(ctx: BBContext) { } } -export async function findMetadata(ctx: BBContext) { +export async function findMetadata(ctx: UserCtx) { ctx.body = await getFullUser(ctx, ctx.params.id) } -export async function setFlag(ctx: BBContext) { +export async function setFlag(ctx: UserCtx) { const userId = ctx.user?._id const { flag, value } = ctx.request.body if (!flag) { @@ -169,7 +83,7 @@ export async function setFlag(ctx: BBContext) { ctx.body = { message: "Flag set successfully" } } -export async function getFlags(ctx: BBContext) { +export async function getFlags(ctx: UserCtx) { const userId = ctx.user?._id const docId = generateUserFlagID(userId!) const db = context.getAppDB() diff --git a/packages/server/src/api/routes/user.ts b/packages/server/src/api/routes/user.ts index 14deb111e6..24f33140a6 100644 --- a/packages/server/src/api/routes/user.ts +++ b/packages/server/src/api/routes/user.ts @@ -32,11 +32,6 @@ router authorized(PermissionType.USER, PermissionLevel.WRITE), controller.destroyMetadata ) - .post( - "/api/users/metadata/sync/:id", - authorized(PermissionType.USER, PermissionLevel.WRITE), - controller.syncUser - ) .post( "/api/users/flags", authorized(PermissionType.USER, PermissionLevel.WRITE), diff --git a/packages/server/src/sdk/app/applications/sync.ts b/packages/server/src/sdk/app/applications/sync.ts index 682fd7b519..3572e047e6 100644 --- a/packages/server/src/sdk/app/applications/sync.ts +++ b/packages/server/src/sdk/app/applications/sync.ts @@ -4,15 +4,114 @@ import { context, docUpdates, constants, + logging, + roles, } from "@budibase/backend-core" +import { User, ContextUser } from "@budibase/types" import sdk from "../../" +import { getGlobalUsers, updateAppRole } from "../../../utilities/global" +import { generateUserMetadataID, InternalTables } from "../../../db/utils" + +type DeletedUser = { _id: string; deleted: boolean } + +async function syncUsersToApp(appId: string, users: (User | DeletedUser)[]) { + if (!(await dbCore.dbExists(appId))) { + return + } + await context.doInAppContext(appId, async () => { + const db = context.getAppDB() + for (let user of users) { + let ctxUser = user as ContextUser + let deletedUser = false + const metadataId = generateUserMetadataID(user._id!) + if ((user as DeletedUser).deleted) { + deletedUser = true + } + + // make sure role is correct + if (!deletedUser) { + ctxUser = updateAppRole(ctxUser, { appId }) + } + let roleId = ctxUser.roleId + if (roleId === roles.BUILTIN_ROLE_IDS.PUBLIC) { + roleId = undefined + } + + let metadata + try { + metadata = await db.get(metadataId) + } catch (err: any) { + if (err.status !== 404) { + throw err + } + // no metadata and user is to be deleted, can skip + // no role - user isn't in app anyway + if (!roleId) { + continue + } else if (!deletedUser) { + // doesn't exist yet, creating it + metadata = { + tableId: InternalTables.USER_METADATA, + } + } + } + + // the user doesn't exist, or doesn't have a role anymore + // get rid of their metadata + if (deletedUser || !roleId) { + await db.remove(metadata) + continue + } + + // assign the roleId for the metadata doc + if (roleId) { + metadata.roleId = roleId + } + + let combined = sdk.users.combineMetadataAndUser(ctxUser, metadata) + // if no combined returned, there are no updates to make + if (combined) { + await db.put(combined) + } + } + }) +} + +async function syncUsersToAllApps(userIds: string[]) { + // list of users, if one has been deleted it will be undefined in array + const users = (await getGlobalUsers(userIds)) as User[] + const finalUsers: (User | DeletedUser)[] = [] + for (let userId of userIds) { + if (!users.find(user => user._id === userId)) { + finalUsers.push({ _id: userId, deleted: true }) + } + } + const devAppIds = await dbCore.getDevAppIDs() + let promises = [] + for (let devAppId of devAppIds) { + const prodAppId = dbCore.getProdAppID(devAppId) + for (let appId of [prodAppId, devAppId]) { + promises.push(syncUsersToApp(appId, finalUsers)) + } + } + const resp = await Promise.allSettled(promises) + const failed = resp.filter(promise => promise.status === "rejected") + if (failed.length > 0) { + const reasons = failed.map(fail => (fail as PromiseRejectedResult).reason) + logging.logAlert("Failed to sync users to apps", reasons) + } +} export function initUserGroupSync() { const types = [constants.DocumentType.USER, constants.DocumentType.GROUP] docUpdates.process(types, async update => { - console.log("syncing - ", JSON.stringify(update)) - // TODO: make the sync smarter - await sdk.users.syncGlobalUsers() + const docId = update.id + const isGroup = docId.startsWith(constants.DocumentType.GROUP) + if (isGroup) { + // TODO: get the group, get users in the group then run the function + } else { + await syncUsersToAllApps([docId]) + } }) } @@ -37,32 +136,28 @@ export async function syncApp( // specific case, want to make sure setup is skipped const prodDb = context.getProdAppDB({ skip_setup: true }) const exists = await prodDb.exists() - if (!exists) { - // the database doesn't exist. Don't replicate - return { - message: "App sync not required, app not deployed.", - } - } - const replication = new dbCore.Replication({ - source: prodAppId, - target: appId, - }) let error - try { - const replOpts = replication.appReplicateOpts() - if (opts?.automationOnly) { - replOpts.filter = (doc: any) => - doc._id.startsWith(dbCore.DocumentType.AUTOMATION) + if (exists) { + const replication = new dbCore.Replication({ + source: prodAppId, + target: appId, + }) + try { + const replOpts = replication.appReplicateOpts() + if (opts?.automationOnly) { + replOpts.filter = (doc: any) => + doc._id.startsWith(dbCore.DocumentType.AUTOMATION) + } + await replication.replicate(replOpts) + } catch (err) { + error = err + } finally { + await replication.close() } - await replication.replicate(replOpts) - } catch (err) { - error = err - } finally { - await replication.close() } - // sync the users + // sync the users - kept for safe keeping await sdk.users.syncGlobalUsers() if (error) { diff --git a/packages/server/src/sdk/users/utils.ts b/packages/server/src/sdk/users/utils.ts index 9b9ea04c56..a3982fd45b 100644 --- a/packages/server/src/sdk/users/utils.ts +++ b/packages/server/src/sdk/users/utils.ts @@ -1,12 +1,13 @@ import { getGlobalUsers } from "../../utilities/global" import { context, roles as rolesCore } from "@budibase/backend-core" import { + getGlobalIDFromUserMetadataID, generateUserMetadataID, getUserMetadataParams, InternalTables, } from "../../db/utils" import { isEqual } from "lodash" -import { ContextUser, UserMetadata } from "@budibase/types" +import { ContextUser, UserMetadata, User } from "@budibase/types" export function combineMetadataAndUser( user: ContextUser, @@ -37,6 +38,10 @@ export function combineMetadataAndUser( if (found) { newDoc._rev = found._rev } + // clear fields that shouldn't be in metadata + delete newDoc.password + delete newDoc.forceResetPassword + delete newDoc.roles if (found == null || !isEqual(newDoc, found)) { return { ...found, @@ -60,10 +65,9 @@ export async function rawUserMetadata() { export async function syncGlobalUsers() { // sync user metadata const db = context.getAppDB() - const [users, metadata] = await Promise.all([ - getGlobalUsers(), - rawUserMetadata(), - ]) + const resp = await Promise.all([getGlobalUsers(), rawUserMetadata()]) + const users = resp[0] as User[] + const metadata = resp[1] as UserMetadata[] const toWrite = [] for (let user of users) { const combined = combineMetadataAndUser(user, metadata) @@ -71,5 +75,14 @@ export async function syncGlobalUsers() { toWrite.push(combined) } } + for (let data of metadata) { + if (!data._id) { + continue + } + const globalId = getGlobalIDFromUserMetadataID(data._id) + if (!users.find(user => user._id === globalId)) { + toWrite.push({ ...data, _deleted: true }) + } + } await db.bulkDocs(toWrite) } diff --git a/packages/server/src/utilities/global.ts b/packages/server/src/utilities/global.ts index a75fcc0b30..0debb68d54 100644 --- a/packages/server/src/utilities/global.ts +++ b/packages/server/src/utilities/global.ts @@ -94,16 +94,13 @@ export async function getGlobalUser(userId: string) { return processUser(user, { appId }) } -export async function getGlobalUsers(users?: ContextUser[]) { +export async function getGlobalUsers(userIds?: string[]) { const appId = context.getAppId() const db = tenancy.getGlobalDB() const allGroups = await groups.fetch() let globalUsers - if (users) { - const globalIds = users.map(user => - getGlobalIDFromUserMetadataID(user._id!) - ) - globalUsers = (await db.allDocs(getMultiIDParams(globalIds))).rows.map( + if (userIds) { + globalUsers = (await db.allDocs(getMultiIDParams(userIds))).rows.map( row => row.doc ) } else { @@ -134,7 +131,7 @@ export async function getGlobalUsers(users?: ContextUser[]) { } export async function getGlobalUsersFromMetadata(users: ContextUser[]) { - const globalUsers = await getGlobalUsers(users) + const globalUsers = await getGlobalUsers(users.map(user => user._id!)) return users.map(user => { const globalUser = globalUsers.find( globalUser => globalUser && user._id?.includes(globalUser._id)