Updating the global user sync to be more accurate and also remove old user metadata from apps that users don't have access to anymore.

packages/server/src/api/controllers/user.ts

@@ -1,98 +1,12 @@
 import { generateUserMetadataID, generateUserFlagID } from "../../db/utils"
 import { InternalTables } from "../../db/utils"
-import { getGlobalUsers, getRawGlobalUser } from "../../utilities/global"
+import { getGlobalUsers } from "../../utilities/global"
 import { getFullUser } from "../../utilities/users"
-import {
+import { context } from "@budibase/backend-core"
-  context,
+import { UserCtx } from "@budibase/types"
-  roles as rolesCore,
-  db as dbCore,
-} from "@budibase/backend-core"
-import { BBContext, Ctx, SyncUserRequest, User } from "@budibase/types"
 import sdk from "../../sdk"
 
-export async function syncUser(ctx: Ctx<SyncUserRequest>) {
+export async function fetchMetadata(ctx: UserCtx) {
-  let deleting = false,
-    user: User | any
-  const userId = ctx.params.id
-
-  const previousUser = ctx.request.body?.previousUser
-
-  try {
-    user = (await getRawGlobalUser(userId)) as User
-  } catch (err: any) {
-    if (err && err.status === 404) {
-      user = {}
-      deleting = true
-    } else {
-      throw err
-    }
-  }
-
-  let previousApps = previousUser
-    ? Object.keys(previousUser.roles).map(appId => appId)
-    : []
-
-  const roles = deleting ? {} : user.roles
-  // remove props which aren't useful to metadata
-  delete user.password
-  delete user.forceResetPassword
-  delete user.roles
-  // run through all production appIDs in the users roles
-  let prodAppIds
-  // if they are a builder then get all production app IDs
-  if ((user.builder && user.builder.global) || deleting) {
-    prodAppIds = await dbCore.getProdAppIDs()
-  } else {
-    prodAppIds = Object.entries(roles)
-      .filter(entry => entry[1] !== rolesCore.BUILTIN_ROLE_IDS.PUBLIC)
-      .map(([appId]) => appId)
-  }
-  for (let prodAppId of new Set([...prodAppIds, ...previousApps])) {
-    const roleId = roles[prodAppId]
-    const deleteFromApp = !roleId
-    const devAppId = dbCore.getDevelopmentAppID(prodAppId)
-    for (let appId of [prodAppId, devAppId]) {
-      if (!(await dbCore.dbExists(appId))) {
-        continue
-      }
-      await context.doInAppContext(appId, async () => {
-        const db = context.getAppDB()
-        const metadataId = generateUserMetadataID(userId)
-        let metadata
-        try {
-          metadata = await db.get(metadataId)
-        } catch (err) {
-          if (deleteFromApp) {
-            return
-          }
-          metadata = {
-            tableId: InternalTables.USER_METADATA,
-          }
-        }
-
-        if (deleteFromApp) {
-          await db.remove(metadata)
-          return
-        }
-
-        // assign the roleId for the metadata doc
-        if (roleId) {
-          metadata.roleId = roleId
-        }
-        let combined = sdk.users.combineMetadataAndUser(user, metadata)
-        // if its null then there was no updates required
-        if (combined) {
-          await db.put(combined)
-        }
-      })
-    }
-  }
-  ctx.body = {
-    message: "User synced.",
-  }
-}
-
-export async function fetchMetadata(ctx: BBContext) {
   const global = await getGlobalUsers()
   const metadata = await sdk.users.rawUserMetadata()
   const users = []
@@ -111,7 +25,7 @@ export async function fetchMetadata(ctx: BBContext) {
   ctx.body = users
 }
 
-export async function updateSelfMetadata(ctx: BBContext) {
+export async function updateSelfMetadata(ctx: UserCtx) {
   // overwrite the ID with current users
   ctx.request.body._id = ctx.user?._id
   // make sure no stale rev
@@ -121,7 +35,7 @@ export async function updateSelfMetadata(ctx: BBContext) {
   await updateMetadata(ctx)
 }
 
-export async function updateMetadata(ctx: BBContext) {
+export async function updateMetadata(ctx: UserCtx) {
   const db = context.getAppDB()
   const user = ctx.request.body
   // this isn't applicable to the user
@@ -133,7 +47,7 @@ export async function updateMetadata(ctx: BBContext) {
   ctx.body = await db.put(metadata)
 }
 
-export async function destroyMetadata(ctx: BBContext) {
+export async function destroyMetadata(ctx: UserCtx) {
   const db = context.getAppDB()
   try {
     const dbUser = await db.get(ctx.params.id)
@@ -146,11 +60,11 @@ export async function destroyMetadata(ctx: BBContext) {
   }
 }
 
-export async function findMetadata(ctx: BBContext) {
+export async function findMetadata(ctx: UserCtx) {
   ctx.body = await getFullUser(ctx, ctx.params.id)
 }
 
-export async function setFlag(ctx: BBContext) {
+export async function setFlag(ctx: UserCtx) {
   const userId = ctx.user?._id
   const { flag, value } = ctx.request.body
   if (!flag) {
@@ -169,7 +83,7 @@ export async function setFlag(ctx: BBContext) {
   ctx.body = { message: "Flag set successfully" }
 }
 
-export async function getFlags(ctx: BBContext) {
+export async function getFlags(ctx: UserCtx) {
   const userId = ctx.user?._id
   const docId = generateUserFlagID(userId!)
   const db = context.getAppDB()

packages/server/src/api/routes/user.ts

@@ -32,11 +32,6 @@ router
     authorized(PermissionType.USER, PermissionLevel.WRITE),
     controller.destroyMetadata
   )
-  .post(
-    "/api/users/metadata/sync/:id",
-    authorized(PermissionType.USER, PermissionLevel.WRITE),
-    controller.syncUser
-  )
   .post(
     "/api/users/flags",
     authorized(PermissionType.USER, PermissionLevel.WRITE),

packages/server/src/sdk/app/applications/sync.ts

@@ -4,15 +4,114 @@ import {
   context,
   docUpdates,
   constants,
+  logging,
+  roles,
 } from "@budibase/backend-core"
+import { User, ContextUser } from "@budibase/types"
 import sdk from "../../"
+import { getGlobalUsers, updateAppRole } from "../../../utilities/global"
+import { generateUserMetadataID, InternalTables } from "../../../db/utils"
+
+type DeletedUser = { _id: string; deleted: boolean }
+
+async function syncUsersToApp(appId: string, users: (User | DeletedUser)[]) {
+  if (!(await dbCore.dbExists(appId))) {
+    return
+  }
+  await context.doInAppContext(appId, async () => {
+    const db = context.getAppDB()
+    for (let user of users) {
+      let ctxUser = user as ContextUser
+      let deletedUser = false
+      const metadataId = generateUserMetadataID(user._id!)
+      if ((user as DeletedUser).deleted) {
+        deletedUser = true
+      }
+
+      // make sure role is correct
+      if (!deletedUser) {
+        ctxUser = updateAppRole(ctxUser, { appId })
+      }
+      let roleId = ctxUser.roleId
+      if (roleId === roles.BUILTIN_ROLE_IDS.PUBLIC) {
+        roleId = undefined
+      }
+
+      let metadata
+      try {
+        metadata = await db.get(metadataId)
+      } catch (err: any) {
+        if (err.status !== 404) {
+          throw err
+        }
+        // no metadata and user is to be deleted, can skip
+        // no role - user isn't in app anyway
+        if (!roleId) {
+          continue
+        } else if (!deletedUser) {
+          // doesn't exist yet, creating it
+          metadata = {
+            tableId: InternalTables.USER_METADATA,
+          }
+        }
+      }
+
+      // the user doesn't exist, or doesn't have a role anymore
+      // get rid of their metadata
+      if (deletedUser || !roleId) {
+        await db.remove(metadata)
+        continue
+      }
+
+      // assign the roleId for the metadata doc
+      if (roleId) {
+        metadata.roleId = roleId
+      }
+
+      let combined = sdk.users.combineMetadataAndUser(ctxUser, metadata)
+      // if no combined returned, there are no updates to make
+      if (combined) {
+        await db.put(combined)
+      }
+    }
+  })
+}
+
+async function syncUsersToAllApps(userIds: string[]) {
+  // list of users, if one has been deleted it will be undefined in array
+  const users = (await getGlobalUsers(userIds)) as User[]
+  const finalUsers: (User | DeletedUser)[] = []
+  for (let userId of userIds) {
+    if (!users.find(user => user._id === userId)) {
+      finalUsers.push({ _id: userId, deleted: true })
+    }
+  }
+  const devAppIds = await dbCore.getDevAppIDs()
+  let promises = []
+  for (let devAppId of devAppIds) {
+    const prodAppId = dbCore.getProdAppID(devAppId)
+    for (let appId of [prodAppId, devAppId]) {
+      promises.push(syncUsersToApp(appId, finalUsers))
+    }
+  }
+  const resp = await Promise.allSettled(promises)
+  const failed = resp.filter(promise => promise.status === "rejected")
+  if (failed.length > 0) {
+    const reasons = failed.map(fail => (fail as PromiseRejectedResult).reason)
+    logging.logAlert("Failed to sync users to apps", reasons)
+  }
+}
 
 export function initUserGroupSync() {
   const types = [constants.DocumentType.USER, constants.DocumentType.GROUP]
   docUpdates.process(types, async update => {
-    console.log("syncing - ", JSON.stringify(update))
+    const docId = update.id
-    // TODO: make the sync smarter
+    const isGroup = docId.startsWith(constants.DocumentType.GROUP)
-    await sdk.users.syncGlobalUsers()
+    if (isGroup) {
+      // TODO: get the group, get users in the group then run the function
+    } else {
+      await syncUsersToAllApps([docId])
+    }
   })
 }
 
@@ -37,32 +136,28 @@ export async function syncApp(
   // specific case, want to make sure setup is skipped
   const prodDb = context.getProdAppDB({ skip_setup: true })
   const exists = await prodDb.exists()
-  if (!exists) {
-    // the database doesn't exist. Don't replicate
-    return {
-      message: "App sync not required, app not deployed.",
-    }
-  }
 
-  const replication = new dbCore.Replication({
-    source: prodAppId,
-    target: appId,
-  })
   let error
-  try {
+  if (exists) {
-    const replOpts = replication.appReplicateOpts()
+    const replication = new dbCore.Replication({
-    if (opts?.automationOnly) {
+      source: prodAppId,
-      replOpts.filter = (doc: any) =>
+      target: appId,
-        doc._id.startsWith(dbCore.DocumentType.AUTOMATION)
+    })
+    try {
+      const replOpts = replication.appReplicateOpts()
+      if (opts?.automationOnly) {
+        replOpts.filter = (doc: any) =>
+          doc._id.startsWith(dbCore.DocumentType.AUTOMATION)
+      }
+      await replication.replicate(replOpts)
+    } catch (err) {
+      error = err
+    } finally {
+      await replication.close()
     }
-    await replication.replicate(replOpts)
-  } catch (err) {
-    error = err
-  } finally {
-    await replication.close()
   }
 
-  // sync the users
+  // sync the users - kept for safe keeping
   await sdk.users.syncGlobalUsers()
 
   if (error) {

packages/server/src/sdk/users/utils.ts

@@ -1,12 +1,13 @@
 import { getGlobalUsers } from "../../utilities/global"
 import { context, roles as rolesCore } from "@budibase/backend-core"
 import {
+  getGlobalIDFromUserMetadataID,
   generateUserMetadataID,
   getUserMetadataParams,
   InternalTables,
 } from "../../db/utils"
 import { isEqual } from "lodash"
-import { ContextUser, UserMetadata } from "@budibase/types"
+import { ContextUser, UserMetadata, User } from "@budibase/types"
 
 export function combineMetadataAndUser(
   user: ContextUser,
@@ -37,6 +38,10 @@ export function combineMetadataAndUser(
   if (found) {
     newDoc._rev = found._rev
   }
+  // clear fields that shouldn't be in metadata
+  delete newDoc.password
+  delete newDoc.forceResetPassword
+  delete newDoc.roles
   if (found == null || !isEqual(newDoc, found)) {
     return {
       ...found,
@@ -60,10 +65,9 @@ export async function rawUserMetadata() {
 export async function syncGlobalUsers() {
   // sync user metadata
   const db = context.getAppDB()
-  const [users, metadata] = await Promise.all([
+  const resp = await Promise.all([getGlobalUsers(), rawUserMetadata()])
-    getGlobalUsers(),
+  const users = resp[0] as User[]
-    rawUserMetadata(),
+  const metadata = resp[1] as UserMetadata[]
-  ])
   const toWrite = []
   for (let user of users) {
     const combined = combineMetadataAndUser(user, metadata)
@@ -71,5 +75,14 @@ export async function syncGlobalUsers() {
       toWrite.push(combined)
     }
   }
+  for (let data of metadata) {
+    if (!data._id) {
+      continue
+    }
+    const globalId = getGlobalIDFromUserMetadataID(data._id)
+    if (!users.find(user => user._id === globalId)) {
+      toWrite.push({ ...data, _deleted: true })
+    }
+  }
   await db.bulkDocs(toWrite)
 }

packages/server/src/utilities/global.ts

@@ -94,16 +94,13 @@ export async function getGlobalUser(userId: string) {
   return processUser(user, { appId })
 }
 
-export async function getGlobalUsers(users?: ContextUser[]) {
+export async function getGlobalUsers(userIds?: string[]) {
   const appId = context.getAppId()
   const db = tenancy.getGlobalDB()
   const allGroups = await groups.fetch()
   let globalUsers
-  if (users) {
+  if (userIds) {
-    const globalIds = users.map(user =>
+    globalUsers = (await db.allDocs(getMultiIDParams(userIds))).rows.map(
-      getGlobalIDFromUserMetadataID(user._id!)
-    )
-    globalUsers = (await db.allDocs(getMultiIDParams(globalIds))).rows.map(
       row => row.doc
     )
   } else {
@@ -134,7 +131,7 @@ export async function getGlobalUsers(users?: ContextUser[]) {
 }
 
 export async function getGlobalUsersFromMetadata(users: ContextUser[]) {
-  const globalUsers = await getGlobalUsers(users)
+  const globalUsers = await getGlobalUsers(users.map(user => user._id!))
   return users.map(user => {
     const globalUser = globalUsers.find(
       globalUser => globalUser && user._id?.includes(globalUser._id)