Adding group and user tests for user sync.

packages/backend-core/src/db/views.ts

@@ -42,7 +42,11 @@ async function removeDeprecated(db: Database, viewName: ViewName) {
   }
 }
 
-export async function createView(db: any, viewJs: string, viewName: string) {
+export async function createView(
+  db: any,
+  viewJs: string,
+  viewName: string
+): Promise<void> {
   let designDoc
   try {
     designDoc = (await db.get(DESIGN_DB)) as DesignDocument
@@ -57,7 +61,15 @@ export async function createView(db: any, viewJs: string, viewName: string) {
     ...designDoc.views,
     [viewName]: view,
   }
-  await db.put(designDoc)
+  try {
+    await db.put(designDoc)
+  } catch (err: any) {
+    if (err.status === 409) {
+      return await createView(db, viewJs, viewName)
+    } else {
+      throw err
+    }
+  }
 }
 
 export const createNewUserEmailView = async () => {
@@ -135,6 +147,10 @@ export const queryView = async <T>(
       await removeDeprecated(db, viewName)
       await createFunc()
       return queryView(viewName, params, db, createFunc, opts)
+    } else if (err.status === 409) {
+      // can happen when multiple queries occur at once, view couldn't be created
+      // other design docs being updated, re-run
+      return queryView(viewName, params, db, createFunc, opts)
     } else {
       throw err
     }

packages/server/src/sdk/app/applications/sync.ts

@@ -7,15 +7,19 @@ import {
   logging,
   roles,
 } from "@budibase/backend-core"
-import { User, ContextUser } from "@budibase/types"
+import { User, ContextUser, UserGroup } from "@budibase/types"
 import { sdk as proSdk } from "@budibase/pro"
 import sdk from "../../"
-import { getGlobalUsers, updateAppRole } from "../../../utilities/global"
+import { getGlobalUsers, processUser } from "../../../utilities/global"
 import { generateUserMetadataID, InternalTables } from "../../../db/utils"
 
 type DeletedUser = { _id: string; deleted: boolean }
 
-async function syncUsersToApp(appId: string, users: (User | DeletedUser)[]) {
+async function syncUsersToApp(
+  appId: string,
+  users: (User | DeletedUser)[],
+  groups: UserGroup[]
+) {
   if (!(await dbCore.dbExists(appId))) {
     return
   }
@@ -31,7 +35,7 @@ async function syncUsersToApp(appId: string, users: (User | DeletedUser)[]) {
 
       // make sure role is correct
       if (!deletedUser) {
-        ctxUser = updateAppRole(ctxUser, { appId })
+        ctxUser = await processUser(ctxUser, { appId, groups })
       }
       let roleId = ctxUser.roleId
       if (roleId === roles.BUILTIN_ROLE_IDS.PUBLIC) {
@@ -80,7 +84,10 @@ async function syncUsersToApp(appId: string, users: (User | DeletedUser)[]) {
 
 async function syncUsersToAllApps(userIds: string[]) {
   // list of users, if one has been deleted it will be undefined in array
-  const users = (await getGlobalUsers(userIds)) as User[]
+  const users = (await getGlobalUsers(userIds, {
+    noProcessing: true,
+  })) as User[]
+  const groups = await proSdk.groups.fetch()
   const finalUsers: (User | DeletedUser)[] = []
   for (let userId of userIds) {
     const user = users.find(user => user._id === userId)
@@ -95,7 +102,7 @@ async function syncUsersToAllApps(userIds: string[]) {
   for (let devAppId of devAppIds) {
     const prodAppId = dbCore.getProdAppID(devAppId)
     for (let appId of [prodAppId, devAppId]) {
-      promises.push(syncUsersToApp(appId, finalUsers))
+      promises.push(syncUsersToApp(appId, finalUsers, groups))
     }
   }
   const resp = await Promise.allSettled(promises)
@@ -106,24 +113,32 @@ async function syncUsersToAllApps(userIds: string[]) {
   }
 }
 
-export function initUserGroupSync(updateCb?: () => void) {
+export function initUserGroupSync(updateCb?: (docId: string) => void) {
   const types = [constants.DocumentType.USER, constants.DocumentType.GROUP]
   docUpdates.process(types, async update => {
-    const docId = update.id
+    try {
-    const isGroup = docId.startsWith(constants.DocumentType.GROUP)
+      const docId = update.id
-    let userIds: string[]
+      const isGroup = docId.startsWith(constants.DocumentType.GROUP)
-    if (isGroup) {
+      let userIds: string[]
-      const group = await proSdk.groups.get(docId)
+      if (isGroup) {
-      userIds = group.users?.map(user => user._id) || []
+        const group = await proSdk.groups.get(docId)
-    } else {
+        userIds = group.users?.map(user => user._id) || []
-      userIds = [docId]
+      } else {
-    }
+        userIds = [docId]
-    if (userIds.length > 0) {
+      }
-      await syncUsersToAllApps(userIds)
+      if (userIds.length > 0) {
-    }
+        await syncUsersToAllApps(userIds)
-    // used to tracking when updates have occurred
+      }
-    if (updateCb) {
+      if (updateCb) {
-      updateCb()
+        updateCb(docId)
+      }
+    } catch (err: any) {
+      // if something not found - no changes to perform
+      if (err?.status === 404) {
+        return
+      } else {
+        logging.logAlert("Failed to perform user/group app sync", err)
+      }
     }
   })
 }

packages/server/src/sdk/app/applications/tests/sync.spec.ts

@@ -1,26 +1,32 @@
 import TestConfiguration from "../../../../tests/utilities/TestConfiguration"
-import { events, context, roles, db as dbCore } from "@budibase/backend-core"
+import { events, context, roles, constants } from "@budibase/backend-core"
 import { initUserGroupSync } from "../sync"
 import { rawUserMetadata } from "../../../users/utils"
 import EventEmitter from "events"
-import { UserMetadata, UserRoles } from "@budibase/types"
+import { UserGroup, UserMetadata, UserRoles, User } from "@budibase/types"
 
 const config = new TestConfiguration()
-let app
+let app, group: UserGroup, groupUser: User
 const ROLE_ID = roles.BUILTIN_ROLE_IDS.BASIC
 
 const emitter = new EventEmitter()
 
-function updateCb() {
+function updateCb(docId: string) {
-  emitter.emit("update")
+  const isGroup = docId.startsWith(constants.DocumentType.GROUP)
+  if (isGroup) {
+    emitter.emit("update-group")
+  } else {
+    emitter.emit("update-user")
+  }
 }
 
-function waitForUpdate() {
+function waitForUpdate(opts: { group?: boolean }) {
   return new Promise<void>((resolve, reject) => {
     const timeout = setTimeout(() => {
       reject()
     }, 5000)
-    emitter.on("update", () => {
+    const event = opts?.group ? "update-group" : "update-user"
+    emitter.on(event, () => {
       clearTimeout(timeout)
       resolve()
     })
@@ -32,38 +38,89 @@ beforeAll(async () => {
   initUserGroupSync(updateCb)
 })
 
-async function createUser(email: string, roles: UserRoles, appId?: string) {
+async function createUser(email: string, roles: UserRoles) {
   const user = await config.createUser({ email, roles })
-  await context.doInContext(appId || config.appId!, async () => {
+  await context.doInContext(config.appId!, async () => {
     await events.user.created(user)
   })
+  return user
+}
+
+async function removeUserRole(user: User) {
+  const final = await config.globalUser({
+    ...user,
+    id: user._id,
+    roles: {},
+    builder: false,
+    admin: false,
+  })
+  await context.doInContext(config.appId!, async () => {
+    await events.user.updated(final)
+  })
 }
 
-async function getUserMetadata(appId?: string): Promise<UserMetadata[]> {
+async function createGroupAndUser(email: string) {
-  return context.doInContext(appId || config.appId!, async () => {
+  groupUser = await config.createUser({
+    email,
+    roles: {},
+    builder: false,
+    admin: false,
+  })
+  group = await config.createGroup()
+  await config.addUserToGroup(group._id!, groupUser._id!)
+}
+
+async function removeUserFromGroup() {
+  await config.removeUserFromGroup(group._id!, groupUser._id!)
+  return context.doInContext(config.appId!, async () => {
+    await events.user.updated(groupUser)
+  })
+}
+
+async function getUserMetadata(): Promise<UserMetadata[]> {
+  return context.doInContext(config.appId!, async () => {
     return await rawUserMetadata()
   })
 }
 
-function buildRoles(appId?: string) {
+function buildRoles() {
-  const prodAppId = dbCore.getProdAppID(appId || config.appId!)
+  return { [config.prodAppId!]: ROLE_ID }
-  return { [prodAppId]: ROLE_ID }
 }
 
 describe("app user/group sync", () => {
-  it("should be able to sync a new user", async () => {
+  const groupEmail = "test2@test.com",
-    const email = "test@test.com"
+    normalEmail = "test@test.com"
-    await createUser(email, buildRoles())
+  async function checkEmail(
-    await waitForUpdate()
+    email: string,
+    opts?: { group?: boolean; notFound?: boolean }
+  ) {
+    await waitForUpdate(opts || {})
     const metadata = await getUserMetadata()
-    expect(metadata.find(data => data.email === email)).toBeDefined()
+    const found = metadata.find(data => data.email === email)
+    if (opts?.notFound) {
+      expect(found).toBeUndefined()
+    } else {
+      expect(found).toBeDefined()
+    }
+  }
+
+  it("should be able to sync a new user, add then remove", async () => {
+    const user = await createUser(normalEmail, buildRoles())
+    await checkEmail(normalEmail)
+    await removeUserRole(user)
+    await checkEmail(normalEmail, { notFound: true })
   })
 
   it("should be able to sync a group", async () => {
-
+    await createGroupAndUser(groupEmail)
+    await checkEmail(groupEmail, { group: true })
   })
 
-  it("should be able to remove user", async () => {
+  it("should be able to remove user from group", async () => {
-
+    if (!group) {
+      await createGroupAndUser(groupEmail)
+    }
+    await removeUserFromGroup()
+    await checkEmail(groupEmail, { notFound: true })
   })
 })

packages/server/src/tests/utilities/TestConfiguration.ts

@@ -49,6 +49,7 @@ import {
   SearchFilters,
   UserRoles,
 } from "@budibase/types"
+import { BUILTIN_ROLE_IDS } from "@budibase/backend-core/src/security/roles"
 
 type DefaultUserValues = {
   globalUserId: string
@@ -306,6 +307,33 @@ class TestConfiguration {
     }
   }
 
+  async createGroup(roleId: string = BUILTIN_ROLE_IDS.BASIC) {
+    return context.doInTenant(this.tenantId!, async () => {
+      const baseGroup = structures.userGroups.userGroup()
+      baseGroup.roles = {
+        [this.prodAppId]: roleId,
+      }
+      const { id, rev } = await pro.sdk.groups.save(baseGroup)
+      return {
+        _id: id,
+        _rev: rev,
+        ...baseGroup,
+      }
+    })
+  }
+
+  async addUserToGroup(groupId: string, userId: string) {
+    return context.doInTenant(this.tenantId!, async () => {
+      await pro.sdk.groups.addUsers(groupId, [userId])
+    })
+  }
+
+  async removeUserFromGroup(groupId: string, userId: string) {
+    return context.doInTenant(this.tenantId!, async () => {
+      await pro.sdk.groups.removeUsers(groupId, [userId])
+    })
+  }
+
   async login({ roleId, userId, builder, prodApp = false }: any = {}) {
     const appId = prodApp ? this.prodAppId : this.appId
     return context.doInAppContext(appId, async () => {

packages/server/src/utilities/global.ts

@@ -9,6 +9,7 @@ import {
 import env from "../environment"
 import { groups } from "@budibase/pro"
 import { UserCtx, ContextUser, User, UserGroup } from "@budibase/types"
+import { global } from "yargs"
 
 export function updateAppRole(
   user: ContextUser,
@@ -16,7 +17,7 @@ export function updateAppRole(
 ) {
   appId = appId || context.getAppId()
 
-  if (!user || !user.roles) {
+  if (!user || (!user.roles && !user.userGroups)) {
     return user
   }
   // if in an multi-tenancy environment make sure roles are never updated
@@ -27,7 +28,7 @@ export function updateAppRole(
     return user
   }
   // always use the deployed app
-  if (appId) {
+  if (appId && user.roles) {
     user.roleId = user.roles[dbCore.getProdAppID(appId)]
   }
   // if a role wasn't found then either set as admin (builder) or public (everyone else)
@@ -60,7 +61,7 @@ async function checkGroupRoles(
   return user
 }
 
-async function processUser(
+export async function processUser(
   user: ContextUser,
   opts: { appId?: string; groups?: UserGroup[] } = {}
 ) {
@@ -94,10 +95,12 @@ export async function getGlobalUser(userId: string) {
   return processUser(user, { appId })
 }
 
-export async function getGlobalUsers(userIds?: string[]) {
+export async function getGlobalUsers(
+  userIds?: string[],
+  opts?: { noProcessing?: boolean }
+) {
   const appId = context.getAppId()
   const db = tenancy.getGlobalDB()
-  const allGroups = await groups.fetch()
   let globalUsers
   if (userIds) {
     globalUsers = (await db.allDocs(getMultiIDParams(userIds))).rows.map(
@@ -123,11 +126,16 @@ export async function getGlobalUsers(userIds?: string[]) {
     return globalUsers
   }
 
-  // pass in the groups, meaning we don't actually need to retrieve them for
+  if (opts?.noProcessing) {
-  // each user individually
+    return globalUsers
-  return Promise.all(
+  } else {
-    globalUsers.map(user => processUser(user, { groups: allGroups }))
+    // pass in the groups, meaning we don't actually need to retrieve them for
-  )
+    // each user individually
+    const allGroups = await groups.fetch()
+    return Promise.all(
+      globalUsers.map(user => processUser(user, { groups: allGroups }))
+    )
+  }
 }
 
 export async function getGlobalUsersFromMetadata(users: ContextUser[]) {