MirrorSave
/
budibase
mirror of https://github.com/Budibase/budibase.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #10287 from Budibase/fix/BUDI-6754 

Improving user sync into app user metadata tables
This commit is contained in:
Michael Drury 2023-04-17 17:46:04 +01:00 committed by GitHub
parent acab893fc0 b4e6cbce7a
commit 6bf70725d4
37 changed files with 667 additions and 313 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
packages/backend-core/src/db/views.ts
View File

@ -42,7 +42,11 @@ async function removeDeprecated(db: Database, viewName: ViewName) {
}
}
export async function createView(db: any, viewJs: string, viewName: string) {
export async function createView(
db: any,
viewJs: string,
viewName: string
): Promise<void> {
let designDoc
try {
designDoc = (await db.get(DESIGN_DB)) as DesignDocument
@ -57,7 +61,15 @@ export async function createView(db: any, viewJs: string, viewName: string) {
...designDoc.views,
[viewName]: view,
}
await db.put(designDoc)
try {
await db.put(designDoc)
} catch (err: any) {
if (err.status === 409) {
return await createView(db, viewJs, viewName)
} else {
throw err
}
}
}
export const createNewUserEmailView = async () => {
@ -135,6 +147,10 @@ export const queryView = async <T>(
await removeDeprecated(db, viewName)
await createFunc()
return queryView(viewName, params, db, createFunc, opts)
} else if (err.status === 409) {
// can happen when multiple queries occur at once, view couldn't be created
// other design docs being updated, re-run
return queryView(viewName, params, db, createFunc, opts)
} else {
throw err
}

29
packages/backend-core/src/docUpdates/index.ts Normal file
View File

@ -0,0 +1,29 @@
import { asyncEventQueue, init as initQueue } from "../events/asyncEvents"
import {
ProcessorMap,
default as DocumentUpdateProcessor,
} from "../events/processors/async/DocumentUpdateProcessor"
let processingPromise: Promise<void>
let documentProcessor: DocumentUpdateProcessor
export function init(processors: ProcessorMap) {
if (!asyncEventQueue) {
initQueue()
}
if (!documentProcessor) {
documentProcessor = new DocumentUpdateProcessor(processors)
}
// if not processing in this instance, kick it off
if (!processingPromise) {
processingPromise = asyncEventQueue.process(async job => {
const { event, identity, properties, timestamp } = job.data
await documentProcessor.processEvent(
event,
identity,
properties,
timestamp
)
})
}
}

2
packages/backend-core/src/events/asyncEvents/index.ts Normal file
View File

@ -0,0 +1,2 @@
export * from "./queue"
export * from "./publisher"

12
packages/backend-core/src/events/asyncEvents/publisher.ts Normal file
View File

@ -0,0 +1,12 @@
import { AsyncEvents } from "@budibase/types"
import { EventPayload, asyncEventQueue, init } from "./queue"
export async function publishAsyncEvent(payload: EventPayload) {
if (!asyncEventQueue) {
init()
}
const { event, identity } = payload
if (AsyncEvents.indexOf(event) !== -1 && identity.tenantId) {
await asyncEventQueue.add(payload)
}
}

22
packages/backend-core/src/events/asyncEvents/queue.ts Normal file
View File

@ -0,0 +1,22 @@
import BullQueue from "bull"
import { createQueue, JobQueue } from "../../queue"
import { Event, Identity } from "@budibase/types"
export interface EventPayload {
event: Event
identity: Identity
properties: any
timestamp?: string | number
}
export let asyncEventQueue: BullQueue.Queue
export function init() {
asyncEventQueue = createQueue<EventPayload>(JobQueue.SYSTEM_EVENT_QUEUE)
}
export async function shutdown() {
if (asyncEventQueue) {
await asyncEventQueue.close()
}
}

56
packages/backend-core/src/events/documentId.ts Normal file
View File

@ -0,0 +1,56 @@
import {
Event,
UserCreatedEvent,
UserUpdatedEvent,
UserDeletedEvent,
UserPermissionAssignedEvent,
UserPermissionRemovedEvent,
GroupCreatedEvent,
GroupUpdatedEvent,
GroupDeletedEvent,
GroupUsersAddedEvent,
GroupUsersDeletedEvent,
GroupPermissionsEditedEvent,
} from "@budibase/types"
const getEventProperties: Record<
string,
(properties: any) => string | undefined
> = {
[Event.USER_CREATED]: (properties: UserCreatedEvent) => properties.userId,
[Event.USER_UPDATED]: (properties: UserUpdatedEvent) => properties.userId,
[Event.USER_DELETED]: (properties: UserDeletedEvent) => properties.userId,
[Event.USER_PERMISSION_ADMIN_ASSIGNED]: (
properties: UserPermissionAssignedEvent
) => properties.userId,
[Event.USER_PERMISSION_ADMIN_REMOVED]: (
properties: UserPermissionRemovedEvent
) => properties.userId,
[Event.USER_PERMISSION_BUILDER_ASSIGNED]: (
properties: UserPermissionAssignedEvent
) => properties.userId,
[Event.USER_PERMISSION_BUILDER_REMOVED]: (
properties: UserPermissionRemovedEvent
) => properties.userId,
[Event.USER_GROUP_CREATED]: (properties: GroupCreatedEvent) =>
properties.groupId,
[Event.USER_GROUP_UPDATED]: (properties: GroupUpdatedEvent) =>
properties.groupId,
[Event.USER_GROUP_DELETED]: (properties: GroupDeletedEvent) =>
properties.groupId,
[Event.USER_GROUP_USERS_ADDED]: (properties: GroupUsersAddedEvent) =>
properties.groupId,
[Event.USER_GROUP_USERS_REMOVED]: (properties: GroupUsersDeletedEvent) =>
properties.groupId,
[Event.USER_GROUP_PERMISSIONS_EDITED]: (
properties: GroupPermissionsEditedEvent
) => properties.groupId,
}
export function getDocumentId(event: Event, properties: any) {
const extractor = getEventProperties[event]
if (!extractor) {
throw new Error("Event does not have a method of document ID extraction")
}
return extractor(properties)
}

11
packages/backend-core/src/events/events.ts
View File

@ -1,7 +1,8 @@
import { Event, AuditedEventFriendlyName } from "@budibase/types"
import { Event } from "@budibase/types"
import { processors } from "./processors"
import identification from "./identification"
import * as backfill from "./backfill"
import { publishAsyncEvent } from "./asyncEvents"
export const publishEvent = async (
event: Event,
@ -14,6 +15,14 @@ export const publishEvent = async (
const backfilling = await backfill.isBackfillingEvent(event)
// no backfill - send the event and exit
if (!backfilling) {
// send off async events if required
await publishAsyncEvent({
event,
identity,
properties,
timestamp,
})
// now handle the main sync event processing pipeline
await processors.processEvent(event, identity, properties, timestamp)
return
}

1
packages/backend-core/src/events/identification.ts
View File

@ -65,6 +65,7 @@ const getCurrentIdentity = async (): Promise<Identity> => {
hosting,
installationId,
tenantId,
realTenantId: context.getTenantId(),
environment,
}
} else if (identityType === IdentityType.USER) {

2
packages/backend-core/src/events/index.ts
View File

@ -6,6 +6,8 @@ export * as backfillCache from "./backfill"
import { processors } from "./processors"
export function initAsyncEvents() {}
export const shutdown = () => {
processors.shutdown()
console.log("Events shutdown")

12
packages/backend-core/src/events/processors/Processors.ts
View File

@ -25,7 +25,9 @@ export default class Processor implements EventProcessor {
timestamp?: string | number
): Promise<void> {
for (const eventProcessor of this.processors) {
await eventProcessor.identify(identity, timestamp)
if (eventProcessor.identify) {
await eventProcessor.identify(identity, timestamp)
}
}
}
@ -34,13 +36,17 @@ export default class Processor implements EventProcessor {
timestamp?: string | number
): Promise<void> {
for (const eventProcessor of this.processors) {
await eventProcessor.identifyGroup(identity, timestamp)
if (eventProcessor.identifyGroup) {
await eventProcessor.identifyGroup(identity, timestamp)
}
}
}
shutdown() {
for (const eventProcessor of this.processors) {
eventProcessor.shutdown()
if (eventProcessor.shutdown) {
eventProcessor.shutdown()
}
}
}
}

43
packages/backend-core/src/events/processors/async/DocumentUpdateProcessor.ts Normal file
View File

@ -0,0 +1,43 @@
import { EventProcessor } from "../types"
import { Event, Identity, DocUpdateEvent } from "@budibase/types"
import { doInTenant } from "../../../context"
import { getDocumentId } from "../../documentId"
import { shutdown } from "../../asyncEvents"
export type Processor = (update: DocUpdateEvent) => Promise<void>
export type ProcessorMap = { events: Event[]; processor: Processor }[]
export default class DocumentUpdateProcessor implements EventProcessor {
processors: ProcessorMap = []
constructor(processors: ProcessorMap) {
this.processors = processors
}
async processEvent(
event: Event,
identity: Identity,
properties: any,
timestamp?: string | number
) {
const tenantId = identity.realTenantId
const docId = getDocumentId(event, properties)
if (!tenantId || !docId) {
return
}
for (let { events, processor } of this.processors) {
if (events.includes(event)) {
await doInTenant(tenantId, async () => {
await processor({
id: docId,
tenantId,
})
})
}
}
}
shutdown() {
return shutdown()
}
}

19
packages/backend-core/src/events/processors/types.ts
View File

@ -1,18 +1 @@
import { Event, Identity, Group } from "@budibase/types"
export enum EventProcessorType {
POSTHOG = "posthog",
LOGGING = "logging",
}
export interface EventProcessor {
processEvent(
event: Event,
identity: Identity,
properties: any,
timestamp?: string | number
): Promise<void>
identify(identity: Identity, timestamp?: string | number): Promise<void>
identifyGroup(group: Group, timestamp?: string | number): Promise<void>
shutdown(): void
}
export { EventProcessor } from "@budibase/types"

1
packages/backend-core/src/index.ts
View File

@ -27,6 +27,7 @@ export * as errors from "./errors"
export * as timers from "./timers"
export { default as env } from "./environment"
export * as blacklist from "./blacklist"
export * as docUpdates from "./docUpdates"
export { SearchParams } from "./db"
// Add context to tenancy for backwards compatibility
// only do this for external usages to prevent internal

1
packages/backend-core/src/queue/constants.ts
View File

@ -2,4 +2,5 @@ export enum JobQueue {
AUTOMATION = "automationQueue",
APP_BACKUP = "appBackupQueue",
AUDIT_LOG = "auditLogQueue",
SYSTEM_EVENT_QUEUE = "systemEventQueue",
}

19
packages/server/src/api/controllers/row/internal.ts
View File

@ -30,7 +30,6 @@ import { finaliseRow, updateRelatedFormula } from "./staticFormula"
import { csv, json, jsonWithSchema, Format } from "../view/exporters"
import { apiFileReturn } from "../../../utilities/fileSystem"
import {
Ctx,
UserCtx,
Database,
LinkDocumentValue,
@ -72,7 +71,7 @@ async function getView(db: Database, viewName: string) {
return viewInfo
}
async function getRawTableData(ctx: Ctx, db: Database, tableId: string) {
async function getRawTableData(ctx: UserCtx, db: Database, tableId: string) {
let rows
if (tableId === InternalTables.USER_METADATA) {
await userController.fetchMetadata(ctx)
@ -188,7 +187,7 @@ export async function save(ctx: UserCtx) {
})
}
export async function fetchView(ctx: Ctx) {
export async function fetchView(ctx: UserCtx) {
const viewName = decodeURIComponent(ctx.params.viewName)
// if this is a table view being looked for just transfer to that
@ -255,7 +254,7 @@ export async function fetchView(ctx: Ctx) {
return rows
}
export async function fetch(ctx: Ctx) {
export async function fetch(ctx: UserCtx) {
const db = context.getAppDB()
const tableId = ctx.params.tableId
@ -264,7 +263,7 @@ export async function fetch(ctx: Ctx) {
return outputProcessing(table, rows)
}
export async function find(ctx: Ctx) {
export async function find(ctx: UserCtx) {
const db = dbCore.getDB(ctx.appId)
const table = await db.get(ctx.params.tableId)
let row = await utils.findRow(ctx, ctx.params.tableId, ctx.params.rowId)
@ -272,7 +271,7 @@ export async function find(ctx: Ctx) {
return row
}
export async function destroy(ctx: Ctx) {
export async function destroy(ctx: UserCtx) {
const db = context.getAppDB()
const { _id } = ctx.request.body
let row = await db.get(_id)
@ -308,7 +307,7 @@ export async function destroy(ctx: Ctx) {
return { response, row }
}
export async function bulkDestroy(ctx: Ctx) {
export async function bulkDestroy(ctx: UserCtx) {
const db = context.getAppDB()
const tableId = ctx.params.tableId
const table = await db.get(tableId)
@ -347,7 +346,7 @@ export async function bulkDestroy(ctx: Ctx) {
return { response: { ok: true }, rows: processedRows }
}
export async function search(ctx: Ctx) {
export async function search(ctx: UserCtx) {
// Fetch the whole table when running in cypress, as search doesn't work
if (!env.COUCH_DB_URL && env.isCypress()) {
return { rows: await fetch(ctx) }
@ -387,7 +386,7 @@ export async function search(ctx: Ctx) {
return response
}
export async function exportRows(ctx: Ctx) {
export async function exportRows(ctx: UserCtx) {
const db = context.getAppDB()
const table = await db.get(ctx.params.tableId)
const rowIds = ctx.request.body.rows
@ -439,7 +438,7 @@ export async function exportRows(ctx: Ctx) {
}
}
export async function fetchEnrichedRow(ctx: Ctx) {
export async function fetchEnrichedRow(ctx: UserCtx) {
const db = context.getAppDB()
const tableId = ctx.params.tableId
const rowId = ctx.params.rowId

4
packages/server/src/api/controllers/row/utils.ts
View File

@ -5,7 +5,7 @@ import { context } from "@budibase/backend-core"
import { makeExternalQuery } from "../../../integrations/base/query"
import { Row, Table } from "@budibase/types"
import { Format } from "../view/exporters"
import { Ctx } from "@budibase/types"
import { UserCtx } from "@budibase/types"
import sdk from "../../../sdk"
const validateJs = require("validate.js")
const { cloneDeep } = require("lodash/fp")
@ -26,7 +26,7 @@ export async function getDatasourceAndQuery(json: any) {
return makeExternalQuery(datasource, json)
}
export async function findRow(ctx: Ctx, tableId: string, rowId: string) {
export async function findRow(ctx: UserCtx, tableId: string, rowId: string) {
const db = context.getAppDB()
let row
// TODO remove special user case in future

106
packages/server/src/api/controllers/user.ts
View File

@ -1,98 +1,12 @@
import { generateUserMetadataID, generateUserFlagID } from "../../db/utils"
import { InternalTables } from "../../db/utils"
import { getGlobalUsers, getRawGlobalUser } from "../../utilities/global"
import { getGlobalUsers } from "../../utilities/global"
import { getFullUser } from "../../utilities/users"
import {
context,
roles as rolesCore,
db as dbCore,
} from "@budibase/backend-core"
import { BBContext, Ctx, SyncUserRequest, User } from "@budibase/types"
import { context } from "@budibase/backend-core"
import { UserCtx } from "@budibase/types"
import sdk from "../../sdk"
export async function syncUser(ctx: Ctx<SyncUserRequest>) {
let deleting = false,
user: User | any
const userId = ctx.params.id
const previousUser = ctx.request.body?.previousUser
try {
user = (await getRawGlobalUser(userId)) as User
} catch (err: any) {
if (err && err.status === 404) {
user = {}
deleting = true
} else {
throw err
}
}
let previousApps = previousUser
? Object.keys(previousUser.roles).map(appId => appId)
: []
const roles = deleting ? {} : user.roles
// remove props which aren't useful to metadata
delete user.password
delete user.forceResetPassword
delete user.roles
// run through all production appIDs in the users roles
let prodAppIds
// if they are a builder then get all production app IDs
if ((user.builder && user.builder.global) || deleting) {
prodAppIds = await dbCore.getProdAppIDs()
} else {
prodAppIds = Object.entries(roles)
.filter(entry => entry[1] !== rolesCore.BUILTIN_ROLE_IDS.PUBLIC)
.map(([appId]) => appId)
}
for (let prodAppId of new Set([...prodAppIds, ...previousApps])) {
const roleId = roles[prodAppId]
const deleteFromApp = !roleId
const devAppId = dbCore.getDevelopmentAppID(prodAppId)
for (let appId of [prodAppId, devAppId]) {
if (!(await dbCore.dbExists(appId))) {
continue
}
await context.doInAppContext(appId, async () => {
const db = context.getAppDB()
const metadataId = generateUserMetadataID(userId)
let metadata
try {
metadata = await db.get(metadataId)
} catch (err) {
if (deleteFromApp) {
return
}
metadata = {
tableId: InternalTables.USER_METADATA,
}
}
if (deleteFromApp) {
await db.remove(metadata)
return
}
// assign the roleId for the metadata doc
if (roleId) {
metadata.roleId = roleId
}
let combined = sdk.users.combineMetadataAndUser(user, metadata)
// if its null then there was no updates required
if (combined) {
await db.put(combined)
}
})
}
}
ctx.body = {
message: "User synced.",
}
}
export async function fetchMetadata(ctx: BBContext) {
export async function fetchMetadata(ctx: UserCtx) {
const global = await getGlobalUsers()
const metadata = await sdk.users.rawUserMetadata()
const users = []
@ -111,7 +25,7 @@ export async function fetchMetadata(ctx: BBContext) {
ctx.body = users
}
export async function updateSelfMetadata(ctx: BBContext) {
export async function updateSelfMetadata(ctx: UserCtx) {
// overwrite the ID with current users
ctx.request.body._id = ctx.user?._id
// make sure no stale rev
@ -121,7 +35,7 @@ export async function updateSelfMetadata(ctx: BBContext) {
await updateMetadata(ctx)
}
export async function updateMetadata(ctx: BBContext) {
export async function updateMetadata(ctx: UserCtx) {
const db = context.getAppDB()
const user = ctx.request.body
// this isn't applicable to the user
@ -133,7 +47,7 @@ export async function updateMetadata(ctx: BBContext) {
ctx.body = await db.put(metadata)
}
export async function destroyMetadata(ctx: BBContext) {
export async function destroyMetadata(ctx: UserCtx) {
const db = context.getAppDB()
try {
const dbUser = await db.get(ctx.params.id)
@ -146,11 +60,11 @@ export async function destroyMetadata(ctx: BBContext) {
}
}
export async function findMetadata(ctx: BBContext) {
export async function findMetadata(ctx: UserCtx) {
ctx.body = await getFullUser(ctx, ctx.params.id)
}
export async function setFlag(ctx: BBContext) {
export async function setFlag(ctx: UserCtx) {
const userId = ctx.user?._id
const { flag, value } = ctx.request.body
if (!flag) {
@ -169,7 +83,7 @@ export async function setFlag(ctx: BBContext) {
ctx.body = { message: "Flag set successfully" }
}
export async function getFlags(ctx: BBContext) {
export async function getFlags(ctx: UserCtx) {
const userId = ctx.user?._id
const docId = generateUserFlagID(userId!)
const db = context.getAppDB()

37
packages/server/src/api/routes/tests/user.spec.js
View File

@ -205,41 +205,4 @@ describe("/users", () => {
expect(res.body.message).toEqual("Flag set successfully")
})
})
describe("syncUser", () => {
it("should sync the user", async () => {
let user = await config.createUser()
await config.createApp("New App")
let res = await request
.post(`/api/users/metadata/sync/${user._id}`)
.set(config.defaultHeaders())
.expect(200)
.expect("Content-Type", /json/)
expect(res.body.message).toEqual("User synced.")
})
it("should sync the user when a previous user is specified", async () => {
const app1 = await config.createApp("App 1")
const app2 = await config.createApp("App 2")
let user = await config.createUser({
builder: false,
admin: true,
roles: { [app1.appId]: "ADMIN" },
})
let res = await request
.post(`/api/users/metadata/sync/${user._id}`)
.set(config.defaultHeaders())
.send({
previousUser: {
...user,
roles: { ...user.roles, [app2.appId]: "BASIC" },
},
})
.expect(200)
.expect("Content-Type", /json/)
expect(res.body.message).toEqual("User synced.")
})
})
})

5
packages/server/src/api/routes/user.ts
View File

@ -32,11 +32,6 @@ router
authorized(PermissionType.USER, PermissionLevel.WRITE),
controller.destroyMetadata
)
.post(
"/api/users/metadata/sync/:id",
authorized(PermissionType.USER, PermissionLevel.WRITE),
controller.syncUser
)
.post(
"/api/users/flags",
authorized(PermissionType.USER, PermissionLevel.WRITE),

1
packages/server/src/events/docUpdates/index.ts Normal file
View File

@ -0,0 +1 @@
export * from "./processors"

14
packages/server/src/events/docUpdates/processors.ts Normal file
View File

@ -0,0 +1,14 @@
import userGroupProcessor from "./syncUsers"
import { docUpdates } from "@budibase/backend-core"
export type UpdateCallback = (docId: string) => void
let started = false
export function init(updateCb?: UpdateCallback) {
if (started) {
return
}
const processors = [userGroupProcessor(updateCb)]
docUpdates.init(processors)
started = true
}

35
packages/server/src/events/docUpdates/syncUsers.ts Normal file
View File

@ -0,0 +1,35 @@
import { constants, logging } from "@budibase/backend-core"
import { sdk as proSdk } from "@budibase/pro"
import { DocUpdateEvent, UserGroupSyncEvents } from "@budibase/types"
import { syncUsersToAllApps } from "../../sdk/app/applications/sync"
import { UpdateCallback } from "./processors"
export default function process(updateCb?: UpdateCallback) {
const processor = async (update: DocUpdateEvent) => {
try {
const docId = update.id
const isGroup = docId.startsWith(constants.DocumentType.GROUP)
let userIds: string[]
if (isGroup) {
const group = await proSdk.groups.get(docId)
userIds = group.users?.map(user => user._id) || []
} else {
userIds = [docId]
}
if (userIds.length > 0) {
await syncUsersToAllApps(userIds)
}
if (updateCb) {
updateCb(docId)
}
} catch (err: any) {
// if something not found - no changes to perform
if (err?.status === 404) {
return
} else {
logging.logAlert("Failed to perform user/group app sync", err)
}
}
}
return { events: UserGroupSyncEvents, processor }
}

1
packages/server/src/events/index.ts
View File

@ -2,4 +2,5 @@ import BudibaseEmitter from "./BudibaseEmitter"
const emitter = new BudibaseEmitter()
export { init } from "./docUpdates"
export default emitter

151
packages/server/src/sdk/app/applications/sync.ts
View File

@ -1,6 +1,117 @@
import env from "../../../environment"
import { db as dbCore, context } from "@budibase/backend-core"
import {
db as dbCore,
context,
docUpdates,
constants,
logging,
roles,
} from "@budibase/backend-core"
import { User, ContextUser, UserGroup } from "@budibase/types"
import { sdk as proSdk } from "@budibase/pro"
import sdk from "../../"
import { getGlobalUsers, processUser } from "../../../utilities/global"
import { generateUserMetadataID, InternalTables } from "../../../db/utils"
type DeletedUser = { _id: string; deleted: boolean }
async function syncUsersToApp(
appId: string,
users: (User | DeletedUser)[],
groups: UserGroup[]
) {
if (!(await dbCore.dbExists(appId))) {
return
}
await context.doInAppContext(appId, async () => {
const db = context.getAppDB()
for (let user of users) {
let ctxUser = user as ContextUser
let deletedUser = false
const metadataId = generateUserMetadataID(user._id!)
if ((user as DeletedUser).deleted) {
deletedUser = true
}
// make sure role is correct
if (!deletedUser) {
ctxUser = await processUser(ctxUser, { appId, groups })
}
let roleId = ctxUser.roleId
if (roleId === roles.BUILTIN_ROLE_IDS.PUBLIC) {
roleId = undefined
}
let metadata
try {
metadata = await db.get(metadataId)
} catch (err: any) {
if (err.status !== 404) {
throw err
}
// no metadata and user is to be deleted, can skip
// no role - user isn't in app anyway
if (!roleId) {
continue
} else if (!deletedUser) {
// doesn't exist yet, creating it
metadata = {
tableId: InternalTables.USER_METADATA,
}
}
}
// the user doesn't exist, or doesn't have a role anymore
// get rid of their metadata
if (deletedUser || !roleId) {
await db.remove(metadata)
continue
}
// assign the roleId for the metadata doc
if (roleId) {
metadata.roleId = roleId
}
let combined = sdk.users.combineMetadataAndUser(ctxUser, metadata)
// if no combined returned, there are no updates to make
if (combined) {
await db.put(combined)
}
}
})
}
export async function syncUsersToAllApps(userIds: string[]) {
// list of users, if one has been deleted it will be undefined in array
const users = (await getGlobalUsers(userIds, {
noProcessing: true,
})) as User[]
const groups = await proSdk.groups.fetch()
const finalUsers: (User | DeletedUser)[] = []
for (let userId of userIds) {
const user = users.find(user => user._id === userId)
if (!user) {
finalUsers.push({ _id: userId, deleted: true })
} else {
finalUsers.push(user)
}
}
const devAppIds = await dbCore.getDevAppIDs()
let promises = []
for (let devAppId of devAppIds) {
const prodAppId = dbCore.getProdAppID(devAppId)
for (let appId of [prodAppId, devAppId]) {
promises.push(syncUsersToApp(appId, finalUsers, groups))
}
}
const resp = await Promise.allSettled(promises)
const failed = resp.filter(promise => promise.status === "rejected")
if (failed.length > 0) {
const reasons = failed.map(fail => (fail as PromiseRejectedResult).reason)
logging.logAlert("Failed to sync users to apps", reasons)
}
}
export async function syncApp(
appId: string,
@ -23,32 +134,28 @@ export async function syncApp(
// specific case, want to make sure setup is skipped
const prodDb = context.getProdAppDB({ skip_setup: true })
const exists = await prodDb.exists()
if (!exists) {
// the database doesn't exist. Don't replicate
return {
message: "App sync not required, app not deployed.",
}
}
const replication = new dbCore.Replication({
source: prodAppId,
target: appId,
})
let error
try {
const replOpts = replication.appReplicateOpts()
if (opts?.automationOnly) {
replOpts.filter = (doc: any) =>
doc._id.startsWith(dbCore.DocumentType.AUTOMATION)
if (exists) {
const replication = new dbCore.Replication({
source: prodAppId,
target: appId,
})
try {
const replOpts = replication.appReplicateOpts()
if (opts?.automationOnly) {
replOpts.filter = (doc: any) =>
doc._id.startsWith(dbCore.DocumentType.AUTOMATION)
}
await replication.replicate(replOpts)
} catch (err) {
error = err
} finally {
await replication.close()
}
await replication.replicate(replOpts)
} catch (err) {
error = err
} finally {
await replication.close()
}
// sync the users
// sync the users - kept for safe keeping
await sdk.users.syncGlobalUsers()
if (error) {

137
packages/server/src/sdk/app/applications/tests/sync.spec.ts Normal file
View File

@ -0,0 +1,137 @@
import TestConfiguration from "../../../../tests/utilities/TestConfiguration"
import { events, context, roles, constants } from "@budibase/backend-core"
import { init } from "../../../../events"
import { rawUserMetadata } from "../../../users/utils"
import EventEmitter from "events"
import { UserGroup, UserMetadata, UserRoles, User } from "@budibase/types"
const config = new TestConfiguration()
let app, group: UserGroup, groupUser: User
const ROLE_ID = roles.BUILTIN_ROLE_IDS.BASIC
const emitter = new EventEmitter()
function updateCb(docId: string) {
const isGroup = docId.startsWith(constants.DocumentType.GROUP)
if (isGroup) {
emitter.emit("update-group")
} else {
emitter.emit("update-user")
}
}
init(updateCb)
function waitForUpdate(opts: { group?: boolean }) {
return new Promise<void>((resolve, reject) => {
const timeout = setTimeout(() => {
reject()
}, 5000)
const event = opts?.group ? "update-group" : "update-user"
emitter.on(event, () => {
clearTimeout(timeout)
resolve()
})
})
}
beforeAll(async () => {
app = await config.init("syncApp")
})
async function createUser(email: string, roles: UserRoles, builder?: boolean) {
const user = await config.createUser({
email,
roles,
builder: builder || false,
admin: false,
})
await context.doInContext(config.appId!, async () => {
await events.user.created(user)
})
return user
}
async function removeUserRole(user: User) {
const final = await config.globalUser({
...user,
id: user._id,
roles: {},
builder: false,
admin: false,
})
await context.doInContext(config.appId!, async () => {
await events.user.updated(final)
})
}
async function createGroupAndUser(email: string) {
groupUser = await config.createUser({
email,
roles: {},
builder: false,
admin: false,
})
group = await config.createGroup()
await config.addUserToGroup(group._id!, groupUser._id!)
}
async function removeUserFromGroup() {
await config.removeUserFromGroup(group._id!, groupUser._id!)
return context.doInContext(config.appId!, async () => {
await events.user.updated(groupUser)
})
}
async function getUserMetadata(): Promise<UserMetadata[]> {
return context.doInContext(config.appId!, async () => {
return await rawUserMetadata()
})
}
function buildRoles() {
return { [config.prodAppId!]: ROLE_ID }
}
describe("app user/group sync", () => {
const groupEmail = "test2@test.com",
normalEmail = "test@test.com"
async function checkEmail(
email: string,
opts?: { group?: boolean; notFound?: boolean }
) {
await waitForUpdate(opts || {})
const metadata = await getUserMetadata()
const found = metadata.find(data => data.email === email)
if (opts?.notFound) {
expect(found).toBeUndefined()
} else {
expect(found).toBeDefined()
}
}
it("should be able to sync a new user, add then remove", async () => {
const user = await createUser(normalEmail, buildRoles())
await checkEmail(normalEmail)
await removeUserRole(user)
await checkEmail(normalEmail, { notFound: true })
})
it("should be able to sync a group", async () => {
await createGroupAndUser(groupEmail)
await checkEmail(groupEmail, { group: true })
})
it("should be able to remove user from group", async () => {
if (!group) {
await createGroupAndUser(groupEmail)
}
await removeUserFromGroup()
await checkEmail(groupEmail, { notFound: true })
})
it("should be able to handle builder users", async () => {
await createUser("test3@test.com", {}, true)
await checkEmail("test3@test.com")
})
})

33
packages/server/src/sdk/users/tests/utils.spec.ts
View File

@ -121,38 +121,7 @@ describe("syncGlobalUsers", () => {
await syncGlobalUsers()
const metadata = await rawUserMetadata()
expect(metadata).toHaveLength(1)
})
})
})
it("app users are removed when app is removed from user group", async () => {
await config.doInTenant(async () => {
const group = await proSdk.groups.save(structures.userGroups.userGroup())
const user1 = await config.createUser({ admin: false, builder: false })
const user2 = await config.createUser({ admin: false, builder: false })
await proSdk.groups.updateGroupApps(group.id, {
appsToAdd: [
{ appId: config.prodAppId!, roleId: roles.BUILTIN_ROLE_IDS.BASIC },
],
})
await proSdk.groups.addUsers(group.id, [user1._id, user2._id])
await config.doInContext(config.appId, async () => {
await syncGlobalUsers()
expect(await rawUserMetadata()).toHaveLength(3)
await proSdk.groups.removeUsers(group.id, [user1._id])
await syncGlobalUsers()
const metadata = await rawUserMetadata()
expect(metadata).toHaveLength(2)
expect(metadata).not.toContainEqual(
expect.objectContaining({
_id: db.generateUserMetadataID(user1._id),
})
)
expect(metadata).toHaveLength(0)
})
})
})

28
packages/server/src/sdk/users/utils.ts
View File

@ -1,12 +1,13 @@
import { getGlobalUsers } from "../../utilities/global"
import { context, roles as rolesCore } from "@budibase/backend-core"
import {
getGlobalIDFromUserMetadataID,
generateUserMetadataID,
getUserMetadataParams,
InternalTables,
} from "../../db/utils"
import { isEqual } from "lodash"
import { ContextUser, UserMetadata } from "@budibase/types"
import { ContextUser, UserMetadata, User } from "@budibase/types"
export function combineMetadataAndUser(
user: ContextUser,
@ -37,6 +38,10 @@ export function combineMetadataAndUser(
if (found) {
newDoc._rev = found._rev
}
// clear fields that shouldn't be in metadata
delete newDoc.password
delete newDoc.forceResetPassword
delete newDoc.roles
if (found == null || !isEqual(newDoc, found)) {
return {
...found,
@ -60,10 +65,9 @@ export async function rawUserMetadata() {
export async function syncGlobalUsers() {
// sync user metadata
const db = context.getAppDB()
const [users, metadata] = await Promise.all([
getGlobalUsers(),
rawUserMetadata(),
])
const resp = await Promise.all([getGlobalUsers(), rawUserMetadata()])
const users = resp[0] as User[]
const metadata = resp[1] as UserMetadata[]
const toWrite = []
for (let user of users) {
const combined = combineMetadataAndUser(user, metadata)
@ -71,5 +75,19 @@ export async function syncGlobalUsers() {
toWrite.push(combined)
}
}
let foundEmails: string[] = []
for (let data of metadata) {
if (!data._id) {
continue
}
const alreadyExisting = data.email && foundEmails.indexOf(data.email) !== -1
const globalId = getGlobalIDFromUserMetadataID(data._id)
if (!users.find(user => user._id === globalId) || alreadyExisting) {
toWrite.push({ ...data, _deleted: true })
}
if (data.email) {
foundEmails.push(data.email)
}
}
await db.bulkDocs(toWrite)
}

3
packages/server/src/startup.ts
View File

@ -10,7 +10,7 @@ import fs from "fs"
import { watch } from "./watch"
import * as automations from "./automations"
import * as fileSystem from "./utilities/fileSystem"
import eventEmitter from "./events"
import { default as eventEmitter, init as eventInit } from "./events"
import * as migrations from "./migrations"
import * as bullboard from "./automations/bullboard"
import * as pro from "@budibase/pro"
@ -63,6 +63,7 @@ export async function startup(app?: any, server?: any) {
eventEmitter.emitPort(env.PORT)
fileSystem.init()
await redis.init()
eventInit()
// run migrations on startup if not done via http
// not recommended in a clustered environment

28
packages/server/src/tests/utilities/TestConfiguration.ts
View File

@ -49,6 +49,7 @@ import {
SearchFilters,
UserRoles,
} from "@budibase/types"
import { BUILTIN_ROLE_IDS } from "@budibase/backend-core/src/security/roles"
type DefaultUserValues = {
globalUserId: string
@ -306,6 +307,33 @@ class TestConfiguration {
}
}
async createGroup(roleId: string = BUILTIN_ROLE_IDS.BASIC) {
return context.doInTenant(this.tenantId!, async () => {
const baseGroup = structures.userGroups.userGroup()
baseGroup.roles = {
[this.prodAppId]: roleId,
}
const { id, rev } = await pro.sdk.groups.save(baseGroup)
return {
_id: id,
_rev: rev,
...baseGroup,
}
})
}
async addUserToGroup(groupId: string, userId: string) {
return context.doInTenant(this.tenantId!, async () => {
await pro.sdk.groups.addUsers(groupId, [userId])
})
}
async removeUserFromGroup(groupId: string, userId: string) {
return context.doInTenant(this.tenantId!, async () => {
await pro.sdk.groups.removeUsers(groupId, [userId])
})
}
async login({ roleId, userId, builder, prodApp = false }: any = {}) {
const appId = prodApp ? this.prodAppId : this.appId
return context.doInAppContext(appId, async () => {

37
packages/server/src/utilities/global.ts
View File

@ -9,6 +9,7 @@ import {
import env from "../environment"
import { groups } from "@budibase/pro"
import { UserCtx, ContextUser, User, UserGroup } from "@budibase/types"
import { global } from "yargs"
export function updateAppRole(
user: ContextUser,
@ -16,7 +17,7 @@ export function updateAppRole(
) {
appId = appId || context.getAppId()
if (!user || !user.roles) {
if (!user || (!user.roles && !user.userGroups)) {
return user
}
// if in an multi-tenancy environment make sure roles are never updated
@ -27,7 +28,7 @@ export function updateAppRole(
return user
}
// always use the deployed app
if (appId) {
if (appId && user.roles) {
user.roleId = user.roles[dbCore.getProdAppID(appId)]
}
// if a role wasn't found then either set as admin (builder) or public (everyone else)
@ -60,7 +61,7 @@ async function checkGroupRoles(
return user
}
async function processUser(
export async function processUser(
user: ContextUser,
opts: { appId?: string; groups?: UserGroup[] } = {}
) {
@ -94,16 +95,15 @@ export async function getGlobalUser(userId: string) {
return processUser(user, { appId })
}
export async function getGlobalUsers(users?: ContextUser[]) {
export async function getGlobalUsers(
userIds?: string[],
opts?: { noProcessing?: boolean }
) {
const appId = context.getAppId()
const db = tenancy.getGlobalDB()
const allGroups = await groups.fetch()
let globalUsers
if (users) {
const globalIds = users.map(user =>
getGlobalIDFromUserMetadataID(user._id!)
)
globalUsers = (await db.allDocs(getMultiIDParams(globalIds))).rows.map(
if (userIds) {
globalUsers = (await db.allDocs(getMultiIDParams(userIds))).rows.map(
row => row.doc
)
} else {
@ -126,15 +126,20 @@ export async function getGlobalUsers(users?: ContextUser[]) {
return globalUsers
}
// pass in the groups, meaning we don't actually need to retrieve them for
// each user individually
return Promise.all(
globalUsers.map(user => processUser(user, { groups: allGroups }))
)
if (opts?.noProcessing) {
return globalUsers
} else {
// pass in the groups, meaning we don't actually need to retrieve them for
// each user individually
const allGroups = await groups.fetch()
return Promise.all(
globalUsers.map(user => processUser(user, { groups: allGroups }))
)
}
}
export async function getGlobalUsersFromMetadata(users: ContextUser[]) {
const globalUsers = await getGlobalUsers(users)
const globalUsers = await getGlobalUsers(users.map(user => user._id!))
return users.map(user => {
const globalUser = globalUsers.find(
globalUser => globalUser && user._id?.includes(globalUser._id)

1
packages/types/src/documents/app/user.ts
View File

@ -2,4 +2,5 @@ import { Document } from "../document"
export interface UserMetadata extends Document {
roleId: string
email?: string
}

37
packages/types/src/sdk/events/event.ts
View File

@ -1,4 +1,5 @@
import { Hosting } from "../hosting"
import { Group, Identity } from "./identification"
export enum Event {
// USER
@ -186,6 +187,24 @@ export enum Event {
AUDIT_LOGS_DOWNLOADED = "audit_log:downloaded",
}
export const UserGroupSyncEvents: Event[] = [
Event.USER_CREATED,
Event.USER_UPDATED,
Event.USER_DELETED,
Event.USER_PERMISSION_ADMIN_ASSIGNED,
Event.USER_PERMISSION_ADMIN_REMOVED,
Event.USER_PERMISSION_BUILDER_ASSIGNED,
Event.USER_PERMISSION_BUILDER_REMOVED,
Event.USER_GROUP_CREATED,
Event.USER_GROUP_UPDATED,
Event.USER_GROUP_DELETED,
Event.USER_GROUP_USERS_ADDED,
Event.USER_GROUP_USERS_REMOVED,
Event.USER_GROUP_PERMISSIONS_EDITED,
]
export const AsyncEvents: Event[] = [...UserGroupSyncEvents]
// all events that are not audited have been added to this record as undefined, this means
// that Typescript can protect us against new events being added and auditing of those
// events not being considered. This might be a little ugly, but provides a level of
@ -383,3 +402,21 @@ export interface BaseEvent {
}
export type TableExportFormat = "json" | "csv"
export type DocUpdateEvent = {
id: string
tenantId: string
appId?: string
}
export interface EventProcessor {
processEvent(
event: Event,
identity: Identity,
properties: any,
timestamp?: string | number
): Promise<void>
identify?(identity: Identity, timestamp?: string | number): Promise<void>
identifyGroup?(group: Group, timestamp?: string | number): Promise<void>
shutdown?(): void
}

2
packages/types/src/sdk/events/identification.ts
View File

@ -46,6 +46,8 @@ export interface Identity {
environment: string
installationId?: string
tenantId?: string
// usable - no unique format
realTenantId?: string
hostInfo?: HostInfo
}

10
packages/worker/src/sdk/users/users.ts
View File

@ -1,5 +1,4 @@
import env from "../../environment"
import * as apps from "../../utilities/appService"
import * as eventHelpers from "./events"
import {
accounts,
@ -30,7 +29,6 @@ import {
PlatformUser,
PlatformUserByEmail,
RowResponse,
SearchUsersRequest,
User,
SaveUserOpts,
Account,
@ -280,9 +278,6 @@ export const save = async (
await platform.users.addUser(tenantId, builtUser._id!, builtUser.email)
await cache.user.invalidateUser(response.id)
// let server know to sync user
await apps.syncUserInApps(_id, dbUser)
await Promise.all(groupPromises)
// finally returned the saved user from the db
@ -432,7 +427,6 @@ export const bulkCreate = async (
// instead of relying on looping tenant creation
await platform.users.addUser(tenantId, user._id, user.email)
await eventHelpers.handleSaveEvents(user, undefined)
await apps.syncUserInApps(user._id)
}
const saved = usersToBulkSave.map(user => {
@ -571,8 +565,6 @@ export const destroy = async (id: string) => {
await eventHelpers.handleDeleteEvents(dbUser)
await cache.user.invalidateUser(userId)
await sessions.invalidateSessions(userId, { reason: "deletion" })
// let server know to sync user
await apps.syncUserInApps(userId, dbUser)
}
const bulkDeleteProcessing = async (dbUser: User) => {
@ -581,8 +573,6 @@ const bulkDeleteProcessing = async (dbUser: User) => {
await eventHelpers.handleDeleteEvents(dbUser)
await cache.user.invalidateUser(userId)
await sessions.invalidateSessions(userId, { reason: "bulk-deletion" })
// let server know to sync user
await apps.syncUserInApps(userId, dbUser)
}
export const invite = async (

46
packages/worker/src/utilities/appService.ts
View File

@ -1,46 +0,0 @@
import fetch from "node-fetch"
import {
constants,
tenancy,
logging,
env as coreEnv,
} from "@budibase/backend-core"
import { checkSlashesInUrl } from "../utilities"
import env from "../environment"
import { SyncUserRequest, User } from "@budibase/types"
async function makeAppRequest(url: string, method: string, body: any) {
if (env.isTest()) {
return
}
const request: any = { headers: {} }
request.headers[constants.Header.API_KEY] = coreEnv.INTERNAL_API_KEY
if (tenancy.isTenantIdSet()) {
request.headers[constants.Header.TENANT_ID] = tenancy.getTenantId()
}
if (body) {
request.headers["Content-Type"] = "application/json"
request.body = JSON.stringify(body)
}
request.method = method
// add x-budibase-correlation-id header
logging.correlation.setHeader(request.headers)
return fetch(checkSlashesInUrl(env.APPS_URL + url), request)
}
export async function syncUserInApps(userId: string, previousUser?: User) {
const body: SyncUserRequest = {
previousUser,
}
const response = await makeAppRequest(
`/api/users/metadata/sync/${userId}`,
"POST",
body
)
if (response && response.status !== 200) {
throw "Unable to sync user."
}
}

2
qa-core/src/internal-api/tests/applications/publish.spec.ts
View File

@ -36,7 +36,7 @@ describe("Internal API - Application creation, update, publish and delete", () =
const [syncResponse, sync] = await config.api.apps.sync(app.appId!)
expect(sync).toEqual({
message: "App sync not required, app not deployed.",
message: "App sync completed successfully.",
})
})

12
yarn.lock
View File

@ -1486,15 +1486,15 @@
pouchdb-promise "^6.0.4"
through2 "^2.0.0"
"@budibase/pro@2.5.5-alpha.4":
version "2.5.5-alpha.4"
resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-2.5.5-alpha.4.tgz#949a8c050300bbbd6d6729b1826c81d7dca1045d"
integrity sha512-7ndB99Mr74lXGbIw2g6NjCXszRsxgoS4Ean3RM7joXu90Gb+aSL/uxCDZbHzDbXlysOCN62CcIt3i4RdXE42eA==
"@budibase/pro@2.5.6-alpha.1":
version "2.5.6-alpha.1"
resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-2.5.6-alpha.1.tgz#a471d8b6b9c5f7e648d6a78c20bea942583c373f"
integrity sha512-L4TtyGUH5LTTHC+Zsk9ozh/ZNQZ1hTcDAJ6ijOW6MJTZSRnOZN9X2u3KAegvmbmNT49adsCr3RcOy+EmDLD/9A==
dependencies:
"@budibase/backend-core" "2.5.5-alpha.4"
"@budibase/backend-core" "2.5.6-alpha.1"
"@budibase/shared-core" "2.4.44-alpha.1"
"@budibase/string-templates" "2.4.44-alpha.1"
"@budibase/types" "2.5.5-alpha.4"
"@budibase/types" "2.5.6-alpha.1"
"@koa/router" "8.0.8"
bull "4.10.1"
joi "17.6.0"