From 03f7fb37ed5381ca59b51e45d1570c906c69694e Mon Sep 17 00:00:00 2001 From: jvcalderon Date: Thu, 18 Jan 2024 11:14:25 +0100 Subject: [PATCH 01/80] Calculate creators count when group role changes --- packages/account-portal | 2 +- packages/backend-core/src/index.ts | 1 + packages/backend-core/src/users/db.ts | 15 +++-- .../backend-core/src/users/test/utils.spec.ts | 67 +++++++++++++++++++ packages/backend-core/src/users/users.ts | 3 +- packages/backend-core/src/users/utils.ts | 35 +++++++++- packages/pro | 2 +- packages/server/src/middleware/authorized.ts | 2 +- .../shared-core/src/sdk/documents/users.ts | 2 +- 9 files changed, 117 insertions(+), 12 deletions(-) create mode 100644 packages/backend-core/src/users/test/utils.spec.ts diff --git a/packages/account-portal b/packages/account-portal index 1bc0128714..b23fb3b179 160000 --- a/packages/account-portal +++ b/packages/account-portal @@ -1 +1 @@ -Subproject commit 1bc012871496ff55e376931b620075b565e34d09 +Subproject commit b23fb3b17961fb04badd9487913a683fcf26dbe6 diff --git a/packages/backend-core/src/index.ts b/packages/backend-core/src/index.ts index 7bf26f3688..8946e37486 100644 --- a/packages/backend-core/src/index.ts +++ b/packages/backend-core/src/index.ts @@ -2,6 +2,7 @@ export * as configs from "./configs" export * as events from "./events" export * as migrations from "./migrations" export * as users from "./users" +export * as usersUtils from "./users/utils" export * as roles from "./security/roles" export * as permissions from "./security/permissions" export * as accounts from "./accounts" diff --git a/packages/backend-core/src/users/db.ts b/packages/backend-core/src/users/db.ts index 4d0d216603..136cb4b8ad 100644 --- a/packages/backend-core/src/users/db.ts +++ b/packages/backend-core/src/users/db.ts @@ -251,7 +251,8 @@ export class UserDB { } const change = dbUser ? 0 : 1 // no change if there is existing user - const creatorsChange = isCreator(dbUser) !== isCreator(user) ? 1 : 0 + const creatorsChange = + (await isCreator(dbUser)) !== (await isCreator(user)) ? 1 : 0 return UserDB.quotas.addUsers(change, creatorsChange, async () => { await validateUniqueUser(email, tenantId) @@ -335,7 +336,7 @@ export class UserDB { } newUser.userGroups = groups || [] newUsers.push(newUser) - if (isCreator(newUser)) { + if (await isCreator(newUser)) { newCreators.push(newUser) } } @@ -432,12 +433,16 @@ export class UserDB { _deleted: true, })) const dbResponse = await usersCore.bulkUpdateGlobalUsers(toDelete) - const creatorsToDelete = usersToDelete.filter(isCreator) + + const creatorsEval = await Promise.all(usersToDelete.map(isCreator)) + const creatorsToDeleteCount = creatorsEval.filter( + creator => !!creator + ).length for (let user of usersToDelete) { await bulkDeleteProcessing(user) } - await UserDB.quotas.removeUsers(toDelete.length, creatorsToDelete.length) + await UserDB.quotas.removeUsers(toDelete.length, creatorsToDeleteCount) // Build Response // index users by id @@ -486,7 +491,7 @@ export class UserDB { await db.remove(userId, dbUser._rev) - const creatorsToDelete = isCreator(dbUser) ? 1 : 0 + const creatorsToDelete = (await isCreator(dbUser)) ? 1 : 0 await UserDB.quotas.removeUsers(1, creatorsToDelete) await eventHelpers.handleDeleteEvents(dbUser) await cache.user.invalidateUser(userId) diff --git a/packages/backend-core/src/users/test/utils.spec.ts b/packages/backend-core/src/users/test/utils.spec.ts new file mode 100644 index 0000000000..0fe27f57a6 --- /dev/null +++ b/packages/backend-core/src/users/test/utils.spec.ts @@ -0,0 +1,67 @@ +import { User, UserGroup } from "@budibase/types" +import { generator, structures } from "../../../tests" +import { DBTestConfiguration } from "../../../tests/extra" +import { getGlobalDB } from "../../context" +import { isCreator } from "../utils" + +const config = new DBTestConfiguration() + +describe("Users", () => { + it("User is a creator if it is configured as a global builder", async () => { + const user: User = structures.users.user({ builder: { global: true } }) + expect(await isCreator(user)).toBe(true) + }) + + it("User is a creator if it is configured as a global admin", async () => { + const user: User = structures.users.user({ admin: { global: true } }) + expect(await isCreator(user)).toBe(true) + }) + + it("User is a creator if it is configured with creator permission", async () => { + const user: User = structures.users.user({ builder: { creator: true } }) + expect(await isCreator(user)).toBe(true) + }) + + it("User is a creator if it is a builder in some application", async () => { + const user: User = structures.users.user({ builder: { apps: ["app1"] } }) + expect(await isCreator(user)).toBe(true) + }) + + it("User is a creator if it has CREATOR permission in some application", async () => { + const user: User = structures.users.user({ roles: { app1: "CREATOR" } }) + expect(await isCreator(user)).toBe(true) + }) + + it("User is a creator if it has ADMIN permission in some application", async () => { + const user: User = structures.users.user({ roles: { app1: "ADMIN" } }) + expect(await isCreator(user)).toBe(true) + }) + + it("User is a creator if it remains to a group with ADMIN permissions", async () => { + const usersInGroup = 10 + const groupId = "gr_17abffe89e0b40268e755b952f101a59" + const group: UserGroup = { + ...structures.userGroups.userGroup(), + ...{ _id: groupId, roles: { app1: "ADMIN" } }, + } + const users: User[] = [] + for (const _ of Array.from({ length: usersInGroup })) { + const userId = `us_${generator.guid()}` + const user: User = structures.users.user({ + _id: userId, + userGroups: [groupId], + }) + users.push(user) + } + + await config.doInTenant(async () => { + const db = getGlobalDB() + await db.put(group) + for (let user of users) { + await db.put(user) + const creator = await isCreator(user) + expect(creator).toBe(true) + } + }) + }) +}) diff --git a/packages/backend-core/src/users/users.ts b/packages/backend-core/src/users/users.ts index cc2b4fc27f..638da4a5b1 100644 --- a/packages/backend-core/src/users/users.ts +++ b/packages/backend-core/src/users/users.ts @@ -309,7 +309,8 @@ export async function getCreatorCount() { let creators = 0 async function iterate(startPage?: string) { const page = await paginatedUsers({ bookmark: startPage }) - creators += page.data.filter(isCreator).length + const creatorsEval = await Promise.all(page.data.map(isCreator)) + creators += creatorsEval.filter(creator => !!creator).length if (page.hasNextPage) { await iterate(page.nextPage) } diff --git a/packages/backend-core/src/users/utils.ts b/packages/backend-core/src/users/utils.ts index 0ef4b77998..348ad1532f 100644 --- a/packages/backend-core/src/users/utils.ts +++ b/packages/backend-core/src/users/utils.ts @@ -1,4 +1,4 @@ -import { CloudAccount } from "@budibase/types" +import { CloudAccount, ContextUser, User, UserGroup } from "@budibase/types" import * as accountSdk from "../accounts" import env from "../environment" import { getPlatformUser } from "./lookup" @@ -6,17 +6,48 @@ import { EmailUnavailableError } from "../errors" import { getTenantId } from "../context" import { sdk } from "@budibase/shared-core" import { getAccountByTenantId } from "../accounts" +import { BUILTIN_ROLE_IDS } from "../security/roles" +import * as context from "../context" // extract from shared-core to make easily accessible from backend-core export const isBuilder = sdk.users.isBuilder export const isAdmin = sdk.users.isAdmin -export const isCreator = sdk.users.isCreator export const isGlobalBuilder = sdk.users.isGlobalBuilder export const isAdminOrBuilder = sdk.users.isAdminOrBuilder export const hasAdminPermissions = sdk.users.hasAdminPermissions export const hasBuilderPermissions = sdk.users.hasBuilderPermissions export const hasAppBuilderPermissions = sdk.users.hasAppBuilderPermissions +export async function isCreator(user?: User | ContextUser) { + const isCreatorByUserDefinition = sdk.users.isCreator(user) + if (!isCreatorByUserDefinition && user) { + return await isCreatorByGroupMembership(user) + } + return isCreatorByUserDefinition +} + +async function isCreatorByGroupMembership(user?: User | ContextUser) { + const userGroups = user?.userGroups || [] + if (userGroups.length > 0) { + const db = context.getGlobalDB() + const groups: UserGroup[] = [] + for (let groupId of userGroups) { + try { + const group = await db.get(groupId) + groups.push(group) + } catch (e: any) { + if (e.error !== "not_found") { + throw e + } + } + } + return groups.some(group => + Object.values(group.roles || {}).includes(BUILTIN_ROLE_IDS.ADMIN) + ) + } + return false +} + export async function validateUniqueUser(email: string, tenantId: string) { // check budibase users in other tenants if (env.MULTI_TENANCY) { diff --git a/packages/pro b/packages/pro index 9d80daaa5b..8c466d6ef2 160000 --- a/packages/pro +++ b/packages/pro @@ -1 +1 @@ -Subproject commit 9d80daaa5b79da68730d6c5f497f629c47a78ef8 +Subproject commit 8c466d6ef2a0c09b843ef63276793ab5af2e96f7 diff --git a/packages/server/src/middleware/authorized.ts b/packages/server/src/middleware/authorized.ts index cba765a887..fe7f6bb6fe 100644 --- a/packages/server/src/middleware/authorized.ts +++ b/packages/server/src/middleware/authorized.ts @@ -34,7 +34,7 @@ const checkAuthorized = async ( const isCreatorApi = permType === PermissionType.CREATOR const isBuilderApi = permType === PermissionType.BUILDER const isGlobalBuilder = users.isGlobalBuilder(ctx.user) - const isCreator = users.isCreator(ctx.user) + const isCreator = await users.isCreator(ctx.user) const isBuilder = appId ? users.isBuilder(ctx.user, appId) : users.hasBuilderPermissions(ctx.user) diff --git a/packages/shared-core/src/sdk/documents/users.ts b/packages/shared-core/src/sdk/documents/users.ts index 1aaf44ff7c..11e80dcf29 100644 --- a/packages/shared-core/src/sdk/documents/users.ts +++ b/packages/shared-core/src/sdk/documents/users.ts @@ -70,7 +70,7 @@ export function hasAppCreatorPermissions(user?: User | ContextUser): boolean { return _.flow( _.get("roles"), _.values, - _.find(x => x === "CREATOR"), + _.find(x => ["CREATOR", "ADMIN"].includes(x)), x => !!x )(user) } From ce0b1f8df3d9d11d3cf8c25daa1944aa257c5ded Mon Sep 17 00:00:00 2001 From: jvcalderon Date: Thu, 18 Jan 2024 11:22:16 +0100 Subject: [PATCH 02/80] Account portal submodule hash to master --- packages/account-portal | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/account-portal b/packages/account-portal index b23fb3b179..1bc0128714 160000 --- a/packages/account-portal +++ b/packages/account-portal @@ -1 +1 @@ -Subproject commit b23fb3b17961fb04badd9487913a683fcf26dbe6 +Subproject commit 1bc012871496ff55e376931b620075b565e34d09 From 64d3114c9f62f2b8f4c7f1382cce49ece836efd9 Mon Sep 17 00:00:00 2001 From: jvcalderon Date: Thu, 18 Jan 2024 12:03:26 +0100 Subject: [PATCH 03/80] Refactor: usersUtils -> userUtils --- packages/backend-core/src/index.ts | 2 +- packages/pro | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/backend-core/src/index.ts b/packages/backend-core/src/index.ts index 8946e37486..8001017092 100644 --- a/packages/backend-core/src/index.ts +++ b/packages/backend-core/src/index.ts @@ -2,7 +2,7 @@ export * as configs from "./configs" export * as events from "./events" export * as migrations from "./migrations" export * as users from "./users" -export * as usersUtils from "./users/utils" +export * as userUtils from "./users/utils" export * as roles from "./security/roles" export * as permissions from "./security/permissions" export * as accounts from "./accounts" diff --git a/packages/pro b/packages/pro index 8c466d6ef2..1857cbbd07 160000 --- a/packages/pro +++ b/packages/pro @@ -1 +1 @@ -Subproject commit 8c466d6ef2a0c09b843ef63276793ab5af2e96f7 +Subproject commit 1857cbbd071a855a56ed70877e55d5566c7f29e1 From 9befe6aa9d19098ccbb5314a1fa70d0f92ce8cc3 Mon Sep 17 00:00:00 2001 From: jvcalderon Date: Thu, 18 Jan 2024 12:32:19 +0100 Subject: [PATCH 04/80] Update pro submodule --- packages/pro | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/pro b/packages/pro index 1857cbbd07..3d815d2629 160000 --- a/packages/pro +++ b/packages/pro @@ -1 +1 @@ -Subproject commit 1857cbbd071a855a56ed70877e55d5566c7f29e1 +Subproject commit 3d815d262939c53092e5aeb0b033d6b481bb5583 From e47b19ef366b2df72ef5e0c72c82cbbe89dfd862 Mon Sep 17 00:00:00 2001 From: jvcalderon Date: Thu, 18 Jan 2024 12:36:06 +0100 Subject: [PATCH 05/80] Update pro submodule --- packages/pro | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/pro b/packages/pro index 3d815d2629..e8511333e0 160000 --- a/packages/pro +++ b/packages/pro @@ -1 +1 @@ -Subproject commit 3d815d262939c53092e5aeb0b033d6b481bb5583 +Subproject commit e8511333e0ef84f89a4d5eb041534438744dc2fd From ede73efd81e03f49ee74c3df02541f6924769843 Mon Sep 17 00:00:00 2001 From: jvcalderon Date: Thu, 18 Jan 2024 12:59:36 +0100 Subject: [PATCH 06/80] Upgrade pro submodule --- packages/pro | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/pro b/packages/pro index e8511333e0..35032a6aea 160000 --- a/packages/pro +++ b/packages/pro @@ -1 +1 @@ -Subproject commit e8511333e0ef84f89a4d5eb041534438744dc2fd +Subproject commit 35032a6aeab96437c64b7892ca0ca585e11a6605 From a825aa191f44ac437de07cb4ff688b0a9d0e583c Mon Sep 17 00:00:00 2001 From: jvcalderon Date: Thu, 18 Jan 2024 16:37:25 +0100 Subject: [PATCH 07/80] Upgrade pro submodule --- packages/pro | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/pro b/packages/pro index 35032a6aea..b91d3933e4 160000 --- a/packages/pro +++ b/packages/pro @@ -1 +1 @@ -Subproject commit 35032a6aeab96437c64b7892ca0ca585e11a6605 +Subproject commit b91d3933e453fb86b1ea66c8d92d44420fb6d2e7 From 00fda6af4127263a04a1558da9ef5648d4f1fb68 Mon Sep 17 00:00:00 2001 From: Peter Clement Date: Thu, 18 Jan 2024 16:38:34 +0000 Subject: [PATCH 08/80] update emitter to account for a new chaining property --- .../server/src/events/AutomationEmitter.ts | 37 ++++++++++++------- packages/types/src/documents/app/app.ts | 6 +++ 2 files changed, 29 insertions(+), 14 deletions(-) diff --git a/packages/server/src/events/AutomationEmitter.ts b/packages/server/src/events/AutomationEmitter.ts index 9d476d5088..3297fc7263 100644 --- a/packages/server/src/events/AutomationEmitter.ts +++ b/packages/server/src/events/AutomationEmitter.ts @@ -1,18 +1,9 @@ import { rowEmission, tableEmission } from "./utils" import mainEmitter from "./index" import env from "../environment" -import { Table, Row } from "@budibase/types" +import { Table, Row, DocumentType, App } from "@budibase/types" +import { context } from "@budibase/backend-core" -// max number of automations that can chain on top of each other -// TODO: in future make this configurable at the automation level -const MAX_AUTOMATION_CHAIN = env.SELF_HOSTED ? 5 : 0 - -/** - * Special emitter which takes the count of automation runs which have occurred and blocks an - * automation from running if it has reached the maximum number of chained automations runs. - * This essentially "fakes" the normal emitter to add some functionality in-between to stop automations - * from getting stuck endlessly chaining. - */ class AutomationEmitter { chainCount: number metadata: { automationChainCount: number } @@ -24,7 +15,23 @@ class AutomationEmitter { } } - emitRow(eventName: string, appId: string, row: Row, table?: Table) { + async getMaxAutomationChain() { + const db = context.getAppDB() + const appMetadata = await db.get(DocumentType.APP_METADATA) + let chainAutomations = appMetadata?.automations?.chainAutomations + + if (chainAutomations === true) { + return 5 + } else if (chainAutomations === undefined && env.SELF_HOSTED) { + return 5 + } else { + return 0 + } + } + + async emitRow(eventName: string, appId: string, row: Row, table?: Table) { + let MAX_AUTOMATION_CHAIN = await this.getMaxAutomationChain() + // don't emit even if we've reached max automation chain if (this.chainCount >= MAX_AUTOMATION_CHAIN) { return @@ -39,9 +46,11 @@ class AutomationEmitter { }) } - emitTable(eventName: string, appId: string, table?: Table) { + async emitTable(eventName: string, appId: string, table?: Table) { + let MAX_AUTOMATION_CHAIN = await this.getMaxAutomationChain() + // don't emit even if we've reached max automation chain - if (this.chainCount > MAX_AUTOMATION_CHAIN) { + if (this.chainCount >= MAX_AUTOMATION_CHAIN) { return } diff --git a/packages/types/src/documents/app/app.ts b/packages/types/src/documents/app/app.ts index 5bbdd86515..e1174d4fe4 100644 --- a/packages/types/src/documents/app/app.ts +++ b/packages/types/src/documents/app/app.ts @@ -23,6 +23,7 @@ export interface App extends Document { automationErrors?: AppMetadataErrors icon?: AppIcon features?: AppFeatures + automations: AutomationSettings } export interface AppInstance { @@ -67,4 +68,9 @@ export interface AppIcon { export interface AppFeatures { componentValidation?: boolean disableUserMetadata?: boolean + enableAutomationChaining?: boolean +} + +export interface AutomationSettings { + chainAutomations?: boolean } From b05484bb49c6f941ec57eff3525b5bebe049c24c Mon Sep 17 00:00:00 2001 From: Peter Clement Date: Thu, 18 Jan 2024 16:38:51 +0000 Subject: [PATCH 09/80] frontend toggle for new automation chaining and update settings ux --- .../src/builderStore/store/frontend.js | 1 + .../components/start/ChooseIconModal.svelte | 1 + .../settings/automation-history/index.svelte | 46 +++++++++++++++++-- 3 files changed, 45 insertions(+), 3 deletions(-) diff --git a/packages/builder/src/builderStore/store/frontend.js b/packages/builder/src/builderStore/store/frontend.js index b05b127b1c..af48afb1bc 100644 --- a/packages/builder/src/builderStore/store/frontend.js +++ b/packages/builder/src/builderStore/store/frontend.js @@ -65,6 +65,7 @@ const INITIAL_FRONTEND_STATE = { features: { componentValidation: false, disableUserMetadata: false, + enableAutomationChaining: false, }, errors: [], hasAppPackage: false, diff --git a/packages/builder/src/components/start/ChooseIconModal.svelte b/packages/builder/src/components/start/ChooseIconModal.svelte index 1ffaa35e34..1f26cb93ec 100644 --- a/packages/builder/src/components/start/ChooseIconModal.svelte +++ b/packages/builder/src/components/start/ChooseIconModal.svelte @@ -49,6 +49,7 @@ return } try { + console.log(app.instance._id) await apps.update(app.instance._id, { icon: { name, color }, }) diff --git a/packages/builder/src/pages/builder/app/[application]/settings/automation-history/index.svelte b/packages/builder/src/pages/builder/app/[application]/settings/automation-history/index.svelte index 373a47aa2e..4ff3409b09 100644 --- a/packages/builder/src/pages/builder/app/[application]/settings/automation-history/index.svelte +++ b/packages/builder/src/pages/builder/app/[application]/settings/automation-history/index.svelte @@ -8,6 +8,8 @@ Body, Heading, Divider, + Toggle, + notifications, } from "@budibase/bbui" import DateTimeRenderer from "components/common/renderers/DateTimeRenderer.svelte" import StatusRenderer from "./_components/StatusRenderer.svelte" @@ -16,7 +18,7 @@ import { createPaginationStore } from "helpers/pagination" import { getContext, onDestroy, onMount } from "svelte" import dayjs from "dayjs" - import { auth, licensing, admin } from "stores/portal" + import { auth, licensing, admin, apps } from "stores/portal" import { Constants } from "@budibase/frontend-core" import Portal from "svelte-portal" @@ -35,9 +37,13 @@ let timeRange = null let loaded = false + $: app = $apps.find(app => app.devId === $store.appId) $: licensePlan = $auth.user?.license?.plan $: page = $pageInfo.page $: fetchLogs(automationId, status, page, timeRange) + $: isCloud = $admin.cloud + + $: chainAutomations = app?.automations?.chainAutomations ?? !isCloud const timeOptions = [ { value: "90-d", label: "Past 90 days" }, @@ -124,6 +130,18 @@ sidePanel.open() } + async function save({ detail }) { + try { + await apps.update($store.appId, { + automations: { + chainAutomations: detail, + }, + }) + } catch (error) { + notifications.error("Error updating automation chaining setting") + } + } + onMount(async () => { await automationStore.actions.fetch() const params = new URLSearchParams(window.location.search) @@ -150,11 +168,30 @@ - Automation History - View the automations your app has executed + Automations + See your automation history and edit advanced settings + + Chain automations + Allow automations to trigger from other automations +
+ { + save(e) + }} + value={chainAutomations} + /> +
+
+ + + + History + Free plan stores up to 1 day of automation history +