From a908125ea3716e17795e3d4a2f1e6ff771d36399 Mon Sep 17 00:00:00 2001 From: Jonny McCullagh Date: Mon, 20 Feb 2023 19:56:04 +0000 Subject: [PATCH 01/15] add metrics endpoint to api --- .../src/api/controllers/public/metrics.ts | 122 ++++++++++++++++++ .../server/src/api/routes/public/index.ts | 2 + .../server/src/api/routes/public/metrics.ts | 28 ++++ 3 files changed, 152 insertions(+) create mode 100644 packages/server/src/api/controllers/public/metrics.ts create mode 100644 packages/server/src/api/routes/public/metrics.ts diff --git a/packages/server/src/api/controllers/public/metrics.ts b/packages/server/src/api/controllers/public/metrics.ts new file mode 100644 index 0000000000..264df7f415 --- /dev/null +++ b/packages/server/src/api/controllers/public/metrics.ts @@ -0,0 +1,122 @@ +import { App, BBContext } from "@budibase/types" +import { db as dbCore, context } from "@budibase/backend-core" +import os from "os" + +export async function fetch(ctx: BBContext) { + const allDatabases = await dbCore.getAllDbs() + const devAppIDs = await dbCore.getDevAppIDs({ idsOnly: true }) + const prodAppIDs = await dbCore.getProdAppIDs({ idsOnly: true }) + const allAppIds = await dbCore.getAllApps({ idsOnly: true }) + + var outputString = "" + + const freeMem = os.freemem() + const totalMem = os.totalmem() + const usedMem = totalMem - freeMem + const uptime = os.uptime() + + // **** budibase_os_uptime **** + outputString += convertToOpenMetrics( + "budibase_os_uptime", + "Time in seconds that the host operating system has been up", + "counter", + uptime + ) + + // **** budibase_os_free_mem **** + outputString += convertToOpenMetrics( + "budibase_os_free_mem", + "Bytes of memory free for usage on the host operating system", + "gauge", + freeMem + ) + + // **** budibase_os_total_mem **** + outputString += convertToOpenMetrics( + "budibase_os_total_mem", + "Total bytes of memory on the host operating system", + "gauge", + totalMem + ) + + // **** budibase_os_used_mem **** + outputString += convertToOpenMetrics( + "budibase_os_used_mem", + "Total bytes of memory in use on the host operating system", + "gauge", + usedMem + ) + + // **** budibase_os_load1 **** + outputString += convertToOpenMetrics( + "budibase_os_load1", + "Host operating system load average", + "gauge", + os.loadavg()[0] + ) + + // **** budibase_os_load5 **** + outputString += convertToOpenMetrics( + "budibase_os_load5", + "Host operating system load average", + "gauge", + os.loadavg()[1] + ) + // **** budibase_os_load15 **** + outputString += convertToOpenMetrics( + "budibase_os_load15", + "Host operating system load average", + "gauge", + os.loadavg()[2] + ) + + // **** budibase_tenant_app_count **** + outputString += convertToOpenMetrics( + "budibase_tenant_app_count", + "The number of apps created by a user", + "gauge", + allAppIds.length + ) + + // **** budibase_tenant_production_app_count **** + outputString += convertToOpenMetrics( + "budibase_tenant_production_app_count", + "The number of apps a user has published", + "gauge", + prodAppIDs.length + ) + + // **** budibase_tenant_dev_app_count **** + outputString += convertToOpenMetrics( + "budibase_tenant_dev_app_count", + "The number of apps a user has unpublished in development", + "gauge", + devAppIDs.length + ) + + // **** budibase_tenant_db_count **** + outputString += convertToOpenMetrics( + "budibase_tenant_db_count", + "The number of couchdb databases including global tables such as _users", + "gauge", + allDatabases.length + ) + + ctx.body = outputString +} + +export function convertToOpenMetrics( + metricName: string, + metricHelp: string, + metricType: string, + metricValue: number +) { + return ` + # HELP ${metricName} ${metricHelp}. + # TYPE ${metricName} ${metricType} + ${metricName} ${metricValue}` +} + +export default { + fetch, +} diff --git a/packages/server/src/api/routes/public/index.ts b/packages/server/src/api/routes/public/index.ts index 79e7731752..e67c10fe02 100644 --- a/packages/server/src/api/routes/public/index.ts +++ b/packages/server/src/api/routes/public/index.ts @@ -1,4 +1,5 @@ import appEndpoints from "./applications" +import metricEndpoints from "./metrics" import queryEndpoints from "./queries" import tableEndpoints from "./tables" import rowEndpoints from "./rows" @@ -120,6 +121,7 @@ function applyRoutes( } applyRoutes(appEndpoints, PermissionType.APP, "appId") +applyRoutes(metricEndpoints, PermissionType.APP, "appId") applyRoutes(tableEndpoints, PermissionType.TABLE, "tableId") applyRoutes(userEndpoints, PermissionType.USER, "userId") applyRoutes(queryEndpoints, PermissionType.QUERY, "queryId") diff --git a/packages/server/src/api/routes/public/metrics.ts b/packages/server/src/api/routes/public/metrics.ts new file mode 100644 index 0000000000..67cae14388 --- /dev/null +++ b/packages/server/src/api/routes/public/metrics.ts @@ -0,0 +1,28 @@ +import controller from "../../controllers/public/metrics" +import Endpoint from "./utils/Endpoint" + +const read = [] + +/** + * @openapi + * /metrics: + * get: + * operationId: getById + * summary: Retrieve Budibase tenant metrics + * description: Output metrics in openMetrics format compatible with Prometheus + * tags: + * - metrics + * responses: + * 200: + * description: Returns tenant metrics. + * content: + * text/plain: + * schema: + * $ref: '#/components/schemas/tableOutput' + * examples: + * table: + * $ref: '#/components/examples/table' + */ +read.push(new Endpoint("get", "/metrics", controller.fetch)) + +export default { read } From dde1237ed220e47202d7a29f5b92b8c7c15b4698 Mon Sep 17 00:00:00 2001 From: Jonny McCullagh Date: Wed, 22 Feb 2023 21:10:35 +0000 Subject: [PATCH 02/15] allUsers function & more metrics --- packages/backend-core/src/users.ts | 12 ++ .../src/api/controllers/public/metrics.ts | 147 ++++++++++++++++-- .../server/src/api/routes/public/metrics.ts | 7 +- 3 files changed, 150 insertions(+), 16 deletions(-) diff --git a/packages/backend-core/src/users.ts b/packages/backend-core/src/users.ts index 1720a79a83..7502ece47a 100644 --- a/packages/backend-core/src/users.ts +++ b/packages/backend-core/src/users.ts @@ -5,6 +5,8 @@ import { generateAppUserID, queryGlobalView, UNICODE_MAX, + DocumentType, + SEPARATOR, } from "./db" import { BulkDocsResponse, User } from "@budibase/types" import { getGlobalDB } from "./context" @@ -19,6 +21,16 @@ export const bulkGetGlobalUsersById = async (userIds: string[]) => { ).rows.map(row => row.doc) as User[] } +export const getAllUserIds = async () => { + const db = getGlobalDB() + const startKey = `${DocumentType.USER}${SEPARATOR}` + const response = await db.allDocs({ + startkey: startKey, + endkey: `${startKey}${UNICODE_MAX}`, + }) + return response.rows.map(row => row.id) +} + export const bulkUpdateGlobalUsers = async (users: User[]) => { const db = getGlobalDB() return (await db.bulkDocs(users)) as BulkDocsResponse diff --git a/packages/server/src/api/controllers/public/metrics.ts b/packages/server/src/api/controllers/public/metrics.ts index 264df7f415..65f8849f7f 100644 --- a/packages/server/src/api/controllers/public/metrics.ts +++ b/packages/server/src/api/controllers/public/metrics.ts @@ -1,19 +1,43 @@ import { App, BBContext } from "@budibase/types" -import { db as dbCore, context } from "@budibase/backend-core" +import { users as userCore, db as dbCore } from "@budibase/backend-core" +import { quotas, licensing } from "@budibase/pro" + import os from "os" export async function fetch(ctx: BBContext) { + // *** OPERATING SYSTEM *** + const freeMem = os.freemem() + const totalMem = os.totalmem() + const usedMem = totalMem - freeMem + const uptime = os.uptime() + + // *** APPS *** const allDatabases = await dbCore.getAllDbs() const devAppIDs = await dbCore.getDevAppIDs({ idsOnly: true }) const prodAppIDs = await dbCore.getProdAppIDs({ idsOnly: true }) const allAppIds = await dbCore.getAllApps({ idsOnly: true }) - var outputString = "" + // *** USERS *** + const usersObject = await userCore.getAllUserIds() - const freeMem = os.freemem() - const totalMem = os.totalmem() - const usedMem = totalMem - freeMem - const uptime = os.uptime() + // *** QUOTAS *** + const usage = await quotas.getQuotaUsage() + const license = await licensing.cache.getCachedLicense() + const appsQuotaUsage = usage.usageQuota.apps + const rowsQuotaUsage = usage.usageQuota.rows + const pluginsQuotaUsage = usage.usageQuota.plugins + const userGroupsQuotaUsage = usage.usageQuota.userGroups + const queryQuotaUsage = usage.monthly.current.queries + const automationsQuotaUsage = usage.monthly.current.automations + const appsQuotaLimit = license.quotas.usage.static.apps.value + const rowsQuotaLimit = license.quotas.usage.static.rows.value + const userGroupsQuotaLimit = license.quotas.usage.static.userGroups.value + const pluginsQuotaLimit = license.quotas.usage.static.plugins.value + const queryQuotaLimit = license.quotas.usage.monthly.queries.value + const automationsQuotaLimit = license.quotas.usage.monthly.automations.value + + // *** BUILD THE OUTPUT STRING *** + var outputString = "" // **** budibase_os_uptime **** outputString += convertToOpenMetrics( @@ -70,6 +94,14 @@ export async function fetch(ctx: BBContext) { os.loadavg()[2] ) + // **** budibase_tenant_user_count **** + outputString += convertToOpenMetrics( + "budibase_tenant_user_count", + "The number of users created", + "gauge", + usersObject.length + ) + // **** budibase_tenant_app_count **** outputString += convertToOpenMetrics( "budibase_tenant_app_count", @@ -102,6 +134,102 @@ export async function fetch(ctx: BBContext) { allDatabases.length ) + // **** budibase_quota_usage_apps **** + outputString += convertToOpenMetrics( + "budibase_quota_usage_apps", + "The number of apps created", + "gauge", + appsQuotaUsage + ) + + // **** budibase_quota_limit_apps **** + outputString += convertToOpenMetrics( + "budibase_quota_limit_apps", + "The limit on the number of apps that can be created", + "gauge", + appsQuotaLimit == -1 ? 3000000 : appsQuotaLimit + ) + + // **** budibase_quota_usage_rows **** + outputString += convertToOpenMetrics( + "budibase_quota_usage_rows", + "The number of database rows used from the quota", + "gauge", + rowsQuotaUsage + ) + + // **** budibase_quota_limit_rows **** + outputString += convertToOpenMetrics( + "budibase_quota_limit_rows", + "The limit on the number of rows that can be created", + "gauge", + rowsQuotaLimit == -1 ? 3000000 : rowsQuotaLimit + ) + + // **** budibase_quota_usage_plugins **** + outputString += convertToOpenMetrics( + "budibase_quota_usage_plugins", + "The number of plugins in use", + "gauge", + pluginsQuotaUsage + ) + + // **** budibase_quota_limit_plugins **** + outputString += convertToOpenMetrics( + "budibase_quota_limit_plugins", + "The limit on the number of plugins that can be created", + "gauge", + pluginsQuotaLimit == -1 ? 3000000 : pluginsQuotaLimit + ) + + // **** budibase_quota_usage_user_groups **** + outputString += convertToOpenMetrics( + "budibase_quota_usage_user_groups", + "The number of user groups created", + "gauge", + userGroupsQuotaUsage + ) + + // **** budibase_quota_limit_user_groups **** + outputString += convertToOpenMetrics( + "budibase_quota_limit_user_groups", + "The limit on the number of user groups that can be created", + "gauge", + userGroupsQuotaLimit == -1 ? 3000000 : userGroupsQuotaLimit + ) + + // **** budibase_quota_usage_queries **** + outputString += convertToOpenMetrics( + "budibase_quota_usage_queries", + "The number of queries used in the current month", + "gauge", + queryQuotaUsage + ) + + // **** budibase_quota_limit_queries **** + outputString += convertToOpenMetrics( + "budibase_quota_limit_queries", + "The limit on the number of queries for the current month", + "gauge", + queryQuotaLimit == -1 ? 3000000 : queryQuotaLimit + ) + + // **** budibase_quota_usage_automations **** + outputString += convertToOpenMetrics( + "budibase_quota_usage_automations", + "The number of automations used in the current month", + "gauge", + automationsQuotaUsage + ) + + // **** budibase_quota_limit_automations **** + outputString += convertToOpenMetrics( + "budibase_quota_limit_automations", + "The limit on the number of automations that can be created", + "gauge", + automationsQuotaLimit == -1 ? 3000000 : automationsQuotaLimit + ) + ctx.body = outputString } @@ -111,10 +239,9 @@ export function convertToOpenMetrics( metricType: string, metricValue: number ) { - return ` - # HELP ${metricName} ${metricHelp}. - # TYPE ${metricName} ${metricType} - ${metricName} ${metricValue}` + return `# HELP ${metricName} ${metricHelp}. +# TYPE ${metricName} ${metricType} +${metricName} ${metricValue}\n` } export default { diff --git a/packages/server/src/api/routes/public/metrics.ts b/packages/server/src/api/routes/public/metrics.ts index 67cae14388..35745e3b13 100644 --- a/packages/server/src/api/routes/public/metrics.ts +++ b/packages/server/src/api/routes/public/metrics.ts @@ -9,7 +9,7 @@ const read = [] * get: * operationId: getById * summary: Retrieve Budibase tenant metrics - * description: Output metrics in openMetrics format compatible with Prometheus + * description: Output metrics in OpenMetrics format compatible with Prometheus * tags: * - metrics * responses: @@ -17,11 +17,6 @@ const read = [] * description: Returns tenant metrics. * content: * text/plain: - * schema: - * $ref: '#/components/schemas/tableOutput' - * examples: - * table: - * $ref: '#/components/examples/table' */ read.push(new Endpoint("get", "/metrics", controller.fetch)) From 057abb5351b63fe78d44d8ddd43892fe7f95f70e Mon Sep 17 00:00:00 2001 From: Mateus Badan de Pieri Date: Mon, 27 Feb 2023 15:04:30 +0000 Subject: [PATCH 03/15] move component to bbui --- .../common/inputs => bbui/src/Input}/CopyInput.svelte | 0 .../src/components/automation/Shared/WebhookDisplay.svelte | 2 +- packages/builder/src/components/deploy/DeployModal.svelte | 2 +- packages/builder/src/components/settings/APIKeyModal.svelte | 2 +- 4 files changed, 3 insertions(+), 3 deletions(-) rename packages/{builder/src/components/common/inputs => bbui/src/Input}/CopyInput.svelte (100%) diff --git a/packages/builder/src/components/common/inputs/CopyInput.svelte b/packages/bbui/src/Input/CopyInput.svelte similarity index 100% rename from packages/builder/src/components/common/inputs/CopyInput.svelte rename to packages/bbui/src/Input/CopyInput.svelte diff --git a/packages/builder/src/components/automation/Shared/WebhookDisplay.svelte b/packages/builder/src/components/automation/Shared/WebhookDisplay.svelte index 9ba4140b51..b0def87c90 100644 --- a/packages/builder/src/components/automation/Shared/WebhookDisplay.svelte +++ b/packages/builder/src/components/automation/Shared/WebhookDisplay.svelte @@ -1,5 +1,5 @@