From 51f2a3b86c44dc4ad7ac85c68b0c82838203ad60 Mon Sep 17 00:00:00 2001 From: Rory Powell Date: Tue, 4 Oct 2022 10:31:40 +0100 Subject: [PATCH 1/5] Types for sending quota triggered email --- packages/types/src/api/account/index.ts | 1 + packages/types/src/api/account/license.ts | 11 +++++++++++ packages/types/src/documents/account/account.ts | 2 ++ .../TestConfiguration/InternalAPIClient.ts | 6 ++---- .../internal-api/TestConfiguration/applications.ts | 8 ++------ .../config/internal-api/TestConfiguration/auth.ts | 6 +++--- .../config/internal-api/fixtures/applications.ts | 9 ++++----- .../tests/internal-api/applications/create.spec.ts | 13 +++++++------ 8 files changed, 32 insertions(+), 24 deletions(-) create mode 100644 packages/types/src/api/account/license.ts diff --git a/packages/types/src/api/account/index.ts b/packages/types/src/api/account/index.ts index 50c6bf22c6..0cbc487bcc 100644 --- a/packages/types/src/api/account/index.ts +++ b/packages/types/src/api/account/index.ts @@ -1 +1,2 @@ export * from "./user" +export * from "./license" diff --git a/packages/types/src/api/account/license.ts b/packages/types/src/api/account/license.ts new file mode 100644 index 0000000000..67791e5d6f --- /dev/null +++ b/packages/types/src/api/account/license.ts @@ -0,0 +1,11 @@ +import { QuotaUsage } from "../../documents" + +export interface GetLicenseRequest { + quotaUsage: QuotaUsage +} + +export interface QuotaUsageTriggeredRequest { + percentage: number + name: string + resetDate: string +} diff --git a/packages/types/src/documents/account/account.ts b/packages/types/src/documents/account/account.ts index e7dcf2d89f..eb12f7754b 100644 --- a/packages/types/src/documents/account/account.ts +++ b/packages/types/src/documents/account/account.ts @@ -1,4 +1,5 @@ import { Feature, Hosting, PlanType, Quotas } from "../../sdk" +import { QuotaUsage } from "../global" export interface CreateAccount { email: string @@ -42,6 +43,7 @@ export interface Account extends CreateAccount { licenseKey?: string licenseKeyActivatedAt?: number licenseOverrides?: LicenseOverrides + quotaUsage?: QuotaUsage } export interface PasswordAccount extends Account { diff --git a/qa-core/src/config/internal-api/TestConfiguration/InternalAPIClient.ts b/qa-core/src/config/internal-api/TestConfiguration/InternalAPIClient.ts index bfcbb9f4e2..dafc2b1ff2 100644 --- a/qa-core/src/config/internal-api/TestConfiguration/InternalAPIClient.ts +++ b/qa-core/src/config/internal-api/TestConfiguration/InternalAPIClient.ts @@ -16,9 +16,7 @@ class InternalAPIClient { constructor(appId?: string) { if (!env.BUDIBASE_SERVER_URL) { - throw new Error( - "Must set BUDIBASE_SERVER_URL env var" - ) + throw new Error("Must set BUDIBASE_SERVER_URL env var") } this.host = `${env.BUDIBASE_SERVER_URL}/api` this.appId = appId @@ -55,4 +53,4 @@ class InternalAPIClient { put = this.apiCall("PUT") } -export default InternalAPIClient \ No newline at end of file +export default InternalAPIClient diff --git a/qa-core/src/config/internal-api/TestConfiguration/applications.ts b/qa-core/src/config/internal-api/TestConfiguration/applications.ts index 10e4a6657b..0c51487122 100644 --- a/qa-core/src/config/internal-api/TestConfiguration/applications.ts +++ b/qa-core/src/config/internal-api/TestConfiguration/applications.ts @@ -1,6 +1,4 @@ -import { - Application, -} from "@budibase/server/api/controllers/public/mapping/types" +import { Application } from "@budibase/server/api/controllers/public/mapping/types" import { App } from "@budibase/types" import { Response } from "node-fetch" import InternalAPIClient from "./InternalAPIClient" @@ -37,9 +35,7 @@ export default class AppApi { return [response, json] } - async create( - body: any - ): Promise<[Response, Partial]> { + async create(body: any): Promise<[Response, Partial]> { const response = await this.api.post(`/applications`, { body }) const json = await response.json() return [response, json] diff --git a/qa-core/src/config/internal-api/TestConfiguration/auth.ts b/qa-core/src/config/internal-api/TestConfiguration/auth.ts index 6ac53f24b6..d83c859ab3 100644 --- a/qa-core/src/config/internal-api/TestConfiguration/auth.ts +++ b/qa-core/src/config/internal-api/TestConfiguration/auth.ts @@ -9,11 +9,11 @@ export default class AuthApi { } async login(): Promise<[Response, any]> { - const response = await this.api.post(`/global/auth/default/login`, { + const response = await this.api.post(`/global/auth/default/login`, { body: { username: process.env.BB_ADMIN_USER_EMAIL, - password: process.env.BB_ADMIN_USER_PASSWORD - } + password: process.env.BB_ADMIN_USER_PASSWORD, + }, }) const cookie = response.headers.get("set-cookie") this.api.cookie = cookie as any diff --git a/qa-core/src/config/internal-api/fixtures/applications.ts b/qa-core/src/config/internal-api/fixtures/applications.ts index dfad7e0b46..9076a05e1b 100644 --- a/qa-core/src/config/internal-api/fixtures/applications.ts +++ b/qa-core/src/config/internal-api/fixtures/applications.ts @@ -1,10 +1,9 @@ import generator from "../../generator" -import { - Application, -} from "@budibase/server/api/controllers/public/mapping/types" +import { Application } from "@budibase/server/api/controllers/public/mapping/types" - -const generate = (overrides: Partial = {}): Partial => ({ +const generate = ( + overrides: Partial = {} +): Partial => ({ name: generator.word(), url: `/${generator.word()}`, ...overrides, diff --git a/qa-core/src/tests/internal-api/applications/create.spec.ts b/qa-core/src/tests/internal-api/applications/create.spec.ts index 81d43d9c91..2c934e0bd7 100644 --- a/qa-core/src/tests/internal-api/applications/create.spec.ts +++ b/qa-core/src/tests/internal-api/applications/create.spec.ts @@ -24,14 +24,14 @@ describe("Internal API - /applications endpoints", () => { useTemplate: "true", templateName: "Near Miss Register", templateKey: "app/near-miss-register", - templateFile: undefined + templateFile: undefined, }) } it("GET - fetch applications", async () => { await config.applications.create({ ...generateApp(), - useTemplate: false + useTemplate: false, }) const [response, apps] = await config.applications.fetch() expect(response).toHaveStatusCode(200) @@ -57,7 +57,7 @@ describe("Internal API - /applications endpoints", () => { expect(publish).toEqual({ _id: expect.any(String), appUrl: app.url, - status: "SUCCESS" + status: "SUCCESS", }) }) @@ -70,7 +70,8 @@ describe("Internal API - /applications endpoints", () => { config.applications.api.appId = app.appId // check preview renders - const [previewResponse, previewRenders] = await config.applications.canRender() + const [previewResponse, previewRenders] = + await config.applications.canRender() expect(previewResponse).toHaveStatusCode(200) expect(previewRenders).toBe(true) @@ -79,8 +80,8 @@ describe("Internal API - /applications endpoints", () => { // check published app renders config.applications.api.appId = db.getProdAppID(app.appId) - const [publishedAppResponse, publishedAppRenders] = await config.applications.canRender() + const [publishedAppResponse, publishedAppRenders] = + await config.applications.canRender() expect(publishedAppRenders).toBe(true) }) - }) From aff6e5cbbb0dbed12d0d3993466d576c1f8104f8 Mon Sep 17 00:00:00 2001 From: Rory Powell Date: Thu, 6 Oct 2022 16:03:47 +0100 Subject: [PATCH 2/5] Process quota emails in account-portal --- packages/backend-core/package.json | 2 + .../tests/utilities/structures/accounts.ts | 23 +++++++ .../tests/utilities/structures/common.ts | 1 + .../tests/utilities/structures/index.ts | 7 ++ .../tests/utilities/structures/licenses.ts | 18 +++++ packages/backend-core/yarn.lock | 10 +++ .../src/api/controllers/deploy/index.ts | 1 - packages/types/src/api/account/license.ts | 6 -- .../types/src/documents/account/account.ts | 36 +++++++++- packages/types/src/documents/global/quotas.ts | 13 ++-- packages/types/src/sdk/licensing/quota.ts | 35 ++++++---- .../src/api/controllers/system/tenants.js | 58 ---------------- .../src/api/controllers/system/tenants.ts | 66 +++++++++++++++++++ packages/worker/src/sdk/users/events.ts | 1 - 14 files changed, 190 insertions(+), 87 deletions(-) create mode 100644 packages/backend-core/tests/utilities/structures/accounts.ts create mode 100644 packages/backend-core/tests/utilities/structures/common.ts create mode 100644 packages/backend-core/tests/utilities/structures/licenses.ts delete mode 100644 packages/worker/src/api/controllers/system/tenants.js create mode 100644 packages/worker/src/api/controllers/system/tenants.ts diff --git a/packages/backend-core/package.json b/packages/backend-core/package.json index a07db6fca0..ed31e3c0c8 100644 --- a/packages/backend-core/package.json +++ b/packages/backend-core/package.json @@ -62,6 +62,7 @@ ] }, "devDependencies": { + "@types/chance": "1.1.3", "@types/jest": "27.5.1", "@types/koa": "2.0.52", "@types/lodash": "4.14.180", @@ -73,6 +74,7 @@ "@types/tar-fs": "2.0.1", "@types/uuid": "8.3.4", "ioredis-mock": "5.8.0", + "chance": "1.1.3", "jest": "27.5.1", "koa": "2.7.0", "nodemon": "2.0.16", diff --git a/packages/backend-core/tests/utilities/structures/accounts.ts b/packages/backend-core/tests/utilities/structures/accounts.ts new file mode 100644 index 0000000000..5d23962575 --- /dev/null +++ b/packages/backend-core/tests/utilities/structures/accounts.ts @@ -0,0 +1,23 @@ +import { generator, uuid } from "." +import { AuthType, CloudAccount, Hosting } from "@budibase/types" +import * as db from "../../../src/db/utils" + +export const cloudAccount = (): CloudAccount => { + return { + accountId: uuid(), + createdAt: Date.now(), + verified: true, + verificationSent: true, + tier: "", + email: generator.email(), + tenantId: generator.word(), + hosting: Hosting.CLOUD, + authType: AuthType.PASSWORD, + password: generator.word(), + tenantName: generator.word(), + name: generator.name(), + size: "10+", + profession: "Software Engineer", + budibaseUserId: db.generateGlobalUserID(), + } +} diff --git a/packages/backend-core/tests/utilities/structures/common.ts b/packages/backend-core/tests/utilities/structures/common.ts new file mode 100644 index 0000000000..51ae220254 --- /dev/null +++ b/packages/backend-core/tests/utilities/structures/common.ts @@ -0,0 +1 @@ +export { v4 as uuid } from "uuid" diff --git a/packages/backend-core/tests/utilities/structures/index.ts b/packages/backend-core/tests/utilities/structures/index.ts index 12b6ab7ad6..68064b9715 100644 --- a/packages/backend-core/tests/utilities/structures/index.ts +++ b/packages/backend-core/tests/utilities/structures/index.ts @@ -1 +1,8 @@ +export * from "./common" + +import Chance from "chance" +export const generator = new Chance() + export * as koa from "./koa" +export * as accounts from "./accounts" +export * as licenses from "./licenses" diff --git a/packages/backend-core/tests/utilities/structures/licenses.ts b/packages/backend-core/tests/utilities/structures/licenses.ts new file mode 100644 index 0000000000..a541e91860 --- /dev/null +++ b/packages/backend-core/tests/utilities/structures/licenses.ts @@ -0,0 +1,18 @@ +import { AccountPlan, License, PlanType, Quotas } from "@budibase/types" + +const newPlan = (type: PlanType = PlanType.FREE): AccountPlan => { + return { + type, + } +} + +export const newLicense = (opts: { + quotas: Quotas + planType?: PlanType +}): License => { + return { + features: [], + quotas: opts.quotas, + plan: newPlan(opts.planType), + } +} diff --git a/packages/backend-core/yarn.lock b/packages/backend-core/yarn.lock index 2e62aea734..6bc9b63728 100644 --- a/packages/backend-core/yarn.lock +++ b/packages/backend-core/yarn.lock @@ -663,6 +663,11 @@ "@types/connect" "*" "@types/node" "*" +"@types/chance@1.1.3": + version "1.1.3" + resolved "https://registry.yarnpkg.com/@types/chance/-/chance-1.1.3.tgz#d19fe9391288d60fdccd87632bfc9ab2b4523fea" + integrity sha512-X6c6ghhe4/sQh4XzcZWSFaTAUOda38GQHmq9BUanYkOE/EO7ZrkazwKmtsj3xzTjkLWmwULE++23g3d3CCWaWw== + "@types/connect@*": version "3.4.35" resolved "https://registry.yarnpkg.com/@types/connect/-/connect-3.4.35.tgz#5fcf6ae445e4021d1fc2219a4873cc73a3bb2ad1" @@ -1555,6 +1560,11 @@ chalk@^4.0.0, chalk@^4.1.0: ansi-styles "^4.1.0" supports-color "^7.1.0" +chance@1.1.3: + version "1.1.3" + resolved "https://registry.yarnpkg.com/chance/-/chance-1.1.3.tgz#414f08634ee479c7a316b569050ea20751b82dd3" + integrity sha512-XeJsdoVAzDb1WRPRuMBesRSiWpW1uNTo5Fd7mYxPJsAfgX71+jfuCOHOdbyBz2uAUZ8TwKcXgWk3DMedFfJkbg== + char-regex@^1.0.2: version "1.0.2" resolved "https://registry.yarnpkg.com/char-regex/-/char-regex-1.0.2.tgz#d744358226217f981ed58f479b1d6bcc29545dcf" diff --git a/packages/server/src/api/controllers/deploy/index.ts b/packages/server/src/api/controllers/deploy/index.ts index 5edf862706..a51e7ad6ec 100644 --- a/packages/server/src/api/controllers/deploy/index.ts +++ b/packages/server/src/api/controllers/deploy/index.ts @@ -17,7 +17,6 @@ import { getProdAppDB, getDevAppDB, } from "@budibase/backend-core/context" -import { quotas } from "@budibase/pro" import { events } from "@budibase/backend-core" // the max time we can wait for an invalidation to complete before considering it failed diff --git a/packages/types/src/api/account/license.ts b/packages/types/src/api/account/license.ts index 67791e5d6f..80481eb8f5 100644 --- a/packages/types/src/api/account/license.ts +++ b/packages/types/src/api/account/license.ts @@ -3,9 +3,3 @@ import { QuotaUsage } from "../../documents" export interface GetLicenseRequest { quotaUsage: QuotaUsage } - -export interface QuotaUsageTriggeredRequest { - percentage: number - name: string - resetDate: string -} diff --git a/packages/types/src/documents/account/account.ts b/packages/types/src/documents/account/account.ts index eb12f7754b..e3fcf75ecf 100644 --- a/packages/types/src/documents/account/account.ts +++ b/packages/types/src/documents/account/account.ts @@ -1,5 +1,12 @@ -import { Feature, Hosting, PlanType, Quotas } from "../../sdk" -import { QuotaUsage } from "../global" +import { + Feature, + Hosting, + MonthlyQuotaName, + PlanType, + Quotas, + StaticQuotaName, +} from "../../sdk" +import { MonthlyUsage, QuotaUsage, StaticUsage } from "../global" export interface CreateAccount { email: string @@ -43,7 +50,7 @@ export interface Account extends CreateAccount { licenseKey?: string licenseKeyActivatedAt?: number licenseOverrides?: LicenseOverrides - quotaUsage?: QuotaUsage + quotaUsage?: AccountQuotaUsage } export interface PasswordAccount extends Account { @@ -86,3 +93,26 @@ export interface OAuthTokens { accessToken: string refreshToken: string } + +export type QuotaTriggers = { + [key: string]: string | null +} + +export interface AccountStaticUsage extends StaticUsage { + triggers?: { + [key in StaticQuotaName]?: QuotaTriggers + } +} + +export interface AccountMonthlyUsage extends MonthlyUsage { + triggers?: { + [key in MonthlyQuotaName]?: QuotaTriggers + } +} + +export interface AccountQuotaUsage extends QuotaUsage { + usageQuota: AccountStaticUsage + monthly: { + [key: string]: AccountMonthlyUsage + } +} diff --git a/packages/types/src/documents/global/quotas.ts b/packages/types/src/documents/global/quotas.ts index eb1d77c228..d9cc83b46a 100644 --- a/packages/types/src/documents/global/quotas.ts +++ b/packages/types/src/documents/global/quotas.ts @@ -24,7 +24,14 @@ export interface UsageBreakdown { } } -export type MonthlyUsage = { +export interface StaticUsage { + [StaticQuotaName.APPS]: number + [StaticQuotaName.PLUGINS]: number + [StaticQuotaName.USER_GROUPS]: number + [StaticQuotaName.ROWS]: number +} + +export interface MonthlyUsage { [MonthlyQuotaName.QUERIES]: number [MonthlyQuotaName.AUTOMATIONS]: number [MonthlyQuotaName.DAY_PASSES]: number @@ -34,9 +41,7 @@ export type MonthlyUsage = { } export interface BaseQuotaUsage { - usageQuota: { - [key in StaticQuotaName]: number - } + usageQuota: StaticUsage monthly: { [key: string]: MonthlyUsage } diff --git a/packages/types/src/sdk/licensing/quota.ts b/packages/types/src/sdk/licensing/quota.ts index 49dd561db0..6e58734925 100644 --- a/packages/types/src/sdk/licensing/quota.ts +++ b/packages/types/src/sdk/licensing/quota.ts @@ -61,26 +61,33 @@ export type PlanQuotas = { [PlanType.ENTERPRISE]: Quotas } +export type MonthlyQuotas = { + [MonthlyQuotaName.QUERIES]: Quota + [MonthlyQuotaName.AUTOMATIONS]: Quota + [MonthlyQuotaName.DAY_PASSES]: Quota +} + +export type StaticQuotas = { + [StaticQuotaName.ROWS]: Quota + [StaticQuotaName.APPS]: Quota + [StaticQuotaName.USER_GROUPS]: Quota + [StaticQuotaName.PLUGINS]: Quota +} + +export type ConstantQuotas = { + [ConstantQuotaName.AUTOMATION_LOG_RETENTION_DAYS]: Quota +} + export type Quotas = { [QuotaType.USAGE]: { - [QuotaUsageType.MONTHLY]: { - [MonthlyQuotaName.QUERIES]: Quota - [MonthlyQuotaName.AUTOMATIONS]: Quota - [MonthlyQuotaName.DAY_PASSES]: Quota - } - [QuotaUsageType.STATIC]: { - [StaticQuotaName.ROWS]: Quota - [StaticQuotaName.APPS]: Quota - [StaticQuotaName.USER_GROUPS]: Quota - [StaticQuotaName.PLUGINS]: Quota - } - } - [QuotaType.CONSTANT]: { - [ConstantQuotaName.AUTOMATION_LOG_RETENTION_DAYS]: Quota + [QuotaUsageType.MONTHLY]: MonthlyQuotas + [QuotaUsageType.STATIC]: StaticQuotas } + [QuotaType.CONSTANT]: ConstantQuotas } export interface Quota { name: string value: number + triggers: number[] } diff --git a/packages/worker/src/api/controllers/system/tenants.js b/packages/worker/src/api/controllers/system/tenants.js deleted file mode 100644 index c54a3d9834..0000000000 --- a/packages/worker/src/api/controllers/system/tenants.js +++ /dev/null @@ -1,58 +0,0 @@ -const { StaticDatabases, doWithDB } = require("@budibase/backend-core/db") -const { getTenantId } = require("@budibase/backend-core/tenancy") -const { deleteTenant } = require("@budibase/backend-core/deprovision") -const { quotas } = require("@budibase/pro") - -exports.exists = async ctx => { - const tenantId = ctx.request.params - ctx.body = { - exists: await doWithDB(StaticDatabases.PLATFORM_INFO.name, async db => { - let exists = false - try { - const tenantsDoc = await db.get( - StaticDatabases.PLATFORM_INFO.docs.tenants - ) - if (tenantsDoc) { - exists = tenantsDoc.tenantIds.indexOf(tenantId) !== -1 - } - } catch (err) { - // if error it doesn't exist - } - return exists - }), - } -} - -exports.fetch = async ctx => { - ctx.body = await doWithDB(StaticDatabases.PLATFORM_INFO.name, async db => { - let tenants = [] - try { - const tenantsDoc = await db.get( - StaticDatabases.PLATFORM_INFO.docs.tenants - ) - if (tenantsDoc) { - tenants = tenantsDoc.tenantIds - } - } catch (err) { - // if error it doesn't exist - } - return tenants - }) -} - -exports.delete = async ctx => { - const tenantId = getTenantId() - - if (ctx.params.tenantId !== tenantId) { - ctx.throw(403, "Unauthorized") - } - - try { - await deleteTenant(tenantId) - await quotas.bustCache() - ctx.status = 204 - } catch (err) { - ctx.log.error(err) - throw err - } -} diff --git a/packages/worker/src/api/controllers/system/tenants.ts b/packages/worker/src/api/controllers/system/tenants.ts new file mode 100644 index 0000000000..d6e6261c22 --- /dev/null +++ b/packages/worker/src/api/controllers/system/tenants.ts @@ -0,0 +1,66 @@ +const { StaticDatabases, doWithDB } = require("@budibase/backend-core/db") +const { getTenantId } = require("@budibase/backend-core/tenancy") +const { deleteTenant } = require("@budibase/backend-core/deprovision") +import { quotas } from "@budibase/pro" + +export const exists = async (ctx: any) => { + const tenantId = ctx.request.params + ctx.body = { + exists: await doWithDB( + StaticDatabases.PLATFORM_INFO.name, + async (db: any) => { + let exists = false + try { + const tenantsDoc = await db.get( + StaticDatabases.PLATFORM_INFO.docs.tenants + ) + if (tenantsDoc) { + exists = tenantsDoc.tenantIds.indexOf(tenantId) !== -1 + } + } catch (err) { + // if error it doesn't exist + } + return exists + } + ), + } +} + +export const fetch = async (ctx: any) => { + ctx.body = await doWithDB( + StaticDatabases.PLATFORM_INFO.name, + async (db: any) => { + let tenants = [] + try { + const tenantsDoc = await db.get( + StaticDatabases.PLATFORM_INFO.docs.tenants + ) + if (tenantsDoc) { + tenants = tenantsDoc.tenantIds + } + } catch (err) { + // if error it doesn't exist + } + return tenants + } + ) +} + +const _delete = async (ctx: any) => { + const tenantId = getTenantId() + + if (ctx.params.tenantId !== tenantId) { + ctx.throw(403, "Unauthorized") + } + + try { + await deleteTenant(tenantId) + await quotas.bustCache() + ctx.status = 204 + } catch (err) { + ctx.log.error(err) + throw err + } +} + +export { _delete as delete } diff --git a/packages/worker/src/sdk/users/events.ts b/packages/worker/src/sdk/users/events.ts index 0094c6fd84..3046442393 100644 --- a/packages/worker/src/sdk/users/events.ts +++ b/packages/worker/src/sdk/users/events.ts @@ -1,7 +1,6 @@ import env from "../../environment" import { events, accounts, tenancy } from "@budibase/backend-core" import { User, UserRoles, CloudAccount } from "@budibase/types" -import { users as pro } from "@budibase/pro" export const handleDeleteEvents = async (user: any) => { await events.user.deleted(user) From 783c012a2676b9bfd06fedb245dd75f20965545f Mon Sep 17 00:00:00 2001 From: Rory Powell Date: Fri, 7 Oct 2022 13:57:10 +0100 Subject: [PATCH 3/5] Move trigger logic inside pro --- packages/types/src/api/account/license.ts | 6 +++++ .../types/src/documents/account/account.ts | 25 +------------------ packages/types/src/documents/global/quotas.ts | 17 +++++++++++++ 3 files changed, 24 insertions(+), 24 deletions(-) diff --git a/packages/types/src/api/account/license.ts b/packages/types/src/api/account/license.ts index 80481eb8f5..40ee79c3e3 100644 --- a/packages/types/src/api/account/license.ts +++ b/packages/types/src/api/account/license.ts @@ -3,3 +3,9 @@ import { QuotaUsage } from "../../documents" export interface GetLicenseRequest { quotaUsage: QuotaUsage } + +export interface QuotaTriggeredRequest { + percentage: number + name: string + resetDate?: string +} diff --git a/packages/types/src/documents/account/account.ts b/packages/types/src/documents/account/account.ts index e3fcf75ecf..70c3061c3f 100644 --- a/packages/types/src/documents/account/account.ts +++ b/packages/types/src/documents/account/account.ts @@ -50,7 +50,7 @@ export interface Account extends CreateAccount { licenseKey?: string licenseKeyActivatedAt?: number licenseOverrides?: LicenseOverrides - quotaUsage?: AccountQuotaUsage + quotaUsage?: QuotaUsage } export interface PasswordAccount extends Account { @@ -93,26 +93,3 @@ export interface OAuthTokens { accessToken: string refreshToken: string } - -export type QuotaTriggers = { - [key: string]: string | null -} - -export interface AccountStaticUsage extends StaticUsage { - triggers?: { - [key in StaticQuotaName]?: QuotaTriggers - } -} - -export interface AccountMonthlyUsage extends MonthlyUsage { - triggers?: { - [key in MonthlyQuotaName]?: QuotaTriggers - } -} - -export interface AccountQuotaUsage extends QuotaUsage { - usageQuota: AccountStaticUsage - monthly: { - [key: string]: AccountMonthlyUsage - } -} diff --git a/packages/types/src/documents/global/quotas.ts b/packages/types/src/documents/global/quotas.ts index d9cc83b46a..84e5af3996 100644 --- a/packages/types/src/documents/global/quotas.ts +++ b/packages/types/src/documents/global/quotas.ts @@ -24,17 +24,27 @@ export interface UsageBreakdown { } } +export type QuotaTriggers = { + [key: string]: string | undefined +} + export interface StaticUsage { [StaticQuotaName.APPS]: number [StaticQuotaName.PLUGINS]: number [StaticQuotaName.USER_GROUPS]: number [StaticQuotaName.ROWS]: number + triggers: { + [key in StaticQuotaName]?: QuotaTriggers + } } export interface MonthlyUsage { [MonthlyQuotaName.QUERIES]: number [MonthlyQuotaName.AUTOMATIONS]: number [MonthlyQuotaName.DAY_PASSES]: number + triggers: { + [key in MonthlyQuotaName]?: QuotaTriggers + } breakdown?: { [key in BreakdownQuotaName]?: UsageBreakdown } @@ -56,6 +66,13 @@ export interface QuotaUsage extends BaseQuotaUsage { } } +export type SetUsageValues = { + total: number + app?: number + breakdown?: number + triggers?: QuotaTriggers +} + export type UsageValues = { total: number app?: number From 84685d3340968809a692b5a99194919c16c0188e Mon Sep 17 00:00:00 2001 From: Rory Powell Date: Mon, 10 Oct 2022 08:21:17 +0100 Subject: [PATCH 4/5] Add locking framework --- packages/backend-core/src/index.ts | 1 + packages/backend-core/src/pkg/redis.ts | 2 + packages/backend-core/src/redis/init.js | 25 ++++---- packages/backend-core/src/redis/redlock.ts | 74 ++++++++++++++++++++-- packages/backend-core/src/redis/utils.js | 1 + packages/server/src/migrations/index.ts | 47 +++++--------- packages/types/src/sdk/index.ts | 1 + packages/types/src/sdk/locks.ts | 31 +++++++++ packages/worker/src/migrations/index.ts | 47 +++++--------- 9 files changed, 149 insertions(+), 80 deletions(-) create mode 100644 packages/types/src/sdk/locks.ts diff --git a/packages/backend-core/src/index.ts b/packages/backend-core/src/index.ts index 83b23b479d..42cad17620 100644 --- a/packages/backend-core/src/index.ts +++ b/packages/backend-core/src/index.ts @@ -37,6 +37,7 @@ const core = { db, ...dbConstants, redis, + locks: redis.redlock, objectStore, utils, users, diff --git a/packages/backend-core/src/pkg/redis.ts b/packages/backend-core/src/pkg/redis.ts index 65ab186d9a..297c2b54f4 100644 --- a/packages/backend-core/src/pkg/redis.ts +++ b/packages/backend-core/src/pkg/redis.ts @@ -3,9 +3,11 @@ import Client from "../redis" import utils from "../redis/utils" import clients from "../redis/init" +import * as redlock from "../redis/redlock" export = { Client, utils, clients, + redlock, } diff --git a/packages/backend-core/src/redis/init.js b/packages/backend-core/src/redis/init.js index 8e5d10f838..3150ef2c1c 100644 --- a/packages/backend-core/src/redis/init.js +++ b/packages/backend-core/src/redis/init.js @@ -1,27 +1,23 @@ const Client = require("./index") const utils = require("./utils") -const { getRedlock } = require("./redlock") -let userClient, sessionClient, appClient, cacheClient, writethroughClient -let migrationsRedlock - -// turn retry off so that only one instance can ever hold the lock -const migrationsRedlockConfig = { retryCount: 0 } +let userClient, + sessionClient, + appClient, + cacheClient, + writethroughClient, + lockClient async function init() { userClient = await new Client(utils.Databases.USER_CACHE).init() sessionClient = await new Client(utils.Databases.SESSIONS).init() appClient = await new Client(utils.Databases.APP_METADATA).init() cacheClient = await new Client(utils.Databases.GENERIC_CACHE).init() + lockClient = await new Client(utils.Databases.LOCKS).init() writethroughClient = await new Client( utils.Databases.WRITE_THROUGH, utils.SelectableDatabases.WRITE_THROUGH ).init() - // pass the underlying ioredis client to redlock - migrationsRedlock = getRedlock( - cacheClient.getClient(), - migrationsRedlockConfig - ) } process.on("exit", async () => { @@ -30,6 +26,7 @@ process.on("exit", async () => { if (appClient) await appClient.finish() if (cacheClient) await cacheClient.finish() if (writethroughClient) await writethroughClient.finish() + if (lockClient) await lockClient.finish() }) module.exports = { @@ -63,10 +60,10 @@ module.exports = { } return writethroughClient }, - getMigrationsRedlock: async () => { - if (!migrationsRedlock) { + getLockClient: async () => { + if (!lockClient) { await init() } - return migrationsRedlock + return lockClient }, } diff --git a/packages/backend-core/src/redis/redlock.ts b/packages/backend-core/src/redis/redlock.ts index beef375b55..abb13b2534 100644 --- a/packages/backend-core/src/redis/redlock.ts +++ b/packages/backend-core/src/redis/redlock.ts @@ -1,14 +1,37 @@ -import Redlock from "redlock" +import Redlock, { Options } from "redlock" +import { getLockClient } from "./init" +import { LockOptions, LockType } from "@budibase/types" +import * as tenancy from "../tenancy" -export const getRedlock = (redisClient: any, opts = { retryCount: 10 }) => { - return new Redlock([redisClient], { +let noRetryRedlock: Redlock | undefined + +const getClient = async (type: LockType): Promise => { + switch (type) { + case LockType.TRY_ONCE: { + if (!noRetryRedlock) { + noRetryRedlock = await newRedlock(OPTIONS.TRY_ONCE) + } + return noRetryRedlock + } + default: { + throw new Error(`Could not get redlock client: ${type}`) + } + } +} + +export const OPTIONS = { + TRY_ONCE: { + // immediately throws an error if the lock is already held + retryCount: 0, + }, + DEFAULT: { // the expected clock drift; for more details // see http://redis.io/topics/distlock driftFactor: 0.01, // multiplied by lock ttl to determine drift time // the max number of times Redlock will attempt // to lock a resource before erroring - retryCount: opts.retryCount, + retryCount: 10, // the time in ms between attempts retryDelay: 200, // time in ms @@ -16,6 +39,45 @@ export const getRedlock = (redisClient: any, opts = { retryCount: 10 }) => { // the max time in ms randomly added to retries // to improve performance under high contention // see https://www.awsarchitectureblog.com/2015/03/backoff.html - retryJitter: 200, // time in ms - }) + retryJitter: 100, // time in ms + }, +} + +export const newRedlock = async (opts: Options = {}) => { + let options = { ...OPTIONS.DEFAULT, ...opts } + const redisWrapper = await getLockClient() + const client = redisWrapper.getClient() + return new Redlock([client], options) +} + +export const doWithLock = async (opts: LockOptions, task: any) => { + const redlock = await getClient(opts.type) + let lock + try { + // aquire lock + let name: string = `${tenancy.getTenantId()}_${opts.name}` + if (opts.nameSuffix) { + name = name + `_${opts.nameSuffix}` + } + lock = await redlock.lock(name, opts.ttl) + // perform locked task + return task() + } catch (e: any) { + // lock limit exceeded + if (e.name === "LockError") { + if (opts.type === LockType.TRY_ONCE) { + // don't throw for try-once locks, they will always error + // due to retry count (0) exceeded + return + } else { + throw e + } + } else { + throw e + } + } finally { + if (lock) { + await lock.unlock() + } + } } diff --git a/packages/backend-core/src/redis/utils.js b/packages/backend-core/src/redis/utils.js index 90b3561f31..af719197b5 100644 --- a/packages/backend-core/src/redis/utils.js +++ b/packages/backend-core/src/redis/utils.js @@ -28,6 +28,7 @@ exports.Databases = { LICENSES: "license", GENERIC_CACHE: "data_cache", WRITE_THROUGH: "writeThrough", + LOCKS: "locks", } /** diff --git a/packages/server/src/migrations/index.ts b/packages/server/src/migrations/index.ts index cb1e6d1c82..275a954a78 100644 --- a/packages/server/src/migrations/index.ts +++ b/packages/server/src/migrations/index.ts @@ -1,5 +1,11 @@ -import { migrations, redis } from "@budibase/backend-core" -import { Migration, MigrationOptions, MigrationName } from "@budibase/types" +import { locks, migrations } from "@budibase/backend-core" +import { + Migration, + MigrationOptions, + MigrationName, + LockType, + LockName, +} from "@budibase/types" import env from "../environment" // migration functions @@ -86,33 +92,14 @@ export const migrate = async (options?: MigrationOptions) => { } const migrateWithLock = async (options?: MigrationOptions) => { - // get a new lock client - const redlock = await redis.clients.getMigrationsRedlock() - // lock for 15 minutes - const ttl = 1000 * 60 * 15 - - let migrationLock - - // acquire lock - try { - migrationLock = await redlock.lock("migrations", ttl) - } catch (e: any) { - if (e.name === "LockError") { - return - } else { - throw e + await locks.doWithLock( + { + type: LockType.TRY_ONCE, + name: LockName.MIGRATIONS, + ttl: 1000 * 60 * 15, // auto expire the migration lock after 15 minutes + }, + async () => { + await migrations.runMigrations(MIGRATIONS, options) } - } - - // run migrations - try { - await migrations.runMigrations(MIGRATIONS, options) - } finally { - // release lock - try { - await migrationLock.unlock() - } catch (e) { - console.error("unable to release migration lock") - } - } + ) } diff --git a/packages/types/src/sdk/index.ts b/packages/types/src/sdk/index.ts index bae566b42e..0c374dd105 100644 --- a/packages/types/src/sdk/index.ts +++ b/packages/types/src/sdk/index.ts @@ -7,3 +7,4 @@ export * from "./datasources" export * from "./search" export * from "./koa" export * from "./auth" +export * from "./locks" diff --git a/packages/types/src/sdk/locks.ts b/packages/types/src/sdk/locks.ts new file mode 100644 index 0000000000..3aa067bea1 --- /dev/null +++ b/packages/types/src/sdk/locks.ts @@ -0,0 +1,31 @@ +export enum LockType { + /** + * If this lock is already held the attempted operation will not be performed. + * No retries will take place and no error will be thrown. + */ + TRY_ONCE = "try_once", +} + +export enum LockName { + MIGRATIONS = "migrations", + TRIGGER_QUOTA = "trigger_quota", +} + +export interface LockOptions { + /** + * The lock type determines which client to use + */ + type: LockType + /** + * The name for the lock + */ + name: LockName + /** + * The ttl to auto-expire the lock if not unlocked manually + */ + ttl: number + /** + * The suffix to add to the lock name for additional uniqueness + */ + nameSuffix?: string +} diff --git a/packages/worker/src/migrations/index.ts b/packages/worker/src/migrations/index.ts index 6900596216..19ef076a52 100644 --- a/packages/worker/src/migrations/index.ts +++ b/packages/worker/src/migrations/index.ts @@ -1,5 +1,11 @@ -import { migrations, redis } from "@budibase/backend-core" -import { Migration, MigrationOptions, MigrationName } from "@budibase/types" +import { migrations, locks } from "@budibase/backend-core" +import { + Migration, + MigrationOptions, + MigrationName, + LockType, + LockName, +} from "@budibase/types" import env from "../environment" // migration functions @@ -42,33 +48,14 @@ export const migrate = async (options?: MigrationOptions) => { } const migrateWithLock = async (options?: MigrationOptions) => { - // get a new lock client - const redlock = await redis.clients.getMigrationsRedlock() - // lock for 15 minutes - const ttl = 1000 * 60 * 15 - - let migrationLock - - // acquire lock - try { - migrationLock = await redlock.lock("migrations", ttl) - } catch (e: any) { - if (e.name === "LockError") { - return - } else { - throw e + await locks.doWithLock( + { + type: LockType.TRY_ONCE, + name: LockName.MIGRATIONS, + ttl: 1000 * 60 * 15, // auto expire the migration lock after 15 minutes + }, + async () => { + await migrations.runMigrations(MIGRATIONS, options) } - } - - // run migrations - try { - await migrations.runMigrations(MIGRATIONS, options) - } finally { - // release lock - try { - await migrationLock.unlock() - } catch (e) { - console.error("unable to release migration lock") - } - } + ) } From f745be1b9c472f94ef7ee5fa04a01abc64ce6010 Mon Sep 17 00:00:00 2001 From: Rory Powell Date: Wed, 12 Oct 2022 11:38:16 +0100 Subject: [PATCH 5/5] Improve documentation on quota.triggers[] --- packages/types/src/sdk/licensing/quota.ts | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/packages/types/src/sdk/licensing/quota.ts b/packages/types/src/sdk/licensing/quota.ts index 6e58734925..74777d4590 100644 --- a/packages/types/src/sdk/licensing/quota.ts +++ b/packages/types/src/sdk/licensing/quota.ts @@ -89,5 +89,12 @@ export type Quotas = { export interface Quota { name: string value: number + /** + * Array of whole numbers (1-100) that dictate the percentage that this quota should trigger + * at in relation to the corresponding usage inside budibase. + * + * Triggering results in a budibase installation sending a request to account-portal, + * which can have subsequent effects such as sending emails to users. + */ triggers: number[] }