MirrorSave
/
budibase
mirror of https://github.com/Budibase/budibase.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Integrate usage quotas with licensing

This commit is contained in:
Rory Powell 2022-03-08 14:21:41 +00:00
parent 995f7c4b58
commit d6092b9133
16 changed files with 210 additions and 324 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

25
packages/backend-core/src/db/utils.js
View File

@ -425,33 +425,8 @@ async function getScopedConfig(db, params) {
return configDoc && configDoc.config ? configDoc.config : configDoc
}
function generateNewUsageQuotaDoc() {
return {
_id: StaticDatabases.GLOBAL.docs.usageQuota,
quotaReset: Date.now() + 2592000000,
usageQuota: {
automationRuns: 0,
rows: 0,
storage: 0,
apps: 0,
users: 0,
views: 0,
emails: 0,
},
usageLimits: {
automationRuns: 1000,
rows: 4000,
apps: 4,
storage: 1000,
users: 10,
emails: 50,
},
}
}
exports.Replication = Replication
exports.getScopedConfig = getScopedConfig
exports.generateConfigID = generateConfigID
exports.getConfigParams = getConfigParams
exports.getScopedFullConfig = getScopedFullConfig
exports.generateNewUsageQuotaDoc = generateNewUsageQuotaDoc

2
packages/backend-core/src/environment.js
View File

@ -28,6 +28,8 @@ module.exports = {
SELF_HOSTED: !!parseInt(process.env.SELF_HOSTED),
COOKIE_DOMAIN: process.env.COOKIE_DOMAIN,
PLATFORM_URL: process.env.PLATFORM_URL,
USE_QUOTAS: process.env.USE_QUOTAS,
EXCLUDE_QUOTAS_TENANTS: process.env.EXCLUDE_QUOTAS_TENANTS,
isTest,
_set(key, value) {
process.env[key] = value

2
packages/server/nodemon.json
View File

@ -1,5 +1,5 @@
{
"watch": ["src", "../backend-core"],
"watch": ["src", "../backend-core", "../../../budibase-pro/packages/pro"],
"ext": "js,ts,json",
"ignore": ["src/**/*.spec.ts", "src/**/*.spec.js"],
"exec": "ts-node src/index.ts"

8
packages/server/src/api/controllers/table/internal.js
View File

@ -7,7 +7,7 @@ const {
getTable,
handleDataImport,
} = require("./utils")
const usageQuota = require("../../../utilities/usageQuota")
const { quotas, StaticQuotaName, QuotaUsageType } = require("@budibase/pro")
const { getAppDB } = require("@budibase/backend-core/context")
const env = require("../../../environment")
const { cleanupAttachments } = require("../../../utilities/rowProcessor")
@ -120,7 +120,11 @@ exports.destroy = async function (ctx) {
})
)
await db.bulkDocs(rows.rows.map(row => ({ ...row.doc, _deleted: true })))
await usageQuota.update(usageQuota.Properties.ROW, -rows.rows.length)
await quotas.updateUsage(
-rows.rows.length,
StaticQuotaName.ROWS,
QuotaUsageType.STATIC
)
// update linked rows
await linkRows.updateLinks({

17
packages/server/src/api/controllers/table/utils.js
View File

@ -24,7 +24,7 @@ const {
} = require("../../../integrations/utils")
const { getViews, saveView } = require("../view/utils")
const viewTemplate = require("../view/viewBuilder")
const usageQuota = require("../../../utilities/usageQuota")
const { quotas, StaticQuotaName, QuotaUsageType } = require("@budibase/pro")
const { getAppDB } = require("@budibase/backend-core/context")
const { cloneDeep } = require("lodash/fp")
@ -143,11 +143,20 @@ exports.handleDataImport = async (user, table, dataImport) => {
finalData.push(row)
}
await usageQuota.update(usageQuota.Properties.ROW, finalData.length, {
await quotas.updateUsage(
finalData.length,
StaticQuotaName.ROWS,
QuotaUsageType.STATIC,
{
dryRun: true,
})
}
)
await db.bulkDocs(finalData)
await usageQuota.update(usageQuota.Properties.ROW, finalData.length)
await quotas.updateUsage(
finalData.length,
StaticQuotaName.ROWS,
QuotaUsageType.STATIC
)
let response = await db.put(table)
table._rev = response._rev
return table

6
packages/server/src/app.ts
View File

@ -1,12 +1,6 @@
// need to load environment first
import { ExtendableContext } from "koa"
import * as env from "./environment"
// temp for testing
import * as poc from "./pro-poc"
poc.run()
const CouchDB = require("./db")
require("@budibase/backend-core").init(CouchDB)
const Koa = require("koa")

8
packages/server/src/automations/steps/createRow.js
View File

@ -1,6 +1,6 @@
const rowController = require("../../api/controllers/row")
const automationUtils = require("../automationUtils")
const usage = require("../../utilities/usageQuota")
const { quotas, StaticQuotaName, QuotaUsageType } = require("@budibase/pro")
const { buildCtx } = require("./utils")
exports.definition = {
@ -81,9 +81,11 @@ exports.run = async function ({ inputs, appId, emitter }) {
inputs.row.tableId,
inputs.row
)
await usage.update(usage.Properties.ROW, 1, { dryRun: true })
await quotas.updateUsage(1, StaticQuotaName.ROWS, QuotaUsageType.STATIC, {
dryRun: true,
})
await rowController.save(ctx)
await usage.update(usage.Properties.ROW, 1)
await quotas.updateUsage(1, StaticQuotaName.ROWS, QuotaUsageType.STATIC)
return {
row: inputs.row,
response: ctx.body,

4
packages/server/src/automations/steps/deleteRow.js
View File

@ -1,5 +1,5 @@
const rowController = require("../../api/controllers/row")
const usage = require("../../utilities/usageQuota")
const { quotas, StaticQuotaName, QuotaUsageType } = require("@budibase/pro")
const { buildCtx } = require("./utils")
const automationUtils = require("../automationUtils")
@ -73,7 +73,7 @@ exports.run = async function ({ inputs, appId, emitter }) {
})
try {
await usage.update(usage.Properties.ROW, -1)
await quotas.updateUsage(-1, StaticQuotaName.ROWS, QuotaUsageType.STATIC)
await rowController.destroy(ctx)
return {
response: ctx.body,

164
packages/server/src/middleware/usageQuota.js
View File

@ -1,164 +0,0 @@
const usageQuota = require("../utilities/usageQuota")
const { getUniqueRows } = require("../utilities/usageQuota/rows")
const {
isExternalTable,
isRowId: isExternalRowId,
} = require("../integrations/utils")
const { getAppDB } = require("@budibase/backend-core/context")
// currently only counting new writes and deletes
const METHOD_MAP = {
POST: 1,
DELETE: -1,
}
const DOMAIN_MAP = {
rows: usageQuota.Properties.ROW,
// upload: usageQuota.Properties.UPLOAD, // doesn't work yet
// views: usageQuota.Properties.VIEW, // doesn't work yet
// users: usageQuota.Properties.USER, // doesn't work yet
applications: usageQuota.Properties.APPS,
// this will not be updated by endpoint calls
// instead it will be updated by triggerInfo
// automationRuns: usageQuota.Properties.AUTOMATION, // doesn't work yet
}
function getProperty(url) {
for (let domain of Object.keys(DOMAIN_MAP)) {
if (url.indexOf(domain) !== -1) {
return DOMAIN_MAP[domain]
}
}
}
module.exports = async (ctx, next) => {
if (!usageQuota.useQuotas()) {
return next()
}
let usage = METHOD_MAP[ctx.req.method]
const property = getProperty(ctx.req.url)
if (usage == null || property == null) {
return next()
}
// post request could be a save of a pre-existing entry
if (ctx.request.body && ctx.request.body._id && ctx.request.body._rev) {
const usageId = ctx.request.body._id
try {
if (ctx.appId) {
const db = getAppDB()
await db.get(usageId)
}
return next()
} catch (err) {
if (
isExternalTable(usageId) ||
(ctx.request.body.tableId &&
isExternalTable(ctx.request.body.tableId)) ||
isExternalRowId(usageId)
) {
return next()
} else {
ctx.throw(404, `${usageId} does not exist`)
}
}
}
// update usage for uploads to be the total size
if (property === usageQuota.Properties.UPLOAD) {
const files =
ctx.request.files.file.length > 1
? Array.from(ctx.request.files.file)
: [ctx.request.files.file]
usage = files.map(file => file.size).reduce((total, size) => total + size)
}
try {
await performRequest(ctx, next, property, usage)
} catch (err) {
ctx.throw(400, err)
}
}
const performRequest = async (ctx, next, property, usage) => {
const usageContext = {
skipNext: false,
skipUsage: false,
[usageQuota.Properties.APPS]: {},
}
if (usage === -1) {
if (PRE_DELETE[property]) {
await PRE_DELETE[property](ctx, usageContext)
}
} else {
if (PRE_CREATE[property]) {
await PRE_CREATE[property](ctx, usageContext)
}
}
// run the request
if (!usageContext.skipNext) {
await usageQuota.update(property, usage, { dryRun: true })
await next()
}
if (usage === -1) {
if (POST_DELETE[property]) {
await POST_DELETE[property](ctx, usageContext)
}
} else {
if (POST_CREATE[property]) {
await POST_CREATE[property](ctx, usageContext)
}
}
// update the usage
if (!usageContext.skipUsage) {
await usageQuota.update(property, usage)
}
}
const appPreDelete = async (ctx, usageContext) => {
if (ctx.query.unpublish) {
// don't run usage decrement for unpublish
usageContext.skipUsage = true
return
}
// store the row count to delete
const rows = await getUniqueRows([ctx.appId])
if (rows.length) {
usageContext[usageQuota.Properties.APPS] = { rowCount: rows.length }
}
}
const appPostDelete = async (ctx, usageContext) => {
// delete the app rows from usage
const rowCount = usageContext[usageQuota.Properties.APPS].rowCount
if (rowCount) {
await usageQuota.update(usageQuota.Properties.ROW, -rowCount)
}
}
const appPostCreate = async ctx => {
// app import & template creation
if (ctx.request.body.useTemplate === "true") {
const rows = await getUniqueRows([ctx.response.body.appId])
const rowCount = rows ? rows.length : 0
await usageQuota.update(usageQuota.Properties.ROW, rowCount)
}
}
const PRE_DELETE = {
[usageQuota.Properties.APPS]: appPreDelete,
}
const POST_DELETE = {
[usageQuota.Properties.APPS]: appPostDelete,
}
const PRE_CREATE = {}
const POST_CREATE = {
[usageQuota.Properties.APPS]: appPostCreate,
}

170
packages/server/src/middleware/usageQuota.ts Normal file
View File

@ -0,0 +1,170 @@
import { quotas, StaticQuotaName, QuotaUsageType } from "@budibase/pro"
const { getUniqueRows } = require("../utilities/usageQuota/rows")
const {
isExternalTable,
isRowId: isExternalRowId,
} = require("../integrations/utils")
const { getAppDB } = require("@budibase/backend-core/context")
// currently only counting new writes and deletes
const METHOD_MAP: any = {
POST: 1,
DELETE: -1,
}
const DOMAIN_MAP: any = {
rows: {
name: StaticQuotaName.ROWS,
type: QuotaUsageType.STATIC,
},
applications: {
name: StaticQuotaName.APPS,
type: QuotaUsageType.STATIC,
},
}
function getQuotaInfo(url: string) {
for (let domain of Object.keys(DOMAIN_MAP)) {
if (url.indexOf(domain) !== -1) {
return DOMAIN_MAP[domain]
}
}
}
module.exports = async (ctx: any, next: any) => {
if (!quotas.useQuotas()) {
return next()
}
let usage = METHOD_MAP[ctx.req.method]
const quotaInfo = getQuotaInfo(ctx.req.url)
if (usage == null || quotaInfo == null) {
return next()
}
// post request could be a save of a pre-existing entry
if (ctx.request.body && ctx.request.body._id && ctx.request.body._rev) {
const usageId = ctx.request.body._id
try {
if (ctx.appId) {
const db = getAppDB()
await db.get(usageId)
}
return next()
} catch (err) {
if (
isExternalTable(usageId) ||
(ctx.request.body.tableId &&
isExternalTable(ctx.request.body.tableId)) ||
isExternalRowId(usageId)
) {
return next()
} else {
ctx.throw(404, `${usageId} does not exist`)
}
}
}
try {
await performRequest(ctx, next, quotaInfo, usage)
} catch (err) {
ctx.throw(400, err)
}
}
const performRequest = async (
ctx: any,
next: any,
quotaInfo: any,
usage: number
) => {
const usageContext = {
skipNext: false,
skipUsage: false,
[StaticQuotaName.APPS]: {},
}
const quotaName = quotaInfo.name
if (usage === -1) {
if (PRE_DELETE[quotaName]) {
await PRE_DELETE[quotaName](ctx, usageContext)
}
} else {
if (PRE_CREATE[quotaName]) {
await PRE_CREATE[quotaName](ctx, usageContext)
}
}
// run the request
if (!usageContext.skipNext) {
await quotas.updateUsage(usage, quotaName, quotaInfo.type, { dryRun: true })
await next()
}
if (usage === -1) {
if (POST_DELETE[quotaName]) {
await POST_DELETE[quotaName](ctx, usageContext)
}
} else {
if (POST_CREATE[quotaName]) {
await POST_CREATE[quotaName](ctx)
}
}
// update the usage
if (!usageContext.skipUsage) {
await quotas.updateUsage(usage, quotaName, quotaInfo.type)
}
}
const appPreDelete = async (ctx: any, usageContext: any) => {
if (ctx.query.unpublish) {
// don't run usage decrement for unpublish
usageContext.skipUsage = true
return
}
// store the row count to delete
const rows = await getUniqueRows([ctx.appId])
if (rows.length) {
usageContext[StaticQuotaName.APPS] = { rowCount: rows.length }
}
}
const appPostDelete = async (ctx: any, usageContext: any) => {
// delete the app rows from usage
const rowCount = usageContext[StaticQuotaName.ROWS].rowCount
if (rowCount) {
await quotas.updateUsage(
-rowCount,
StaticQuotaName.ROWS,
QuotaUsageType.STATIC
)
}
}
const appPostCreate = async (ctx: any) => {
// app import & template creation
if (ctx.request.body.useTemplate === "true") {
const rows = await getUniqueRows([ctx.response.body.appId])
const rowCount = rows ? rows.length : 0
await quotas.updateUsage(
rowCount,
StaticQuotaName.ROWS,
QuotaUsageType.STATIC
)
}
}
const PRE_DELETE: any = {
[StaticQuotaName.APPS]: appPreDelete,
}
const POST_DELETE: any = {
[StaticQuotaName.APPS]: appPostDelete,
}
const PRE_CREATE: any = {}
const POST_CREATE: any = {
[StaticQuotaName.APPS]: appPostCreate,
}

4
packages/server/src/migrations/functions/usageQuotas/index.ts
View File

@ -1,7 +1,7 @@
const { useQuotas } = require("../../../utilities/usageQuota")
import { quotas } from "@budibase/pro"
export const runQuotaMigration = async (migration: Function) => {
if (!useQuotas()) {
if (!quotas.useQuotas()) {
return
}
await migration()

9
packages/server/src/migrations/functions/usageQuotas/syncApps.ts
View File

@ -1,9 +1,8 @@
import { getGlobalDB, getTenantId } from "@budibase/backend-core/tenancy"
import { getTenantId } from "@budibase/backend-core/tenancy"
import { getAllApps } from "@budibase/backend-core/db"
import { getUsageQuotaDoc } from "../../../utilities/usageQuota"
import { quotas, QuotaUsageType, StaticQuotaName } from "@budibase/pro"
export const run = async () => {
const db = getGlobalDB()
// get app count
// @ts-ignore
const devApps = await getAllApps({ dev: true })
@ -12,7 +11,5 @@ export const run = async () => {
// sync app count
const tenantId = getTenantId()
console.log(`[Tenant: ${tenantId}] Syncing app count: ${appCount}`)
const usageDoc = await getUsageQuotaDoc(db)
usageDoc.usageQuota.apps = appCount
await db.put(usageDoc)
await quotas.setUsage(appCount, StaticQuotaName.APPS, QuotaUsageType.STATIC)
}

9
packages/server/src/migrations/functions/usageQuotas/syncRows.ts
View File

@ -1,10 +1,9 @@
import { getGlobalDB, getTenantId } from "@budibase/backend-core/tenancy"
import { getTenantId } from "@budibase/backend-core/tenancy"
import { getAllApps } from "@budibase/backend-core/db"
import { getUsageQuotaDoc } from "../../../utilities/usageQuota"
import { quotas, QuotaUsageType, StaticQuotaName } from "@budibase/pro"
import { getUniqueRows } from "../../../utilities/usageQuota/rows"
export const run = async () => {
const db = getGlobalDB()
// get all rows in all apps
// @ts-ignore
const allApps = await getAllApps({ all: true })
@ -16,7 +15,5 @@ export const run = async () => {
// sync row count
const tenantId = getTenantId()
console.log(`[Tenant: ${tenantId}] Syncing row count: ${rowCount}`)
const usageDoc = await getUsageQuotaDoc(db)
usageDoc.usageQuota.rows = rowCount
await db.put(usageDoc)
await quotas.setUsage(rowCount, StaticQuotaName.ROWS, QuotaUsageType.STATIC)
}

2
packages/server/src/migrations/index.ts
View File

@ -28,7 +28,7 @@ export interface Migration {
*/
export interface MigrationOptions {
tenantIds?: string[]
forced?: {
force?: {
[type: string]: string[]
}
}

9
packages/server/src/threads/automation.js
View File

@ -1,13 +1,11 @@
require("./utils").threadSetup()
const env = require("../environment")
const actions = require("../automations/actions")
const automationUtils = require("../automations/automationUtils")
const AutomationEmitter = require("../events/AutomationEmitter")
const { processObject } = require("@budibase/string-templates")
const { DEFAULT_TENANT_ID } = require("@budibase/backend-core/constants")
const { DocumentTypes, isDevAppID } = require("../db/utils")
const { DocumentTypes } = require("../db/utils")
const { doInTenant } = require("@budibase/backend-core/tenancy")
const usage = require("../utilities/usageQuota")
const { definitions: triggerDefs } = require("../automations/triggerInfo")
const { doInAppContext, getAppDB } = require("@budibase/backend-core/context")
@ -120,11 +118,6 @@ class Orchestrator {
return err
}
}
// Increment quota for automation runs
if (!env.SELF_HOSTED && !isDevAppID(this._appId)) {
await usage.update(usage.Properties.AUTOMATION, 1)
}
return this.executionOutput
}
}

93
packages/server/src/utilities/usageQuota/index.js
View File

@ -1,93 +0,0 @@
const env = require("../../environment")
const { getGlobalDB, getTenantId } = require("@budibase/backend-core/tenancy")
const {
StaticDatabases,
generateNewUsageQuotaDoc,
} = require("@budibase/backend-core/db")
exports.useQuotas = () => {
// check if quotas are enabled
if (env.USE_QUOTAS) {
// check if there are any tenants without limits
if (env.EXCLUDE_QUOTAS_TENANTS) {
const excludedTenants = env.EXCLUDE_QUOTAS_TENANTS.replace(
/\s/g,
""
).split(",")
const tenantId = getTenantId()
if (excludedTenants.includes(tenantId)) {
return false
}
}
return true
}
return false
}
exports.Properties = {
ROW: "rows",
UPLOAD: "storage", // doesn't work yet
VIEW: "views", // doesn't work yet
USER: "users", // doesn't work yet
AUTOMATION: "automationRuns", // doesn't work yet
APPS: "apps",
EMAILS: "emails", // doesn't work yet
}
exports.getUsageQuotaDoc = async db => {
let quota
try {
quota = await db.get(StaticDatabases.GLOBAL.docs.usageQuota)
} catch (err) {
// doc doesn't exist. Create it
quota = generateNewUsageQuotaDoc()
const response = await db.put(quota)
quota._rev = response.rev
}
return quota
}
/**
* Given a specified tenantId this will add to the usage object for the specified property.
* @param {string} property The property which is to be added to (within the nested usageQuota object).
* @param {number} usage The amount (this can be negative) to adjust the number by.
* @param {object} opts optional - options such as dryRun, to check what update will do.
* @returns {Promise<void>} When this completes the API key will now be up to date - the quota period may have
* also been reset after this call.
*/
exports.update = async (property, usage, opts = { dryRun: false }) => {
if (!exports.useQuotas()) {
return
}
try {
const db = getGlobalDB()
const quota = await exports.getUsageQuotaDoc(db)
// increment the quota
quota.usageQuota[property] += usage
if (
quota.usageQuota[property] > quota.usageLimits[property] &&
usage > 0 // allow for decrementing usage when the quota is already exceeded
) {
throw new Error(
`You have exceeded your usage quota of ${quota.usageLimits[property]} ${property}.`
)
}
if (quota.usageQuota[property] < 0) {
// never go negative if the quota has previously been exceeded
quota.usageQuota[property] = 0
}
// update the usage quotas
if (!opts.dryRun) {
await db.put(quota)
}
} catch (err) {
console.error(`Error updating usage quotas for ${property}`, err)
throw err
}
}