MirrorSave
/
budibase
mirror of https://github.com/Budibase/budibase.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

automation runs quotas

This commit is contained in:
Martin McKeaveney 2021-09-23 23:25:25 +01:00
parent 4a2028c354
commit c49b88b57a
12 changed files with 184 additions and 159 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
packages/server/src/api/routes/application.js
View File

@ -22,6 +22,11 @@ router
authorized(BUILDER),
controller.revertClient
)
.delete("/api/applications/:appId", authorized(BUILDER), usage, controller.delete)
.delete(
"/api/applications/:appId",
authorized(BUILDER),
usage,
controller.delete
)
module.exports = router

5
packages/server/src/api/routes/user.js
View File

@ -5,7 +5,6 @@ const {
PermissionLevels,
PermissionTypes,
} = require("@budibase/auth/permissions")
const usage = require("../../middleware/usageQuota")
const router = Router()
@ -28,13 +27,13 @@ router
.post(
"/api/users/metadata/self",
authorized(PermissionTypes.USER, PermissionLevels.WRITE),
usage,
// usage,
controller.updateSelfMetadata
)
.delete(
"/api/users/metadata/:id",
authorized(PermissionTypes.USER, PermissionLevels.WRITE),
usage,
// usage,
controller.destroyMetadata
)

4
packages/server/src/automations/steps/createRow.js
View File

@ -60,7 +60,7 @@ exports.definition = {
},
}
exports.run = async function ({ inputs, appId, tenantId, emitter }) {
exports.run = async function ({ inputs, appId, emitter }) {
if (inputs.row == null || inputs.row.tableId == null) {
return {
success: false,
@ -84,7 +84,7 @@ exports.run = async function ({ inputs, appId, tenantId, emitter }) {
inputs.row
)
if (env.USE_QUOTAS) {
await usage.update(tenantId, usage.Properties.ROW, 1)
await usage.update(usage.Properties.ROW, 1)
}
await rowController.save(ctx)
return {

4
packages/server/src/automations/steps/deleteRow.js
View File

@ -52,7 +52,7 @@ exports.definition = {
},
}
exports.run = async function ({ inputs, appId, apiKey, emitter }) {
exports.run = async function ({ inputs, appId, emitter }) {
if (inputs.id == null || inputs.revision == null) {
return {
success: false,
@ -74,7 +74,7 @@ exports.run = async function ({ inputs, appId, apiKey, emitter }) {
try {
if (env.isProd()) {
await usage.update(apiKey, usage.Properties.ROW, -1)
await usage.update(usage.Properties.ROW, -1)
}
await rowController.destroy(ctx)
return {

7
packages/server/src/automations/thread.js
View File

@ -6,6 +6,8 @@ const { DEFAULT_TENANT_ID } = require("@budibase/auth").constants
const CouchDB = require("../db")
const { DocumentTypes } = require("../db/utils")
const { doInTenant } = require("@budibase/auth/tenancy")
const env = require("../environment")
const usage = require("../utilities/usageQuota")
const FILTER_STEP_ID = actions.ACTION_DEFINITIONS.FILTER.stepId
@ -80,7 +82,6 @@ class Orchestrator {
return stepFn({
inputs: step.inputs,
appId: this._appId,
apiKey: automation.apiKey,
emitter: this._emitter,
context: this._context,
})
@ -95,6 +96,10 @@ class Orchestrator {
return err
}
}
// TODO: don't count test runs
if (!env.SELF_HOSTED) {
usage.update(usage.Properties.AUTOMATION, 1)
}
return this.executionOutput
}
}

6
packages/server/src/integrations/mysql.ts
View File

@ -12,7 +12,11 @@ import { getSqlQuery } from "./utils"
module MySQLModule {
const mysql = require("mysql")
const Sql = require("./base/sql")
const { buildExternalTableId, convertType, copyExistingPropsOver } = require("./utils")
const {
buildExternalTableId,
convertType,
copyExistingPropsOver,
} = require("./utils")
const { FieldTypes } = require("../constants")
interface MySQLConfig {

18
packages/server/src/integrations/postgres.ts
View File

@ -12,7 +12,11 @@ module PostgresModule {
const { Pool } = require("pg")
const Sql = require("./base/sql")
const { FieldTypes } = require("../constants")
const { buildExternalTableId, convertType, copyExistingPropsOver } = require("./utils")
const {
buildExternalTableId,
convertType,
copyExistingPropsOver,
} = require("./utils")
interface PostgresConfig {
host: string
@ -179,10 +183,16 @@ module PostgresModule {
}
const type: string = convertType(column.data_type, TYPE_MAP)
const identity = !!(column.identity_generation || column.identity_start || column.identity_increment)
const hasDefault = typeof column.column_default === "string" &&
const identity = !!(
column.identity_generation ||
column.identity_start ||
column.identity_increment
)
const hasDefault =
typeof column.column_default === "string" &&
column.column_default.startsWith("nextval")
const isGenerated = column.is_generated && column.is_generated !== "NEVER"
const isGenerated =
column.is_generated && column.is_generated !== "NEVER"
const isAuto: boolean = hasDefault || identity || isGenerated
tables[tableName].schema[columnName] = {
autocolumn: isAuto,

6
packages/server/src/integrations/utils.ts
View File

@ -84,7 +84,11 @@ export function isIsoDateString(str: string) {
}
// add the existing relationships from the entities if they exist, to prevent them from being overridden
export function copyExistingPropsOver(tableName: string, tables: { [key: string]: any }, entities: { [key: string]: any }) {
export function copyExistingPropsOver(
tableName: string,
tables: { [key: string]: any },
entities: { [key: string]: any }
) {
if (entities && entities[tableName]) {
if (entities[tableName].primaryDisplay) {
tables[tableName].primaryDisplay = entities[tableName].primaryDisplay

2
packages/server/src/middleware/usageQuota.js
View File

@ -1,6 +1,6 @@
const CouchDB = require("../db")
const usageQuota = require("../utilities/usageQuota")
const env = require("../environment")
// const env = require("../environment")
// currently only counting new writes and deletes
const METHOD_MAP = {

45
packages/server/src/utilities/usageQuota.js
View File

@ -1,5 +1,4 @@
const env = require("../environment")
// const env = require("../environment")
const { getGlobalDB } = require("@budibase/auth/tenancy")
function getNewQuotaReset() {
@ -12,44 +11,44 @@ exports.Properties = {
VIEW: "views",
USER: "users",
AUTOMATION: "automationRuns",
APPS: "apps"
APPS: "apps",
}
/**
* Given a specified tenantId this will add to the usage object for the specified property.
* @param {string} tenantId The tenant to update the usage quotas for.
* @param {string} property The property which is to be added to (within the nested usageQuota object).
* @param {number} usage The amount (this can be negative) to adjust the number by.
* @returns {Promise<void>} When this completes the API key will now be up to date - the quota period may have
* also been reset after this call.
*/
exports.update = async (tenantId, property, usage) => {
exports.update = async (property, usage) => {
// if (!env.USE_QUOTAS) {
// return
// }
try {
const db = getGlobalDB()
const quota = await db.get("usage_quota")
// TODO: check if the quota needs reset
if (Date.now() >= quota.quotaReset) {
quota.quotaReset = getNewQuotaReset()
for (let prop of Object.keys(quota.usageQuota)) {
quota.usageQuota[prop] = 0
}
}
const db = getGlobalDB()
const quota = await db.get("usage_quota")
// TODO: check if the quota needs reset
if (Date.now() >= quota.quotaReset) {
quota.quotaReset = getNewQuotaReset()
for (let prop of Object.keys(quota.usageQuota)) {
quota.usageQuota[prop] = 0
}
}
// increment the quota
quota.usageQuota[property] += usage
// increment the quota
quota.usageQuota[property] += usage
if (quota.usageQuota[property] >= quota.usageLimits[property]) {
throw new Error(`You have exceeded your usage quota of ${quota.usageLimits[property]} ${property}.`)
}
if (quota.usageQuota[property] >= quota.usageLimits[property]) {
throw new Error(
`You have exceeded your usage quota of ${quota.usageLimits[property]} ${property}.`
)
}
// update the usage quotas
await db.put(quota)
// update the usage quotas
await db.put(quota)
} catch (err) {
console.error(`Error updating usage quotas for ${property}`, err)
console.error(`Error updating usage quotas for ${property}`, err)
throw err
}
}

196
packages/server/src/utilities/usageQuota.old.js
View File

@ -1,105 +1,105 @@
const env = require("../environment")
const { apiKeyTable } = require("../db/dynamoClient")
// const env = require("../environment")
// const { apiKeyTable } = require("../db/dynamoClient")
const DEFAULT_USAGE = {
rows: 0,
storage: 0,
views: 0,
automationRuns: 0,
users: 0,
}
// const DEFAULT_USAGE = {
// rows: 0,
// storage: 0,
// views: 0,
// automationRuns: 0,
// users: 0,
// }
const DEFAULT_PLAN = {
rows: 1000,
// 1 GB
storage: 8589934592,
views: 10,
automationRuns: 100,
users: 10000,
}
// const DEFAULT_PLAN = {
// rows: 1000,
// // 1 GB
// storage: 8589934592,
// views: 10,
// automationRuns: 100,
// users: 10000,
// }
function buildUpdateParams(key, property, usage) {
return {
primary: key,
condition:
"attribute_exists(#quota) AND attribute_exists(#limits) AND #quota.#prop < #limits.#prop AND #quotaReset > :now",
expression: "ADD #quota.#prop :usage",
names: {
"#quota": "usageQuota",
"#prop": property,
"#limits": "usageLimits",
"#quotaReset": "quotaReset",
},
values: {
":usage": usage,
":now": Date.now(),
},
}
}
// function buildUpdateParams(key, property, usage) {
// return {
// primary: key,
// condition:
// "attribute_exists(#quota) AND attribute_exists(#limits) AND #quota.#prop < #limits.#prop AND #quotaReset > :now",
// expression: "ADD #quota.#prop :usage",
// names: {
// "#quota": "usageQuota",
// "#prop": property,
// "#limits": "usageLimits",
// "#quotaReset": "quotaReset",
// },
// values: {
// ":usage": usage,
// ":now": Date.now(),
// },
// }
// }
function getNewQuotaReset() {
return Date.now() + 2592000000
}
// function getNewQuotaReset() {
// return Date.now() + 2592000000
// }
exports.Properties = {
ROW: "rows",
UPLOAD: "storage",
VIEW: "views",
USER: "users",
AUTOMATION: "automationRuns",
}
// exports.Properties = {
// ROW: "rows",
// UPLOAD: "storage",
// VIEW: "views",
// USER: "users",
// AUTOMATION: "automationRuns",
// }
exports.getAPIKey = async appId => {
if (!env.USE_QUOTAS) {
return { apiKey: null }
}
return apiKeyTable.get({ primary: appId })
}
// exports.getAPIKey = async appId => {
// if (!env.USE_QUOTAS) {
// return { apiKey: null }
// }
// return apiKeyTable.get({ primary: appId })
// }
/**
* Given a specified API key this will add to the usage object for the specified property.
* @param {string} apiKey The API key which is to be updated.
* @param {string} property The property which is to be added to (within the nested usageQuota object).
* @param {number} usage The amount (this can be negative) to adjust the number by.
* @returns {Promise<void>} When this completes the API key will now be up to date - the quota period may have
* also been reset after this call.
*/
exports.update = async (apiKey, property, usage) => {
if (!env.USE_QUOTAS) {
return
}
try {
await apiKeyTable.update(buildUpdateParams(apiKey, property, usage))
} catch (err) {
// conditional check means the condition failed, need to check why
if (err.code === "ConditionalCheckFailedException") {
// get the API key so we can check it
const keyObj = await apiKeyTable.get({ primary: apiKey })
// the usage quota or usage limits didn't exist
if (keyObj && (keyObj.usageQuota == null || keyObj.usageLimits == null)) {
keyObj.usageQuota =
keyObj.usageQuota == null ? DEFAULT_USAGE : keyObj.usageQuota
keyObj.usageLimits =
keyObj.usageLimits == null ? DEFAULT_PLAN : keyObj.usageLimits
keyObj.quotaReset = getNewQuotaReset()
await apiKeyTable.put({ item: keyObj })
return
}
// we have in fact breached the reset period
else if (keyObj && keyObj.quotaReset <= Date.now()) {
// update the quota reset period and reset the values for all properties
keyObj.quotaReset = getNewQuotaReset()
for (let prop of Object.keys(keyObj.usageQuota)) {
if (prop === property) {
keyObj.usageQuota[prop] = usage > 0 ? usage : 0
} else {
keyObj.usageQuota[prop] = 0
}
}
await apiKeyTable.put({ item: keyObj })
return
}
}
throw err
}
}
// /**
// * Given a specified API key this will add to the usage object for the specified property.
// * @param {string} apiKey The API key which is to be updated.
// * @param {string} property The property which is to be added to (within the nested usageQuota object).
// * @param {number} usage The amount (this can be negative) to adjust the number by.
// * @returns {Promise<void>} When this completes the API key will now be up to date - the quota period may have
// * also been reset after this call.
// */
// exports.update = async (apiKey, property, usage) => {
// if (!env.USE_QUOTAS) {
// return
// }
// try {
// await apiKeyTable.update(buildUpdateParams(apiKey, property, usage))
// } catch (err) {
// // conditional check means the condition failed, need to check why
// if (err.code === "ConditionalCheckFailedException") {
// // get the API key so we can check it
// const keyObj = await apiKeyTable.get({ primary: apiKey })
// // the usage quota or usage limits didn't exist
// if (keyObj && (keyObj.usageQuota == null || keyObj.usageLimits == null)) {
// keyObj.usageQuota =
// keyObj.usageQuota == null ? DEFAULT_USAGE : keyObj.usageQuota
// keyObj.usageLimits =
// keyObj.usageLimits == null ? DEFAULT_PLAN : keyObj.usageLimits
// keyObj.quotaReset = getNewQuotaReset()
// await apiKeyTable.put({ item: keyObj })
// return
// }
// // we have in fact breached the reset period
// else if (keyObj && keyObj.quotaReset <= Date.now()) {
// // update the quota reset period and reset the values for all properties
// keyObj.quotaReset = getNewQuotaReset()
// for (let prop of Object.keys(keyObj.usageQuota)) {
// if (prop === property) {
// keyObj.usageQuota[prop] = usage > 0 ? usage : 0
// } else {
// keyObj.usageQuota[prop] = 0
// }
// }
// await apiKeyTable.put({ item: keyObj })
// return
// }
// }
// throw err
// }
// }

43
packages/worker/src/api/controllers/global/users.js
View File

@ -1,7 +1,6 @@
const {
generateGlobalUserID,
getGlobalUserParams,
StaticDatabases,
} = require("@budibase/auth/db")
const { hash, getGlobalUserByEmail } = require("@budibase/auth").utils
@ -18,7 +17,7 @@ const {
tryAddTenant,
updateTenantId,
} = require("@budibase/auth/tenancy")
const env = require("../../../environment")
// const env = require("../../../environment")
const PLATFORM_INFO_DB = StaticDatabases.PLATFORM_INFO.name
@ -139,28 +138,28 @@ exports.adminUser = async ctx => {
include_docs: true,
})
)
// write usage quotas for cloud
// if (!env.SELF_HOSTED) {
await db.post({
_id: "usage_quota",
quotaReset: Date.now() + 2592000000,
usageQuota: {
automationRuns: 0,
rows: 0,
storage: 0,
apps: 0,
users: 0,
views: 0,
},
usageLimits: {
automationRuns: 1000,
rows: 4000,
apps: 4,
// storage: 1000,
// users: 10
},
})
await db.post({
_id: "usage_quota",
quotaReset: Date.now() + 2592000000,
usageQuota: {
automationRuns: 0,
rows: 0,
storage: 0,
apps: 0,
users: 0,
views: 0,
},
usageLimits: {
automationRuns: 1000,
rows: 4000,
apps: 4,
storage: 1000,
users: 10
},
})
// }
if (response.rows.some(row => row.doc.admin)) {