Merge branch 'api-usage-tracking' of github.com:Budibase/budibase into deployment-api-usage

This commit is contained in:
Martin McKeaveney 2020-10-09 11:21:03 +01:00
commit 05206b6049
24 changed files with 410 additions and 37 deletions

View File

@ -46,7 +46,7 @@
"@koa/router": "^8.0.0",
"@sendgrid/mail": "^7.1.1",
"@sentry/node": "^5.19.2",
"aws-sdk": "^2.706.0",
"aws-sdk": "^2.767.0",
"bcryptjs": "^2.4.3",
"chmodr": "^1.2.0",
"csvtojson": "^2.0.10",

View File

@ -0,0 +1,55 @@
// THIS will create API Keys and App Ids input in a local Dynamo instance if it is running
const dynamoClient = require("../src/db/dynamoClient")
if (process.argv[2] == null || process.argv[3] == null) {
console.error(
"Inputs incorrect format, was expecting: node createApiKeyAndAppId.js <API_KEY> <APP_ID>"
)
process.exit(-1)
}
const FAKE_STRING = "fakestring"
// set fake credentials for local dynamo to actually work
process.env.AWS_ACCESS_KEY_ID = "KEY_ID"
process.env.AWS_SECRET_ACCESS_KEY = "SECRET_KEY"
dynamoClient.init("http://localhost:8333")
async function run() {
await dynamoClient.apiKeyTable.put({
item: {
pk: process.argv[2],
accountId: FAKE_STRING,
trackingId: FAKE_STRING,
quotaReset: Date.now() + 2592000000,
usageQuota: {
automationRuns: 0,
records: 0,
storage: 0,
users: 0,
views: 0,
},
usageLimits: {
automationRuns: 10,
records: 10,
storage: 1000,
users: 10,
views: 10,
},
},
})
await dynamoClient.apiKeyTable.put({
item: {
pk: process.argv[3],
apiKey: process.argv[2],
},
})
}
run()
.then(() => {
console.log("Records should have been created.")
})
.catch(err => {
console.error("Cannot create records - " + err)
})

View File

@ -2,6 +2,8 @@ const jwt = require("jsonwebtoken")
const CouchDB = require("../../db")
const ClientDb = require("../../db/clientDb")
const bcrypt = require("../../utilities/bcrypt")
const environment = require("../../environment")
const { getAPIKey } = require("../../utilities/usageQuota")
const { generateUserID } = require("../../db/utils")
exports.authenticate = async ctx => {
@ -51,6 +53,10 @@ exports.authenticate = async ctx => {
appId: ctx.user.appId,
instanceId,
}
// if in cloud add the user api key
if (environment.CLOUD) {
payload.apiKey = getAPIKey(ctx.user.appId)
}
const token = jwt.sign(payload, ctx.config.jwtSecret, {
expiresIn: "1 day",

View File

@ -33,6 +33,7 @@ function cleanAutomationInputs(automation) {
exports.create = async function(ctx) {
const db = new CouchDB(ctx.user.instanceId)
let automation = ctx.request.body
automation.appId = ctx.user.appId
automation._id = generateAutomationID()
@ -54,6 +55,7 @@ exports.create = async function(ctx) {
exports.update = async function(ctx) {
const db = new CouchDB(ctx.user.instanceId)
let automation = ctx.request.body
automation.appId = ctx.user.appId
automation = cleanAutomationInputs(automation)
const response = await db.put(automation)

View File

@ -33,17 +33,9 @@ exports.save = async function(ctx) {
views: {},
...rest,
}
// get the model in its previous state for differencing
let oldModel
let oldModelId = ctx.request.body._id
if (oldModelId) {
// if it errors then the oldModelId is invalid - can't diff it
try {
oldModel = await db.get(oldModelId)
} catch (err) {
oldModel = null
}
}
// if the model obj had an _id then it will have been retrieved
const oldModel = ctx.preExisting
// rename record fields when table column is renamed
const { _rename } = modelToSave

View File

@ -77,6 +77,9 @@ exports.save = async function(ctx) {
record._id = generateRecordID(record.modelId)
}
// if the record obj had an _id then it will have been retrieved
const existingRecord = ctx.preExisting
const model = await db.get(record.modelId)
record = coerceRecordValues(record, model)
@ -95,8 +98,6 @@ exports.save = async function(ctx) {
return
}
const existingRecord = record._rev && (await db.get(record._id))
// make sure link records are up to date
record = await linkRecords.updateLinks({
instanceId,

View File

@ -1,6 +1,7 @@
const Router = require("@koa/router")
const recordController = require("../controllers/record")
const authorized = require("../../middleware/authorized")
const usage = require("../../middleware/usageQuota")
const { READ_MODEL, WRITE_MODEL } = require("../../utilities/accessLevels")
const router = Router()
@ -25,6 +26,7 @@ router
.post(
"/api/:modelId/records",
authorized(WRITE_MODEL, ctx => ctx.params.modelId),
usage,
recordController.save
)
.patch(
@ -40,6 +42,7 @@ router
.delete(
"/api/:modelId/records/:recordId/:revId",
authorized(WRITE_MODEL, ctx => ctx.params.modelId),
usage,
recordController.destroy
)

View File

@ -4,6 +4,7 @@ const { budibaseTempDir } = require("../../utilities/budibaseDir")
const env = require("../../environment")
const authorized = require("../../middleware/authorized")
const { BUILDER } = require("../../utilities/accessLevels")
const usage = require("../../middleware/usageQuota")
const router = Router()
@ -28,7 +29,7 @@ router
authorized(BUILDER),
controller.performLocalFileProcessing
)
.post("/api/attachments/upload", controller.uploadFile)
.post("/api/attachments/upload", usage, controller.uploadFile)
.get("/componentlibrary", controller.serveComponentLibrary)
.get("/assets/:file*", controller.serveAppAsset)
.get("/attachments/:file*", controller.serveAttachment)

View File

@ -40,6 +40,9 @@ exports.defaultHeaders = (appId, instanceId) => {
}
exports.createModel = async (request, appId, instanceId, model) => {
if (model != null && model._id) {
delete model._id
}
model = model || {
name: "TestModel",
type: "model",

View File

@ -2,6 +2,7 @@ const Router = require("@koa/router")
const controller = require("../controllers/user")
const authorized = require("../../middleware/authorized")
const { USER_MANAGEMENT, LIST_USERS } = require("../../utilities/accessLevels")
const usage = require("../../middleware/usageQuota")
const router = Router()
@ -9,10 +10,11 @@ router
.get("/api/users", authorized(LIST_USERS), controller.fetch)
.get("/api/users/:username", authorized(USER_MANAGEMENT), controller.find)
.put("/api/users/", authorized(USER_MANAGEMENT), controller.update)
.post("/api/users", authorized(USER_MANAGEMENT), controller.create)
.post("/api/users", authorized(USER_MANAGEMENT), usage, controller.create)
.delete(
"/api/users/:username",
authorized(USER_MANAGEMENT),
usage,
controller.destroy
)

View File

@ -3,6 +3,7 @@ const viewController = require("../controllers/view")
const recordController = require("../controllers/record")
const authorized = require("../../middleware/authorized")
const { BUILDER, READ_VIEW } = require("../../utilities/accessLevels")
const usage = require("../../middleware/usageQuota")
const router = Router()
@ -13,8 +14,13 @@ router
recordController.fetchView
)
.get("/api/views", authorized(BUILDER), viewController.fetch)
.delete("/api/views/:viewName", authorized(BUILDER), viewController.destroy)
.post("/api/views", authorized(BUILDER), viewController.save)
.delete(
"/api/views/:viewName",
authorized(BUILDER),
usage,
viewController.destroy
)
.post("/api/views", authorized(BUILDER), usage, viewController.save)
.post("/api/views/export", authorized(BUILDER), viewController.exportView)
.get(
"/api/views/export/download/:fileName",

View File

@ -3,6 +3,7 @@ const actions = require("./actions")
const environment = require("../environment")
const workerFarm = require("worker-farm")
const singleThread = require("./thread")
const { getAPIKey, update, Properties } = require("../utilities/usageQuota")
let workers = workerFarm(require.resolve("./thread"))
@ -18,16 +19,33 @@ function runWorker(job) {
})
}
async function updateQuota(automation) {
const appId = automation.appId
const apiObj = await getAPIKey(appId)
// this will fail, causing automation to escape if limits reached
await update(apiObj.apiKey, Properties.AUTOMATION, 1)
return apiObj.apiKey
}
/**
* This module is built purely to kick off the worker farm and manage the inputs/outputs
*/
module.exports.init = function() {
actions.init().then(() => {
triggers.automationQueue.process(async job => {
if (environment.BUDIBASE_ENVIRONMENT === "PRODUCTION") {
await runWorker(job)
} else {
await singleThread(job)
try {
if (environment.CLOUD && job.data.automation) {
job.data.automation.apiKey = await updateQuota(job.data.automation)
}
if (environment.BUDIBASE_ENVIRONMENT === "PRODUCTION") {
await runWorker(job)
} else {
await singleThread(job)
}
} catch (err) {
console.error(
`${job.data.automation.appId} automation ${job.data.automation._id} was unable to run - ${err}`
)
}
})
})

View File

@ -1,5 +1,7 @@
const recordController = require("../../api/controllers/record")
const automationUtils = require("../automationUtils")
const environment = require("../../environment")
const usage = require("../../utilities/usageQuota")
module.exports.definition = {
name: "Create Row",
@ -56,7 +58,7 @@ module.exports.definition = {
},
}
module.exports.run = async function({ inputs, instanceId }) {
module.exports.run = async function({ inputs, instanceId, apiKey }) {
// TODO: better logging of when actions are missed due to missing parameters
if (inputs.record == null || inputs.record.modelId == null) {
return
@ -78,6 +80,9 @@ module.exports.run = async function({ inputs, instanceId }) {
}
try {
if (environment.CLOUD) {
await usage.update(apiKey, usage.Properties.RECORD, 1)
}
await recordController.save(ctx)
return {
record: inputs.record,

View File

@ -1,5 +1,7 @@
const accessLevels = require("../../utilities/accessLevels")
const userController = require("../../api/controllers/user")
const environment = require("../../environment")
const usage = require("../../utilities/usageQuota")
module.exports.definition = {
description: "Create a new user",
@ -56,7 +58,7 @@ module.exports.definition = {
},
}
module.exports.run = async function({ inputs, instanceId }) {
module.exports.run = async function({ inputs, instanceId, apiKey }) {
const { username, password, accessLevelId } = inputs
const ctx = {
user: {
@ -68,6 +70,9 @@ module.exports.run = async function({ inputs, instanceId }) {
}
try {
if (environment.CLOUD) {
await usage.update(apiKey, usage.Properties.USER, 1)
}
await userController.create(ctx)
return {
response: ctx.body,

View File

@ -1,4 +1,6 @@
const recordController = require("../../api/controllers/record")
const environment = require("../../environment")
const usage = require("../../utilities/usageQuota")
module.exports.definition = {
description: "Delete a row from your database",
@ -48,7 +50,7 @@ module.exports.definition = {
},
}
module.exports.run = async function({ inputs, instanceId }) {
module.exports.run = async function({ inputs, instanceId, apiKey }) {
// TODO: better logging of when actions are missed due to missing parameters
if (inputs.id == null || inputs.revision == null) {
return
@ -63,6 +65,9 @@ module.exports.run = async function({ inputs, instanceId }) {
}
try {
if (environment.CLOUD) {
await usage.update(apiKey, usage.Properties.RECORD, -1)
}
await recordController.destroy(ctx)
return {
response: ctx.body,

View File

@ -62,6 +62,7 @@ class Orchestrator {
const outputs = await stepFn({
inputs: step.inputs,
instanceId: this._instanceId,
apiKey: automation.apiKey,
})
if (step.stepId === FILTER_STEP_ID && !outputs.success) {
break

View File

@ -0,0 +1,128 @@
let _ = require("lodash")
let environment = require("../environment")
const AWS_REGION = environment.AWS_REGION ? environment.AWS_REGION : "eu-west-1"
const TableInfo = {
API_KEYS: {
name: "beta-api-key-table",
primary: "pk",
},
USERS: {
name: "prod-budi-table",
primary: "pk",
sort: "sk",
},
}
let docClient = null
class Table {
constructor(tableInfo) {
if (!tableInfo.name || !tableInfo.primary) {
throw "Table info must specify a name and a primary key"
}
this._name = tableInfo.name
this._primary = tableInfo.primary
this._sort = tableInfo.sort
}
async get({ primary, sort, otherProps }) {
let params = {
TableName: this._name,
Key: {
[this._primary]: primary,
},
}
if (this._sort && sort) {
params.Key[this._sort] = sort
}
if (otherProps) {
params = _.merge(params, otherProps)
}
let response = await docClient.get(params).promise()
return response.Item
}
async update({
primary,
sort,
expression,
condition,
names,
values,
exists,
otherProps,
}) {
let params = {
TableName: this._name,
Key: {
[this._primary]: primary,
},
ExpressionAttributeNames: names,
ExpressionAttributeValues: values,
UpdateExpression: expression,
}
if (condition) {
params.ConditionExpression = condition
}
if (this._sort && sort) {
params.Key[this._sort] = sort
}
if (exists) {
params.ExpressionAttributeNames["#PRIMARY"] = this._primary
if (params.ConditionExpression) {
params.ConditionExpression += " AND "
}
params.ConditionExpression += "attribute_exists(#PRIMARY)"
}
if (otherProps) {
params = _.merge(params, otherProps)
}
return docClient.update(params).promise()
}
async put({ item, otherProps }) {
if (
item[this._primary] == null ||
(this._sort && item[this._sort] == null)
) {
throw "Cannot put item without primary and sort key (if required)"
}
let params = {
TableName: this._name,
Item: item,
}
if (otherProps) {
params = _.merge(params, otherProps)
}
return docClient.put(params).promise()
}
}
exports.init = endpoint => {
let AWS = require("aws-sdk")
AWS.config.update({
region: AWS_REGION,
})
let docClientParams = {
correctClockSkew: true,
}
if (endpoint) {
docClientParams.endpoint = endpoint
} else if (environment.DYNAMO_ENDPOINT) {
docClientParams.endpoint = environment.DYNAMO_ENDPOINT
}
docClient = new AWS.DynamoDB.DocumentClient(docClientParams)
}
exports.apiKeyTable = new Table(TableInfo.API_KEYS)
exports.userTable = new Table(TableInfo.USERS)
if (environment.CLOUD) {
exports.init(`https://dynamodb.${AWS_REGION}.amazonaws.com`)
} else {
process.env.AWS_ACCESS_KEY_ID = "KEY_ID"
process.env.AWS_SECRET_ACCESS_KEY = "SECRET_KEY"
exports.init("http://localhost:8333")
}

View File

@ -11,4 +11,7 @@ module.exports = {
AUTOMATION_BUCKET: process.env.AUTOMATION_BUCKET,
BUDIBASE_ENVIRONMENT: process.env.BUDIBASE_ENVIRONMENT,
SENDGRID_API_KEY: process.env.SENDGRID_API_KEY,
CLOUD: process.env.CLOUD,
DYNAMO_ENDPOINT: process.env.DYNAMO_ENDPOINT,
AWS_REGION: process.env.AWS_REGION,
}

View File

@ -20,6 +20,7 @@ module.exports = async (ctx, next) => {
if (builderToken) {
try {
const jwtPayload = jwt.verify(builderToken, ctx.config.jwtSecret)
ctx.apiKey = jwtPayload.apiKey
ctx.isAuthenticated = jwtPayload.accessLevelId === BUILDER_LEVEL_ID
ctx.user = {
...jwtPayload,

View File

@ -16,8 +16,7 @@ module.exports = (permName, getItemId) => async (ctx, next) => {
}
if (ctx.user.accessLevel._id === BUILDER_LEVEL_ID) {
await next()
return
return next()
}
if (permName === BUILDER) {
@ -28,8 +27,7 @@ module.exports = (permName, getItemId) => async (ctx, next) => {
const permissionId = ({ name, itemId }) => name + (itemId ? `-${itemId}` : "")
if (ctx.user.accessLevel._id === ADMIN_LEVEL_ID) {
await next()
return
return next()
}
const thisPermissionId = permissionId({
@ -42,8 +40,7 @@ module.exports = (permName, getItemId) => async (ctx, next) => {
ctx.user.accessLevel._id === POWERUSER_LEVEL_ID &&
!adminPermissions.map(permissionId).includes(thisPermissionId)
) {
await next()
return
return next()
}
if (
@ -51,8 +48,7 @@ module.exports = (permName, getItemId) => async (ctx, next) => {
.map(permissionId)
.includes(thisPermissionId)
) {
await next()
return
return next()
}
ctx.throw(403, "Not Authorized")

View File

@ -0,0 +1,63 @@
const CouchDB = require("../db")
const usageQuota = require("../utilities/usageQuota")
const environment = require("../environment")
// currently only counting new writes and deletes
const METHOD_MAP = {
POST: 1,
DELETE: -1,
}
const DOMAIN_MAP = {
records: usageQuota.Properties.RECORD,
upload: usageQuota.Properties.UPLOAD,
views: usageQuota.Properties.VIEW,
users: usageQuota.Properties.USER,
// this will not be updated by endpoint calls
// instead it will be updated by triggers
automationRuns: usageQuota.Properties.AUTOMATION,
}
function getProperty(url) {
for (let domain of Object.keys(DOMAIN_MAP)) {
if (url.indexOf(domain) !== -1) {
return DOMAIN_MAP[domain]
}
}
}
module.exports = async (ctx, next) => {
const db = new CouchDB(ctx.user.instanceId)
let usage = METHOD_MAP[ctx.req.method]
const property = getProperty(ctx.req.url)
if (usage == null || property == null) {
return next()
}
// post request could be a save of a pre-existing entry
if (ctx.request.body && ctx.request.body._id) {
try {
ctx.preExisting = await db.get(ctx.request.body._id)
return next()
} catch (err) {
ctx.throw(404, `${ctx.request.body._id} does not exist`)
return
}
}
// update usage for uploads to be the total size
if (property === usageQuota.Properties.UPLOAD) {
const files =
ctx.request.files.file.length > 1
? Array.from(ctx.request.files.file)
: [ctx.request.files.file]
usage = files.map(file => file.size).reduce((total, size) => total + size)
}
if (!environment.CLOUD) {
return next()
}
try {
await usageQuota.update(ctx.apiKey, property, usage)
return next()
} catch (err) {
ctx.throw(403, err)
}
}

View File

@ -8,7 +8,9 @@ module.exports = (ctx, appId, instanceId) => {
instanceId,
appId,
}
if (process.env.BUDIBASE_API_KEY) {
builderUser.apiKey = process.env.BUDIBASE_API_KEY
}
const token = jwt.sign(builderUser, ctx.config.jwtSecret, {
expiresIn: "30 days",
})

View File

@ -0,0 +1,73 @@
const environment = require("../environment")
const { apiKeyTable } = require("../db/dynamoClient")
function buildUpdateParams(key, property, usage) {
return {
primary: key,
condition: "#quota.#prop < #limits.#prop AND #quotaReset > :now",
expression: "ADD #quota.#prop :usage",
names: {
"#quota": "usageQuota",
"#prop": property,
"#limits": "usageLimits",
"#quotaReset": "quotaReset",
},
values: {
":usage": usage,
":now": Date.now(),
},
}
}
// a normalised month in milliseconds
const QUOTA_RESET = 2592000000
exports.Properties = {
RECORD: "records",
UPLOAD: "storage",
VIEW: "views",
USER: "users",
AUTOMATION: "automationRuns",
}
exports.getAPIKey = async appId => {
return apiKeyTable.get({ primary: appId })
}
/**
* Given a specified API key this will add to the usage object for the specified property.
* @param {string} apiKey The API key which is to be updated.
* @param {string} property The property which is to be added to (within the nested usageQuota object).
* @param {number} usage The amount (this can be negative) to adjust the number by.
* @returns {Promise<void>} When this completes the API key will now be up to date - the quota period may have
* also been reset after this call.
*/
exports.update = async (apiKey, property, usage) => {
// don't try validate in builder
if (!environment.CLOUD) {
return
}
try {
await apiKeyTable.update(buildUpdateParams(apiKey, property, usage))
} catch (err) {
if (err.code === "ConditionalCheckFailedException") {
// get the API key so we can check it
const keyObj = await apiKeyTable.get({ primary: apiKey })
// we have infact breached the reset period
if (keyObj && keyObj.quotaReset <= Date.now()) {
// update the quota reset period and reset the values for all properties
keyObj.quotaReset = Date.now() + QUOTA_RESET
for (let prop of Object.keys(keyObj.usageQuota)) {
if (prop === property) {
keyObj.usageQuota[prop] = usage > 0 ? usage : 0
} else {
keyObj.usageQuota[prop] = 0
}
}
await apiKeyTable.put({ item: keyObj })
return
}
throw "Resource limits have been reached"
}
}
}

View File

@ -946,9 +946,10 @@ available-typed-arrays@^1.0.0, available-typed-arrays@^1.0.2:
dependencies:
array-filter "^1.0.0"
aws-sdk@^2.706.0:
version "2.706.0"
resolved "https://registry.yarnpkg.com/aws-sdk/-/aws-sdk-2.706.0.tgz#09f65e9a91ecac5a635daf934082abae30eca953"
aws-sdk@^2.767.0:
version "2.767.0"
resolved "https://registry.yarnpkg.com/aws-sdk/-/aws-sdk-2.767.0.tgz#9863c8bfd5990106b95f38e9345a547fee782470"
integrity sha512-soPZxjNpat0CtuIqm54GO/FDT4SZTlQG0icSptWYfMFYdkXe8b0tJqaPssNn9TzlgoWDCNTdaoepM6TN0rNHkQ==
dependencies:
buffer "4.9.2"
events "1.1.1"
@ -1081,6 +1082,7 @@ bluebird-lst@^1.0.9:
bluebird@^3.5.1, bluebird@^3.5.5:
version "3.7.2"
resolved "https://registry.yarnpkg.com/bluebird/-/bluebird-3.7.2.tgz#9f229c15be272454ffa973ace0dbee79a1b0c36f"
integrity sha512-XpNj6GDQzdfW+r2Wnn7xiSAd7TM3jzkxGXBGTtWKuSXv1xUV+azxAm8jdWZN06QTQk+2N2XB9jRDkvbmQmcRtg==
boolean@^3.0.0, boolean@^3.0.1:
version "3.0.1"