Merge pull request #678 from Budibase/api-usage-tracking
Api usage tracking
This commit is contained in:
commit
dc3c0bd4c3
|
@ -46,7 +46,7 @@
|
||||||
"@koa/router": "^8.0.0",
|
"@koa/router": "^8.0.0",
|
||||||
"@sendgrid/mail": "^7.1.1",
|
"@sendgrid/mail": "^7.1.1",
|
||||||
"@sentry/node": "^5.19.2",
|
"@sentry/node": "^5.19.2",
|
||||||
"aws-sdk": "^2.706.0",
|
"aws-sdk": "^2.767.0",
|
||||||
"bcryptjs": "^2.4.3",
|
"bcryptjs": "^2.4.3",
|
||||||
"chmodr": "^1.2.0",
|
"chmodr": "^1.2.0",
|
||||||
"csvtojson": "^2.0.10",
|
"csvtojson": "^2.0.10",
|
||||||
|
|
|
@ -0,0 +1,55 @@
|
||||||
|
// THIS will create API Keys and App Ids input in a local Dynamo instance if it is running
|
||||||
|
const dynamoClient = require("../src/db/dynamoClient")
|
||||||
|
|
||||||
|
if (process.argv[2] == null || process.argv[3] == null) {
|
||||||
|
console.error(
|
||||||
|
"Inputs incorrect format, was expecting: node createApiKeyAndAppId.js <API_KEY> <APP_ID>"
|
||||||
|
)
|
||||||
|
process.exit(-1)
|
||||||
|
}
|
||||||
|
|
||||||
|
const FAKE_STRING = "fakestring"
|
||||||
|
|
||||||
|
// set fake credentials for local dynamo to actually work
|
||||||
|
process.env.AWS_ACCESS_KEY_ID = "KEY_ID"
|
||||||
|
process.env.AWS_SECRET_ACCESS_KEY = "SECRET_KEY"
|
||||||
|
dynamoClient.init("http://localhost:8333")
|
||||||
|
|
||||||
|
async function run() {
|
||||||
|
await dynamoClient.apiKeyTable.put({
|
||||||
|
item: {
|
||||||
|
pk: process.argv[2],
|
||||||
|
accountId: FAKE_STRING,
|
||||||
|
trackingId: FAKE_STRING,
|
||||||
|
quotaReset: Date.now() + 2592000000,
|
||||||
|
usageQuota: {
|
||||||
|
automationRuns: 0,
|
||||||
|
records: 0,
|
||||||
|
storage: 0,
|
||||||
|
users: 0,
|
||||||
|
views: 0,
|
||||||
|
},
|
||||||
|
usageLimits: {
|
||||||
|
automationRuns: 10,
|
||||||
|
records: 10,
|
||||||
|
storage: 1000,
|
||||||
|
users: 10,
|
||||||
|
views: 10,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
})
|
||||||
|
await dynamoClient.apiKeyTable.put({
|
||||||
|
item: {
|
||||||
|
pk: process.argv[3],
|
||||||
|
apiKey: process.argv[2],
|
||||||
|
},
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
run()
|
||||||
|
.then(() => {
|
||||||
|
console.log("Records should have been created.")
|
||||||
|
})
|
||||||
|
.catch(err => {
|
||||||
|
console.error("Cannot create records - " + err)
|
||||||
|
})
|
|
@ -2,6 +2,8 @@ const jwt = require("jsonwebtoken")
|
||||||
const CouchDB = require("../../db")
|
const CouchDB = require("../../db")
|
||||||
const ClientDb = require("../../db/clientDb")
|
const ClientDb = require("../../db/clientDb")
|
||||||
const bcrypt = require("../../utilities/bcrypt")
|
const bcrypt = require("../../utilities/bcrypt")
|
||||||
|
const environment = require("../../environment")
|
||||||
|
const { getAPIKey } = require("../../utilities/usageQuota")
|
||||||
const { generateUserID } = require("../../db/utils")
|
const { generateUserID } = require("../../db/utils")
|
||||||
|
|
||||||
exports.authenticate = async ctx => {
|
exports.authenticate = async ctx => {
|
||||||
|
@ -51,6 +53,10 @@ exports.authenticate = async ctx => {
|
||||||
appId: ctx.user.appId,
|
appId: ctx.user.appId,
|
||||||
instanceId,
|
instanceId,
|
||||||
}
|
}
|
||||||
|
// if in cloud add the user api key
|
||||||
|
if (environment.CLOUD) {
|
||||||
|
payload.apiKey = await getAPIKey(ctx.user.appId)
|
||||||
|
}
|
||||||
|
|
||||||
const token = jwt.sign(payload, ctx.config.jwtSecret, {
|
const token = jwt.sign(payload, ctx.config.jwtSecret, {
|
||||||
expiresIn: "1 day",
|
expiresIn: "1 day",
|
||||||
|
|
|
@ -33,6 +33,7 @@ function cleanAutomationInputs(automation) {
|
||||||
exports.create = async function(ctx) {
|
exports.create = async function(ctx) {
|
||||||
const db = new CouchDB(ctx.user.instanceId)
|
const db = new CouchDB(ctx.user.instanceId)
|
||||||
let automation = ctx.request.body
|
let automation = ctx.request.body
|
||||||
|
automation.appId = ctx.user.appId
|
||||||
|
|
||||||
automation._id = generateAutomationID()
|
automation._id = generateAutomationID()
|
||||||
|
|
||||||
|
@ -54,6 +55,7 @@ exports.create = async function(ctx) {
|
||||||
exports.update = async function(ctx) {
|
exports.update = async function(ctx) {
|
||||||
const db = new CouchDB(ctx.user.instanceId)
|
const db = new CouchDB(ctx.user.instanceId)
|
||||||
let automation = ctx.request.body
|
let automation = ctx.request.body
|
||||||
|
automation.appId = ctx.user.appId
|
||||||
|
|
||||||
automation = cleanAutomationInputs(automation)
|
automation = cleanAutomationInputs(automation)
|
||||||
const response = await db.put(automation)
|
const response = await db.put(automation)
|
||||||
|
|
|
@ -33,17 +33,9 @@ exports.save = async function(ctx) {
|
||||||
views: {},
|
views: {},
|
||||||
...rest,
|
...rest,
|
||||||
}
|
}
|
||||||
// get the model in its previous state for differencing
|
|
||||||
let oldModel
|
// if the model obj had an _id then it will have been retrieved
|
||||||
let oldModelId = ctx.request.body._id
|
const oldModel = ctx.preExisting
|
||||||
if (oldModelId) {
|
|
||||||
// if it errors then the oldModelId is invalid - can't diff it
|
|
||||||
try {
|
|
||||||
oldModel = await db.get(oldModelId)
|
|
||||||
} catch (err) {
|
|
||||||
oldModel = null
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// rename record fields when table column is renamed
|
// rename record fields when table column is renamed
|
||||||
const { _rename } = modelToSave
|
const { _rename } = modelToSave
|
||||||
|
|
|
@ -77,6 +77,9 @@ exports.save = async function(ctx) {
|
||||||
record._id = generateRecordID(record.modelId)
|
record._id = generateRecordID(record.modelId)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// if the record obj had an _id then it will have been retrieved
|
||||||
|
const existingRecord = ctx.preExisting
|
||||||
|
|
||||||
const model = await db.get(record.modelId)
|
const model = await db.get(record.modelId)
|
||||||
|
|
||||||
record = coerceRecordValues(record, model)
|
record = coerceRecordValues(record, model)
|
||||||
|
@ -95,8 +98,6 @@ exports.save = async function(ctx) {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
const existingRecord = record._rev && (await db.get(record._id))
|
|
||||||
|
|
||||||
// make sure link records are up to date
|
// make sure link records are up to date
|
||||||
record = await linkRecords.updateLinks({
|
record = await linkRecords.updateLinks({
|
||||||
instanceId,
|
instanceId,
|
||||||
|
|
|
@ -1,6 +1,7 @@
|
||||||
const Router = require("@koa/router")
|
const Router = require("@koa/router")
|
||||||
const recordController = require("../controllers/record")
|
const recordController = require("../controllers/record")
|
||||||
const authorized = require("../../middleware/authorized")
|
const authorized = require("../../middleware/authorized")
|
||||||
|
const usage = require("../../middleware/usageQuota")
|
||||||
const { READ_MODEL, WRITE_MODEL } = require("../../utilities/accessLevels")
|
const { READ_MODEL, WRITE_MODEL } = require("../../utilities/accessLevels")
|
||||||
|
|
||||||
const router = Router()
|
const router = Router()
|
||||||
|
@ -25,6 +26,7 @@ router
|
||||||
.post(
|
.post(
|
||||||
"/api/:modelId/records",
|
"/api/:modelId/records",
|
||||||
authorized(WRITE_MODEL, ctx => ctx.params.modelId),
|
authorized(WRITE_MODEL, ctx => ctx.params.modelId),
|
||||||
|
usage,
|
||||||
recordController.save
|
recordController.save
|
||||||
)
|
)
|
||||||
.patch(
|
.patch(
|
||||||
|
@ -40,6 +42,7 @@ router
|
||||||
.delete(
|
.delete(
|
||||||
"/api/:modelId/records/:recordId/:revId",
|
"/api/:modelId/records/:recordId/:revId",
|
||||||
authorized(WRITE_MODEL, ctx => ctx.params.modelId),
|
authorized(WRITE_MODEL, ctx => ctx.params.modelId),
|
||||||
|
usage,
|
||||||
recordController.destroy
|
recordController.destroy
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
|
@ -4,6 +4,7 @@ const { budibaseTempDir } = require("../../utilities/budibaseDir")
|
||||||
const env = require("../../environment")
|
const env = require("../../environment")
|
||||||
const authorized = require("../../middleware/authorized")
|
const authorized = require("../../middleware/authorized")
|
||||||
const { BUILDER } = require("../../utilities/accessLevels")
|
const { BUILDER } = require("../../utilities/accessLevels")
|
||||||
|
const usage = require("../../middleware/usageQuota")
|
||||||
|
|
||||||
const router = Router()
|
const router = Router()
|
||||||
|
|
||||||
|
@ -28,7 +29,7 @@ router
|
||||||
authorized(BUILDER),
|
authorized(BUILDER),
|
||||||
controller.performLocalFileProcessing
|
controller.performLocalFileProcessing
|
||||||
)
|
)
|
||||||
.post("/api/attachments/upload", controller.uploadFile)
|
.post("/api/attachments/upload", usage, controller.uploadFile)
|
||||||
.get("/componentlibrary", controller.serveComponentLibrary)
|
.get("/componentlibrary", controller.serveComponentLibrary)
|
||||||
.get("/assets/:file*", controller.serveAppAsset)
|
.get("/assets/:file*", controller.serveAppAsset)
|
||||||
.get("/attachments/:file*", controller.serveAttachment)
|
.get("/attachments/:file*", controller.serveAttachment)
|
||||||
|
|
|
@ -40,6 +40,9 @@ exports.defaultHeaders = (appId, instanceId) => {
|
||||||
}
|
}
|
||||||
|
|
||||||
exports.createModel = async (request, appId, instanceId, model) => {
|
exports.createModel = async (request, appId, instanceId, model) => {
|
||||||
|
if (model != null && model._id) {
|
||||||
|
delete model._id
|
||||||
|
}
|
||||||
model = model || {
|
model = model || {
|
||||||
name: "TestModel",
|
name: "TestModel",
|
||||||
type: "model",
|
type: "model",
|
||||||
|
|
|
@ -2,6 +2,7 @@ const Router = require("@koa/router")
|
||||||
const controller = require("../controllers/user")
|
const controller = require("../controllers/user")
|
||||||
const authorized = require("../../middleware/authorized")
|
const authorized = require("../../middleware/authorized")
|
||||||
const { USER_MANAGEMENT, LIST_USERS } = require("../../utilities/accessLevels")
|
const { USER_MANAGEMENT, LIST_USERS } = require("../../utilities/accessLevels")
|
||||||
|
const usage = require("../../middleware/usageQuota")
|
||||||
|
|
||||||
const router = Router()
|
const router = Router()
|
||||||
|
|
||||||
|
@ -9,10 +10,11 @@ router
|
||||||
.get("/api/users", authorized(LIST_USERS), controller.fetch)
|
.get("/api/users", authorized(LIST_USERS), controller.fetch)
|
||||||
.get("/api/users/:username", authorized(USER_MANAGEMENT), controller.find)
|
.get("/api/users/:username", authorized(USER_MANAGEMENT), controller.find)
|
||||||
.put("/api/users/", authorized(USER_MANAGEMENT), controller.update)
|
.put("/api/users/", authorized(USER_MANAGEMENT), controller.update)
|
||||||
.post("/api/users", authorized(USER_MANAGEMENT), controller.create)
|
.post("/api/users", authorized(USER_MANAGEMENT), usage, controller.create)
|
||||||
.delete(
|
.delete(
|
||||||
"/api/users/:username",
|
"/api/users/:username",
|
||||||
authorized(USER_MANAGEMENT),
|
authorized(USER_MANAGEMENT),
|
||||||
|
usage,
|
||||||
controller.destroy
|
controller.destroy
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
|
@ -3,6 +3,7 @@ const viewController = require("../controllers/view")
|
||||||
const recordController = require("../controllers/record")
|
const recordController = require("../controllers/record")
|
||||||
const authorized = require("../../middleware/authorized")
|
const authorized = require("../../middleware/authorized")
|
||||||
const { BUILDER, READ_VIEW } = require("../../utilities/accessLevels")
|
const { BUILDER, READ_VIEW } = require("../../utilities/accessLevels")
|
||||||
|
const usage = require("../../middleware/usageQuota")
|
||||||
|
|
||||||
const router = Router()
|
const router = Router()
|
||||||
|
|
||||||
|
@ -13,8 +14,13 @@ router
|
||||||
recordController.fetchView
|
recordController.fetchView
|
||||||
)
|
)
|
||||||
.get("/api/views", authorized(BUILDER), viewController.fetch)
|
.get("/api/views", authorized(BUILDER), viewController.fetch)
|
||||||
.delete("/api/views/:viewName", authorized(BUILDER), viewController.destroy)
|
.delete(
|
||||||
.post("/api/views", authorized(BUILDER), viewController.save)
|
"/api/views/:viewName",
|
||||||
|
authorized(BUILDER),
|
||||||
|
usage,
|
||||||
|
viewController.destroy
|
||||||
|
)
|
||||||
|
.post("/api/views", authorized(BUILDER), usage, viewController.save)
|
||||||
.post("/api/views/export", authorized(BUILDER), viewController.exportView)
|
.post("/api/views/export", authorized(BUILDER), viewController.exportView)
|
||||||
.get(
|
.get(
|
||||||
"/api/views/export/download/:fileName",
|
"/api/views/export/download/:fileName",
|
||||||
|
|
|
@ -3,6 +3,7 @@ const actions = require("./actions")
|
||||||
const environment = require("../environment")
|
const environment = require("../environment")
|
||||||
const workerFarm = require("worker-farm")
|
const workerFarm = require("worker-farm")
|
||||||
const singleThread = require("./thread")
|
const singleThread = require("./thread")
|
||||||
|
const { getAPIKey, update, Properties } = require("../utilities/usageQuota")
|
||||||
|
|
||||||
let workers = workerFarm(require.resolve("./thread"))
|
let workers = workerFarm(require.resolve("./thread"))
|
||||||
|
|
||||||
|
@ -18,16 +19,33 @@ function runWorker(job) {
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async function updateQuota(automation) {
|
||||||
|
const appId = automation.appId
|
||||||
|
const apiObj = await getAPIKey(appId)
|
||||||
|
// this will fail, causing automation to escape if limits reached
|
||||||
|
await update(apiObj.apiKey, Properties.AUTOMATION, 1)
|
||||||
|
return apiObj.apiKey
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* This module is built purely to kick off the worker farm and manage the inputs/outputs
|
* This module is built purely to kick off the worker farm and manage the inputs/outputs
|
||||||
*/
|
*/
|
||||||
module.exports.init = function() {
|
module.exports.init = function() {
|
||||||
actions.init().then(() => {
|
actions.init().then(() => {
|
||||||
triggers.automationQueue.process(async job => {
|
triggers.automationQueue.process(async job => {
|
||||||
if (environment.BUDIBASE_ENVIRONMENT === "PRODUCTION") {
|
try {
|
||||||
await runWorker(job)
|
if (environment.CLOUD && job.data.automation) {
|
||||||
} else {
|
job.data.automation.apiKey = await updateQuota(job.data.automation)
|
||||||
await singleThread(job)
|
}
|
||||||
|
if (environment.BUDIBASE_ENVIRONMENT === "PRODUCTION") {
|
||||||
|
await runWorker(job)
|
||||||
|
} else {
|
||||||
|
await singleThread(job)
|
||||||
|
}
|
||||||
|
} catch (err) {
|
||||||
|
console.error(
|
||||||
|
`${job.data.automation.appId} automation ${job.data.automation._id} was unable to run - ${err}`
|
||||||
|
)
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
})
|
})
|
||||||
|
|
|
@ -1,5 +1,7 @@
|
||||||
const recordController = require("../../api/controllers/record")
|
const recordController = require("../../api/controllers/record")
|
||||||
const automationUtils = require("../automationUtils")
|
const automationUtils = require("../automationUtils")
|
||||||
|
const environment = require("../../environment")
|
||||||
|
const usage = require("../../utilities/usageQuota")
|
||||||
|
|
||||||
module.exports.definition = {
|
module.exports.definition = {
|
||||||
name: "Create Row",
|
name: "Create Row",
|
||||||
|
@ -56,7 +58,7 @@ module.exports.definition = {
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
module.exports.run = async function({ inputs, instanceId }) {
|
module.exports.run = async function({ inputs, instanceId, apiKey }) {
|
||||||
// TODO: better logging of when actions are missed due to missing parameters
|
// TODO: better logging of when actions are missed due to missing parameters
|
||||||
if (inputs.record == null || inputs.record.modelId == null) {
|
if (inputs.record == null || inputs.record.modelId == null) {
|
||||||
return
|
return
|
||||||
|
@ -78,6 +80,9 @@ module.exports.run = async function({ inputs, instanceId }) {
|
||||||
}
|
}
|
||||||
|
|
||||||
try {
|
try {
|
||||||
|
if (environment.CLOUD) {
|
||||||
|
await usage.update(apiKey, usage.Properties.RECORD, 1)
|
||||||
|
}
|
||||||
await recordController.save(ctx)
|
await recordController.save(ctx)
|
||||||
return {
|
return {
|
||||||
record: inputs.record,
|
record: inputs.record,
|
||||||
|
|
|
@ -1,5 +1,7 @@
|
||||||
const accessLevels = require("../../utilities/accessLevels")
|
const accessLevels = require("../../utilities/accessLevels")
|
||||||
const userController = require("../../api/controllers/user")
|
const userController = require("../../api/controllers/user")
|
||||||
|
const environment = require("../../environment")
|
||||||
|
const usage = require("../../utilities/usageQuota")
|
||||||
|
|
||||||
module.exports.definition = {
|
module.exports.definition = {
|
||||||
description: "Create a new user",
|
description: "Create a new user",
|
||||||
|
@ -56,7 +58,7 @@ module.exports.definition = {
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
module.exports.run = async function({ inputs, instanceId }) {
|
module.exports.run = async function({ inputs, instanceId, apiKey }) {
|
||||||
const { username, password, accessLevelId } = inputs
|
const { username, password, accessLevelId } = inputs
|
||||||
const ctx = {
|
const ctx = {
|
||||||
user: {
|
user: {
|
||||||
|
@ -68,6 +70,9 @@ module.exports.run = async function({ inputs, instanceId }) {
|
||||||
}
|
}
|
||||||
|
|
||||||
try {
|
try {
|
||||||
|
if (environment.CLOUD) {
|
||||||
|
await usage.update(apiKey, usage.Properties.USER, 1)
|
||||||
|
}
|
||||||
await userController.create(ctx)
|
await userController.create(ctx)
|
||||||
return {
|
return {
|
||||||
response: ctx.body,
|
response: ctx.body,
|
||||||
|
|
|
@ -1,4 +1,6 @@
|
||||||
const recordController = require("../../api/controllers/record")
|
const recordController = require("../../api/controllers/record")
|
||||||
|
const environment = require("../../environment")
|
||||||
|
const usage = require("../../utilities/usageQuota")
|
||||||
|
|
||||||
module.exports.definition = {
|
module.exports.definition = {
|
||||||
description: "Delete a row from your database",
|
description: "Delete a row from your database",
|
||||||
|
@ -48,7 +50,7 @@ module.exports.definition = {
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
module.exports.run = async function({ inputs, instanceId }) {
|
module.exports.run = async function({ inputs, instanceId, apiKey }) {
|
||||||
// TODO: better logging of when actions are missed due to missing parameters
|
// TODO: better logging of when actions are missed due to missing parameters
|
||||||
if (inputs.id == null || inputs.revision == null) {
|
if (inputs.id == null || inputs.revision == null) {
|
||||||
return
|
return
|
||||||
|
@ -63,6 +65,9 @@ module.exports.run = async function({ inputs, instanceId }) {
|
||||||
}
|
}
|
||||||
|
|
||||||
try {
|
try {
|
||||||
|
if (environment.CLOUD) {
|
||||||
|
await usage.update(apiKey, usage.Properties.RECORD, -1)
|
||||||
|
}
|
||||||
await recordController.destroy(ctx)
|
await recordController.destroy(ctx)
|
||||||
return {
|
return {
|
||||||
response: ctx.body,
|
response: ctx.body,
|
||||||
|
|
|
@ -62,6 +62,7 @@ class Orchestrator {
|
||||||
const outputs = await stepFn({
|
const outputs = await stepFn({
|
||||||
inputs: step.inputs,
|
inputs: step.inputs,
|
||||||
instanceId: this._instanceId,
|
instanceId: this._instanceId,
|
||||||
|
apiKey: automation.apiKey,
|
||||||
})
|
})
|
||||||
if (step.stepId === FILTER_STEP_ID && !outputs.success) {
|
if (step.stepId === FILTER_STEP_ID && !outputs.success) {
|
||||||
break
|
break
|
||||||
|
|
|
@ -0,0 +1,128 @@
|
||||||
|
let _ = require("lodash")
|
||||||
|
let environment = require("../environment")
|
||||||
|
|
||||||
|
const AWS_REGION = environment.AWS_REGION ? environment.AWS_REGION : "eu-west-1"
|
||||||
|
|
||||||
|
const TableInfo = {
|
||||||
|
API_KEYS: {
|
||||||
|
name: "beta-api-key-table",
|
||||||
|
primary: "pk",
|
||||||
|
},
|
||||||
|
USERS: {
|
||||||
|
name: "prod-budi-table",
|
||||||
|
primary: "pk",
|
||||||
|
sort: "sk",
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
let docClient = null
|
||||||
|
|
||||||
|
class Table {
|
||||||
|
constructor(tableInfo) {
|
||||||
|
if (!tableInfo.name || !tableInfo.primary) {
|
||||||
|
throw "Table info must specify a name and a primary key"
|
||||||
|
}
|
||||||
|
this._name = tableInfo.name
|
||||||
|
this._primary = tableInfo.primary
|
||||||
|
this._sort = tableInfo.sort
|
||||||
|
}
|
||||||
|
|
||||||
|
async get({ primary, sort, otherProps }) {
|
||||||
|
let params = {
|
||||||
|
TableName: this._name,
|
||||||
|
Key: {
|
||||||
|
[this._primary]: primary,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
if (this._sort && sort) {
|
||||||
|
params.Key[this._sort] = sort
|
||||||
|
}
|
||||||
|
if (otherProps) {
|
||||||
|
params = _.merge(params, otherProps)
|
||||||
|
}
|
||||||
|
let response = await docClient.get(params).promise()
|
||||||
|
return response.Item
|
||||||
|
}
|
||||||
|
|
||||||
|
async update({
|
||||||
|
primary,
|
||||||
|
sort,
|
||||||
|
expression,
|
||||||
|
condition,
|
||||||
|
names,
|
||||||
|
values,
|
||||||
|
exists,
|
||||||
|
otherProps,
|
||||||
|
}) {
|
||||||
|
let params = {
|
||||||
|
TableName: this._name,
|
||||||
|
Key: {
|
||||||
|
[this._primary]: primary,
|
||||||
|
},
|
||||||
|
ExpressionAttributeNames: names,
|
||||||
|
ExpressionAttributeValues: values,
|
||||||
|
UpdateExpression: expression,
|
||||||
|
}
|
||||||
|
if (condition) {
|
||||||
|
params.ConditionExpression = condition
|
||||||
|
}
|
||||||
|
if (this._sort && sort) {
|
||||||
|
params.Key[this._sort] = sort
|
||||||
|
}
|
||||||
|
if (exists) {
|
||||||
|
params.ExpressionAttributeNames["#PRIMARY"] = this._primary
|
||||||
|
if (params.ConditionExpression) {
|
||||||
|
params.ConditionExpression += " AND "
|
||||||
|
}
|
||||||
|
params.ConditionExpression += "attribute_exists(#PRIMARY)"
|
||||||
|
}
|
||||||
|
if (otherProps) {
|
||||||
|
params = _.merge(params, otherProps)
|
||||||
|
}
|
||||||
|
return docClient.update(params).promise()
|
||||||
|
}
|
||||||
|
|
||||||
|
async put({ item, otherProps }) {
|
||||||
|
if (
|
||||||
|
item[this._primary] == null ||
|
||||||
|
(this._sort && item[this._sort] == null)
|
||||||
|
) {
|
||||||
|
throw "Cannot put item without primary and sort key (if required)"
|
||||||
|
}
|
||||||
|
let params = {
|
||||||
|
TableName: this._name,
|
||||||
|
Item: item,
|
||||||
|
}
|
||||||
|
if (otherProps) {
|
||||||
|
params = _.merge(params, otherProps)
|
||||||
|
}
|
||||||
|
return docClient.put(params).promise()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
exports.init = endpoint => {
|
||||||
|
let AWS = require("aws-sdk")
|
||||||
|
AWS.config.update({
|
||||||
|
region: AWS_REGION,
|
||||||
|
})
|
||||||
|
let docClientParams = {
|
||||||
|
correctClockSkew: true,
|
||||||
|
}
|
||||||
|
if (endpoint) {
|
||||||
|
docClientParams.endpoint = endpoint
|
||||||
|
} else if (environment.DYNAMO_ENDPOINT) {
|
||||||
|
docClientParams.endpoint = environment.DYNAMO_ENDPOINT
|
||||||
|
}
|
||||||
|
docClient = new AWS.DynamoDB.DocumentClient(docClientParams)
|
||||||
|
}
|
||||||
|
|
||||||
|
exports.apiKeyTable = new Table(TableInfo.API_KEYS)
|
||||||
|
exports.userTable = new Table(TableInfo.USERS)
|
||||||
|
|
||||||
|
if (environment.CLOUD) {
|
||||||
|
exports.init(`https://dynamodb.${AWS_REGION}.amazonaws.com`)
|
||||||
|
} else {
|
||||||
|
process.env.AWS_ACCESS_KEY_ID = "KEY_ID"
|
||||||
|
process.env.AWS_SECRET_ACCESS_KEY = "SECRET_KEY"
|
||||||
|
exports.init("http://localhost:8333")
|
||||||
|
}
|
|
@ -11,4 +11,7 @@ module.exports = {
|
||||||
AUTOMATION_BUCKET: process.env.AUTOMATION_BUCKET,
|
AUTOMATION_BUCKET: process.env.AUTOMATION_BUCKET,
|
||||||
BUDIBASE_ENVIRONMENT: process.env.BUDIBASE_ENVIRONMENT,
|
BUDIBASE_ENVIRONMENT: process.env.BUDIBASE_ENVIRONMENT,
|
||||||
SENDGRID_API_KEY: process.env.SENDGRID_API_KEY,
|
SENDGRID_API_KEY: process.env.SENDGRID_API_KEY,
|
||||||
|
CLOUD: process.env.CLOUD,
|
||||||
|
DYNAMO_ENDPOINT: process.env.DYNAMO_ENDPOINT,
|
||||||
|
AWS_REGION: process.env.AWS_REGION,
|
||||||
}
|
}
|
||||||
|
|
|
@ -20,6 +20,7 @@ module.exports = async (ctx, next) => {
|
||||||
if (builderToken) {
|
if (builderToken) {
|
||||||
try {
|
try {
|
||||||
const jwtPayload = jwt.verify(builderToken, ctx.config.jwtSecret)
|
const jwtPayload = jwt.verify(builderToken, ctx.config.jwtSecret)
|
||||||
|
ctx.apiKey = jwtPayload.apiKey
|
||||||
ctx.isAuthenticated = jwtPayload.accessLevelId === BUILDER_LEVEL_ID
|
ctx.isAuthenticated = jwtPayload.accessLevelId === BUILDER_LEVEL_ID
|
||||||
ctx.user = {
|
ctx.user = {
|
||||||
...jwtPayload,
|
...jwtPayload,
|
||||||
|
|
|
@ -16,8 +16,7 @@ module.exports = (permName, getItemId) => async (ctx, next) => {
|
||||||
}
|
}
|
||||||
|
|
||||||
if (ctx.user.accessLevel._id === BUILDER_LEVEL_ID) {
|
if (ctx.user.accessLevel._id === BUILDER_LEVEL_ID) {
|
||||||
await next()
|
return next()
|
||||||
return
|
|
||||||
}
|
}
|
||||||
|
|
||||||
if (permName === BUILDER) {
|
if (permName === BUILDER) {
|
||||||
|
@ -28,8 +27,7 @@ module.exports = (permName, getItemId) => async (ctx, next) => {
|
||||||
const permissionId = ({ name, itemId }) => name + (itemId ? `-${itemId}` : "")
|
const permissionId = ({ name, itemId }) => name + (itemId ? `-${itemId}` : "")
|
||||||
|
|
||||||
if (ctx.user.accessLevel._id === ADMIN_LEVEL_ID) {
|
if (ctx.user.accessLevel._id === ADMIN_LEVEL_ID) {
|
||||||
await next()
|
return next()
|
||||||
return
|
|
||||||
}
|
}
|
||||||
|
|
||||||
const thisPermissionId = permissionId({
|
const thisPermissionId = permissionId({
|
||||||
|
@ -42,8 +40,7 @@ module.exports = (permName, getItemId) => async (ctx, next) => {
|
||||||
ctx.user.accessLevel._id === POWERUSER_LEVEL_ID &&
|
ctx.user.accessLevel._id === POWERUSER_LEVEL_ID &&
|
||||||
!adminPermissions.map(permissionId).includes(thisPermissionId)
|
!adminPermissions.map(permissionId).includes(thisPermissionId)
|
||||||
) {
|
) {
|
||||||
await next()
|
return next()
|
||||||
return
|
|
||||||
}
|
}
|
||||||
|
|
||||||
if (
|
if (
|
||||||
|
@ -51,8 +48,7 @@ module.exports = (permName, getItemId) => async (ctx, next) => {
|
||||||
.map(permissionId)
|
.map(permissionId)
|
||||||
.includes(thisPermissionId)
|
.includes(thisPermissionId)
|
||||||
) {
|
) {
|
||||||
await next()
|
return next()
|
||||||
return
|
|
||||||
}
|
}
|
||||||
|
|
||||||
ctx.throw(403, "Not Authorized")
|
ctx.throw(403, "Not Authorized")
|
||||||
|
|
|
@ -0,0 +1,63 @@
|
||||||
|
const CouchDB = require("../db")
|
||||||
|
const usageQuota = require("../utilities/usageQuota")
|
||||||
|
const environment = require("../environment")
|
||||||
|
|
||||||
|
// currently only counting new writes and deletes
|
||||||
|
const METHOD_MAP = {
|
||||||
|
POST: 1,
|
||||||
|
DELETE: -1,
|
||||||
|
}
|
||||||
|
|
||||||
|
const DOMAIN_MAP = {
|
||||||
|
records: usageQuota.Properties.RECORD,
|
||||||
|
upload: usageQuota.Properties.UPLOAD,
|
||||||
|
views: usageQuota.Properties.VIEW,
|
||||||
|
users: usageQuota.Properties.USER,
|
||||||
|
// this will not be updated by endpoint calls
|
||||||
|
// instead it will be updated by triggers
|
||||||
|
automationRuns: usageQuota.Properties.AUTOMATION,
|
||||||
|
}
|
||||||
|
|
||||||
|
function getProperty(url) {
|
||||||
|
for (let domain of Object.keys(DOMAIN_MAP)) {
|
||||||
|
if (url.indexOf(domain) !== -1) {
|
||||||
|
return DOMAIN_MAP[domain]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
module.exports = async (ctx, next) => {
|
||||||
|
const db = new CouchDB(ctx.user.instanceId)
|
||||||
|
let usage = METHOD_MAP[ctx.req.method]
|
||||||
|
const property = getProperty(ctx.req.url)
|
||||||
|
if (usage == null || property == null) {
|
||||||
|
return next()
|
||||||
|
}
|
||||||
|
// post request could be a save of a pre-existing entry
|
||||||
|
if (ctx.request.body && ctx.request.body._id) {
|
||||||
|
try {
|
||||||
|
ctx.preExisting = await db.get(ctx.request.body._id)
|
||||||
|
return next()
|
||||||
|
} catch (err) {
|
||||||
|
ctx.throw(404, `${ctx.request.body._id} does not exist`)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// update usage for uploads to be the total size
|
||||||
|
if (property === usageQuota.Properties.UPLOAD) {
|
||||||
|
const files =
|
||||||
|
ctx.request.files.file.length > 1
|
||||||
|
? Array.from(ctx.request.files.file)
|
||||||
|
: [ctx.request.files.file]
|
||||||
|
usage = files.map(file => file.size).reduce((total, size) => total + size)
|
||||||
|
}
|
||||||
|
if (!environment.CLOUD) {
|
||||||
|
return next()
|
||||||
|
}
|
||||||
|
try {
|
||||||
|
await usageQuota.update(ctx.apiKey, property, usage)
|
||||||
|
return next()
|
||||||
|
} catch (err) {
|
||||||
|
ctx.throw(403, err)
|
||||||
|
}
|
||||||
|
}
|
|
@ -8,7 +8,9 @@ module.exports = (ctx, appId, instanceId) => {
|
||||||
instanceId,
|
instanceId,
|
||||||
appId,
|
appId,
|
||||||
}
|
}
|
||||||
|
if (process.env.BUDIBASE_API_KEY) {
|
||||||
|
builderUser.apiKey = process.env.BUDIBASE_API_KEY
|
||||||
|
}
|
||||||
const token = jwt.sign(builderUser, ctx.config.jwtSecret, {
|
const token = jwt.sign(builderUser, ctx.config.jwtSecret, {
|
||||||
expiresIn: "30 days",
|
expiresIn: "30 days",
|
||||||
})
|
})
|
||||||
|
|
|
@ -0,0 +1,103 @@
|
||||||
|
const environment = require("../environment")
|
||||||
|
const { apiKeyTable } = require("../db/dynamoClient")
|
||||||
|
|
||||||
|
const DEFAULT_USAGE = {
|
||||||
|
records: 0,
|
||||||
|
storage: 0,
|
||||||
|
views: 0,
|
||||||
|
automationRuns: 0,
|
||||||
|
users: 0,
|
||||||
|
}
|
||||||
|
|
||||||
|
const DEFAULT_PLAN = {
|
||||||
|
records: 1000,
|
||||||
|
// 1 GB
|
||||||
|
storage: 8589934592,
|
||||||
|
views: 10,
|
||||||
|
automationRuns: 100,
|
||||||
|
users: 10000,
|
||||||
|
}
|
||||||
|
|
||||||
|
function buildUpdateParams(key, property, usage) {
|
||||||
|
return {
|
||||||
|
primary: key,
|
||||||
|
condition:
|
||||||
|
"#quota.#prop < #limits.#prop AND #quotaReset > :now AND attribute_exists(#quota) AND attribute_exists(#limits)",
|
||||||
|
expression: "ADD #quota.#prop :usage",
|
||||||
|
names: {
|
||||||
|
"#quota": "usageQuota",
|
||||||
|
"#prop": property,
|
||||||
|
"#limits": "usageLimits",
|
||||||
|
"#quotaReset": "quotaReset",
|
||||||
|
},
|
||||||
|
values: {
|
||||||
|
":usage": usage,
|
||||||
|
":now": Date.now(),
|
||||||
|
},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function getNewQuotaReset() {
|
||||||
|
return Date.now() + 2592000000
|
||||||
|
}
|
||||||
|
|
||||||
|
exports.Properties = {
|
||||||
|
RECORD: "records",
|
||||||
|
UPLOAD: "storage",
|
||||||
|
VIEW: "views",
|
||||||
|
USER: "users",
|
||||||
|
AUTOMATION: "automationRuns",
|
||||||
|
}
|
||||||
|
|
||||||
|
exports.getAPIKey = async appId => {
|
||||||
|
return apiKeyTable.get({ primary: appId })
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Given a specified API key this will add to the usage object for the specified property.
|
||||||
|
* @param {string} apiKey The API key which is to be updated.
|
||||||
|
* @param {string} property The property which is to be added to (within the nested usageQuota object).
|
||||||
|
* @param {number} usage The amount (this can be negative) to adjust the number by.
|
||||||
|
* @returns {Promise<void>} When this completes the API key will now be up to date - the quota period may have
|
||||||
|
* also been reset after this call.
|
||||||
|
*/
|
||||||
|
exports.update = async (apiKey, property, usage) => {
|
||||||
|
// don't try validate in builder
|
||||||
|
if (!environment.CLOUD) {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
try {
|
||||||
|
await apiKeyTable.update(buildUpdateParams(apiKey, property, usage))
|
||||||
|
} catch (err) {
|
||||||
|
// conditional check means the condition failed, need to check why
|
||||||
|
if (err.code === "ConditionalCheckFailedException") {
|
||||||
|
// get the API key so we can check it
|
||||||
|
const keyObj = await apiKeyTable.get({ primary: apiKey })
|
||||||
|
// the usage quota or usage limits didn't exist
|
||||||
|
if (keyObj && (keyObj.usageQuota == null || keyObj.usageLimits == null)) {
|
||||||
|
keyObj.usageQuota =
|
||||||
|
keyObj.usageQuota == null ? DEFAULT_USAGE : keyObj.usageQuota
|
||||||
|
keyObj.usageLimits =
|
||||||
|
keyObj.usageLimits == null ? DEFAULT_PLAN : keyObj.usageLimits
|
||||||
|
keyObj.quotaReset = getNewQuotaReset()
|
||||||
|
await apiKeyTable.put({ item: keyObj })
|
||||||
|
return
|
||||||
|
}
|
||||||
|
// we have infact breached the reset period
|
||||||
|
else if (keyObj && keyObj.quotaReset <= Date.now()) {
|
||||||
|
// update the quota reset period and reset the values for all properties
|
||||||
|
keyObj.quotaReset = getNewQuotaReset()
|
||||||
|
for (let prop of Object.keys(keyObj.usageQuota)) {
|
||||||
|
if (prop === property) {
|
||||||
|
keyObj.usageQuota[prop] = usage > 0 ? usage : 0
|
||||||
|
} else {
|
||||||
|
keyObj.usageQuota[prop] = 0
|
||||||
|
}
|
||||||
|
}
|
||||||
|
await apiKeyTable.put({ item: keyObj })
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
|
throw "Resource limits have been reached"
|
||||||
|
}
|
||||||
|
}
|
|
@ -946,9 +946,10 @@ available-typed-arrays@^1.0.0, available-typed-arrays@^1.0.2:
|
||||||
dependencies:
|
dependencies:
|
||||||
array-filter "^1.0.0"
|
array-filter "^1.0.0"
|
||||||
|
|
||||||
aws-sdk@^2.706.0:
|
aws-sdk@^2.767.0:
|
||||||
version "2.706.0"
|
version "2.767.0"
|
||||||
resolved "https://registry.yarnpkg.com/aws-sdk/-/aws-sdk-2.706.0.tgz#09f65e9a91ecac5a635daf934082abae30eca953"
|
resolved "https://registry.yarnpkg.com/aws-sdk/-/aws-sdk-2.767.0.tgz#9863c8bfd5990106b95f38e9345a547fee782470"
|
||||||
|
integrity sha512-soPZxjNpat0CtuIqm54GO/FDT4SZTlQG0icSptWYfMFYdkXe8b0tJqaPssNn9TzlgoWDCNTdaoepM6TN0rNHkQ==
|
||||||
dependencies:
|
dependencies:
|
||||||
buffer "4.9.2"
|
buffer "4.9.2"
|
||||||
events "1.1.1"
|
events "1.1.1"
|
||||||
|
@ -1081,6 +1082,7 @@ bluebird-lst@^1.0.9:
|
||||||
bluebird@^3.5.1, bluebird@^3.5.5:
|
bluebird@^3.5.1, bluebird@^3.5.5:
|
||||||
version "3.7.2"
|
version "3.7.2"
|
||||||
resolved "https://registry.yarnpkg.com/bluebird/-/bluebird-3.7.2.tgz#9f229c15be272454ffa973ace0dbee79a1b0c36f"
|
resolved "https://registry.yarnpkg.com/bluebird/-/bluebird-3.7.2.tgz#9f229c15be272454ffa973ace0dbee79a1b0c36f"
|
||||||
|
integrity sha512-XpNj6GDQzdfW+r2Wnn7xiSAd7TM3jzkxGXBGTtWKuSXv1xUV+azxAm8jdWZN06QTQk+2N2XB9jRDkvbmQmcRtg==
|
||||||
|
|
||||||
boolean@^3.0.0, boolean@^3.0.1:
|
boolean@^3.0.0, boolean@^3.0.1:
|
||||||
version "3.0.1"
|
version "3.0.1"
|
||||||
|
|
Loading…
Reference in New Issue