From 552c31a53e2676c8a9c9b80bb30482926a9c69a2 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Thu, 8 Oct 2020 17:34:41 +0100 Subject: [PATCH] Updates for API usage after testing against local Dynamo. --- .../server/scripts/createApiKeyAndAppId.js | 55 +++++++++++++++++++ packages/server/src/automations/index.js | 9 +-- .../src/automations/steps/createRecord.js | 7 ++- .../src/automations/steps/createUser.js | 7 ++- .../src/automations/steps/deleteRecord.js | 7 ++- packages/server/src/automations/thread.js | 1 + packages/server/src/db/dynamoClient.js | 30 ++++++++-- packages/server/src/middleware/authorized.js | 12 ++-- packages/server/src/middleware/usageQuota.js | 7 ++- .../src/utilities/builder/setBuilderToken.js | 4 +- packages/server/src/utilities/usageQuota.js | 21 +++---- 11 files changed, 127 insertions(+), 33 deletions(-) create mode 100644 packages/server/scripts/createApiKeyAndAppId.js diff --git a/packages/server/scripts/createApiKeyAndAppId.js b/packages/server/scripts/createApiKeyAndAppId.js new file mode 100644 index 0000000000..f6a6c1fcee --- /dev/null +++ b/packages/server/scripts/createApiKeyAndAppId.js @@ -0,0 +1,55 @@ +// THIS will create API Keys and App Ids input in a local Dynamo instance if it is running +const dynamoClient = require("../src/db/dynamoClient") + +if (process.argv[2] == null || process.argv[3] == null) { + console.error( + "Inputs incorrect format, was expecting: node createApiKeyAndAppId.js " + ) + process.exit(-1) +} + +const FAKE_STRING = "fakestring" + +// set fake credentials for local dynamo to actually work +process.env.AWS_ACCESS_KEY_ID = "KEY_ID" +process.env.AWS_SECRET_ACCESS_KEY = "SECRET_KEY" +dynamoClient.init("http://localhost:8333") + +async function run() { + await dynamoClient.apiKeyTable.put({ + item: { + pk: process.argv[2], + accountId: FAKE_STRING, + trackingId: FAKE_STRING, + quotaReset: Date.now() + 2592000000, + usageQuota: { + automationRuns: 0, + records: 0, + storage: 0, + users: 0, + views: 0, + }, + usageLimits: { + automationRuns: 10, + records: 10, + storage: 1000, + users: 10, + views: 10, + }, + }, + }) + await dynamoClient.apiKeyTable.put({ + item: { + pk: process.argv[3], + apiKey: process.argv[2], + }, + }) +} + +run() + .then(() => { + console.log("Records should have been created.") + }) + .catch(err => { + console.error("Cannot create records - " + err) + }) diff --git a/packages/server/src/automations/index.js b/packages/server/src/automations/index.js index 882f24391b..f407e35a78 100644 --- a/packages/server/src/automations/index.js +++ b/packages/server/src/automations/index.js @@ -21,9 +21,10 @@ function runWorker(job) { async function updateQuota(automation) { const appId = automation.appId - const apiKey = await getAPIKey(appId) + const apiObj = await getAPIKey(appId) // this will fail, causing automation to escape if limits reached - await update(apiKey, Properties.AUTOMATION, 1) + await update(apiObj.apiKey, Properties.AUTOMATION, 1) + return apiObj.apiKey } /** @@ -33,8 +34,8 @@ module.exports.init = function() { actions.init().then(() => { triggers.automationQueue.process(async job => { try { - if (environment.CLOUD) { - await updateQuota(job.data.automation) + if (environment.CLOUD && job.data.automation) { + job.data.automation.apiKey = await updateQuota(job.data.automation) } if (environment.BUDIBASE_ENVIRONMENT === "PRODUCTION") { await runWorker(job) diff --git a/packages/server/src/automations/steps/createRecord.js b/packages/server/src/automations/steps/createRecord.js index d0d6a36f5a..e268f218e2 100644 --- a/packages/server/src/automations/steps/createRecord.js +++ b/packages/server/src/automations/steps/createRecord.js @@ -1,5 +1,7 @@ const recordController = require("../../api/controllers/record") const automationUtils = require("../automationUtils") +const environment = require("../../environment") +const usage = require("../../utilities/usageQuota") module.exports.definition = { name: "Create Row", @@ -56,7 +58,7 @@ module.exports.definition = { }, } -module.exports.run = async function({ inputs, instanceId }) { +module.exports.run = async function({ inputs, instanceId, apiKey }) { // TODO: better logging of when actions are missed due to missing parameters if (inputs.record == null || inputs.record.modelId == null) { return @@ -78,6 +80,9 @@ module.exports.run = async function({ inputs, instanceId }) { } try { + if (environment.CLOUD) { + await usage.update(apiKey, usage.Properties.RECORD, 1) + } await recordController.save(ctx) return { record: inputs.record, diff --git a/packages/server/src/automations/steps/createUser.js b/packages/server/src/automations/steps/createUser.js index de2b6ca1ad..f0bea286d7 100644 --- a/packages/server/src/automations/steps/createUser.js +++ b/packages/server/src/automations/steps/createUser.js @@ -1,5 +1,7 @@ const accessLevels = require("../../utilities/accessLevels") const userController = require("../../api/controllers/user") +const environment = require("../../environment") +const usage = require("../../utilities/usageQuota") module.exports.definition = { description: "Create a new user", @@ -56,7 +58,7 @@ module.exports.definition = { }, } -module.exports.run = async function({ inputs, instanceId }) { +module.exports.run = async function({ inputs, instanceId, apiKey }) { const { username, password, accessLevelId } = inputs const ctx = { user: { @@ -68,6 +70,9 @@ module.exports.run = async function({ inputs, instanceId }) { } try { + if (environment.CLOUD) { + await usage.update(apiKey, usage.Properties.USER, 1) + } await userController.create(ctx) return { response: ctx.body, diff --git a/packages/server/src/automations/steps/deleteRecord.js b/packages/server/src/automations/steps/deleteRecord.js index 0a02099bd4..6126895da6 100644 --- a/packages/server/src/automations/steps/deleteRecord.js +++ b/packages/server/src/automations/steps/deleteRecord.js @@ -1,4 +1,6 @@ const recordController = require("../../api/controllers/record") +const environment = require("../../environment") +const usage = require("../../utilities/usageQuota") module.exports.definition = { description: "Delete a row from your database", @@ -48,7 +50,7 @@ module.exports.definition = { }, } -module.exports.run = async function({ inputs, instanceId }) { +module.exports.run = async function({ inputs, instanceId, apiKey }) { // TODO: better logging of when actions are missed due to missing parameters if (inputs.id == null || inputs.revision == null) { return @@ -63,6 +65,9 @@ module.exports.run = async function({ inputs, instanceId }) { } try { + if (environment.CLOUD) { + await usage.update(apiKey, usage.Properties.RECORD, -1) + } await recordController.destroy(ctx) return { response: ctx.body, diff --git a/packages/server/src/automations/thread.js b/packages/server/src/automations/thread.js index fa826afbe9..5362597cfd 100644 --- a/packages/server/src/automations/thread.js +++ b/packages/server/src/automations/thread.js @@ -62,6 +62,7 @@ class Orchestrator { const outputs = await stepFn({ inputs: step.inputs, instanceId: this._instanceId, + apiKey: automation.apiKey, }) if (step.stepId === FILTER_STEP_ID && !outputs.success) { break diff --git a/packages/server/src/db/dynamoClient.js b/packages/server/src/db/dynamoClient.js index 9f30b151da..6250486bf7 100644 --- a/packages/server/src/db/dynamoClient.js +++ b/packages/server/src/db/dynamoClient.js @@ -1,6 +1,8 @@ let _ = require("lodash") let environment = require("../environment") +const AWS_REGION = environment.AWS_REGION ? environment.AWS_REGION : "eu-west-1" + const TableInfo = { API_KEYS: { name: "beta-api-key-table", @@ -49,6 +51,7 @@ class Table { condition, names, values, + exists, otherProps, }) { let params = { @@ -66,6 +69,13 @@ class Table { if (this._sort && sort) { params.Key[this._sort] = sort } + if (exists) { + params.ExpressionAttributeNames["#PRIMARY"] = this._primary + if (params.ConditionExpression) { + params.ConditionExpression += " AND " + } + params.ConditionExpression += "attribute_exists(#PRIMARY)" + } if (otherProps) { params = _.merge(params, otherProps) } @@ -90,15 +100,17 @@ class Table { } } -exports.init = () => { - if (!environment.CLOUD) { - return - } +exports.init = endpoint => { let AWS = require("aws-sdk") + AWS.config.update({ + region: AWS_REGION, + }) let docClientParams = { correctClockSkew: true, } - if (environment.DYNAMO_ENDPOINT) { + if (endpoint) { + docClientParams.endpoint = endpoint + } else if (environment.DYNAMO_ENDPOINT) { docClientParams.endpoint = environment.DYNAMO_ENDPOINT } docClient = new AWS.DynamoDB.DocumentClient(docClientParams) @@ -106,3 +118,11 @@ exports.init = () => { exports.apiKeyTable = new Table(TableInfo.API_KEYS) exports.userTable = new Table(TableInfo.USERS) + +if (environment.CLOUD) { + exports.init(`https://dynamodb.${AWS_REGION}.amazonaws.com`) +} else { + process.env.AWS_ACCESS_KEY_ID = "KEY_ID" + process.env.AWS_SECRET_ACCESS_KEY = "SECRET_KEY" + exports.init("http://localhost:8333") +} diff --git a/packages/server/src/middleware/authorized.js b/packages/server/src/middleware/authorized.js index b452d63cf5..4cce4c4670 100644 --- a/packages/server/src/middleware/authorized.js +++ b/packages/server/src/middleware/authorized.js @@ -16,8 +16,7 @@ module.exports = (permName, getItemId) => async (ctx, next) => { } if (ctx.user.accessLevel._id === BUILDER_LEVEL_ID) { - await next() - return + return next() } if (permName === BUILDER) { @@ -28,8 +27,7 @@ module.exports = (permName, getItemId) => async (ctx, next) => { const permissionId = ({ name, itemId }) => name + (itemId ? `-${itemId}` : "") if (ctx.user.accessLevel._id === ADMIN_LEVEL_ID) { - await next() - return + return next() } const thisPermissionId = permissionId({ @@ -42,8 +40,7 @@ module.exports = (permName, getItemId) => async (ctx, next) => { ctx.user.accessLevel._id === POWERUSER_LEVEL_ID && !adminPermissions.map(permissionId).includes(thisPermissionId) ) { - await next() - return + return next() } if ( @@ -51,8 +48,7 @@ module.exports = (permName, getItemId) => async (ctx, next) => { .map(permissionId) .includes(thisPermissionId) ) { - await next() - return + return next() } ctx.throw(403, "Not Authorized") diff --git a/packages/server/src/middleware/usageQuota.js b/packages/server/src/middleware/usageQuota.js index 74df19a015..e82305dc12 100644 --- a/packages/server/src/middleware/usageQuota.js +++ b/packages/server/src/middleware/usageQuota.js @@ -1,5 +1,6 @@ const CouchDB = require("../db") const usageQuota = require("../utilities/usageQuota") +const environment = require("../environment") // currently only counting new writes and deletes const METHOD_MAP = { @@ -29,11 +30,9 @@ module.exports = async (ctx, next) => { const db = new CouchDB(ctx.user.instanceId) let usage = METHOD_MAP[ctx.req.method] const property = getProperty(ctx.req.url) - console.log(ctx.req.url) if (usage == null || property == null) { return next() } - console.log(`${usage} to ${property}`) // post request could be a save of a pre-existing entry if (ctx.request.body && ctx.request.body._id) { try { @@ -52,8 +51,12 @@ module.exports = async (ctx, next) => { : [ctx.request.files.file] usage = files.map(file => file.size).reduce((total, size) => total + size) } + if (!environment.CLOUD) { + return next() + } try { await usageQuota.update(ctx.apiKey, property, usage) + return next() } catch (err) { ctx.throw(403, err) } diff --git a/packages/server/src/utilities/builder/setBuilderToken.js b/packages/server/src/utilities/builder/setBuilderToken.js index 12622d5522..d43a9543e7 100644 --- a/packages/server/src/utilities/builder/setBuilderToken.js +++ b/packages/server/src/utilities/builder/setBuilderToken.js @@ -8,7 +8,9 @@ module.exports = (ctx, appId, instanceId) => { instanceId, appId, } - + if (process.env.BUDIBASE_API_KEY) { + builderUser.apiKey = process.env.BUDIBASE_API_KEY + } const token = jwt.sign(builderUser, ctx.config.jwtSecret, { expiresIn: "30 days", }) diff --git a/packages/server/src/utilities/usageQuota.js b/packages/server/src/utilities/usageQuota.js index 7ba0005213..f16fb6aba8 100644 --- a/packages/server/src/utilities/usageQuota.js +++ b/packages/server/src/utilities/usageQuota.js @@ -4,12 +4,12 @@ const { apiKeyTable } = require("../db/dynamoClient") function buildUpdateParams(key, property, usage) { return { primary: key, - condition: "#quota.#prop + :usage < #limits.model AND #quotaReset < :now", + condition: "#quota.#prop < #limits.#prop AND #quotaReset > :now", expression: "ADD #quota.#prop :usage", names: { "#quota": "usageQuota", "#prop": property, - "#limits": "limits", + "#limits": "usageLimits", "#quotaReset": "quotaReset", }, values: { @@ -50,21 +50,22 @@ exports.update = async (apiKey, property, usage) => { try { await apiKeyTable.update(buildUpdateParams(apiKey, property, usage)) } catch (err) { - if (err.code !== "ConditionalCheckFailedException") { + if (err.code === "ConditionalCheckFailedException") { // get the API key so we can check it - let apiKey = await apiKeyTable.get({ primary: apiKey }) + const keyObj = await apiKeyTable.get({ primary: apiKey }) // we have infact breached the reset period - if (apiKey && apiKey.quotaReset >= Date.now()) { + if (keyObj && keyObj.quotaReset <= Date.now()) { // update the quota reset period and reset the values for all properties - apiKey.quotaReset = Date.now() + QUOTA_RESET - for (let prop of Object.keys(apiKey.usageQuota)) { + keyObj.quotaReset = Date.now() + QUOTA_RESET + for (let prop of Object.keys(keyObj.usageQuota)) { if (prop === property) { - apiKey.usageQuota[prop] = usage > 0 ? usage : 0 + keyObj.usageQuota[prop] = usage > 0 ? usage : 0 } else { - apiKey.usageQuota[prop] = 0 + keyObj.usageQuota[prop] = 0 } } - await apiKeyTable.put({ item: apiKey }) + await apiKeyTable.put({ item: keyObj }) + return } throw "Resource limits have been reached" }