From fc2e2a88040e8cfb0c091631f1498d3e7a9e4e9a Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Tue, 6 Oct 2020 19:13:41 +0100 Subject: [PATCH 01/21] First work towards implementing Dynamo usage in the server when running in the cloud; this is for tracking usage against API keys. --- packages/server/package.json | 2 +- packages/server/src/api/controllers/auth.js | 6 + packages/server/src/api/controllers/model.js | 14 +- packages/server/src/api/controllers/record.js | 5 +- packages/server/src/api/routes/model.js | 4 +- packages/server/src/api/routes/record.js | 3 + .../src/api/routes/tests/couchTestUtils.js | 3 + packages/server/src/db/dynamoClient.js | 108 ++++++++ packages/server/src/environment.js | 3 + .../server/src/middleware/authenticated.js | 1 + packages/server/src/middleware/usageQuota.js | 77 ++++++ packages/server/yarn.lock | 256 +++++++++++++++++- 12 files changed, 461 insertions(+), 21 deletions(-) create mode 100644 packages/server/src/db/dynamoClient.js create mode 100644 packages/server/src/middleware/usageQuota.js diff --git a/packages/server/package.json b/packages/server/package.json index 1875e44285..6905685324 100644 --- a/packages/server/package.json +++ b/packages/server/package.json @@ -46,7 +46,7 @@ "@koa/router": "^8.0.0", "@sendgrid/mail": "^7.1.1", "@sentry/node": "^5.19.2", - "aws-sdk": "^2.706.0", + "aws-sdk": "^2.767.0", "bcryptjs": "^2.4.3", "chmodr": "^1.2.0", "dotenv": "^8.2.0", diff --git a/packages/server/src/api/controllers/auth.js b/packages/server/src/api/controllers/auth.js index 828a88bb9b..b7a7c116b9 100644 --- a/packages/server/src/api/controllers/auth.js +++ b/packages/server/src/api/controllers/auth.js @@ -2,6 +2,8 @@ const jwt = require("jsonwebtoken") const CouchDB = require("../../db") const ClientDb = require("../../db/clientDb") const bcrypt = require("../../utilities/bcrypt") +const environment = require("../../environment") +const { apiKeyTable } = require("../../db/dynamoClient") const { generateUserID } = require("../../db/utils") exports.authenticate = async ctx => { @@ -51,6 +53,10 @@ exports.authenticate = async ctx => { appId: ctx.user.appId, instanceId, } + // if in cloud add the user api key + if (environment.CLOUD) { + payload.apiKey = await apiKeyTable.get({ primary: ctx.user.appId }) + } const token = jwt.sign(payload, ctx.config.jwtSecret, { expiresIn: "1 day", diff --git a/packages/server/src/api/controllers/model.js b/packages/server/src/api/controllers/model.js index 2593002d72..29bf2dd90f 100644 --- a/packages/server/src/api/controllers/model.js +++ b/packages/server/src/api/controllers/model.js @@ -30,17 +30,9 @@ exports.save = async function(ctx) { views: {}, ...ctx.request.body, } - // get the model in its previous state for differencing - let oldModel - let oldModelId = ctx.request.body._id - if (oldModelId) { - // if it errors then the oldModelId is invalid - can't diff it - try { - oldModel = await db.get(oldModelId) - } catch (err) { - oldModel = null - } - } + + // if the model obj had an _id then it will have been retrieved + const oldModel = ctx.preExisting // rename record fields when table column is renamed const { _rename } = modelToSave diff --git a/packages/server/src/api/controllers/record.js b/packages/server/src/api/controllers/record.js index 45d983eb2b..7422cc9fc8 100644 --- a/packages/server/src/api/controllers/record.js +++ b/packages/server/src/api/controllers/record.js @@ -70,6 +70,9 @@ exports.save = async function(ctx) { record._id = generateRecordID(record.modelId) } + // if the record obj had an _id then it will have been retrieved + const existingRecord = ctx.preExisting + const model = await db.get(record.modelId) const validateResult = await validate({ @@ -86,8 +89,6 @@ exports.save = async function(ctx) { return } - const existingRecord = record._rev && (await db.get(record._id)) - // make sure link records are up to date record = await linkRecords.updateLinks({ instanceId, diff --git a/packages/server/src/api/routes/model.js b/packages/server/src/api/routes/model.js index 00eb46d515..118870cc61 100644 --- a/packages/server/src/api/routes/model.js +++ b/packages/server/src/api/routes/model.js @@ -1,6 +1,7 @@ const Router = require("@koa/router") const modelController = require("../controllers/model") const authorized = require("../../middleware/authorized") +const usage = require("../../middleware/usageQuota") const { BUILDER, READ_MODEL } = require("../../utilities/accessLevels") const router = Router() @@ -12,10 +13,11 @@ router authorized(READ_MODEL, ctx => ctx.params.id), modelController.find ) - .post("/api/models", authorized(BUILDER), modelController.save) + .post("/api/models", authorized(BUILDER), usage, modelController.save) .delete( "/api/models/:modelId/:revId", authorized(BUILDER), + usage, modelController.destroy ) diff --git a/packages/server/src/api/routes/record.js b/packages/server/src/api/routes/record.js index ddc26a55af..b3b5e9ed56 100644 --- a/packages/server/src/api/routes/record.js +++ b/packages/server/src/api/routes/record.js @@ -1,6 +1,7 @@ const Router = require("@koa/router") const recordController = require("../controllers/record") const authorized = require("../../middleware/authorized") +const usage = require("../../middleware/usageQuota") const { READ_MODEL, WRITE_MODEL } = require("../../utilities/accessLevels") const router = Router() @@ -25,6 +26,7 @@ router .post( "/api/:modelId/records", authorized(WRITE_MODEL, ctx => ctx.params.modelId), + usage, recordController.save ) .patch( @@ -40,6 +42,7 @@ router .delete( "/api/:modelId/records/:recordId/:revId", authorized(WRITE_MODEL, ctx => ctx.params.modelId), + usage, recordController.destroy ) diff --git a/packages/server/src/api/routes/tests/couchTestUtils.js b/packages/server/src/api/routes/tests/couchTestUtils.js index a22a2a427f..c269c14e4d 100644 --- a/packages/server/src/api/routes/tests/couchTestUtils.js +++ b/packages/server/src/api/routes/tests/couchTestUtils.js @@ -40,6 +40,9 @@ exports.defaultHeaders = (appId, instanceId) => { } exports.createModel = async (request, appId, instanceId, model) => { + if (model != null && model._id) { + delete model._id + } model = model || { name: "TestModel", type: "model", diff --git a/packages/server/src/db/dynamoClient.js b/packages/server/src/db/dynamoClient.js new file mode 100644 index 0000000000..9f30b151da --- /dev/null +++ b/packages/server/src/db/dynamoClient.js @@ -0,0 +1,108 @@ +let _ = require("lodash") +let environment = require("../environment") + +const TableInfo = { + API_KEYS: { + name: "beta-api-key-table", + primary: "pk", + }, + USERS: { + name: "prod-budi-table", + primary: "pk", + sort: "sk", + }, +} + +let docClient = null + +class Table { + constructor(tableInfo) { + if (!tableInfo.name || !tableInfo.primary) { + throw "Table info must specify a name and a primary key" + } + this._name = tableInfo.name + this._primary = tableInfo.primary + this._sort = tableInfo.sort + } + + async get({ primary, sort, otherProps }) { + let params = { + TableName: this._name, + Key: { + [this._primary]: primary, + }, + } + if (this._sort && sort) { + params.Key[this._sort] = sort + } + if (otherProps) { + params = _.merge(params, otherProps) + } + let response = await docClient.get(params).promise() + return response.Item + } + + async update({ + primary, + sort, + expression, + condition, + names, + values, + otherProps, + }) { + let params = { + TableName: this._name, + Key: { + [this._primary]: primary, + }, + ExpressionAttributeNames: names, + ExpressionAttributeValues: values, + UpdateExpression: expression, + } + if (condition) { + params.ConditionExpression = condition + } + if (this._sort && sort) { + params.Key[this._sort] = sort + } + if (otherProps) { + params = _.merge(params, otherProps) + } + return docClient.update(params).promise() + } + + async put({ item, otherProps }) { + if ( + item[this._primary] == null || + (this._sort && item[this._sort] == null) + ) { + throw "Cannot put item without primary and sort key (if required)" + } + let params = { + TableName: this._name, + Item: item, + } + if (otherProps) { + params = _.merge(params, otherProps) + } + return docClient.put(params).promise() + } +} + +exports.init = () => { + if (!environment.CLOUD) { + return + } + let AWS = require("aws-sdk") + let docClientParams = { + correctClockSkew: true, + } + if (environment.DYNAMO_ENDPOINT) { + docClientParams.endpoint = environment.DYNAMO_ENDPOINT + } + docClient = new AWS.DynamoDB.DocumentClient(docClientParams) +} + +exports.apiKeyTable = new Table(TableInfo.API_KEYS) +exports.userTable = new Table(TableInfo.USERS) diff --git a/packages/server/src/environment.js b/packages/server/src/environment.js index bd08f191aa..f24d495397 100644 --- a/packages/server/src/environment.js +++ b/packages/server/src/environment.js @@ -11,4 +11,7 @@ module.exports = { AUTOMATION_BUCKET: process.env.AUTOMATION_BUCKET, BUDIBASE_ENVIRONMENT: process.env.BUDIBASE_ENVIRONMENT, SENDGRID_API_KEY: process.env.SENDGRID_API_KEY, + CLOUD: process.env.CLOUD, + DYNAMO_ENDPOINT: process.env.DYNAMO_ENDPOINT, + AWS_REGION: process.env.AWS_REGION, } diff --git a/packages/server/src/middleware/authenticated.js b/packages/server/src/middleware/authenticated.js index 53cb0b2c13..126d616e3b 100644 --- a/packages/server/src/middleware/authenticated.js +++ b/packages/server/src/middleware/authenticated.js @@ -20,6 +20,7 @@ module.exports = async (ctx, next) => { if (builderToken) { try { const jwtPayload = jwt.verify(builderToken, ctx.config.jwtSecret) + ctx.apiKey = jwtPayload.apiKey ctx.isAuthenticated = jwtPayload.accessLevelId === BUILDER_LEVEL_ID ctx.user = { ...jwtPayload, diff --git a/packages/server/src/middleware/usageQuota.js b/packages/server/src/middleware/usageQuota.js new file mode 100644 index 0000000000..1d119f25c8 --- /dev/null +++ b/packages/server/src/middleware/usageQuota.js @@ -0,0 +1,77 @@ +const CouchDB = require("../db") +const environment = require("../environment") +const { apiKeyTable } = require("../db/dynamoClient") + +// a normalised month in milliseconds +const QUOTA_RESET = 2592000000 + +// currently only counting new writes and deletes +const METHOD_MAP = { + POST: 1, + DELETE: -1, +} + +const DOMAIN_MAP = { + models: "model", + records: "record", +} + +function buildUpdateParams(key, property, usage) { + return { + primary: key, + condition: "#quota.#prop + :usage < #limits.model AND #quotaReset < :now", + expression: "ADD #quota.#prop :usage", + names: { + "#quota": "usageQuota", + "#prop": property, + "#limits": "limits", + "#quotaReset": "quotaReset", + }, + values: { + ":usage": usage, + ":now": Date.now(), + }, + } +} + +module.exports = async (ctx, next) => { + const db = new CouchDB(ctx.user.instanceId) + const usage = METHOD_MAP[ctx.req.method] + const domainParts = ctx.req.url.split("/") + const property = DOMAIN_MAP[domainParts[domainParts.length - 1]] + if (usage == null || property == null) { + return next() + } + // post request could be a save of a pre-existing entry + if (ctx.request.body && ctx.request.body._id) { + try { + ctx.preExisting = await db.get(ctx.request.body._id) + return next() + } catch (err) { + ctx.throw(404, `${ctx.request.body._id} does not exist`) + return + } + } + // don't try validate in builder + if (!environment.CLOUD) { + return next() + } + try { + await apiKeyTable.update(buildUpdateParams(ctx.apiKey, property, usage)) + } catch (err) { + if (err.code !== "ConditionalCheckFailedException") { + // get the API key so we can check it + let apiKey = await apiKeyTable.get({ primary: ctx.apiKey }) + // we have infact breached the reset period + if (apiKey && apiKey.quotaReset >= Date.now()) { + // update the quota reset period and reset the values for all properties + apiKey.quotaReset = Date.now() + QUOTA_RESET + for (let prop of Object.keys(apiKey.usageQuota)) { + apiKey.usageQuota[prop] = 0 + } + await apiKeyTable.put({ item: apiKey }) + } + ctx.throw(403, `Resource limits have been reached`) + } + } +} \ No newline at end of file diff --git a/packages/server/yarn.lock b/packages/server/yarn.lock index de283385a8..cb22d75c7b 100644 --- a/packages/server/yarn.lock +++ b/packages/server/yarn.lock @@ -172,6 +172,15 @@ lodash "^4.17.13" to-fast-properties "^2.0.0" +"@budibase/client@^0.1.23": + version "0.1.23" + resolved "https://registry.yarnpkg.com/@budibase/client/-/client-0.1.23.tgz#d72d2b26ff3a2d99f2b6c1b71020b1136880937d" + integrity sha512-pZdwdCq5kKLZfZYxasIHBNnqu3BFFrqJLxXMFs0K9ddCVZ0UNons59nn73nFGbeRgNVdWp6yyW71XyMQr8NOEw== + dependencies: + deep-equal "^2.0.1" + mustache "^4.0.1" + regexparam "^1.3.0" + "@cnakazawa/watch@^1.0.3": version "1.0.4" resolved "https://registry.yarnpkg.com/@cnakazawa/watch/-/watch-1.0.4.tgz#f864ae85004d0fcab6f50be9141c4da368d1656a" @@ -867,6 +876,11 @@ array-equal@^1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/array-equal/-/array-equal-1.0.0.tgz#8c2a5ef2472fd9ea742b04c77a75093ba2757c93" +array-filter@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/array-filter/-/array-filter-1.0.0.tgz#baf79e62e6ef4c2a4c0b831232daffec251f9d83" + integrity sha1-uveeYubvTCpMC4MSMtr/7CUfnYM= + array-unique@^0.3.2: version "0.3.2" resolved "https://registry.yarnpkg.com/array-unique/-/array-unique-0.3.2.tgz#a894b75d4bc4f6cd679ef3244a9fd8f46ae2d428" @@ -925,9 +939,17 @@ atomic-sleep@^1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/atomic-sleep/-/atomic-sleep-1.0.0.tgz#eb85b77a601fc932cfe432c5acd364a9e2c9075b" -aws-sdk@^2.706.0: - version "2.706.0" - resolved "https://registry.yarnpkg.com/aws-sdk/-/aws-sdk-2.706.0.tgz#09f65e9a91ecac5a635daf934082abae30eca953" +available-typed-arrays@^1.0.0, available-typed-arrays@^1.0.2: + version "1.0.2" + resolved "https://registry.yarnpkg.com/available-typed-arrays/-/available-typed-arrays-1.0.2.tgz#6b098ca9d8039079ee3f77f7b783c4480ba513f5" + integrity sha512-XWX3OX8Onv97LMk/ftVyBibpGwY5a8SmuxZPzeOxqmuEqUCOM9ZE+uIaD1VNJ5QnvU2UQusvmKbuM1FR8QWGfQ== + dependencies: + array-filter "^1.0.0" + +aws-sdk@^2.767.0: + version "2.767.0" + resolved "https://registry.yarnpkg.com/aws-sdk/-/aws-sdk-2.767.0.tgz#9863c8bfd5990106b95f38e9345a547fee782470" + integrity sha512-soPZxjNpat0CtuIqm54GO/FDT4SZTlQG0icSptWYfMFYdkXe8b0tJqaPssNn9TzlgoWDCNTdaoepM6TN0rNHkQ== dependencies: buffer "4.9.2" events "1.1.1" @@ -1725,6 +1747,26 @@ decompress@^4.2.1: pify "^2.3.0" strip-dirs "^2.0.0" +deep-equal@^2.0.1: + version "2.0.4" + resolved "https://registry.yarnpkg.com/deep-equal/-/deep-equal-2.0.4.tgz#6b0b407a074666033169df3acaf128e1c6f3eab6" + integrity sha512-BUfaXrVoCfgkOQY/b09QdO9L3XNoF2XH0A3aY9IQwQL/ZjLOe8FQgCNVl1wiolhsFo8kFdO9zdPViCPbmaJA5w== + dependencies: + es-abstract "^1.18.0-next.1" + es-get-iterator "^1.1.0" + is-arguments "^1.0.4" + is-date-object "^1.0.2" + is-regex "^1.1.1" + isarray "^2.0.5" + object-is "^1.1.3" + object-keys "^1.1.1" + object.assign "^4.1.1" + regexp.prototype.flags "^1.3.0" + side-channel "^1.0.3" + which-boxed-primitive "^1.0.1" + which-collection "^1.0.1" + which-typed-array "^1.1.2" + deep-equal@~1.0.1: version "1.0.1" resolved "https://registry.yarnpkg.com/deep-equal/-/deep-equal-1.0.1.tgz#f5d260292b660e084eff4cdbc9f08ad3247448b5" @@ -2073,6 +2115,54 @@ es-abstract@^1.17.0-next.1, es-abstract@^1.17.2, es-abstract@^1.17.5: string.prototype.trimleft "^2.1.1" string.prototype.trimright "^2.1.1" +es-abstract@^1.17.4: + version "1.17.7" + resolved "https://registry.yarnpkg.com/es-abstract/-/es-abstract-1.17.7.tgz#a4de61b2f66989fc7421676c1cb9787573ace54c" + integrity sha512-VBl/gnfcJ7OercKA9MVaegWsBHFjV492syMudcnQZvt/Dw8ezpcOHYZXa/J96O8vx+g4x65YKhxOwDUh63aS5g== + dependencies: + es-to-primitive "^1.2.1" + function-bind "^1.1.1" + has "^1.0.3" + has-symbols "^1.0.1" + is-callable "^1.2.2" + is-regex "^1.1.1" + object-inspect "^1.8.0" + object-keys "^1.1.1" + object.assign "^4.1.1" + string.prototype.trimend "^1.0.1" + string.prototype.trimstart "^1.0.1" + +es-abstract@^1.18.0-next.0, es-abstract@^1.18.0-next.1: + version "1.18.0-next.1" + resolved "https://registry.yarnpkg.com/es-abstract/-/es-abstract-1.18.0-next.1.tgz#6e3a0a4bda717e5023ab3b8e90bec36108d22c68" + integrity sha512-I4UGspA0wpZXWENrdA0uHbnhte683t3qT/1VFH9aX2dA5PPSf6QW5HHXf5HImaqPmjXaVeVk4RGWnaylmV7uAA== + dependencies: + es-to-primitive "^1.2.1" + function-bind "^1.1.1" + has "^1.0.3" + has-symbols "^1.0.1" + is-callable "^1.2.2" + is-negative-zero "^2.0.0" + is-regex "^1.1.1" + object-inspect "^1.8.0" + object-keys "^1.1.1" + object.assign "^4.1.1" + string.prototype.trimend "^1.0.1" + string.prototype.trimstart "^1.0.1" + +es-get-iterator@^1.1.0: + version "1.1.0" + resolved "https://registry.yarnpkg.com/es-get-iterator/-/es-get-iterator-1.1.0.tgz#bb98ad9d6d63b31aacdc8f89d5d0ee57bcb5b4c8" + integrity sha512-UfrmHuWQlNMTs35e1ypnvikg6jCz3SK8v8ImvmDsh36fCVUR1MqoFDiyn0/k52C8NqO3YsO8Oe0azeesNuqSsQ== + dependencies: + es-abstract "^1.17.4" + has-symbols "^1.0.1" + is-arguments "^1.0.4" + is-map "^2.0.1" + is-set "^2.0.1" + is-string "^1.0.5" + isarray "^2.0.5" + es-to-primitive@^1.2.1: version "1.2.1" resolved "https://registry.yarnpkg.com/es-to-primitive/-/es-to-primitive-1.2.1.tgz#e55cd4c9cdc188bcefb03b366c736323fc5c898a" @@ -3112,6 +3202,11 @@ is-accessor-descriptor@^1.0.0: dependencies: kind-of "^6.0.0" +is-arguments@^1.0.4: + version "1.0.4" + resolved "https://registry.yarnpkg.com/is-arguments/-/is-arguments-1.0.4.tgz#3faf966c7cba0ff437fb31f6250082fcf0448cf3" + integrity sha512-xPh0Rmt8NE65sNzvyUmWgI1tz3mKq74lGA0mL8LYZcoIzKOzDh6HmrYm3d18k60nHerC8A9Km8kYu87zfSFnLA== + is-arrayish@^0.2.1: version "0.2.1" resolved "https://registry.yarnpkg.com/is-arrayish/-/is-arrayish-0.2.1.tgz#77c99840527aa8ecb1a8ba697b80645a7a926a9d" @@ -3121,12 +3216,22 @@ is-arrayish@^0.3.1: resolved "https://registry.yarnpkg.com/is-arrayish/-/is-arrayish-0.3.2.tgz#4574a2ae56f7ab206896fb431eaeed066fdf8f03" integrity sha512-eVRqCvVlZbuw3GrM63ovNSNAeA1K16kaR/LRY/92w0zxQ5/1YzwblUX652i4Xs9RwAGjW9d9y6X88t8OaAJfWQ== +is-bigint@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/is-bigint/-/is-bigint-1.0.0.tgz#73da8c33208d00f130e9b5e15d23eac9215601c4" + integrity sha512-t5mGUXC/xRheCK431ylNiSkGGpBp8bHENBcENTkDT6ppwPzEVxNGZRvgvmOEfbWkFhA7D2GEuE2mmQTr78sl2g== + is-binary-path@~2.1.0: version "2.1.0" resolved "https://registry.yarnpkg.com/is-binary-path/-/is-binary-path-2.1.0.tgz#ea1f7f3b80f064236e83470f86c09c254fb45b09" dependencies: binary-extensions "^2.0.0" +is-boolean-object@^1.0.0: + version "1.0.1" + resolved "https://registry.yarnpkg.com/is-boolean-object/-/is-boolean-object-1.0.1.tgz#10edc0900dd127697a92f6f9807c7617d68ac48e" + integrity sha512-TqZuVwa/sppcrhUCAYkGBk7w0yxfQQnxq28fjkO53tnK9FQXmdwz2JS5+GjsWQ6RByES1K40nI+yDic5c9/aAQ== + is-buffer@^1.1.5: version "1.1.6" resolved "https://registry.yarnpkg.com/is-buffer/-/is-buffer-1.1.6.tgz#efaa2ea9daa0d7ab2ea13a97b2b8ad51fefbe8be" @@ -3135,6 +3240,11 @@ is-callable@^1.1.4, is-callable@^1.1.5: version "1.1.5" resolved "https://registry.yarnpkg.com/is-callable/-/is-callable-1.1.5.tgz#f7e46b596890456db74e7f6e976cb3273d06faab" +is-callable@^1.2.2: + version "1.2.2" + resolved "https://registry.yarnpkg.com/is-callable/-/is-callable-1.2.2.tgz#c7c6715cd22d4ddb48d3e19970223aceabb080d9" + integrity sha512-dnMqspv5nU3LoewK2N/y7KLtxtakvTuaCsU9FU50/QDmdbHNy/4/JuRtMHqRU22o3q+W89YQndQEeCVwK+3qrA== + is-ci@^2.0.0: version "2.0.0" resolved "https://registry.yarnpkg.com/is-ci/-/is-ci-2.0.0.tgz#6bc6334181810e04b5c22b3d589fdca55026404c" @@ -3157,7 +3267,7 @@ is-data-descriptor@^1.0.0: dependencies: kind-of "^6.0.0" -is-date-object@^1.0.1: +is-date-object@^1.0.1, is-date-object@^1.0.2: version "1.0.2" resolved "https://registry.yarnpkg.com/is-date-object/-/is-date-object-1.0.2.tgz#bda736f2cd8fd06d32844e7743bfa7494c3bfd7e" @@ -3227,15 +3337,30 @@ is-installed-globally@^0.3.1: global-dirs "^2.0.1" is-path-inside "^3.0.1" +is-map@^2.0.1: + version "2.0.1" + resolved "https://registry.yarnpkg.com/is-map/-/is-map-2.0.1.tgz#520dafc4307bb8ebc33b813de5ce7c9400d644a1" + integrity sha512-T/S49scO8plUiAOA2DBTBG3JHpn1yiw0kRp6dgiZ0v2/6twi5eiB0rHtHFH9ZIrvlWc6+4O+m4zg5+Z833aXgw== + is-natural-number@^4.0.1: version "4.0.1" resolved "https://registry.yarnpkg.com/is-natural-number/-/is-natural-number-4.0.1.tgz#ab9d76e1db4ced51e35de0c72ebecf09f734cde8" integrity sha1-q5124dtM7VHjXeDHLr7PCfc0zeg= +is-negative-zero@^2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/is-negative-zero/-/is-negative-zero-2.0.0.tgz#9553b121b0fac28869da9ed459e20c7543788461" + integrity sha1-lVOxIbD6wohp2p7UWeIMdUN4hGE= + is-npm@^4.0.0: version "4.0.0" resolved "https://registry.yarnpkg.com/is-npm/-/is-npm-4.0.0.tgz#c90dd8380696df87a7a6d823c20d0b12bbe3c84d" +is-number-object@^1.0.3: + version "1.0.4" + resolved "https://registry.yarnpkg.com/is-number-object/-/is-number-object-1.0.4.tgz#36ac95e741cf18b283fc1ddf5e83da798e3ec197" + integrity sha512-zohwelOAur+5uXtk8O3GPQ1eAcu4ZX3UwxQhUlfFFMNpUd83gXgjbhJh6HmB6LUNV/ieOLQuDwJO3dWJosUeMw== + is-number@^3.0.0: version "3.0.0" resolved "https://registry.yarnpkg.com/is-number/-/is-number-3.0.0.tgz#24fd6201a4782cf50561c810276afc7d12d71195" @@ -3276,15 +3401,32 @@ is-regex@^1.0.5: dependencies: has "^1.0.3" +is-regex@^1.1.1: + version "1.1.1" + resolved "https://registry.yarnpkg.com/is-regex/-/is-regex-1.1.1.tgz#c6f98aacc546f6cec5468a07b7b153ab564a57b9" + integrity sha512-1+QkEcxiLlB7VEyFtyBg94e08OAsvq7FUBgApTq/w2ymCLyKJgDPsybBENVtA7XCQEgEXxKPonG+mvYRxh/LIg== + dependencies: + has-symbols "^1.0.1" + is-retry-allowed@^1.1.0: version "1.2.0" resolved "https://registry.yarnpkg.com/is-retry-allowed/-/is-retry-allowed-1.2.0.tgz#d778488bd0a4666a3be8a1482b9f2baafedea8b4" integrity sha512-RUbUeKwvm3XG2VYamhJL1xFktgjvPzL0Hq8C+6yrWIswDy3BIXGqCxhxkc30N9jqK311gVU137K8Ei55/zVJRg== +is-set@^2.0.1: + version "2.0.1" + resolved "https://registry.yarnpkg.com/is-set/-/is-set-2.0.1.tgz#d1604afdab1724986d30091575f54945da7e5f43" + integrity sha512-eJEzOtVyenDs1TMzSQ3kU3K+E0GUS9sno+F0OBT97xsgcJsF9nXMBtkT9/kut5JEpM7oL7X/0qxR17K3mcwIAA== + is-stream@^1.1.0: version "1.1.0" resolved "https://registry.yarnpkg.com/is-stream/-/is-stream-1.1.0.tgz#12d4a3dd4e68e0b79ceb8dbc84173ae80d91ca44" +is-string@^1.0.4, is-string@^1.0.5: + version "1.0.5" + resolved "https://registry.yarnpkg.com/is-string/-/is-string-1.0.5.tgz#40493ed198ef3ff477b8c7f92f644ec82a5cd3a6" + integrity sha512-buY6VNRjhQMiF1qWDouloZlQbRhDPCebwxSjxMjxgemYT46YMd2NR0/H+fBhEfWX4A/w9TBJ+ol+okqJKFE6vQ== + is-symbol@^1.0.2: version "1.0.3" resolved "https://registry.yarnpkg.com/is-symbol/-/is-symbol-1.0.3.tgz#38e1014b9e6329be0de9d24a414fd7441ec61937" @@ -3299,10 +3441,30 @@ is-type-of@^1.0.0: is-class-hotfix "~0.0.6" isstream "~0.1.2" +is-typed-array@^1.1.3: + version "1.1.3" + resolved "https://registry.yarnpkg.com/is-typed-array/-/is-typed-array-1.1.3.tgz#a4ff5a5e672e1a55f99c7f54e59597af5c1df04d" + integrity sha512-BSYUBOK/HJibQ30wWkWold5txYwMUXQct9YHAQJr8fSwvZoiglcqB0pd7vEN23+Tsi9IUEjztdOSzl4qLVYGTQ== + dependencies: + available-typed-arrays "^1.0.0" + es-abstract "^1.17.4" + foreach "^2.0.5" + has-symbols "^1.0.1" + is-typedarray@^1.0.0, is-typedarray@~1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/is-typedarray/-/is-typedarray-1.0.0.tgz#e479c80858df0c1b11ddda6940f96011fcda4a9a" +is-weakmap@^2.0.1: + version "2.0.1" + resolved "https://registry.yarnpkg.com/is-weakmap/-/is-weakmap-2.0.1.tgz#5008b59bdc43b698201d18f62b37b2ca243e8cf2" + integrity sha512-NSBR4kH5oVj1Uwvv970ruUkCV7O1mzgVFO4/rev2cLRda9Tm9HrL70ZPut4rOHgY0FNrUu9BCbXA2sdQ+x0chA== + +is-weakset@^2.0.1: + version "2.0.1" + resolved "https://registry.yarnpkg.com/is-weakset/-/is-weakset-2.0.1.tgz#e9a0af88dbd751589f5e50d80f4c98b780884f83" + integrity sha512-pi4vhbhVHGLxohUw7PhGsueT4vRGFoXhP7+RGN0jKIv9+8PWYCQTqtADngrxOm2g46hoH0+g8uZZBzMrvVGDmw== + is-windows@^1.0.2: version "1.0.2" resolved "https://registry.yarnpkg.com/is-windows/-/is-windows-1.0.2.tgz#d1850eb9791ecd18e6182ce12a30f396634bb19d" @@ -3323,6 +3485,11 @@ isarray@1.0.0, isarray@^1.0.0, isarray@~1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/isarray/-/isarray-1.0.0.tgz#bb935d48582cba168c06834957a54a3e07124f11" +isarray@^2.0.5: + version "2.0.5" + resolved "https://registry.yarnpkg.com/isarray/-/isarray-2.0.5.tgz#8af1e4c1221244cc62459faf38940d4e644a5723" + integrity sha512-xHjhDr3cNBK0BzdUJSPXZntQUx/mwMS5Rw4A7lPJ90XGAO6ISP/ePDNuo0vhqOZU+UD5JoodwCAAoZQd3FeAKw== + isbinaryfile@^4.0.6: version "4.0.6" resolved "https://registry.yarnpkg.com/isbinaryfile/-/isbinaryfile-4.0.6.tgz#edcb62b224e2b4710830b67498c8e4e5a4d2610b" @@ -4668,6 +4835,19 @@ object-inspect@^1.7.0: version "1.7.0" resolved "https://registry.yarnpkg.com/object-inspect/-/object-inspect-1.7.0.tgz#f4f6bd181ad77f006b5ece60bd0b6f398ff74a67" +object-inspect@^1.8.0: + version "1.8.0" + resolved "https://registry.yarnpkg.com/object-inspect/-/object-inspect-1.8.0.tgz#df807e5ecf53a609cc6bfe93eac3cc7be5b3a9d0" + integrity sha512-jLdtEOB112fORuypAyl/50VRVIBIdVQOSUUGQHzJ4xBSbit81zRarz7GThkEFZy1RceYrWYcPcBFPQwHyAc1gA== + +object-is@^1.1.3: + version "1.1.3" + resolved "https://registry.yarnpkg.com/object-is/-/object-is-1.1.3.tgz#2e3b9e65560137455ee3bd62aec4d90a2ea1cc81" + integrity sha512-teyqLvFWzLkq5B9ki8FVWA902UER2qkxmdA4nLf+wjOLAWgxzCWZNCxpDq9MvE8MmhWNr+I8w3BN49Vx36Y6Xg== + dependencies: + define-properties "^1.1.3" + es-abstract "^1.18.0-next.1" + object-keys@^1.0.11, object-keys@^1.0.12, object-keys@^1.0.6, object-keys@^1.1.1: version "1.1.1" resolved "https://registry.yarnpkg.com/object-keys/-/object-keys-1.1.1.tgz#1c47f272df277f3b1daf061677d9c82e2322c60e" @@ -4687,6 +4867,16 @@ object.assign@^4.1.0: has-symbols "^1.0.0" object-keys "^1.0.11" +object.assign@^4.1.1: + version "4.1.1" + resolved "https://registry.yarnpkg.com/object.assign/-/object.assign-4.1.1.tgz#303867a666cdd41936ecdedfb1f8f3e32a478cdd" + integrity sha512-VT/cxmx5yaoHSOTSyrCygIDFco+RsibY2NM0a4RdEeY/4KgqezwFtK1yr3U67xYhqJSlASm2pKhLVzPj2lr4bA== + dependencies: + define-properties "^1.1.3" + es-abstract "^1.18.0-next.0" + has-symbols "^1.0.1" + object-keys "^1.1.1" + object.getownpropertydescriptors@^2.1.0: version "2.1.0" resolved "https://registry.yarnpkg.com/object.getownpropertydescriptors/-/object.getownpropertydescriptors-2.1.0.tgz#369bf1f9592d8ab89d712dced5cb81c7c5352649" @@ -5374,6 +5564,19 @@ regex-not@^1.0.0, regex-not@^1.0.2: extend-shallow "^3.0.2" safe-regex "^1.1.0" +regexp.prototype.flags@^1.3.0: + version "1.3.0" + resolved "https://registry.yarnpkg.com/regexp.prototype.flags/-/regexp.prototype.flags-1.3.0.tgz#7aba89b3c13a64509dabcf3ca8d9fbb9bdf5cb75" + integrity sha512-2+Q0C5g951OlYlJz6yu5/M33IcsESLlLfsyIaLJaG4FA2r4yP8MvVMJUUP/fVBkSpbbbZlS5gynbEWLipiiXiQ== + dependencies: + define-properties "^1.1.3" + es-abstract "^1.17.0-next.1" + +regexparam@^1.3.0: + version "1.3.0" + resolved "https://registry.yarnpkg.com/regexparam/-/regexparam-1.3.0.tgz#2fe42c93e32a40eff6235d635e0ffa344b92965f" + integrity sha512-6IQpFBv6e5vz1QAqI+V4k8P2e/3gRrqfCJ9FI+O1FLQTO+Uz6RXZEZOPmTJ6hlGj7gkERzY5BRCv09whKP96/g== + regexpp@^2.0.1: version "2.0.1" resolved "https://registry.yarnpkg.com/regexpp/-/regexpp-2.0.1.tgz#8d19d31cf632482b589049f8281f93dbcba4d07f" @@ -5692,6 +5895,14 @@ shellwords@^0.1.1: version "0.1.1" resolved "https://registry.yarnpkg.com/shellwords/-/shellwords-0.1.1.tgz#d6b9181c1a48d397324c84871efbcfc73fc0654b" +side-channel@^1.0.3: + version "1.0.3" + resolved "https://registry.yarnpkg.com/side-channel/-/side-channel-1.0.3.tgz#cdc46b057550bbab63706210838df5d4c19519c3" + integrity sha512-A6+ByhlLkksFoUepsGxfj5x1gTSrs+OydsRptUxeNCabQpCFUvcwIczgOigI8vhY/OJCnPnyE9rGiwgvr9cS1g== + dependencies: + es-abstract "^1.18.0-next.0" + object-inspect "^1.8.0" + signal-exit@^3.0.0, signal-exit@^3.0.2: version "3.0.3" resolved "https://registry.yarnpkg.com/signal-exit/-/signal-exit-3.0.3.tgz#a1410c2edd8f077b08b4e253c8eacfcaf057461c" @@ -5981,7 +6192,7 @@ string-width@^4.0.0, string-width@^4.1.0, string-width@^4.2.0: is-fullwidth-code-point "^3.0.0" strip-ansi "^6.0.0" -string.prototype.trimend@^1.0.0: +string.prototype.trimend@^1.0.0, string.prototype.trimend@^1.0.1: version "1.0.1" resolved "https://registry.yarnpkg.com/string.prototype.trimend/-/string.prototype.trimend-1.0.1.tgz#85812a6b847ac002270f5808146064c995fb6913" dependencies: @@ -6004,7 +6215,7 @@ string.prototype.trimright@^2.1.1: es-abstract "^1.17.5" string.prototype.trimend "^1.0.0" -string.prototype.trimstart@^1.0.0: +string.prototype.trimstart@^1.0.0, string.prototype.trimstart@^1.0.1: version "1.0.1" resolved "https://registry.yarnpkg.com/string.prototype.trimstart/-/string.prototype.trimstart-1.0.1.tgz#14af6d9f34b053f7cfc89b72f8f2ee14b9039a54" dependencies: @@ -6611,6 +6822,27 @@ whatwg-url@^7.0.0: tr46 "^1.0.1" webidl-conversions "^4.0.2" +which-boxed-primitive@^1.0.1: + version "1.0.1" + resolved "https://registry.yarnpkg.com/which-boxed-primitive/-/which-boxed-primitive-1.0.1.tgz#cbe8f838ebe91ba2471bb69e9edbda67ab5a5ec1" + integrity sha512-7BT4TwISdDGBgaemWU0N0OU7FeAEJ9Oo2P1PHRm/FCWoEi2VLWC9b6xvxAA3C/NMpxg3HXVgi0sMmGbNUbNepQ== + dependencies: + is-bigint "^1.0.0" + is-boolean-object "^1.0.0" + is-number-object "^1.0.3" + is-string "^1.0.4" + is-symbol "^1.0.2" + +which-collection@^1.0.1: + version "1.0.1" + resolved "https://registry.yarnpkg.com/which-collection/-/which-collection-1.0.1.tgz#70eab71ebbbd2aefaf32f917082fc62cdcb70906" + integrity sha512-W8xeTUwaln8i3K/cY1nGXzdnVZlidBcagyNFtBdD5kxnb4TvGKR7FfSIS3mYpwWS1QUCutfKz8IY8RjftB0+1A== + dependencies: + is-map "^2.0.1" + is-set "^2.0.1" + is-weakmap "^2.0.1" + is-weakset "^2.0.1" + which-module@^2.0.0: version "2.0.0" resolved "https://registry.yarnpkg.com/which-module/-/which-module-2.0.0.tgz#d9ef07dce77b9902b8a3a8fa4b31c3e3f7e6e87a" @@ -6620,6 +6852,18 @@ which-pm-runs@^1.0.0: resolved "https://registry.yarnpkg.com/which-pm-runs/-/which-pm-runs-1.0.0.tgz#670b3afbc552e0b55df6b7780ca74615f23ad1cb" integrity sha1-Zws6+8VS4LVd9rd4DKdGFfI60cs= +which-typed-array@^1.1.2: + version "1.1.2" + resolved "https://registry.yarnpkg.com/which-typed-array/-/which-typed-array-1.1.2.tgz#e5f98e56bda93e3dac196b01d47c1156679c00b2" + integrity sha512-KT6okrd1tE6JdZAy3o2VhMoYPh3+J6EMZLyrxBQsZflI1QCZIxMrIYLkosd8Twf+YfknVIHmYQPgJt238p8dnQ== + dependencies: + available-typed-arrays "^1.0.2" + es-abstract "^1.17.5" + foreach "^2.0.5" + function-bind "^1.1.1" + has-symbols "^1.0.1" + is-typed-array "^1.1.3" + which@^1.2.9, which@^1.3.0: version "1.3.1" resolved "https://registry.yarnpkg.com/which/-/which-1.3.1.tgz#a45043d54f5805316da8d62f9f50918d3da70b0a" From f2b7d85b6e1145ad55545a25f0360e6d439a6aa6 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Wed, 7 Oct 2020 17:56:47 +0100 Subject: [PATCH 02/21] Fixing up middleware to handle uploads, views, records, automation runs and users. --- packages/server/src/api/controllers/auth.js | 4 +- .../server/src/api/controllers/automation.js | 2 + packages/server/src/api/routes/model.js | 4 +- packages/server/src/api/routes/static.js | 3 +- packages/server/src/api/routes/user.js | 4 +- packages/server/src/api/routes/view.js | 10 ++- packages/server/src/automations/index.js | 25 +++++-- packages/server/src/middleware/usageQuota.js | 71 +++++++----------- .../server/src/utilities/fileProcessor.js | 18 ++--- packages/server/src/utilities/usageQuota.js | 72 +++++++++++++++++++ 10 files changed, 147 insertions(+), 66 deletions(-) create mode 100644 packages/server/src/utilities/usageQuota.js diff --git a/packages/server/src/api/controllers/auth.js b/packages/server/src/api/controllers/auth.js index b7a7c116b9..c16f76caca 100644 --- a/packages/server/src/api/controllers/auth.js +++ b/packages/server/src/api/controllers/auth.js @@ -3,7 +3,7 @@ const CouchDB = require("../../db") const ClientDb = require("../../db/clientDb") const bcrypt = require("../../utilities/bcrypt") const environment = require("../../environment") -const { apiKeyTable } = require("../../db/dynamoClient") +const { getAPIKey } = require("../../utilities/usageQuota") const { generateUserID } = require("../../db/utils") exports.authenticate = async ctx => { @@ -55,7 +55,7 @@ exports.authenticate = async ctx => { } // if in cloud add the user api key if (environment.CLOUD) { - payload.apiKey = await apiKeyTable.get({ primary: ctx.user.appId }) + payload.apiKey = getAPIKey(ctx.user.appId) } const token = jwt.sign(payload, ctx.config.jwtSecret, { diff --git a/packages/server/src/api/controllers/automation.js b/packages/server/src/api/controllers/automation.js index 5414f3878c..92391da7e9 100644 --- a/packages/server/src/api/controllers/automation.js +++ b/packages/server/src/api/controllers/automation.js @@ -33,6 +33,7 @@ function cleanAutomationInputs(automation) { exports.create = async function(ctx) { const db = new CouchDB(ctx.user.instanceId) let automation = ctx.request.body + automation.appId = ctx.user.appId automation._id = generateAutomationID() @@ -54,6 +55,7 @@ exports.create = async function(ctx) { exports.update = async function(ctx) { const db = new CouchDB(ctx.user.instanceId) let automation = ctx.request.body + automation.appId = ctx.user.appId automation = cleanAutomationInputs(automation) const response = await db.put(automation) diff --git a/packages/server/src/api/routes/model.js b/packages/server/src/api/routes/model.js index 3c667df520..fe782d4cf5 100644 --- a/packages/server/src/api/routes/model.js +++ b/packages/server/src/api/routes/model.js @@ -1,7 +1,6 @@ const Router = require("@koa/router") const modelController = require("../controllers/model") const authorized = require("../../middleware/authorized") -const usage = require("../../middleware/usageQuota") const { BUILDER, READ_MODEL } = require("../../utilities/accessLevels") const router = Router() @@ -13,7 +12,7 @@ router authorized(READ_MODEL, ctx => ctx.params.id), modelController.find ) - .post("/api/models", authorized(BUILDER), usage, modelController.save) + .post("/api/models", authorized(BUILDER), modelController.save) .post( "/api/models/csv/validate", authorized(BUILDER), @@ -22,7 +21,6 @@ router .delete( "/api/models/:modelId/:revId", authorized(BUILDER), - usage, modelController.destroy ) diff --git a/packages/server/src/api/routes/static.js b/packages/server/src/api/routes/static.js index aa136a3d15..5c33900eca 100644 --- a/packages/server/src/api/routes/static.js +++ b/packages/server/src/api/routes/static.js @@ -4,6 +4,7 @@ const { budibaseTempDir } = require("../../utilities/budibaseDir") const env = require("../../environment") const authorized = require("../../middleware/authorized") const { BUILDER } = require("../../utilities/accessLevels") +const usage = require("../../middleware/usageQuota") const router = Router() @@ -28,7 +29,7 @@ router authorized(BUILDER), controller.performLocalFileProcessing ) - .post("/api/attachments/upload", controller.uploadFile) + .post("/api/attachments/upload", usage, controller.uploadFile) .get("/componentlibrary", controller.serveComponentLibrary) .get("/assets/:file*", controller.serveAppAsset) .get("/attachments/:file*", controller.serveAttachment) diff --git a/packages/server/src/api/routes/user.js b/packages/server/src/api/routes/user.js index 532943ea62..5289439e41 100644 --- a/packages/server/src/api/routes/user.js +++ b/packages/server/src/api/routes/user.js @@ -2,6 +2,7 @@ const Router = require("@koa/router") const controller = require("../controllers/user") const authorized = require("../../middleware/authorized") const { USER_MANAGEMENT, LIST_USERS } = require("../../utilities/accessLevels") +const usage = require("../../middleware/usageQuota") const router = Router() @@ -9,10 +10,11 @@ router .get("/api/users", authorized(LIST_USERS), controller.fetch) .get("/api/users/:username", authorized(USER_MANAGEMENT), controller.find) .put("/api/users/", authorized(USER_MANAGEMENT), controller.update) - .post("/api/users", authorized(USER_MANAGEMENT), controller.create) + .post("/api/users", authorized(USER_MANAGEMENT), usage, controller.create) .delete( "/api/users/:username", authorized(USER_MANAGEMENT), + usage, controller.destroy ) diff --git a/packages/server/src/api/routes/view.js b/packages/server/src/api/routes/view.js index 2c88f6d19a..571e4494f1 100644 --- a/packages/server/src/api/routes/view.js +++ b/packages/server/src/api/routes/view.js @@ -3,6 +3,7 @@ const viewController = require("../controllers/view") const recordController = require("../controllers/record") const authorized = require("../../middleware/authorized") const { BUILDER, READ_VIEW } = require("../../utilities/accessLevels") +const usage = require("../../middleware/usageQuota") const router = Router() @@ -13,8 +14,13 @@ router recordController.fetchView ) .get("/api/views", authorized(BUILDER), viewController.fetch) - .delete("/api/views/:viewName", authorized(BUILDER), viewController.destroy) - .post("/api/views", authorized(BUILDER), viewController.save) + .delete( + "/api/views/:viewName", + authorized(BUILDER), + usage, + viewController.destroy + ) + .post("/api/views", authorized(BUILDER), usage, viewController.save) .post("/api/views/export", authorized(BUILDER), viewController.exportView) .get( "/api/views/export/download/:fileName", diff --git a/packages/server/src/automations/index.js b/packages/server/src/automations/index.js index e419985ce2..882f24391b 100644 --- a/packages/server/src/automations/index.js +++ b/packages/server/src/automations/index.js @@ -3,6 +3,7 @@ const actions = require("./actions") const environment = require("../environment") const workerFarm = require("worker-farm") const singleThread = require("./thread") +const { getAPIKey, update, Properties } = require("../utilities/usageQuota") let workers = workerFarm(require.resolve("./thread")) @@ -18,16 +19,32 @@ function runWorker(job) { }) } +async function updateQuota(automation) { + const appId = automation.appId + const apiKey = await getAPIKey(appId) + // this will fail, causing automation to escape if limits reached + await update(apiKey, Properties.AUTOMATION, 1) +} + /** * This module is built purely to kick off the worker farm and manage the inputs/outputs */ module.exports.init = function() { actions.init().then(() => { triggers.automationQueue.process(async job => { - if (environment.BUDIBASE_ENVIRONMENT === "PRODUCTION") { - await runWorker(job) - } else { - await singleThread(job) + try { + if (environment.CLOUD) { + await updateQuota(job.data.automation) + } + if (environment.BUDIBASE_ENVIRONMENT === "PRODUCTION") { + await runWorker(job) + } else { + await singleThread(job) + } + } catch (err) { + console.error( + `${job.data.automation.appId} automation ${job.data.automation._id} was unable to run - ${err}` + ) } }) }) diff --git a/packages/server/src/middleware/usageQuota.js b/packages/server/src/middleware/usageQuota.js index 1d119f25c8..74df19a015 100644 --- a/packages/server/src/middleware/usageQuota.js +++ b/packages/server/src/middleware/usageQuota.js @@ -1,9 +1,5 @@ const CouchDB = require("../db") -const environment = require("../environment") -const { apiKeyTable } = require("../db/dynamoClient") - -// a normalised month in milliseconds -const QUOTA_RESET = 2592000000 +const usageQuota = require("../utilities/usageQuota") // currently only counting new writes and deletes const METHOD_MAP = { @@ -12,36 +8,32 @@ const METHOD_MAP = { } const DOMAIN_MAP = { - models: "model", - records: "record", + records: usageQuota.Properties.RECORD, + upload: usageQuota.Properties.UPLOAD, + views: usageQuota.Properties.VIEW, + users: usageQuota.Properties.USER, + // this will not be updated by endpoint calls + // instead it will be updated by triggers + automationRuns: usageQuota.Properties.AUTOMATION, } -function buildUpdateParams(key, property, usage) { - return { - primary: key, - condition: "#quota.#prop + :usage < #limits.model AND #quotaReset < :now", - expression: "ADD #quota.#prop :usage", - names: { - "#quota": "usageQuota", - "#prop": property, - "#limits": "limits", - "#quotaReset": "quotaReset", - }, - values: { - ":usage": usage, - ":now": Date.now(), - }, +function getProperty(url) { + for (let domain of Object.keys(DOMAIN_MAP)) { + if (url.indexOf(domain) !== -1) { + return DOMAIN_MAP[domain] + } } } module.exports = async (ctx, next) => { const db = new CouchDB(ctx.user.instanceId) - const usage = METHOD_MAP[ctx.req.method] - const domainParts = ctx.req.url.split("/") - const property = DOMAIN_MAP[domainParts[domainParts.length - 1]] + let usage = METHOD_MAP[ctx.req.method] + const property = getProperty(ctx.req.url) + console.log(ctx.req.url) if (usage == null || property == null) { return next() } + console.log(`${usage} to ${property}`) // post request could be a save of a pre-existing entry if (ctx.request.body && ctx.request.body._id) { try { @@ -52,26 +44,17 @@ module.exports = async (ctx, next) => { return } } - // don't try validate in builder - if (!environment.CLOUD) { - return next() + // update usage for uploads to be the total size + if (property === usageQuota.Properties.UPLOAD) { + const files = + ctx.request.files.file.length > 1 + ? Array.from(ctx.request.files.file) + : [ctx.request.files.file] + usage = files.map(file => file.size).reduce((total, size) => total + size) } try { - await apiKeyTable.update(buildUpdateParams(ctx.apiKey, property, usage)) + await usageQuota.update(ctx.apiKey, property, usage) } catch (err) { - if (err.code !== "ConditionalCheckFailedException") { - // get the API key so we can check it - let apiKey = await apiKeyTable.get({ primary: ctx.apiKey }) - // we have infact breached the reset period - if (apiKey && apiKey.quotaReset >= Date.now()) { - // update the quota reset period and reset the values for all properties - apiKey.quotaReset = Date.now() + QUOTA_RESET - for (let prop of Object.keys(apiKey.usageQuota)) { - apiKey.usageQuota[prop] = 0 - } - await apiKeyTable.put({ item: apiKey }) - } - ctx.throw(403, `Resource limits have been reached`) - } + ctx.throw(403, err) } -} \ No newline at end of file +} diff --git a/packages/server/src/utilities/fileProcessor.js b/packages/server/src/utilities/fileProcessor.js index 3e580e9e37..734209733d 100644 --- a/packages/server/src/utilities/fileProcessor.js +++ b/packages/server/src/utilities/fileProcessor.js @@ -1,5 +1,5 @@ const fs = require("fs") -const sharp = require("sharp") +// const sharp = require("sharp") const fsPromises = fs.promises const FORMATS = { @@ -7,14 +7,14 @@ const FORMATS = { } async function processImage(file) { - const imgMeta = await sharp(file.path) - .resize(300) - .toFile(file.outputPath) - - return { - ...file, - ...imgMeta, - } + // const imgMeta = await sharp(file.path) + // .resize(300) + // .toFile(file.outputPath) + // + // return { + // ...file, + // ...imgMeta, + // } } async function process(file) { diff --git a/packages/server/src/utilities/usageQuota.js b/packages/server/src/utilities/usageQuota.js new file mode 100644 index 0000000000..7ba0005213 --- /dev/null +++ b/packages/server/src/utilities/usageQuota.js @@ -0,0 +1,72 @@ +const environment = require("../environment") +const { apiKeyTable } = require("../db/dynamoClient") + +function buildUpdateParams(key, property, usage) { + return { + primary: key, + condition: "#quota.#prop + :usage < #limits.model AND #quotaReset < :now", + expression: "ADD #quota.#prop :usage", + names: { + "#quota": "usageQuota", + "#prop": property, + "#limits": "limits", + "#quotaReset": "quotaReset", + }, + values: { + ":usage": usage, + ":now": Date.now(), + }, + } +} + +// a normalised month in milliseconds +const QUOTA_RESET = 2592000000 + +exports.Properties = { + RECORD: "records", + UPLOAD: "storage", + VIEW: "views", + USER: "users", + AUTOMATION: "automationRuns", +} + +exports.getAPIKey = async appId => { + return apiKeyTable.get({ primary: appId }) +} + +/** + * Given a specified API key this will add to the usage object for the specified property. + * @param {string} apiKey The API key which is to be updated. + * @param {string} property The property which is to be added to (within the nested usageQuota object). + * @param {number} usage The amount (this can be negative) to adjust the number by. + * @returns {Promise} When this completes the API key will now be up to date - the quota period may have + * also been reset after this call. + */ +exports.update = async (apiKey, property, usage) => { + // don't try validate in builder + if (!environment.CLOUD) { + return + } + try { + await apiKeyTable.update(buildUpdateParams(apiKey, property, usage)) + } catch (err) { + if (err.code !== "ConditionalCheckFailedException") { + // get the API key so we can check it + let apiKey = await apiKeyTable.get({ primary: apiKey }) + // we have infact breached the reset period + if (apiKey && apiKey.quotaReset >= Date.now()) { + // update the quota reset period and reset the values for all properties + apiKey.quotaReset = Date.now() + QUOTA_RESET + for (let prop of Object.keys(apiKey.usageQuota)) { + if (prop === property) { + apiKey.usageQuota[prop] = usage > 0 ? usage : 0 + } else { + apiKey.usageQuota[prop] = 0 + } + } + await apiKeyTable.put({ item: apiKey }) + } + throw "Resource limits have been reached" + } + } +} From a482d0a23ead0cf1e44cff730a4e49445f3b96e7 Mon Sep 17 00:00:00 2001 From: Martin McKeaveney Date: Wed, 7 Oct 2020 20:37:55 +0100 Subject: [PATCH 03/21] check that deployment is possible using lambda API --- .../server/src/api/controllers/deploy/aws.js | 12 +++++- .../src/api/controllers/deploy/index.js | 37 +++++++++++++++++-- 2 files changed, 45 insertions(+), 4 deletions(-) diff --git a/packages/server/src/api/controllers/deploy/aws.js b/packages/server/src/api/controllers/deploy/aws.js index 2f89d97742..6b73751695 100644 --- a/packages/server/src/api/controllers/deploy/aws.js +++ b/packages/server/src/api/controllers/deploy/aws.js @@ -21,11 +21,21 @@ async function invalidateCDN(cfDistribution, appId) { .promise() } -exports.fetchTemporaryCredentials = async function() { +/** + * Verifies the users API key and + * Verifies that the deployment fits within the quota of the user, + * @param {String} instanceId - instanceId being deployed + * @param {String} appId - appId being deployed + * @param {quota} quota - current quota being changed with this application + */ +exports.verifyDeployment = async function({ instanceId, appId, quota }) { const response = await fetch(process.env.DEPLOYMENT_CREDENTIALS_URL, { method: "POST", body: JSON.stringify({ apiKey: process.env.BUDIBASE_API_KEY, + instanceId, + appId, + quota, }), }) diff --git a/packages/server/src/api/controllers/deploy/index.js b/packages/server/src/api/controllers/deploy/index.js index 34579b64c7..a155fbbf81 100644 --- a/packages/server/src/api/controllers/deploy/index.js +++ b/packages/server/src/api/controllers/deploy/index.js @@ -1,10 +1,17 @@ const CouchDB = require("pouchdb") const PouchDB = require("../../../db") -const { uploadAppAssets, fetchTemporaryCredentials } = require("./aws") +const { + uploadAppAssets, + verifyDeployment, + determineDeploymentAllowed, +} = require("./aws") +const { getRecordParams } = require("../../db/utils") function replicate(local, remote) { return new Promise((resolve, reject) => { - const replication = local.sync(remote) + const replication = local.sync(remote, { + retry: true, + }) replication.on("complete", () => resolve()) replication.on("error", err => reject(err)) @@ -31,13 +38,37 @@ async function replicateCouch({ instanceId, clientId, credentials }) { await Promise.all(replications) } +async function getCurrentInstanceQuota(instanceId) { + const db = new PouchDB(instanceId) + const records = await db.allDocs( + getRecordParams("", null, { + include_docs: true, + }) + ) + const existingRecords = records.rows.length + return { + records: existingRecords, + } +} + exports.deployApp = async function(ctx) { try { const clientAppLookupDB = new PouchDB("client_app_lookup") const { clientId } = await clientAppLookupDB.get(ctx.user.appId) + const instanceQuota = await getCurrentInstanceQuota(ctx.user.instanceId) + const credentials = await verifyDeployment({ + instanceId: ctx.user.instanceId, + appId: ctx.user.appId, + quota: instanceQuota, + }) + ctx.log.info(`Uploading assets for appID ${ctx.user.appId} assets to s3..`) - const credentials = await fetchTemporaryCredentials() + + if (credentials.errors) { + ctx.throw(500, credentials.errors) + return + } await uploadAppAssets({ clientId, From 212a24348aa199edfc325e3cd010d6ffde1317cb Mon Sep 17 00:00:00 2001 From: Martin McKeaveney Date: Thu, 8 Oct 2020 10:56:32 +0100 Subject: [PATCH 04/21] hitting deployment success endpoint --- .../server/src/api/controllers/deploy/aws.js | 21 +++++++++++++++++++ .../src/api/controllers/deploy/index.js | 7 ++++++- 2 files changed, 27 insertions(+), 1 deletion(-) diff --git a/packages/server/src/api/controllers/deploy/aws.js b/packages/server/src/api/controllers/deploy/aws.js index 6b73751695..b56d3c7788 100644 --- a/packages/server/src/api/controllers/deploy/aws.js +++ b/packages/server/src/api/controllers/deploy/aws.js @@ -21,6 +21,27 @@ async function invalidateCDN(cfDistribution, appId) { .promise() } +exports.updateDeploymentQuota = async function(quota) { + const response = await fetch( + `${process.env.DEPLOYMENT_CREDENTIALS_URL}/deploy/success`, + { + method: "POST", + body: JSON.stringify({ + apiKey: process.env.BUDIBASE_API_KEY, + quota, + }), + } + ) + + if (response.status !== 200) { + throw new Error(`Error updating deployment quota for app`) + } + + const json = await response.json() + + return json +} + /** * Verifies the users API key and * Verifies that the deployment fits within the quota of the user, diff --git a/packages/server/src/api/controllers/deploy/index.js b/packages/server/src/api/controllers/deploy/index.js index a155fbbf81..c5d1169962 100644 --- a/packages/server/src/api/controllers/deploy/index.js +++ b/packages/server/src/api/controllers/deploy/index.js @@ -3,7 +3,7 @@ const PouchDB = require("../../../db") const { uploadAppAssets, verifyDeployment, - determineDeploymentAllowed, + updateDeploymentQuota, } = require("./aws") const { getRecordParams } = require("../../db/utils") @@ -85,6 +85,11 @@ exports.deployApp = async function(ctx) { credentials: credentials.couchDbCreds, }) + const deployedInstanceQuota = await getCurrentInstanceQuota( + ctx.user.instanceId + ) + updateDeploymentQuota(deployedInstanceQuota) + ctx.body = { status: "SUCCESS", completed: Date.now(), From 3f4fa0674d8489cdfc69728840e430a09c019e53 Mon Sep 17 00:00:00 2001 From: Martin McKeaveney Date: Thu, 8 Oct 2020 15:06:27 +0100 Subject: [PATCH 05/21] update deployment quota after deploy --- packages/server/src/api/controllers/deploy/index.js | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/packages/server/src/api/controllers/deploy/index.js b/packages/server/src/api/controllers/deploy/index.js index c5d1169962..7f6e4588a5 100644 --- a/packages/server/src/api/controllers/deploy/index.js +++ b/packages/server/src/api/controllers/deploy/index.js @@ -46,8 +46,12 @@ async function getCurrentInstanceQuota(instanceId) { }) ) const existingRecords = records.rows.length + + const designDoc = await db.get("_design/database") + return { records: existingRecords, + views: Object.keys(designDoc.views).length, } } @@ -88,7 +92,7 @@ exports.deployApp = async function(ctx) { const deployedInstanceQuota = await getCurrentInstanceQuota( ctx.user.instanceId ) - updateDeploymentQuota(deployedInstanceQuota) + await updateDeploymentQuota(deployedInstanceQuota) ctx.body = { status: "SUCCESS", From b400a06027677beeb78928f73d45e10c92615dc4 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Thu, 8 Oct 2020 17:34:41 +0100 Subject: [PATCH 06/21] Updates for API usage after testing against local Dynamo. --- .../server/scripts/createApiKeyAndAppId.js | 55 +++++++++++++++++++ packages/server/src/automations/index.js | 9 +-- .../src/automations/steps/createRecord.js | 7 ++- .../src/automations/steps/createUser.js | 7 ++- .../src/automations/steps/deleteRecord.js | 7 ++- packages/server/src/automations/thread.js | 1 + packages/server/src/db/dynamoClient.js | 30 ++++++++-- packages/server/src/middleware/authorized.js | 12 ++-- packages/server/src/middleware/usageQuota.js | 7 ++- .../src/utilities/builder/setBuilderToken.js | 4 +- packages/server/src/utilities/usageQuota.js | 21 +++---- 11 files changed, 127 insertions(+), 33 deletions(-) create mode 100644 packages/server/scripts/createApiKeyAndAppId.js diff --git a/packages/server/scripts/createApiKeyAndAppId.js b/packages/server/scripts/createApiKeyAndAppId.js new file mode 100644 index 0000000000..f6a6c1fcee --- /dev/null +++ b/packages/server/scripts/createApiKeyAndAppId.js @@ -0,0 +1,55 @@ +// THIS will create API Keys and App Ids input in a local Dynamo instance if it is running +const dynamoClient = require("../src/db/dynamoClient") + +if (process.argv[2] == null || process.argv[3] == null) { + console.error( + "Inputs incorrect format, was expecting: node createApiKeyAndAppId.js " + ) + process.exit(-1) +} + +const FAKE_STRING = "fakestring" + +// set fake credentials for local dynamo to actually work +process.env.AWS_ACCESS_KEY_ID = "KEY_ID" +process.env.AWS_SECRET_ACCESS_KEY = "SECRET_KEY" +dynamoClient.init("http://localhost:8333") + +async function run() { + await dynamoClient.apiKeyTable.put({ + item: { + pk: process.argv[2], + accountId: FAKE_STRING, + trackingId: FAKE_STRING, + quotaReset: Date.now() + 2592000000, + usageQuota: { + automationRuns: 0, + records: 0, + storage: 0, + users: 0, + views: 0, + }, + usageLimits: { + automationRuns: 10, + records: 10, + storage: 1000, + users: 10, + views: 10, + }, + }, + }) + await dynamoClient.apiKeyTable.put({ + item: { + pk: process.argv[3], + apiKey: process.argv[2], + }, + }) +} + +run() + .then(() => { + console.log("Records should have been created.") + }) + .catch(err => { + console.error("Cannot create records - " + err) + }) diff --git a/packages/server/src/automations/index.js b/packages/server/src/automations/index.js index 882f24391b..f407e35a78 100644 --- a/packages/server/src/automations/index.js +++ b/packages/server/src/automations/index.js @@ -21,9 +21,10 @@ function runWorker(job) { async function updateQuota(automation) { const appId = automation.appId - const apiKey = await getAPIKey(appId) + const apiObj = await getAPIKey(appId) // this will fail, causing automation to escape if limits reached - await update(apiKey, Properties.AUTOMATION, 1) + await update(apiObj.apiKey, Properties.AUTOMATION, 1) + return apiObj.apiKey } /** @@ -33,8 +34,8 @@ module.exports.init = function() { actions.init().then(() => { triggers.automationQueue.process(async job => { try { - if (environment.CLOUD) { - await updateQuota(job.data.automation) + if (environment.CLOUD && job.data.automation) { + job.data.automation.apiKey = await updateQuota(job.data.automation) } if (environment.BUDIBASE_ENVIRONMENT === "PRODUCTION") { await runWorker(job) diff --git a/packages/server/src/automations/steps/createRecord.js b/packages/server/src/automations/steps/createRecord.js index d0d6a36f5a..e268f218e2 100644 --- a/packages/server/src/automations/steps/createRecord.js +++ b/packages/server/src/automations/steps/createRecord.js @@ -1,5 +1,7 @@ const recordController = require("../../api/controllers/record") const automationUtils = require("../automationUtils") +const environment = require("../../environment") +const usage = require("../../utilities/usageQuota") module.exports.definition = { name: "Create Row", @@ -56,7 +58,7 @@ module.exports.definition = { }, } -module.exports.run = async function({ inputs, instanceId }) { +module.exports.run = async function({ inputs, instanceId, apiKey }) { // TODO: better logging of when actions are missed due to missing parameters if (inputs.record == null || inputs.record.modelId == null) { return @@ -78,6 +80,9 @@ module.exports.run = async function({ inputs, instanceId }) { } try { + if (environment.CLOUD) { + await usage.update(apiKey, usage.Properties.RECORD, 1) + } await recordController.save(ctx) return { record: inputs.record, diff --git a/packages/server/src/automations/steps/createUser.js b/packages/server/src/automations/steps/createUser.js index de2b6ca1ad..f0bea286d7 100644 --- a/packages/server/src/automations/steps/createUser.js +++ b/packages/server/src/automations/steps/createUser.js @@ -1,5 +1,7 @@ const accessLevels = require("../../utilities/accessLevels") const userController = require("../../api/controllers/user") +const environment = require("../../environment") +const usage = require("../../utilities/usageQuota") module.exports.definition = { description: "Create a new user", @@ -56,7 +58,7 @@ module.exports.definition = { }, } -module.exports.run = async function({ inputs, instanceId }) { +module.exports.run = async function({ inputs, instanceId, apiKey }) { const { username, password, accessLevelId } = inputs const ctx = { user: { @@ -68,6 +70,9 @@ module.exports.run = async function({ inputs, instanceId }) { } try { + if (environment.CLOUD) { + await usage.update(apiKey, usage.Properties.USER, 1) + } await userController.create(ctx) return { response: ctx.body, diff --git a/packages/server/src/automations/steps/deleteRecord.js b/packages/server/src/automations/steps/deleteRecord.js index 0a02099bd4..6126895da6 100644 --- a/packages/server/src/automations/steps/deleteRecord.js +++ b/packages/server/src/automations/steps/deleteRecord.js @@ -1,4 +1,6 @@ const recordController = require("../../api/controllers/record") +const environment = require("../../environment") +const usage = require("../../utilities/usageQuota") module.exports.definition = { description: "Delete a row from your database", @@ -48,7 +50,7 @@ module.exports.definition = { }, } -module.exports.run = async function({ inputs, instanceId }) { +module.exports.run = async function({ inputs, instanceId, apiKey }) { // TODO: better logging of when actions are missed due to missing parameters if (inputs.id == null || inputs.revision == null) { return @@ -63,6 +65,9 @@ module.exports.run = async function({ inputs, instanceId }) { } try { + if (environment.CLOUD) { + await usage.update(apiKey, usage.Properties.RECORD, -1) + } await recordController.destroy(ctx) return { response: ctx.body, diff --git a/packages/server/src/automations/thread.js b/packages/server/src/automations/thread.js index fa826afbe9..5362597cfd 100644 --- a/packages/server/src/automations/thread.js +++ b/packages/server/src/automations/thread.js @@ -62,6 +62,7 @@ class Orchestrator { const outputs = await stepFn({ inputs: step.inputs, instanceId: this._instanceId, + apiKey: automation.apiKey, }) if (step.stepId === FILTER_STEP_ID && !outputs.success) { break diff --git a/packages/server/src/db/dynamoClient.js b/packages/server/src/db/dynamoClient.js index 9f30b151da..6250486bf7 100644 --- a/packages/server/src/db/dynamoClient.js +++ b/packages/server/src/db/dynamoClient.js @@ -1,6 +1,8 @@ let _ = require("lodash") let environment = require("../environment") +const AWS_REGION = environment.AWS_REGION ? environment.AWS_REGION : "eu-west-1" + const TableInfo = { API_KEYS: { name: "beta-api-key-table", @@ -49,6 +51,7 @@ class Table { condition, names, values, + exists, otherProps, }) { let params = { @@ -66,6 +69,13 @@ class Table { if (this._sort && sort) { params.Key[this._sort] = sort } + if (exists) { + params.ExpressionAttributeNames["#PRIMARY"] = this._primary + if (params.ConditionExpression) { + params.ConditionExpression += " AND " + } + params.ConditionExpression += "attribute_exists(#PRIMARY)" + } if (otherProps) { params = _.merge(params, otherProps) } @@ -90,15 +100,17 @@ class Table { } } -exports.init = () => { - if (!environment.CLOUD) { - return - } +exports.init = endpoint => { let AWS = require("aws-sdk") + AWS.config.update({ + region: AWS_REGION, + }) let docClientParams = { correctClockSkew: true, } - if (environment.DYNAMO_ENDPOINT) { + if (endpoint) { + docClientParams.endpoint = endpoint + } else if (environment.DYNAMO_ENDPOINT) { docClientParams.endpoint = environment.DYNAMO_ENDPOINT } docClient = new AWS.DynamoDB.DocumentClient(docClientParams) @@ -106,3 +118,11 @@ exports.init = () => { exports.apiKeyTable = new Table(TableInfo.API_KEYS) exports.userTable = new Table(TableInfo.USERS) + +if (environment.CLOUD) { + exports.init(`https://dynamodb.${AWS_REGION}.amazonaws.com`) +} else { + process.env.AWS_ACCESS_KEY_ID = "KEY_ID" + process.env.AWS_SECRET_ACCESS_KEY = "SECRET_KEY" + exports.init("http://localhost:8333") +} diff --git a/packages/server/src/middleware/authorized.js b/packages/server/src/middleware/authorized.js index b452d63cf5..4cce4c4670 100644 --- a/packages/server/src/middleware/authorized.js +++ b/packages/server/src/middleware/authorized.js @@ -16,8 +16,7 @@ module.exports = (permName, getItemId) => async (ctx, next) => { } if (ctx.user.accessLevel._id === BUILDER_LEVEL_ID) { - await next() - return + return next() } if (permName === BUILDER) { @@ -28,8 +27,7 @@ module.exports = (permName, getItemId) => async (ctx, next) => { const permissionId = ({ name, itemId }) => name + (itemId ? `-${itemId}` : "") if (ctx.user.accessLevel._id === ADMIN_LEVEL_ID) { - await next() - return + return next() } const thisPermissionId = permissionId({ @@ -42,8 +40,7 @@ module.exports = (permName, getItemId) => async (ctx, next) => { ctx.user.accessLevel._id === POWERUSER_LEVEL_ID && !adminPermissions.map(permissionId).includes(thisPermissionId) ) { - await next() - return + return next() } if ( @@ -51,8 +48,7 @@ module.exports = (permName, getItemId) => async (ctx, next) => { .map(permissionId) .includes(thisPermissionId) ) { - await next() - return + return next() } ctx.throw(403, "Not Authorized") diff --git a/packages/server/src/middleware/usageQuota.js b/packages/server/src/middleware/usageQuota.js index 74df19a015..e82305dc12 100644 --- a/packages/server/src/middleware/usageQuota.js +++ b/packages/server/src/middleware/usageQuota.js @@ -1,5 +1,6 @@ const CouchDB = require("../db") const usageQuota = require("../utilities/usageQuota") +const environment = require("../environment") // currently only counting new writes and deletes const METHOD_MAP = { @@ -29,11 +30,9 @@ module.exports = async (ctx, next) => { const db = new CouchDB(ctx.user.instanceId) let usage = METHOD_MAP[ctx.req.method] const property = getProperty(ctx.req.url) - console.log(ctx.req.url) if (usage == null || property == null) { return next() } - console.log(`${usage} to ${property}`) // post request could be a save of a pre-existing entry if (ctx.request.body && ctx.request.body._id) { try { @@ -52,8 +51,12 @@ module.exports = async (ctx, next) => { : [ctx.request.files.file] usage = files.map(file => file.size).reduce((total, size) => total + size) } + if (!environment.CLOUD) { + return next() + } try { await usageQuota.update(ctx.apiKey, property, usage) + return next() } catch (err) { ctx.throw(403, err) } diff --git a/packages/server/src/utilities/builder/setBuilderToken.js b/packages/server/src/utilities/builder/setBuilderToken.js index 12622d5522..d43a9543e7 100644 --- a/packages/server/src/utilities/builder/setBuilderToken.js +++ b/packages/server/src/utilities/builder/setBuilderToken.js @@ -8,7 +8,9 @@ module.exports = (ctx, appId, instanceId) => { instanceId, appId, } - + if (process.env.BUDIBASE_API_KEY) { + builderUser.apiKey = process.env.BUDIBASE_API_KEY + } const token = jwt.sign(builderUser, ctx.config.jwtSecret, { expiresIn: "30 days", }) diff --git a/packages/server/src/utilities/usageQuota.js b/packages/server/src/utilities/usageQuota.js index 7ba0005213..f16fb6aba8 100644 --- a/packages/server/src/utilities/usageQuota.js +++ b/packages/server/src/utilities/usageQuota.js @@ -4,12 +4,12 @@ const { apiKeyTable } = require("../db/dynamoClient") function buildUpdateParams(key, property, usage) { return { primary: key, - condition: "#quota.#prop + :usage < #limits.model AND #quotaReset < :now", + condition: "#quota.#prop < #limits.#prop AND #quotaReset > :now", expression: "ADD #quota.#prop :usage", names: { "#quota": "usageQuota", "#prop": property, - "#limits": "limits", + "#limits": "usageLimits", "#quotaReset": "quotaReset", }, values: { @@ -50,21 +50,22 @@ exports.update = async (apiKey, property, usage) => { try { await apiKeyTable.update(buildUpdateParams(apiKey, property, usage)) } catch (err) { - if (err.code !== "ConditionalCheckFailedException") { + if (err.code === "ConditionalCheckFailedException") { // get the API key so we can check it - let apiKey = await apiKeyTable.get({ primary: apiKey }) + const keyObj = await apiKeyTable.get({ primary: apiKey }) // we have infact breached the reset period - if (apiKey && apiKey.quotaReset >= Date.now()) { + if (keyObj && keyObj.quotaReset <= Date.now()) { // update the quota reset period and reset the values for all properties - apiKey.quotaReset = Date.now() + QUOTA_RESET - for (let prop of Object.keys(apiKey.usageQuota)) { + keyObj.quotaReset = Date.now() + QUOTA_RESET + for (let prop of Object.keys(keyObj.usageQuota)) { if (prop === property) { - apiKey.usageQuota[prop] = usage > 0 ? usage : 0 + keyObj.usageQuota[prop] = usage > 0 ? usage : 0 } else { - apiKey.usageQuota[prop] = 0 + keyObj.usageQuota[prop] = 0 } } - await apiKeyTable.put({ item: apiKey }) + await apiKeyTable.put({ item: keyObj }) + return } throw "Resource limits have been reached" } From 275bd64b1b8ed01e2de2f6f2483dc16fa99a2398 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Thu, 8 Oct 2020 18:36:31 +0100 Subject: [PATCH 07/21] Fixing linting issue. --- packages/server/src/utilities/fileProcessor.js | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/packages/server/src/utilities/fileProcessor.js b/packages/server/src/utilities/fileProcessor.js index 734209733d..3e580e9e37 100644 --- a/packages/server/src/utilities/fileProcessor.js +++ b/packages/server/src/utilities/fileProcessor.js @@ -1,5 +1,5 @@ const fs = require("fs") -// const sharp = require("sharp") +const sharp = require("sharp") const fsPromises = fs.promises const FORMATS = { @@ -7,14 +7,14 @@ const FORMATS = { } async function processImage(file) { - // const imgMeta = await sharp(file.path) - // .resize(300) - // .toFile(file.outputPath) - // - // return { - // ...file, - // ...imgMeta, - // } + const imgMeta = await sharp(file.path) + .resize(300) + .toFile(file.outputPath) + + return { + ...file, + ...imgMeta, + } } async function process(file) { From 8880e7574bb182f840fb422303dfc222606eae7a Mon Sep 17 00:00:00 2001 From: Martin McKeaveney Date: Thu, 8 Oct 2020 20:23:58 +0100 Subject: [PATCH 08/21] tidy up --- .../server/src/api/controllers/deploy/aws.js | 66 ++++++++++--------- .../src/api/controllers/deploy/index.js | 14 ++-- 2 files changed, 40 insertions(+), 40 deletions(-) diff --git a/packages/server/src/api/controllers/deploy/aws.js b/packages/server/src/api/controllers/deploy/aws.js index b56d3c7788..f421e2f4fc 100644 --- a/packages/server/src/api/controllers/deploy/aws.js +++ b/packages/server/src/api/controllers/deploy/aws.js @@ -22,19 +22,20 @@ async function invalidateCDN(cfDistribution, appId) { } exports.updateDeploymentQuota = async function(quota) { - const response = await fetch( - `${process.env.DEPLOYMENT_CREDENTIALS_URL}/deploy/success`, - { - method: "POST", - body: JSON.stringify({ - apiKey: process.env.BUDIBASE_API_KEY, - quota, - }), - } - ) + const DEPLOYMENT_SUCCESS_URL = + process.env.DEPLOYMENT_CREDENTIALS_URL + "deploy/success" + + console.log(DEPLOYMENT_SUCCESS_URL) + const response = await fetch(DEPLOYMENT_SUCCESS_URL, { + method: "POST", + body: JSON.stringify({ + apiKey: process.env.BUDIBASE_API_KEY, + quota, + }), + }) if (response.status !== 200) { - throw new Error(`Error updating deployment quota for app`) + throw new Error(`Error updating deployment quota for API Key`) } const json = await response.json() @@ -163,30 +164,33 @@ exports.uploadAppAssets = async function({ // Upload file attachments const db = new PouchDB(instanceId) - const fileUploads = await db.get("_local/fileuploads") - if (fileUploads) { - for (let file of fileUploads.uploads) { - if (file.uploaded) continue - - const attachmentUpload = prepareUploadForS3({ - file, - s3Key: `assets/${appId}/attachments/${file.processedFileName}`, - s3, - metadata: { accountId }, - }) - - uploads.push(attachmentUpload) - - // mark file as uploaded - file.uploaded = true - } - - db.put(fileUploads) + let fileUploads + try { + fileUploads = await db.get("_local/fileuploads") + } catch (err) { + fileUploads = { _id: "_local/fileuploads", uploads: [] } } + for (let file of fileUploads.uploads) { + if (file.uploaded) continue + + const attachmentUpload = prepareUploadForS3({ + file, + s3Key: `assets/${appId}/attachments/${file.processedFileName}`, + s3, + metadata: { accountId }, + }) + + uploads.push(attachmentUpload) + + // mark file as uploaded + file.uploaded = true + } + + db.put(fileUploads) + try { await Promise.all(uploads) - // TODO: update dynamoDB with a synopsis of the app deployment for historical purposes await invalidateCDN(cfDistribution, appId) } catch (err) { console.error("Error uploading budibase app assets to s3", err) diff --git a/packages/server/src/api/controllers/deploy/index.js b/packages/server/src/api/controllers/deploy/index.js index 7f6e4588a5..2b433140fe 100644 --- a/packages/server/src/api/controllers/deploy/index.js +++ b/packages/server/src/api/controllers/deploy/index.js @@ -5,13 +5,10 @@ const { verifyDeployment, updateDeploymentQuota, } = require("./aws") -const { getRecordParams } = require("../../db/utils") function replicate(local, remote) { return new Promise((resolve, reject) => { - const replication = local.sync(remote, { - retry: true, - }) + const replication = local.sync(remote) replication.on("complete", () => resolve()) replication.on("error", err => reject(err)) @@ -40,11 +37,10 @@ async function replicateCouch({ instanceId, clientId, credentials }) { async function getCurrentInstanceQuota(instanceId) { const db = new PouchDB(instanceId) - const records = await db.allDocs( - getRecordParams("", null, { - include_docs: true, - }) - ) + const records = await db.allDocs({ + startkey: "record:", + endkey: `record:\ufff0`, + }) const existingRecords = records.rows.length const designDoc = await db.get("_design/database") From b21cc23b3b0ba1afb09ab09d651ee3d693e5ffcf Mon Sep 17 00:00:00 2001 From: Martin McKeaveney Date: Thu, 8 Oct 2020 21:11:10 +0100 Subject: [PATCH 09/21] removing retry param --- .../pages/[application]/deploy/index.svelte | 7 +- packages/server/yarn.lock | 249 +----------------- 2 files changed, 9 insertions(+), 247 deletions(-) diff --git a/packages/builder/src/pages/[application]/deploy/index.svelte b/packages/builder/src/pages/[application]/deploy/index.svelte index de649a8dd2..39144df2bf 100644 --- a/packages/builder/src/pages/[application]/deploy/index.svelte +++ b/packages/builder/src/pages/[application]/deploy/index.svelte @@ -1,5 +1,5 @@