From 297be181599c7f8cddbe44ac9f23cd8b145bdeea Mon Sep 17 00:00:00 2001 From: Budibase Release Bot <> Date: Wed, 22 Sep 2021 08:42:24 +0000 Subject: [PATCH 01/19] v0.9.140 --- lerna.json | 2 +- packages/auth/package.json | 2 +- packages/bbui/package.json | 2 +- packages/builder/package.json | 8 ++++---- packages/cli/package.json | 2 +- packages/client/package.json | 6 +++--- packages/server/package.json | 8 ++++---- packages/string-templates/package.json | 2 +- packages/worker/package.json | 6 +++--- 9 files changed, 19 insertions(+), 19 deletions(-) diff --git a/lerna.json b/lerna.json index 9b2b1cac6d..a21b86e032 100644 --- a/lerna.json +++ b/lerna.json @@ -1,5 +1,5 @@ { - "version": "0.9.140-alpha.0", + "version": "0.9.140", "npmClient": "yarn", "packages": [ "packages/*" diff --git a/packages/auth/package.json b/packages/auth/package.json index 448b408742..b02625ef29 100644 --- a/packages/auth/package.json +++ b/packages/auth/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/auth", - "version": "0.9.140-alpha.0", + "version": "0.9.140", "description": "Authentication middlewares for budibase builder and apps", "main": "src/index.js", "author": "Budibase", diff --git a/packages/bbui/package.json b/packages/bbui/package.json index 123d168fee..b542df3bb8 100644 --- a/packages/bbui/package.json +++ b/packages/bbui/package.json @@ -1,7 +1,7 @@ { "name": "@budibase/bbui", "description": "A UI solution used in the different Budibase projects.", - "version": "0.9.140-alpha.0", + "version": "0.9.140", "license": "AGPL-3.0", "svelte": "src/index.js", "module": "dist/bbui.es.js", diff --git a/packages/builder/package.json b/packages/builder/package.json index fb4d050392..10c7572743 100644 --- a/packages/builder/package.json +++ b/packages/builder/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/builder", - "version": "0.9.140-alpha.0", + "version": "0.9.140", "license": "AGPL-3.0", "private": true, "scripts": { @@ -65,10 +65,10 @@ } }, "dependencies": { - "@budibase/bbui": "^0.9.140-alpha.0", - "@budibase/client": "^0.9.140-alpha.0", + "@budibase/bbui": "^0.9.140", + "@budibase/client": "^0.9.140", "@budibase/colorpicker": "1.1.2", - "@budibase/string-templates": "^0.9.140-alpha.0", + "@budibase/string-templates": "^0.9.140", "@sentry/browser": "5.19.1", "@spectrum-css/page": "^3.0.1", "@spectrum-css/vars": "^3.0.1", diff --git a/packages/cli/package.json b/packages/cli/package.json index 7bdf1a394f..f6fadeff6b 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/cli", - "version": "0.9.140-alpha.0", + "version": "0.9.140", "description": "Budibase CLI, for developers, self hosting and migrations.", "main": "src/index.js", "bin": { diff --git a/packages/client/package.json b/packages/client/package.json index 98b0daf581..5ab814a4ac 100644 --- a/packages/client/package.json +++ b/packages/client/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/client", - "version": "0.9.140-alpha.0", + "version": "0.9.140", "license": "MPL-2.0", "module": "dist/budibase-client.js", "main": "dist/budibase-client.js", @@ -19,9 +19,9 @@ "dev:builder": "rollup -cw" }, "dependencies": { - "@budibase/bbui": "^0.9.140-alpha.0", + "@budibase/bbui": "^0.9.140", "@budibase/standard-components": "^0.9.139", - "@budibase/string-templates": "^0.9.140-alpha.0", + "@budibase/string-templates": "^0.9.140", "regexparam": "^1.3.0", "shortid": "^2.2.15", "svelte-spa-router": "^3.0.5" diff --git a/packages/server/package.json b/packages/server/package.json index ad6e55dd6c..743c2d8ff4 100644 --- a/packages/server/package.json +++ b/packages/server/package.json @@ -1,7 +1,7 @@ { "name": "@budibase/server", "email": "hi@budibase.com", - "version": "0.9.140-alpha.0", + "version": "0.9.140", "description": "Budibase Web Server", "main": "src/index.js", "repository": { @@ -64,9 +64,9 @@ "author": "Budibase", "license": "AGPL-3.0-or-later", "dependencies": { - "@budibase/auth": "^0.9.140-alpha.0", - "@budibase/client": "^0.9.140-alpha.0", - "@budibase/string-templates": "^0.9.140-alpha.0", + "@budibase/auth": "^0.9.140", + "@budibase/client": "^0.9.140", + "@budibase/string-templates": "^0.9.140", "@elastic/elasticsearch": "7.10.0", "@koa/router": "8.0.0", "@sendgrid/mail": "7.1.1", diff --git a/packages/string-templates/package.json b/packages/string-templates/package.json index 0ec109eca5..8686a43f51 100644 --- a/packages/string-templates/package.json +++ b/packages/string-templates/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/string-templates", - "version": "0.9.140-alpha.0", + "version": "0.9.140", "description": "Handlebars wrapper for Budibase templating.", "main": "src/index.cjs", "module": "dist/bundle.mjs", diff --git a/packages/worker/package.json b/packages/worker/package.json index eecd3d6959..2d2bf0df80 100644 --- a/packages/worker/package.json +++ b/packages/worker/package.json @@ -1,7 +1,7 @@ { "name": "@budibase/worker", "email": "hi@budibase.com", - "version": "0.9.140-alpha.0", + "version": "0.9.140", "description": "Budibase background service", "main": "src/index.js", "repository": { @@ -25,8 +25,8 @@ "author": "Budibase", "license": "AGPL-3.0-or-later", "dependencies": { - "@budibase/auth": "^0.9.140-alpha.0", - "@budibase/string-templates": "^0.9.140-alpha.0", + "@budibase/auth": "^0.9.140", + "@budibase/string-templates": "^0.9.140", "@koa/router": "^8.0.0", "@techpass/passport-openidconnect": "^0.3.0", "aws-sdk": "^2.811.0", From ecbe015c2a47cd25a3f6670c7be1aa501a48dae7 Mon Sep 17 00:00:00 2001 From: Budibase Release Bot <> Date: Wed, 22 Sep 2021 11:36:15 +0000 Subject: [PATCH 02/19] v0.9.141 --- lerna.json | 2 +- packages/auth/package.json | 2 +- packages/bbui/package.json | 2 +- packages/builder/package.json | 8 ++++---- packages/cli/package.json | 2 +- packages/client/package.json | 6 +++--- packages/server/package.json | 8 ++++---- packages/string-templates/package.json | 2 +- packages/worker/package.json | 6 +++--- 9 files changed, 19 insertions(+), 19 deletions(-) diff --git a/lerna.json b/lerna.json index a21b86e032..483a1abd82 100644 --- a/lerna.json +++ b/lerna.json @@ -1,5 +1,5 @@ { - "version": "0.9.140", + "version": "0.9.141", "npmClient": "yarn", "packages": [ "packages/*" diff --git a/packages/auth/package.json b/packages/auth/package.json index b02625ef29..911163deed 100644 --- a/packages/auth/package.json +++ b/packages/auth/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/auth", - "version": "0.9.140", + "version": "0.9.141", "description": "Authentication middlewares for budibase builder and apps", "main": "src/index.js", "author": "Budibase", diff --git a/packages/bbui/package.json b/packages/bbui/package.json index b542df3bb8..b45081b890 100644 --- a/packages/bbui/package.json +++ b/packages/bbui/package.json @@ -1,7 +1,7 @@ { "name": "@budibase/bbui", "description": "A UI solution used in the different Budibase projects.", - "version": "0.9.140", + "version": "0.9.141", "license": "AGPL-3.0", "svelte": "src/index.js", "module": "dist/bbui.es.js", diff --git a/packages/builder/package.json b/packages/builder/package.json index 10c7572743..6f9ae796f0 100644 --- a/packages/builder/package.json +++ b/packages/builder/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/builder", - "version": "0.9.140", + "version": "0.9.141", "license": "AGPL-3.0", "private": true, "scripts": { @@ -65,10 +65,10 @@ } }, "dependencies": { - "@budibase/bbui": "^0.9.140", - "@budibase/client": "^0.9.140", + "@budibase/bbui": "^0.9.141", + "@budibase/client": "^0.9.141", "@budibase/colorpicker": "1.1.2", - "@budibase/string-templates": "^0.9.140", + "@budibase/string-templates": "^0.9.141", "@sentry/browser": "5.19.1", "@spectrum-css/page": "^3.0.1", "@spectrum-css/vars": "^3.0.1", diff --git a/packages/cli/package.json b/packages/cli/package.json index f6fadeff6b..9f9f0e7454 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/cli", - "version": "0.9.140", + "version": "0.9.141", "description": "Budibase CLI, for developers, self hosting and migrations.", "main": "src/index.js", "bin": { diff --git a/packages/client/package.json b/packages/client/package.json index 5ab814a4ac..3cfdf8e2f3 100644 --- a/packages/client/package.json +++ b/packages/client/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/client", - "version": "0.9.140", + "version": "0.9.141", "license": "MPL-2.0", "module": "dist/budibase-client.js", "main": "dist/budibase-client.js", @@ -19,9 +19,9 @@ "dev:builder": "rollup -cw" }, "dependencies": { - "@budibase/bbui": "^0.9.140", + "@budibase/bbui": "^0.9.141", "@budibase/standard-components": "^0.9.139", - "@budibase/string-templates": "^0.9.140", + "@budibase/string-templates": "^0.9.141", "regexparam": "^1.3.0", "shortid": "^2.2.15", "svelte-spa-router": "^3.0.5" diff --git a/packages/server/package.json b/packages/server/package.json index 743c2d8ff4..1bbb871f1b 100644 --- a/packages/server/package.json +++ b/packages/server/package.json @@ -1,7 +1,7 @@ { "name": "@budibase/server", "email": "hi@budibase.com", - "version": "0.9.140", + "version": "0.9.141", "description": "Budibase Web Server", "main": "src/index.js", "repository": { @@ -64,9 +64,9 @@ "author": "Budibase", "license": "AGPL-3.0-or-later", "dependencies": { - "@budibase/auth": "^0.9.140", - "@budibase/client": "^0.9.140", - "@budibase/string-templates": "^0.9.140", + "@budibase/auth": "^0.9.141", + "@budibase/client": "^0.9.141", + "@budibase/string-templates": "^0.9.141", "@elastic/elasticsearch": "7.10.0", "@koa/router": "8.0.0", "@sendgrid/mail": "7.1.1", diff --git a/packages/string-templates/package.json b/packages/string-templates/package.json index 8686a43f51..e0e73955d3 100644 --- a/packages/string-templates/package.json +++ b/packages/string-templates/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/string-templates", - "version": "0.9.140", + "version": "0.9.141", "description": "Handlebars wrapper for Budibase templating.", "main": "src/index.cjs", "module": "dist/bundle.mjs", diff --git a/packages/worker/package.json b/packages/worker/package.json index 2d2bf0df80..8849499786 100644 --- a/packages/worker/package.json +++ b/packages/worker/package.json @@ -1,7 +1,7 @@ { "name": "@budibase/worker", "email": "hi@budibase.com", - "version": "0.9.140", + "version": "0.9.141", "description": "Budibase background service", "main": "src/index.js", "repository": { @@ -25,8 +25,8 @@ "author": "Budibase", "license": "AGPL-3.0-or-later", "dependencies": { - "@budibase/auth": "^0.9.140", - "@budibase/string-templates": "^0.9.140", + "@budibase/auth": "^0.9.141", + "@budibase/string-templates": "^0.9.141", "@koa/router": "^8.0.0", "@techpass/passport-openidconnect": "^0.3.0", "aws-sdk": "^2.811.0", From 9b989850401bbab65c568a9a94816af5b73e3936 Mon Sep 17 00:00:00 2001 From: Martin McKeaveney Date: Thu, 23 Sep 2021 16:22:12 +0100 Subject: [PATCH 03/19] point logo upload to S3 bucket when not self hosted --- packages/worker/src/api/controllers/global/configs.js | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/packages/worker/src/api/controllers/global/configs.js b/packages/worker/src/api/controllers/global/configs.js index aa83fd695f..c0c300e4db 100644 --- a/packages/worker/src/api/controllers/global/configs.js +++ b/packages/worker/src/api/controllers/global/configs.js @@ -10,6 +10,7 @@ const email = require("../../../utilities/email") const { upload, ObjectStoreBuckets } = require("@budibase/auth").objectStore const CouchDB = require("../../../db") const { getGlobalDB } = require("@budibase/auth/tenancy") +const env = require("../../../environment") exports.save = async function (ctx) { const db = getGlobalDB() @@ -174,7 +175,13 @@ exports.upload = async function (ctx) { const file = ctx.request.files.file const { type, name } = ctx.params - const bucket = ObjectStoreBuckets.GLOBAL + let bucket + if (env.SELF_HOSTED) { + bucket = ObjectStoreBuckets.GLOBAL + } else { + bucket = ObjectStoreBuckets.GLOBAL_CLOUD + } + const key = `${type}/${name}` await upload({ bucket, From ce3f8a364fe951325cba33accc2818d12a06a37b Mon Sep 17 00:00:00 2001 From: Budibase Release Bot <> Date: Thu, 23 Sep 2021 15:48:36 +0000 Subject: [PATCH 04/19] v0.9.142 --- lerna.json | 2 +- packages/auth/package.json | 2 +- packages/bbui/package.json | 2 +- packages/builder/package.json | 8 ++++---- packages/cli/package.json | 2 +- packages/client/package.json | 6 +++--- packages/server/package.json | 8 ++++---- packages/string-templates/package.json | 2 +- packages/worker/package.json | 6 +++--- 9 files changed, 19 insertions(+), 19 deletions(-) diff --git a/lerna.json b/lerna.json index 483a1abd82..a8cda22a63 100644 --- a/lerna.json +++ b/lerna.json @@ -1,5 +1,5 @@ { - "version": "0.9.141", + "version": "0.9.142", "npmClient": "yarn", "packages": [ "packages/*" diff --git a/packages/auth/package.json b/packages/auth/package.json index 911163deed..f62e4cb5e6 100644 --- a/packages/auth/package.json +++ b/packages/auth/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/auth", - "version": "0.9.141", + "version": "0.9.142", "description": "Authentication middlewares for budibase builder and apps", "main": "src/index.js", "author": "Budibase", diff --git a/packages/bbui/package.json b/packages/bbui/package.json index b45081b890..ddede26a86 100644 --- a/packages/bbui/package.json +++ b/packages/bbui/package.json @@ -1,7 +1,7 @@ { "name": "@budibase/bbui", "description": "A UI solution used in the different Budibase projects.", - "version": "0.9.141", + "version": "0.9.142", "license": "AGPL-3.0", "svelte": "src/index.js", "module": "dist/bbui.es.js", diff --git a/packages/builder/package.json b/packages/builder/package.json index 6f9ae796f0..7ab313974b 100644 --- a/packages/builder/package.json +++ b/packages/builder/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/builder", - "version": "0.9.141", + "version": "0.9.142", "license": "AGPL-3.0", "private": true, "scripts": { @@ -65,10 +65,10 @@ } }, "dependencies": { - "@budibase/bbui": "^0.9.141", - "@budibase/client": "^0.9.141", + "@budibase/bbui": "^0.9.142", + "@budibase/client": "^0.9.142", "@budibase/colorpicker": "1.1.2", - "@budibase/string-templates": "^0.9.141", + "@budibase/string-templates": "^0.9.142", "@sentry/browser": "5.19.1", "@spectrum-css/page": "^3.0.1", "@spectrum-css/vars": "^3.0.1", diff --git a/packages/cli/package.json b/packages/cli/package.json index 9f9f0e7454..0340759e45 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/cli", - "version": "0.9.141", + "version": "0.9.142", "description": "Budibase CLI, for developers, self hosting and migrations.", "main": "src/index.js", "bin": { diff --git a/packages/client/package.json b/packages/client/package.json index 3cfdf8e2f3..3d25a1b5d6 100644 --- a/packages/client/package.json +++ b/packages/client/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/client", - "version": "0.9.141", + "version": "0.9.142", "license": "MPL-2.0", "module": "dist/budibase-client.js", "main": "dist/budibase-client.js", @@ -19,9 +19,9 @@ "dev:builder": "rollup -cw" }, "dependencies": { - "@budibase/bbui": "^0.9.141", + "@budibase/bbui": "^0.9.142", "@budibase/standard-components": "^0.9.139", - "@budibase/string-templates": "^0.9.141", + "@budibase/string-templates": "^0.9.142", "regexparam": "^1.3.0", "shortid": "^2.2.15", "svelte-spa-router": "^3.0.5" diff --git a/packages/server/package.json b/packages/server/package.json index 1bbb871f1b..9397ab16f3 100644 --- a/packages/server/package.json +++ b/packages/server/package.json @@ -1,7 +1,7 @@ { "name": "@budibase/server", "email": "hi@budibase.com", - "version": "0.9.141", + "version": "0.9.142", "description": "Budibase Web Server", "main": "src/index.js", "repository": { @@ -64,9 +64,9 @@ "author": "Budibase", "license": "AGPL-3.0-or-later", "dependencies": { - "@budibase/auth": "^0.9.141", - "@budibase/client": "^0.9.141", - "@budibase/string-templates": "^0.9.141", + "@budibase/auth": "^0.9.142", + "@budibase/client": "^0.9.142", + "@budibase/string-templates": "^0.9.142", "@elastic/elasticsearch": "7.10.0", "@koa/router": "8.0.0", "@sendgrid/mail": "7.1.1", diff --git a/packages/string-templates/package.json b/packages/string-templates/package.json index e0e73955d3..44a3290696 100644 --- a/packages/string-templates/package.json +++ b/packages/string-templates/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/string-templates", - "version": "0.9.141", + "version": "0.9.142", "description": "Handlebars wrapper for Budibase templating.", "main": "src/index.cjs", "module": "dist/bundle.mjs", diff --git a/packages/worker/package.json b/packages/worker/package.json index 8849499786..83e53e9b45 100644 --- a/packages/worker/package.json +++ b/packages/worker/package.json @@ -1,7 +1,7 @@ { "name": "@budibase/worker", "email": "hi@budibase.com", - "version": "0.9.141", + "version": "0.9.142", "description": "Budibase background service", "main": "src/index.js", "repository": { @@ -25,8 +25,8 @@ "author": "Budibase", "license": "AGPL-3.0-or-later", "dependencies": { - "@budibase/auth": "^0.9.141", - "@budibase/string-templates": "^0.9.141", + "@budibase/auth": "^0.9.142", + "@budibase/string-templates": "^0.9.142", "@koa/router": "^8.0.0", "@techpass/passport-openidconnect": "^0.3.0", "aws-sdk": "^2.811.0", From 4a2028c354a41845c7a7f145932ad808f136266e Mon Sep 17 00:00:00 2001 From: Martin McKeaveney Date: Thu, 23 Sep 2021 22:40:14 +0100 Subject: [PATCH 05/19] usageQuota middleware writing to couch --- packages/server/src/api/routes/application.js | 5 +- .../server/src/automations/steps/createRow.js | 4 +- packages/server/src/middleware/usageQuota.js | 25 ++-- packages/server/src/utilities/usageQuota.js | 110 +++++------------- .../server/src/utilities/usageQuota.old.js | 105 +++++++++++++++++ .../src/api/controllers/global/users.js | 24 ++++ 6 files changed, 177 insertions(+), 96 deletions(-) create mode 100644 packages/server/src/utilities/usageQuota.old.js diff --git a/packages/server/src/api/routes/application.js b/packages/server/src/api/routes/application.js index c1d39acbd5..ef4aacf708 100644 --- a/packages/server/src/api/routes/application.js +++ b/packages/server/src/api/routes/application.js @@ -2,11 +2,12 @@ const Router = require("@koa/router") const controller = require("../controllers/application") const authorized = require("../../middleware/authorized") const { BUILDER } = require("@budibase/auth/permissions") +const usage = require("../../middleware/usageQuota") const router = Router() router - .post("/api/applications", authorized(BUILDER), controller.create) + .post("/api/applications", authorized(BUILDER), usage, controller.create) .get("/api/applications/:appId/definition", controller.fetchAppDefinition) .get("/api/applications", controller.fetch) .get("/api/applications/:appId/appPackage", controller.fetchAppPackage) @@ -21,6 +22,6 @@ router authorized(BUILDER), controller.revertClient ) - .delete("/api/applications/:appId", authorized(BUILDER), controller.delete) + .delete("/api/applications/:appId", authorized(BUILDER), usage, controller.delete) module.exports = router diff --git a/packages/server/src/automations/steps/createRow.js b/packages/server/src/automations/steps/createRow.js index 9033004578..41e775b3de 100644 --- a/packages/server/src/automations/steps/createRow.js +++ b/packages/server/src/automations/steps/createRow.js @@ -60,7 +60,7 @@ exports.definition = { }, } -exports.run = async function ({ inputs, appId, apiKey, emitter }) { +exports.run = async function ({ inputs, appId, tenantId, emitter }) { if (inputs.row == null || inputs.row.tableId == null) { return { success: false, @@ -84,7 +84,7 @@ exports.run = async function ({ inputs, appId, apiKey, emitter }) { inputs.row ) if (env.USE_QUOTAS) { - await usage.update(apiKey, usage.Properties.ROW, 1) + await usage.update(tenantId, usage.Properties.ROW, 1) } await rowController.save(ctx) return { diff --git a/packages/server/src/middleware/usageQuota.js b/packages/server/src/middleware/usageQuota.js index 4647878721..4ad1092f6c 100644 --- a/packages/server/src/middleware/usageQuota.js +++ b/packages/server/src/middleware/usageQuota.js @@ -13,6 +13,7 @@ const DOMAIN_MAP = { upload: usageQuota.Properties.UPLOAD, views: usageQuota.Properties.VIEW, users: usageQuota.Properties.USER, + applications: usageQuota.Properties.APPS, // this will not be updated by endpoint calls // instead it will be updated by triggerInfo automationRuns: usageQuota.Properties.AUTOMATION, @@ -28,9 +29,9 @@ function getProperty(url) { module.exports = async (ctx, next) => { // if in development or a self hosted cloud usage quotas should not be executed - if (env.isDev() || env.SELF_HOSTED) { - return next() - } + // if (env.isDev() || env.SELF_HOSTED) { + // return next() + // } const db = new CouchDB(ctx.appId) let usage = METHOD_MAP[ctx.req.method] @@ -49,17 +50,17 @@ module.exports = async (ctx, next) => { } // update usage for uploads to be the total size - if (property === usageQuota.Properties.UPLOAD) { - const files = - ctx.request.files.file.length > 1 - ? Array.from(ctx.request.files.file) - : [ctx.request.files.file] - usage = files.map(file => file.size).reduce((total, size) => total + size) - } + // if (property === usageQuota.Properties.UPLOAD) { + // const files = + // ctx.request.files.file.length > 1 + // ? Array.from(ctx.request.files.file) + // : [ctx.request.files.file] + // usage = files.map(file => file.size).reduce((total, size) => total + size) + // } try { - await usageQuota.update(ctx.auth.apiKey, property, usage) + await usageQuota.update(ctx.user.tenantId, property, usage) return next() } catch (err) { - ctx.throw(403, err) + ctx.throw(400, err) } } diff --git a/packages/server/src/utilities/usageQuota.js b/packages/server/src/utilities/usageQuota.js index bfe71a4093..502fc4cad2 100644 --- a/packages/server/src/utilities/usageQuota.js +++ b/packages/server/src/utilities/usageQuota.js @@ -1,41 +1,6 @@ + const env = require("../environment") -const { apiKeyTable } = require("../db/dynamoClient") - -const DEFAULT_USAGE = { - rows: 0, - storage: 0, - views: 0, - automationRuns: 0, - users: 0, -} - -const DEFAULT_PLAN = { - rows: 1000, - // 1 GB - storage: 8589934592, - views: 10, - automationRuns: 100, - users: 10000, -} - -function buildUpdateParams(key, property, usage) { - return { - primary: key, - condition: - "attribute_exists(#quota) AND attribute_exists(#limits) AND #quota.#prop < #limits.#prop AND #quotaReset > :now", - expression: "ADD #quota.#prop :usage", - names: { - "#quota": "usageQuota", - "#prop": property, - "#limits": "usageLimits", - "#quotaReset": "quotaReset", - }, - values: { - ":usage": usage, - ":now": Date.now(), - }, - } -} +const { getGlobalDB } = require("@budibase/auth/tenancy") function getNewQuotaReset() { return Date.now() + 2592000000 @@ -47,59 +12,44 @@ exports.Properties = { VIEW: "views", USER: "users", AUTOMATION: "automationRuns", -} - -exports.getAPIKey = async appId => { - if (!env.USE_QUOTAS) { - return { apiKey: null } - } - return apiKeyTable.get({ primary: appId }) + APPS: "apps" } /** - * Given a specified API key this will add to the usage object for the specified property. - * @param {string} apiKey The API key which is to be updated. + * Given a specified tenantId this will add to the usage object for the specified property. + * @param {string} tenantId The tenant to update the usage quotas for. * @param {string} property The property which is to be added to (within the nested usageQuota object). * @param {number} usage The amount (this can be negative) to adjust the number by. * @returns {Promise} When this completes the API key will now be up to date - the quota period may have * also been reset after this call. */ -exports.update = async (apiKey, property, usage) => { - if (!env.USE_QUOTAS) { - return - } +exports.update = async (tenantId, property, usage) => { + // if (!env.USE_QUOTAS) { + // return + // } try { - await apiKeyTable.update(buildUpdateParams(apiKey, property, usage)) + const db = getGlobalDB() + const quota = await db.get("usage_quota") + // TODO: check if the quota needs reset + if (Date.now() >= quota.quotaReset) { + quota.quotaReset = getNewQuotaReset() + for (let prop of Object.keys(quota.usageQuota)) { + quota.usageQuota[prop] = 0 + } + } + + // increment the quota + quota.usageQuota[property] += usage + + if (quota.usageQuota[property] >= quota.usageLimits[property]) { + throw new Error(`You have exceeded your usage quota of ${quota.usageLimits[property]} ${property}.`) + } + + // update the usage quotas + await db.put(quota) } catch (err) { - // conditional check means the condition failed, need to check why - if (err.code === "ConditionalCheckFailedException") { - // get the API key so we can check it - const keyObj = await apiKeyTable.get({ primary: apiKey }) - // the usage quota or usage limits didn't exist - if (keyObj && (keyObj.usageQuota == null || keyObj.usageLimits == null)) { - keyObj.usageQuota = - keyObj.usageQuota == null ? DEFAULT_USAGE : keyObj.usageQuota - keyObj.usageLimits = - keyObj.usageLimits == null ? DEFAULT_PLAN : keyObj.usageLimits - keyObj.quotaReset = getNewQuotaReset() - await apiKeyTable.put({ item: keyObj }) - return - } - // we have in fact breached the reset period - else if (keyObj && keyObj.quotaReset <= Date.now()) { - // update the quota reset period and reset the values for all properties - keyObj.quotaReset = getNewQuotaReset() - for (let prop of Object.keys(keyObj.usageQuota)) { - if (prop === property) { - keyObj.usageQuota[prop] = usage > 0 ? usage : 0 - } else { - keyObj.usageQuota[prop] = 0 - } - } - await apiKeyTable.put({ item: keyObj }) - return - } - } + console.error(`Error updating usage quotas for ${property}`, err) throw err } + } diff --git a/packages/server/src/utilities/usageQuota.old.js b/packages/server/src/utilities/usageQuota.old.js new file mode 100644 index 0000000000..bfe71a4093 --- /dev/null +++ b/packages/server/src/utilities/usageQuota.old.js @@ -0,0 +1,105 @@ +const env = require("../environment") +const { apiKeyTable } = require("../db/dynamoClient") + +const DEFAULT_USAGE = { + rows: 0, + storage: 0, + views: 0, + automationRuns: 0, + users: 0, +} + +const DEFAULT_PLAN = { + rows: 1000, + // 1 GB + storage: 8589934592, + views: 10, + automationRuns: 100, + users: 10000, +} + +function buildUpdateParams(key, property, usage) { + return { + primary: key, + condition: + "attribute_exists(#quota) AND attribute_exists(#limits) AND #quota.#prop < #limits.#prop AND #quotaReset > :now", + expression: "ADD #quota.#prop :usage", + names: { + "#quota": "usageQuota", + "#prop": property, + "#limits": "usageLimits", + "#quotaReset": "quotaReset", + }, + values: { + ":usage": usage, + ":now": Date.now(), + }, + } +} + +function getNewQuotaReset() { + return Date.now() + 2592000000 +} + +exports.Properties = { + ROW: "rows", + UPLOAD: "storage", + VIEW: "views", + USER: "users", + AUTOMATION: "automationRuns", +} + +exports.getAPIKey = async appId => { + if (!env.USE_QUOTAS) { + return { apiKey: null } + } + return apiKeyTable.get({ primary: appId }) +} + +/** + * Given a specified API key this will add to the usage object for the specified property. + * @param {string} apiKey The API key which is to be updated. + * @param {string} property The property which is to be added to (within the nested usageQuota object). + * @param {number} usage The amount (this can be negative) to adjust the number by. + * @returns {Promise} When this completes the API key will now be up to date - the quota period may have + * also been reset after this call. + */ +exports.update = async (apiKey, property, usage) => { + if (!env.USE_QUOTAS) { + return + } + try { + await apiKeyTable.update(buildUpdateParams(apiKey, property, usage)) + } catch (err) { + // conditional check means the condition failed, need to check why + if (err.code === "ConditionalCheckFailedException") { + // get the API key so we can check it + const keyObj = await apiKeyTable.get({ primary: apiKey }) + // the usage quota or usage limits didn't exist + if (keyObj && (keyObj.usageQuota == null || keyObj.usageLimits == null)) { + keyObj.usageQuota = + keyObj.usageQuota == null ? DEFAULT_USAGE : keyObj.usageQuota + keyObj.usageLimits = + keyObj.usageLimits == null ? DEFAULT_PLAN : keyObj.usageLimits + keyObj.quotaReset = getNewQuotaReset() + await apiKeyTable.put({ item: keyObj }) + return + } + // we have in fact breached the reset period + else if (keyObj && keyObj.quotaReset <= Date.now()) { + // update the quota reset period and reset the values for all properties + keyObj.quotaReset = getNewQuotaReset() + for (let prop of Object.keys(keyObj.usageQuota)) { + if (prop === property) { + keyObj.usageQuota[prop] = usage > 0 ? usage : 0 + } else { + keyObj.usageQuota[prop] = 0 + } + } + await apiKeyTable.put({ item: keyObj }) + return + } + } + throw err + } +} diff --git a/packages/worker/src/api/controllers/global/users.js b/packages/worker/src/api/controllers/global/users.js index 1375240f34..b75c72290d 100644 --- a/packages/worker/src/api/controllers/global/users.js +++ b/packages/worker/src/api/controllers/global/users.js @@ -18,6 +18,7 @@ const { tryAddTenant, updateTenantId, } = require("@budibase/auth/tenancy") +const env = require("../../../environment") const PLATFORM_INFO_DB = StaticDatabases.PLATFORM_INFO.name @@ -138,6 +139,29 @@ exports.adminUser = async ctx => { include_docs: true, }) ) + + // write usage quotas for cloud + // if (!env.SELF_HOSTED) { + await db.post({ + _id: "usage_quota", + quotaReset: Date.now() + 2592000000, + usageQuota: { + automationRuns: 0, + rows: 0, + storage: 0, + apps: 0, + users: 0, + views: 0, + }, + usageLimits: { + automationRuns: 1000, + rows: 4000, + apps: 4, + // storage: 1000, + // users: 10 + }, + }) + // } if (response.rows.some(row => row.doc.admin)) { ctx.throw( From c49b88b57aedde3588f8b96cf7831953ded323d2 Mon Sep 17 00:00:00 2001 From: Martin McKeaveney Date: Thu, 23 Sep 2021 23:25:25 +0100 Subject: [PATCH 06/19] automation runs quotas --- packages/server/src/api/routes/application.js | 7 +- packages/server/src/api/routes/user.js | 5 +- .../server/src/automations/steps/createRow.js | 4 +- .../server/src/automations/steps/deleteRow.js | 4 +- packages/server/src/automations/thread.js | 7 +- packages/server/src/integrations/mysql.ts | 6 +- packages/server/src/integrations/postgres.ts | 18 +- packages/server/src/integrations/utils.ts | 6 +- packages/server/src/middleware/usageQuota.js | 2 +- packages/server/src/utilities/usageQuota.js | 45 ++-- .../server/src/utilities/usageQuota.old.js | 196 +++++++++--------- .../src/api/controllers/global/users.js | 43 ++-- 12 files changed, 184 insertions(+), 159 deletions(-) diff --git a/packages/server/src/api/routes/application.js b/packages/server/src/api/routes/application.js index ef4aacf708..4d67a0f4f4 100644 --- a/packages/server/src/api/routes/application.js +++ b/packages/server/src/api/routes/application.js @@ -22,6 +22,11 @@ router authorized(BUILDER), controller.revertClient ) - .delete("/api/applications/:appId", authorized(BUILDER), usage, controller.delete) + .delete( + "/api/applications/:appId", + authorized(BUILDER), + usage, + controller.delete + ) module.exports = router diff --git a/packages/server/src/api/routes/user.js b/packages/server/src/api/routes/user.js index b3b486fe45..465fef82c8 100644 --- a/packages/server/src/api/routes/user.js +++ b/packages/server/src/api/routes/user.js @@ -5,7 +5,6 @@ const { PermissionLevels, PermissionTypes, } = require("@budibase/auth/permissions") -const usage = require("../../middleware/usageQuota") const router = Router() @@ -28,13 +27,13 @@ router .post( "/api/users/metadata/self", authorized(PermissionTypes.USER, PermissionLevels.WRITE), - usage, + // usage, controller.updateSelfMetadata ) .delete( "/api/users/metadata/:id", authorized(PermissionTypes.USER, PermissionLevels.WRITE), - usage, + // usage, controller.destroyMetadata ) diff --git a/packages/server/src/automations/steps/createRow.js b/packages/server/src/automations/steps/createRow.js index 41e775b3de..47d0b4eb99 100644 --- a/packages/server/src/automations/steps/createRow.js +++ b/packages/server/src/automations/steps/createRow.js @@ -60,7 +60,7 @@ exports.definition = { }, } -exports.run = async function ({ inputs, appId, tenantId, emitter }) { +exports.run = async function ({ inputs, appId, emitter }) { if (inputs.row == null || inputs.row.tableId == null) { return { success: false, @@ -84,7 +84,7 @@ exports.run = async function ({ inputs, appId, tenantId, emitter }) { inputs.row ) if (env.USE_QUOTAS) { - await usage.update(tenantId, usage.Properties.ROW, 1) + await usage.update(usage.Properties.ROW, 1) } await rowController.save(ctx) return { diff --git a/packages/server/src/automations/steps/deleteRow.js b/packages/server/src/automations/steps/deleteRow.js index 0f9648cc51..225f00c5df 100644 --- a/packages/server/src/automations/steps/deleteRow.js +++ b/packages/server/src/automations/steps/deleteRow.js @@ -52,7 +52,7 @@ exports.definition = { }, } -exports.run = async function ({ inputs, appId, apiKey, emitter }) { +exports.run = async function ({ inputs, appId, emitter }) { if (inputs.id == null || inputs.revision == null) { return { success: false, @@ -74,7 +74,7 @@ exports.run = async function ({ inputs, appId, apiKey, emitter }) { try { if (env.isProd()) { - await usage.update(apiKey, usage.Properties.ROW, -1) + await usage.update(usage.Properties.ROW, -1) } await rowController.destroy(ctx) return { diff --git a/packages/server/src/automations/thread.js b/packages/server/src/automations/thread.js index a3e81a2274..37484e50bd 100644 --- a/packages/server/src/automations/thread.js +++ b/packages/server/src/automations/thread.js @@ -6,6 +6,8 @@ const { DEFAULT_TENANT_ID } = require("@budibase/auth").constants const CouchDB = require("../db") const { DocumentTypes } = require("../db/utils") const { doInTenant } = require("@budibase/auth/tenancy") +const env = require("../environment") +const usage = require("../utilities/usageQuota") const FILTER_STEP_ID = actions.ACTION_DEFINITIONS.FILTER.stepId @@ -80,7 +82,6 @@ class Orchestrator { return stepFn({ inputs: step.inputs, appId: this._appId, - apiKey: automation.apiKey, emitter: this._emitter, context: this._context, }) @@ -95,6 +96,10 @@ class Orchestrator { return err } } + // TODO: don't count test runs + if (!env.SELF_HOSTED) { + usage.update(usage.Properties.AUTOMATION, 1) + } return this.executionOutput } } diff --git a/packages/server/src/integrations/mysql.ts b/packages/server/src/integrations/mysql.ts index c5db35ed2a..605e23024b 100644 --- a/packages/server/src/integrations/mysql.ts +++ b/packages/server/src/integrations/mysql.ts @@ -12,7 +12,11 @@ import { getSqlQuery } from "./utils" module MySQLModule { const mysql = require("mysql") const Sql = require("./base/sql") - const { buildExternalTableId, convertType, copyExistingPropsOver } = require("./utils") + const { + buildExternalTableId, + convertType, + copyExistingPropsOver, + } = require("./utils") const { FieldTypes } = require("../constants") interface MySQLConfig { diff --git a/packages/server/src/integrations/postgres.ts b/packages/server/src/integrations/postgres.ts index 23a8685648..707d779c65 100644 --- a/packages/server/src/integrations/postgres.ts +++ b/packages/server/src/integrations/postgres.ts @@ -12,7 +12,11 @@ module PostgresModule { const { Pool } = require("pg") const Sql = require("./base/sql") const { FieldTypes } = require("../constants") - const { buildExternalTableId, convertType, copyExistingPropsOver } = require("./utils") + const { + buildExternalTableId, + convertType, + copyExistingPropsOver, + } = require("./utils") interface PostgresConfig { host: string @@ -179,10 +183,16 @@ module PostgresModule { } const type: string = convertType(column.data_type, TYPE_MAP) - const identity = !!(column.identity_generation || column.identity_start || column.identity_increment) - const hasDefault = typeof column.column_default === "string" && + const identity = !!( + column.identity_generation || + column.identity_start || + column.identity_increment + ) + const hasDefault = + typeof column.column_default === "string" && column.column_default.startsWith("nextval") - const isGenerated = column.is_generated && column.is_generated !== "NEVER" + const isGenerated = + column.is_generated && column.is_generated !== "NEVER" const isAuto: boolean = hasDefault || identity || isGenerated tables[tableName].schema[columnName] = { autocolumn: isAuto, diff --git a/packages/server/src/integrations/utils.ts b/packages/server/src/integrations/utils.ts index 82c35bc2d9..6e3dc6f684 100644 --- a/packages/server/src/integrations/utils.ts +++ b/packages/server/src/integrations/utils.ts @@ -84,7 +84,11 @@ export function isIsoDateString(str: string) { } // add the existing relationships from the entities if they exist, to prevent them from being overridden -export function copyExistingPropsOver(tableName: string, tables: { [key: string]: any }, entities: { [key: string]: any }) { +export function copyExistingPropsOver( + tableName: string, + tables: { [key: string]: any }, + entities: { [key: string]: any } +) { if (entities && entities[tableName]) { if (entities[tableName].primaryDisplay) { tables[tableName].primaryDisplay = entities[tableName].primaryDisplay diff --git a/packages/server/src/middleware/usageQuota.js b/packages/server/src/middleware/usageQuota.js index 4ad1092f6c..d18ffae205 100644 --- a/packages/server/src/middleware/usageQuota.js +++ b/packages/server/src/middleware/usageQuota.js @@ -1,6 +1,6 @@ const CouchDB = require("../db") const usageQuota = require("../utilities/usageQuota") -const env = require("../environment") +// const env = require("../environment") // currently only counting new writes and deletes const METHOD_MAP = { diff --git a/packages/server/src/utilities/usageQuota.js b/packages/server/src/utilities/usageQuota.js index 502fc4cad2..b98fb923cd 100644 --- a/packages/server/src/utilities/usageQuota.js +++ b/packages/server/src/utilities/usageQuota.js @@ -1,5 +1,4 @@ - -const env = require("../environment") +// const env = require("../environment") const { getGlobalDB } = require("@budibase/auth/tenancy") function getNewQuotaReset() { @@ -12,44 +11,44 @@ exports.Properties = { VIEW: "views", USER: "users", AUTOMATION: "automationRuns", - APPS: "apps" + APPS: "apps", } /** * Given a specified tenantId this will add to the usage object for the specified property. - * @param {string} tenantId The tenant to update the usage quotas for. * @param {string} property The property which is to be added to (within the nested usageQuota object). * @param {number} usage The amount (this can be negative) to adjust the number by. * @returns {Promise} When this completes the API key will now be up to date - the quota period may have * also been reset after this call. */ -exports.update = async (tenantId, property, usage) => { +exports.update = async (property, usage) => { // if (!env.USE_QUOTAS) { // return // } try { - const db = getGlobalDB() - const quota = await db.get("usage_quota") - // TODO: check if the quota needs reset - if (Date.now() >= quota.quotaReset) { - quota.quotaReset = getNewQuotaReset() - for (let prop of Object.keys(quota.usageQuota)) { - quota.usageQuota[prop] = 0 - } - } + const db = getGlobalDB() + const quota = await db.get("usage_quota") + // TODO: check if the quota needs reset + if (Date.now() >= quota.quotaReset) { + quota.quotaReset = getNewQuotaReset() + for (let prop of Object.keys(quota.usageQuota)) { + quota.usageQuota[prop] = 0 + } + } - // increment the quota - quota.usageQuota[property] += usage + // increment the quota + quota.usageQuota[property] += usage - if (quota.usageQuota[property] >= quota.usageLimits[property]) { - throw new Error(`You have exceeded your usage quota of ${quota.usageLimits[property]} ${property}.`) - } + if (quota.usageQuota[property] >= quota.usageLimits[property]) { + throw new Error( + `You have exceeded your usage quota of ${quota.usageLimits[property]} ${property}.` + ) + } - // update the usage quotas - await db.put(quota) + // update the usage quotas + await db.put(quota) } catch (err) { - console.error(`Error updating usage quotas for ${property}`, err) + console.error(`Error updating usage quotas for ${property}`, err) throw err } - } diff --git a/packages/server/src/utilities/usageQuota.old.js b/packages/server/src/utilities/usageQuota.old.js index bfe71a4093..39dad89c6d 100644 --- a/packages/server/src/utilities/usageQuota.old.js +++ b/packages/server/src/utilities/usageQuota.old.js @@ -1,105 +1,105 @@ -const env = require("../environment") -const { apiKeyTable } = require("../db/dynamoClient") +// const env = require("../environment") +// const { apiKeyTable } = require("../db/dynamoClient") -const DEFAULT_USAGE = { - rows: 0, - storage: 0, - views: 0, - automationRuns: 0, - users: 0, -} +// const DEFAULT_USAGE = { +// rows: 0, +// storage: 0, +// views: 0, +// automationRuns: 0, +// users: 0, +// } -const DEFAULT_PLAN = { - rows: 1000, - // 1 GB - storage: 8589934592, - views: 10, - automationRuns: 100, - users: 10000, -} +// const DEFAULT_PLAN = { +// rows: 1000, +// // 1 GB +// storage: 8589934592, +// views: 10, +// automationRuns: 100, +// users: 10000, +// } -function buildUpdateParams(key, property, usage) { - return { - primary: key, - condition: - "attribute_exists(#quota) AND attribute_exists(#limits) AND #quota.#prop < #limits.#prop AND #quotaReset > :now", - expression: "ADD #quota.#prop :usage", - names: { - "#quota": "usageQuota", - "#prop": property, - "#limits": "usageLimits", - "#quotaReset": "quotaReset", - }, - values: { - ":usage": usage, - ":now": Date.now(), - }, - } -} +// function buildUpdateParams(key, property, usage) { +// return { +// primary: key, +// condition: +// "attribute_exists(#quota) AND attribute_exists(#limits) AND #quota.#prop < #limits.#prop AND #quotaReset > :now", +// expression: "ADD #quota.#prop :usage", +// names: { +// "#quota": "usageQuota", +// "#prop": property, +// "#limits": "usageLimits", +// "#quotaReset": "quotaReset", +// }, +// values: { +// ":usage": usage, +// ":now": Date.now(), +// }, +// } +// } -function getNewQuotaReset() { - return Date.now() + 2592000000 -} +// function getNewQuotaReset() { +// return Date.now() + 2592000000 +// } -exports.Properties = { - ROW: "rows", - UPLOAD: "storage", - VIEW: "views", - USER: "users", - AUTOMATION: "automationRuns", -} +// exports.Properties = { +// ROW: "rows", +// UPLOAD: "storage", +// VIEW: "views", +// USER: "users", +// AUTOMATION: "automationRuns", +// } -exports.getAPIKey = async appId => { - if (!env.USE_QUOTAS) { - return { apiKey: null } - } - return apiKeyTable.get({ primary: appId }) -} +// exports.getAPIKey = async appId => { +// if (!env.USE_QUOTAS) { +// return { apiKey: null } +// } +// return apiKeyTable.get({ primary: appId }) +// } -/** - * Given a specified API key this will add to the usage object for the specified property. - * @param {string} apiKey The API key which is to be updated. - * @param {string} property The property which is to be added to (within the nested usageQuota object). - * @param {number} usage The amount (this can be negative) to adjust the number by. - * @returns {Promise} When this completes the API key will now be up to date - the quota period may have - * also been reset after this call. - */ -exports.update = async (apiKey, property, usage) => { - if (!env.USE_QUOTAS) { - return - } - try { - await apiKeyTable.update(buildUpdateParams(apiKey, property, usage)) - } catch (err) { - // conditional check means the condition failed, need to check why - if (err.code === "ConditionalCheckFailedException") { - // get the API key so we can check it - const keyObj = await apiKeyTable.get({ primary: apiKey }) - // the usage quota or usage limits didn't exist - if (keyObj && (keyObj.usageQuota == null || keyObj.usageLimits == null)) { - keyObj.usageQuota = - keyObj.usageQuota == null ? DEFAULT_USAGE : keyObj.usageQuota - keyObj.usageLimits = - keyObj.usageLimits == null ? DEFAULT_PLAN : keyObj.usageLimits - keyObj.quotaReset = getNewQuotaReset() - await apiKeyTable.put({ item: keyObj }) - return - } - // we have in fact breached the reset period - else if (keyObj && keyObj.quotaReset <= Date.now()) { - // update the quota reset period and reset the values for all properties - keyObj.quotaReset = getNewQuotaReset() - for (let prop of Object.keys(keyObj.usageQuota)) { - if (prop === property) { - keyObj.usageQuota[prop] = usage > 0 ? usage : 0 - } else { - keyObj.usageQuota[prop] = 0 - } - } - await apiKeyTable.put({ item: keyObj }) - return - } - } - throw err - } -} +// /** +// * Given a specified API key this will add to the usage object for the specified property. +// * @param {string} apiKey The API key which is to be updated. +// * @param {string} property The property which is to be added to (within the nested usageQuota object). +// * @param {number} usage The amount (this can be negative) to adjust the number by. +// * @returns {Promise} When this completes the API key will now be up to date - the quota period may have +// * also been reset after this call. +// */ +// exports.update = async (apiKey, property, usage) => { +// if (!env.USE_QUOTAS) { +// return +// } +// try { +// await apiKeyTable.update(buildUpdateParams(apiKey, property, usage)) +// } catch (err) { +// // conditional check means the condition failed, need to check why +// if (err.code === "ConditionalCheckFailedException") { +// // get the API key so we can check it +// const keyObj = await apiKeyTable.get({ primary: apiKey }) +// // the usage quota or usage limits didn't exist +// if (keyObj && (keyObj.usageQuota == null || keyObj.usageLimits == null)) { +// keyObj.usageQuota = +// keyObj.usageQuota == null ? DEFAULT_USAGE : keyObj.usageQuota +// keyObj.usageLimits = +// keyObj.usageLimits == null ? DEFAULT_PLAN : keyObj.usageLimits +// keyObj.quotaReset = getNewQuotaReset() +// await apiKeyTable.put({ item: keyObj }) +// return +// } +// // we have in fact breached the reset period +// else if (keyObj && keyObj.quotaReset <= Date.now()) { +// // update the quota reset period and reset the values for all properties +// keyObj.quotaReset = getNewQuotaReset() +// for (let prop of Object.keys(keyObj.usageQuota)) { +// if (prop === property) { +// keyObj.usageQuota[prop] = usage > 0 ? usage : 0 +// } else { +// keyObj.usageQuota[prop] = 0 +// } +// } +// await apiKeyTable.put({ item: keyObj }) +// return +// } +// } +// throw err +// } +// } diff --git a/packages/worker/src/api/controllers/global/users.js b/packages/worker/src/api/controllers/global/users.js index b75c72290d..294a835f14 100644 --- a/packages/worker/src/api/controllers/global/users.js +++ b/packages/worker/src/api/controllers/global/users.js @@ -1,7 +1,6 @@ const { generateGlobalUserID, getGlobalUserParams, - StaticDatabases, } = require("@budibase/auth/db") const { hash, getGlobalUserByEmail } = require("@budibase/auth").utils @@ -18,7 +17,7 @@ const { tryAddTenant, updateTenantId, } = require("@budibase/auth/tenancy") -const env = require("../../../environment") +// const env = require("../../../environment") const PLATFORM_INFO_DB = StaticDatabases.PLATFORM_INFO.name @@ -139,28 +138,28 @@ exports.adminUser = async ctx => { include_docs: true, }) ) - + // write usage quotas for cloud // if (!env.SELF_HOSTED) { - await db.post({ - _id: "usage_quota", - quotaReset: Date.now() + 2592000000, - usageQuota: { - automationRuns: 0, - rows: 0, - storage: 0, - apps: 0, - users: 0, - views: 0, - }, - usageLimits: { - automationRuns: 1000, - rows: 4000, - apps: 4, - // storage: 1000, - // users: 10 - }, - }) + await db.post({ + _id: "usage_quota", + quotaReset: Date.now() + 2592000000, + usageQuota: { + automationRuns: 0, + rows: 0, + storage: 0, + apps: 0, + users: 0, + views: 0, + }, + usageLimits: { + automationRuns: 1000, + rows: 4000, + apps: 4, + storage: 1000, + users: 10 + }, + }) // } if (response.rows.some(row => row.doc.admin)) { From 83384faed84d14804ad2ec85bb30e380c1d8af4d Mon Sep 17 00:00:00 2001 From: Martin McKeaveney Date: Thu, 23 Sep 2021 23:26:09 +0100 Subject: [PATCH 07/19] remove old middleware --- .../server/src/utilities/usageQuota.old.js | 105 ------------------ 1 file changed, 105 deletions(-) delete mode 100644 packages/server/src/utilities/usageQuota.old.js diff --git a/packages/server/src/utilities/usageQuota.old.js b/packages/server/src/utilities/usageQuota.old.js deleted file mode 100644 index 39dad89c6d..0000000000 --- a/packages/server/src/utilities/usageQuota.old.js +++ /dev/null @@ -1,105 +0,0 @@ -// const env = require("../environment") -// const { apiKeyTable } = require("../db/dynamoClient") - -// const DEFAULT_USAGE = { -// rows: 0, -// storage: 0, -// views: 0, -// automationRuns: 0, -// users: 0, -// } - -// const DEFAULT_PLAN = { -// rows: 1000, -// // 1 GB -// storage: 8589934592, -// views: 10, -// automationRuns: 100, -// users: 10000, -// } - -// function buildUpdateParams(key, property, usage) { -// return { -// primary: key, -// condition: -// "attribute_exists(#quota) AND attribute_exists(#limits) AND #quota.#prop < #limits.#prop AND #quotaReset > :now", -// expression: "ADD #quota.#prop :usage", -// names: { -// "#quota": "usageQuota", -// "#prop": property, -// "#limits": "usageLimits", -// "#quotaReset": "quotaReset", -// }, -// values: { -// ":usage": usage, -// ":now": Date.now(), -// }, -// } -// } - -// function getNewQuotaReset() { -// return Date.now() + 2592000000 -// } - -// exports.Properties = { -// ROW: "rows", -// UPLOAD: "storage", -// VIEW: "views", -// USER: "users", -// AUTOMATION: "automationRuns", -// } - -// exports.getAPIKey = async appId => { -// if (!env.USE_QUOTAS) { -// return { apiKey: null } -// } -// return apiKeyTable.get({ primary: appId }) -// } - -// /** -// * Given a specified API key this will add to the usage object for the specified property. -// * @param {string} apiKey The API key which is to be updated. -// * @param {string} property The property which is to be added to (within the nested usageQuota object). -// * @param {number} usage The amount (this can be negative) to adjust the number by. -// * @returns {Promise} When this completes the API key will now be up to date - the quota period may have -// * also been reset after this call. -// */ -// exports.update = async (apiKey, property, usage) => { -// if (!env.USE_QUOTAS) { -// return -// } -// try { -// await apiKeyTable.update(buildUpdateParams(apiKey, property, usage)) -// } catch (err) { -// // conditional check means the condition failed, need to check why -// if (err.code === "ConditionalCheckFailedException") { -// // get the API key so we can check it -// const keyObj = await apiKeyTable.get({ primary: apiKey }) -// // the usage quota or usage limits didn't exist -// if (keyObj && (keyObj.usageQuota == null || keyObj.usageLimits == null)) { -// keyObj.usageQuota = -// keyObj.usageQuota == null ? DEFAULT_USAGE : keyObj.usageQuota -// keyObj.usageLimits = -// keyObj.usageLimits == null ? DEFAULT_PLAN : keyObj.usageLimits -// keyObj.quotaReset = getNewQuotaReset() -// await apiKeyTable.put({ item: keyObj }) -// return -// } -// // we have in fact breached the reset period -// else if (keyObj && keyObj.quotaReset <= Date.now()) { -// // update the quota reset period and reset the values for all properties -// keyObj.quotaReset = getNewQuotaReset() -// for (let prop of Object.keys(keyObj.usageQuota)) { -// if (prop === property) { -// keyObj.usageQuota[prop] = usage > 0 ? usage : 0 -// } else { -// keyObj.usageQuota[prop] = 0 -// } -// } -// await apiKeyTable.put({ item: keyObj }) -// return -// } -// } -// throw err -// } -// } From 6c7423e52e32e952b5f46fb1be8c263a86c23374 Mon Sep 17 00:00:00 2001 From: Martin McKeaveney Date: Mon, 27 Sep 2021 10:32:39 +0100 Subject: [PATCH 08/19] ensure automation quotas are not incremented for test runs --- packages/server/src/automations/thread.js | 7 ++++--- packages/worker/src/api/controllers/global/users.js | 2 +- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/packages/server/src/automations/thread.js b/packages/server/src/automations/thread.js index 37484e50bd..ef12494165 100644 --- a/packages/server/src/automations/thread.js +++ b/packages/server/src/automations/thread.js @@ -4,7 +4,7 @@ const AutomationEmitter = require("../events/AutomationEmitter") const { processObject } = require("@budibase/string-templates") const { DEFAULT_TENANT_ID } = require("@budibase/auth").constants const CouchDB = require("../db") -const { DocumentTypes } = require("../db/utils") +const { DocumentTypes, isDevAppID } = require("../db/utils") const { doInTenant } = require("@budibase/auth/tenancy") const env = require("../environment") const usage = require("../utilities/usageQuota") @@ -96,8 +96,9 @@ class Orchestrator { return err } } - // TODO: don't count test runs - if (!env.SELF_HOSTED) { + + // Increment quota for automation runs + if (!env.SELF_HOSTED && !isDevAppID(this._appId)) { usage.update(usage.Properties.AUTOMATION, 1) } return this.executionOutput diff --git a/packages/worker/src/api/controllers/global/users.js b/packages/worker/src/api/controllers/global/users.js index 294a835f14..61c2981280 100644 --- a/packages/worker/src/api/controllers/global/users.js +++ b/packages/worker/src/api/controllers/global/users.js @@ -157,7 +157,7 @@ exports.adminUser = async ctx => { rows: 4000, apps: 4, storage: 1000, - users: 10 + users: 10, }, }) // } From 35d4eac78aa664c9f0c23b7be65392541d6d3e17 Mon Sep 17 00:00:00 2001 From: Martin McKeaveney Date: Mon, 27 Sep 2021 14:57:22 +0100 Subject: [PATCH 09/19] fallback SMTP configuration for cloud --- .../api/controllers/row/ExternalRequest.ts | 2 +- .../src/automations/tests/automation.spec.js | 2 -- .../src/automations/tests/createRow.spec.js | 2 +- .../src/automations/tests/deleteRow.spec.js | 2 +- packages/server/src/definitions/datasource.ts | 4 ++-- packages/server/src/integrations/base/sql.ts | 2 +- packages/server/src/integrations/mysql.ts | 6 +++--- .../src/middleware/tests/usageQuota.spec.js | 6 +++--- packages/server/src/middleware/usageQuota.js | 16 +++++++------- packages/server/src/utilities/usageQuota.js | 4 +++- .../src/api/controllers/global/users.js | 2 ++ packages/worker/src/environment.js | 5 +++++ packages/worker/src/utilities/email.js | 21 ++++++++++++++++--- 13 files changed, 48 insertions(+), 26 deletions(-) diff --git a/packages/server/src/api/controllers/row/ExternalRequest.ts b/packages/server/src/api/controllers/row/ExternalRequest.ts index 12db55efdc..75c3e9b492 100644 --- a/packages/server/src/api/controllers/row/ExternalRequest.ts +++ b/packages/server/src/api/controllers/row/ExternalRequest.ts @@ -546,7 +546,7 @@ module External { }, meta: { table, - } + }, } // can't really use response right now const response = await makeExternalQuery(appId, json) diff --git a/packages/server/src/automations/tests/automation.spec.js b/packages/server/src/automations/tests/automation.spec.js index 83b7b81a75..9444995ca1 100644 --- a/packages/server/src/automations/tests/automation.spec.js +++ b/packages/server/src/automations/tests/automation.spec.js @@ -13,8 +13,6 @@ const { makePartial } = require("../../tests/utilities") const { cleanInputValues } = require("../automationUtils") const setup = require("./utilities") -usageQuota.getAPIKey.mockReturnValue({ apiKey: "test" }) - describe("Run through some parts of the automations system", () => { let config = setup.getConfig() diff --git a/packages/server/src/automations/tests/createRow.spec.js b/packages/server/src/automations/tests/createRow.spec.js index 1004711d87..a04fc7aad4 100644 --- a/packages/server/src/automations/tests/createRow.spec.js +++ b/packages/server/src/automations/tests/createRow.spec.js @@ -46,7 +46,7 @@ describe("test the create row action", () => { await setup.runStep(setup.actions.CREATE_ROW.stepId, { row }) - expect(usageQuota.update).toHaveBeenCalledWith(setup.apiKey, "rows", 1) + expect(usageQuota.update).toHaveBeenCalledWith("rows", 1) }) }) diff --git a/packages/server/src/automations/tests/deleteRow.spec.js b/packages/server/src/automations/tests/deleteRow.spec.js index a3d73d3bf6..21246f22d0 100644 --- a/packages/server/src/automations/tests/deleteRow.spec.js +++ b/packages/server/src/automations/tests/deleteRow.spec.js @@ -37,7 +37,7 @@ describe("test the delete row action", () => { it("check usage quota attempts", async () => { await setup.runInProd(async () => { await setup.runStep(setup.actions.DELETE_ROW.stepId, inputs) - expect(usageQuota.update).toHaveBeenCalledWith(setup.apiKey, "rows", -1) + expect(usageQuota.update).toHaveBeenCalledWith("rows", -1) }) }) diff --git a/packages/server/src/definitions/datasource.ts b/packages/server/src/definitions/datasource.ts index d7d4e77961..2daef8eda7 100644 --- a/packages/server/src/definitions/datasource.ts +++ b/packages/server/src/definitions/datasource.ts @@ -1,4 +1,4 @@ -import {Table} from "./common"; +import { Table } from "./common" export enum Operation { CREATE = "CREATE", @@ -139,7 +139,7 @@ export interface QueryJson { paginate?: PaginationJson body?: object meta?: { - table?: Table, + table?: Table } extra?: { idFilter?: SearchFilters diff --git a/packages/server/src/integrations/base/sql.ts b/packages/server/src/integrations/base/sql.ts index 91af3e1a85..c5e9bdb0bb 100644 --- a/packages/server/src/integrations/base/sql.ts +++ b/packages/server/src/integrations/base/sql.ts @@ -148,7 +148,7 @@ function buildRead(knex: Knex, json: QueryJson, limit: number): KnexQuery { if (!resource) { resource = { fields: [] } } - let selectStatement: string|string[] = "*" + let selectStatement: string | string[] = "*" // handle select if (resource.fields && resource.fields.length > 0) { // select the resources as the format "table.columnName" - this is what is provided diff --git a/packages/server/src/integrations/mysql.ts b/packages/server/src/integrations/mysql.ts index d43ae86bea..c17cca0745 100644 --- a/packages/server/src/integrations/mysql.ts +++ b/packages/server/src/integrations/mysql.ts @@ -108,7 +108,7 @@ module MySQLModule { client: any, query: SqlQuery, connect: boolean = true - ): Promise { + ): Promise { // Node MySQL is callback based, so we must wrap our call in a promise return new Promise((resolve, reject) => { if (connect) { @@ -252,9 +252,9 @@ module MySQLModule { json.extra = { idFilter: { equal: { - [primaryKey]: results.insertId + [primaryKey]: results.insertId, }, - } + }, } return json } diff --git a/packages/server/src/middleware/tests/usageQuota.spec.js b/packages/server/src/middleware/tests/usageQuota.spec.js index 97d9c7794a..d828f2ca60 100644 --- a/packages/server/src/middleware/tests/usageQuota.spec.js +++ b/packages/server/src/middleware/tests/usageQuota.spec.js @@ -39,7 +39,7 @@ class TestConfiguration { if (bool) { env.isDev = () => false env.isProd = () => true - this.ctx.auth = { apiKey: "test" } + this.ctx.user = { tenantId: "test" } } else { env.isDev = () => true env.isProd = () => false @@ -114,7 +114,7 @@ describe("usageQuota middleware", () => { await config.executeMiddleware() - expect(usageQuota.update).toHaveBeenCalledWith("test", "rows", 1) + expect(usageQuota.update).toHaveBeenCalledWith("rows", 1) expect(config.next).toHaveBeenCalled() }) @@ -131,7 +131,7 @@ describe("usageQuota middleware", () => { ]) await config.executeMiddleware() - expect(usageQuota.update).toHaveBeenCalledWith("test", "storage", 10100) + expect(usageQuota.update).toHaveBeenCalledWith("storage", 10100) expect(config.next).toHaveBeenCalled() }) }) \ No newline at end of file diff --git a/packages/server/src/middleware/usageQuota.js b/packages/server/src/middleware/usageQuota.js index d18ffae205..d56a960615 100644 --- a/packages/server/src/middleware/usageQuota.js +++ b/packages/server/src/middleware/usageQuota.js @@ -50,15 +50,15 @@ module.exports = async (ctx, next) => { } // update usage for uploads to be the total size - // if (property === usageQuota.Properties.UPLOAD) { - // const files = - // ctx.request.files.file.length > 1 - // ? Array.from(ctx.request.files.file) - // : [ctx.request.files.file] - // usage = files.map(file => file.size).reduce((total, size) => total + size) - // } + if (property === usageQuota.Properties.UPLOAD) { + const files = + ctx.request.files.file.length > 1 + ? Array.from(ctx.request.files.file) + : [ctx.request.files.file] + usage = files.map(file => file.size).reduce((total, size) => total + size) + } try { - await usageQuota.update(ctx.user.tenantId, property, usage) + await usageQuota.update(property, usage) return next() } catch (err) { ctx.throw(400, err) diff --git a/packages/server/src/utilities/usageQuota.js b/packages/server/src/utilities/usageQuota.js index b98fb923cd..fb3f61f2f6 100644 --- a/packages/server/src/utilities/usageQuota.js +++ b/packages/server/src/utilities/usageQuota.js @@ -12,6 +12,7 @@ exports.Properties = { USER: "users", AUTOMATION: "automationRuns", APPS: "apps", + EMAILS: "emails", } /** @@ -28,7 +29,8 @@ exports.update = async (property, usage) => { try { const db = getGlobalDB() const quota = await db.get("usage_quota") - // TODO: check if the quota needs reset + + // Check if the quota needs reset if (Date.now() >= quota.quotaReset) { quota.quotaReset = getNewQuotaReset() for (let prop of Object.keys(quota.usageQuota)) { diff --git a/packages/worker/src/api/controllers/global/users.js b/packages/worker/src/api/controllers/global/users.js index 61c2981280..c714f2b1ca 100644 --- a/packages/worker/src/api/controllers/global/users.js +++ b/packages/worker/src/api/controllers/global/users.js @@ -151,6 +151,7 @@ exports.adminUser = async ctx => { apps: 0, users: 0, views: 0, + emails: 0, }, usageLimits: { automationRuns: 1000, @@ -158,6 +159,7 @@ exports.adminUser = async ctx => { apps: 4, storage: 1000, users: 10, + emails: 50, }, }) // } diff --git a/packages/worker/src/environment.js b/packages/worker/src/environment.js index 646536f292..5f0556efc4 100644 --- a/packages/worker/src/environment.js +++ b/packages/worker/src/environment.js @@ -33,6 +33,11 @@ module.exports = { INTERNAL_API_KEY: process.env.INTERNAL_API_KEY, MULTI_TENANCY: process.env.MULTI_TENANCY, ACCOUNT_PORTAL_URL: process.env.ACCOUNT_PORTAL_URL, + SMTP_USER: process.env.SMTP_USER, + SMTP_PASSWORD: process.env.SMTP_PASSWORD, + SMTP_HOST: process.env.SMTP_HOST, + SMTP_PORT: process.env.SMTP_PORT, + SMTP_FROM_ADDRESS: process.env.SMTP_FROM_ADDRESS, _set(key, value) { process.env[key] = value module.exports[key] = value diff --git a/packages/worker/src/utilities/email.js b/packages/worker/src/utilities/email.js index d22933ef36..5843cb28ea 100644 --- a/packages/worker/src/utilities/email.js +++ b/packages/worker/src/utilities/email.js @@ -1,4 +1,5 @@ const nodemailer = require("nodemailer") +const env = require("../environment") const { getScopedConfig } = require("@budibase/auth/db") const { EmailTemplatePurpose, TemplateTypes, Configs } = require("../constants") const { getTemplateByPurpose } = require("../constants/templates") @@ -110,7 +111,21 @@ async function getSmtpConfiguration(db, workspaceId = null) { if (workspaceId) { params.workspace = workspaceId } - return getScopedConfig(db, params) + + if (!env.SMTP_FALLBACK_ENABLED) { + return getScopedConfig(db, params) + } else { + // Use an SMTP fallback configuration from env variables + return { + port: env.SMTP_PORT, + host: env.SMTP_HOST, + secure: false, + auth: { + user: env.SMTP_USER, + pass: env.SMTP_PASSWORD, + }, + } + } } /** @@ -118,8 +133,8 @@ async function getSmtpConfiguration(db, workspaceId = null) { * @return {Promise} returns true if there is a configuration that can be used. */ exports.isEmailConfigured = async (workspaceId = null) => { - // when "testing" simply return true - if (TEST_MODE) { + // when "testing" or smtp fallback is enabled simply return true + if (TEST_MODE || env.SMTP_FALLBACK_ENABLED) { return true } const db = getGlobalDB() From 5b4bf74e2c438c0549d6945b8f41a83f3410460d Mon Sep 17 00:00:00 2001 From: Martin McKeaveney Date: Mon, 27 Sep 2021 15:03:48 +0100 Subject: [PATCH 10/19] re-adding env var controls for cloud limit functionality --- packages/server/src/middleware/usageQuota.js | 8 ++-- packages/server/src/utilities/usageQuota.js | 9 ++-- .../src/api/controllers/global/users.js | 48 +++++++++---------- packages/worker/src/utilities/email.js | 6 ++- 4 files changed, 37 insertions(+), 34 deletions(-) diff --git a/packages/server/src/middleware/usageQuota.js b/packages/server/src/middleware/usageQuota.js index d56a960615..3a244ef5bc 100644 --- a/packages/server/src/middleware/usageQuota.js +++ b/packages/server/src/middleware/usageQuota.js @@ -1,6 +1,6 @@ const CouchDB = require("../db") const usageQuota = require("../utilities/usageQuota") -// const env = require("../environment") +const env = require("../environment") // currently only counting new writes and deletes const METHOD_MAP = { @@ -29,9 +29,9 @@ function getProperty(url) { module.exports = async (ctx, next) => { // if in development or a self hosted cloud usage quotas should not be executed - // if (env.isDev() || env.SELF_HOSTED) { - // return next() - // } + if (env.isDev() || env.SELF_HOSTED) { + return next() + } const db = new CouchDB(ctx.appId) let usage = METHOD_MAP[ctx.req.method] diff --git a/packages/server/src/utilities/usageQuota.js b/packages/server/src/utilities/usageQuota.js index fb3f61f2f6..4e645ec67d 100644 --- a/packages/server/src/utilities/usageQuota.js +++ b/packages/server/src/utilities/usageQuota.js @@ -1,4 +1,4 @@ -// const env = require("../environment") +const env = require("../environment") const { getGlobalDB } = require("@budibase/auth/tenancy") function getNewQuotaReset() { @@ -23,9 +23,10 @@ exports.Properties = { * also been reset after this call. */ exports.update = async (property, usage) => { - // if (!env.USE_QUOTAS) { - // return - // } + if (!env.USE_QUOTAS) { + return + } + try { const db = getGlobalDB() const quota = await db.get("usage_quota") diff --git a/packages/worker/src/api/controllers/global/users.js b/packages/worker/src/api/controllers/global/users.js index c714f2b1ca..e00fd3ecc0 100644 --- a/packages/worker/src/api/controllers/global/users.js +++ b/packages/worker/src/api/controllers/global/users.js @@ -17,7 +17,7 @@ const { tryAddTenant, updateTenantId, } = require("@budibase/auth/tenancy") -// const env = require("../../../environment") +const env = require("../../../environment") const PLATFORM_INFO_DB = StaticDatabases.PLATFORM_INFO.name @@ -140,29 +140,29 @@ exports.adminUser = async ctx => { ) // write usage quotas for cloud - // if (!env.SELF_HOSTED) { - await db.post({ - _id: "usage_quota", - quotaReset: Date.now() + 2592000000, - usageQuota: { - automationRuns: 0, - rows: 0, - storage: 0, - apps: 0, - users: 0, - views: 0, - emails: 0, - }, - usageLimits: { - automationRuns: 1000, - rows: 4000, - apps: 4, - storage: 1000, - users: 10, - emails: 50, - }, - }) - // } + if (!env.SELF_HOSTED) { + await db.post({ + _id: "usage_quota", + quotaReset: Date.now() + 2592000000, + usageQuota: { + automationRuns: 0, + rows: 0, + storage: 0, + apps: 0, + users: 0, + views: 0, + emails: 0, + }, + usageLimits: { + automationRuns: 1000, + rows: 4000, + apps: 4, + storage: 1000, + users: 10, + emails: 50, + }, + }) + } if (response.rows.some(row => row.doc.admin)) { ctx.throw( diff --git a/packages/worker/src/utilities/email.js b/packages/worker/src/utilities/email.js index 5843cb28ea..cd29170348 100644 --- a/packages/worker/src/utilities/email.js +++ b/packages/worker/src/utilities/email.js @@ -112,8 +112,10 @@ async function getSmtpConfiguration(db, workspaceId = null) { params.workspace = workspaceId } - if (!env.SMTP_FALLBACK_ENABLED) { - return getScopedConfig(db, params) + const customConfig = getScopedConfig(db, params) + + if (customConfig && !env.SMTP_FALLBACK_ENABLED) { + return customConfig } else { // Use an SMTP fallback configuration from env variables return { From c25bd4861696714c17d8b66e415a779809ac49c2 Mon Sep 17 00:00:00 2001 From: Martin McKeaveney Date: Mon, 27 Sep 2021 15:10:06 +0100 Subject: [PATCH 11/19] SMTP config fallback logic --- packages/worker/src/utilities/email.js | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/packages/worker/src/utilities/email.js b/packages/worker/src/utilities/email.js index cd29170348..5fd8c05939 100644 --- a/packages/worker/src/utilities/email.js +++ b/packages/worker/src/utilities/email.js @@ -114,10 +114,12 @@ async function getSmtpConfiguration(db, workspaceId = null) { const customConfig = getScopedConfig(db, params) - if (customConfig && !env.SMTP_FALLBACK_ENABLED) { + if (customConfig) { return customConfig - } else { - // Use an SMTP fallback configuration from env variables + } + + // Use an SMTP fallback configuration from env variables + if (env.SMTP_FALLBACK_ENABLED) { return { port: env.SMTP_PORT, host: env.SMTP_HOST, From f9525b906538b5c6ae2f63d46fe3d543feb1d140 Mon Sep 17 00:00:00 2001 From: Budibase Staging Release Bot <> Date: Mon, 27 Sep 2021 14:25:00 +0000 Subject: [PATCH 12/19] v0.9.140-alpha.12 --- lerna.json | 2 +- packages/auth/package.json | 2 +- packages/bbui/package.json | 2 +- packages/builder/package.json | 8 ++++---- packages/cli/package.json | 2 +- packages/client/package.json | 6 +++--- packages/server/package.json | 8 ++++---- packages/string-templates/package.json | 2 +- packages/worker/package.json | 6 +++--- 9 files changed, 19 insertions(+), 19 deletions(-) diff --git a/lerna.json b/lerna.json index e4dd4f4661..f3ea43dcc1 100644 --- a/lerna.json +++ b/lerna.json @@ -1,5 +1,5 @@ { - "version": "0.9.140-alpha.11", + "version": "0.9.140-alpha.12", "npmClient": "yarn", "packages": [ "packages/*" diff --git a/packages/auth/package.json b/packages/auth/package.json index 3be63010df..d42cb7b2c3 100644 --- a/packages/auth/package.json +++ b/packages/auth/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/auth", - "version": "0.9.140-alpha.11", + "version": "0.9.140-alpha.12", "description": "Authentication middlewares for budibase builder and apps", "main": "src/index.js", "author": "Budibase", diff --git a/packages/bbui/package.json b/packages/bbui/package.json index 23f75a9565..e9aba805a9 100644 --- a/packages/bbui/package.json +++ b/packages/bbui/package.json @@ -1,7 +1,7 @@ { "name": "@budibase/bbui", "description": "A UI solution used in the different Budibase projects.", - "version": "0.9.140-alpha.11", + "version": "0.9.140-alpha.12", "license": "AGPL-3.0", "svelte": "src/index.js", "module": "dist/bbui.es.js", diff --git a/packages/builder/package.json b/packages/builder/package.json index 4c0c6f1248..67ea2da45a 100644 --- a/packages/builder/package.json +++ b/packages/builder/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/builder", - "version": "0.9.140-alpha.11", + "version": "0.9.140-alpha.12", "license": "AGPL-3.0", "private": true, "scripts": { @@ -65,10 +65,10 @@ } }, "dependencies": { - "@budibase/bbui": "^0.9.140-alpha.11", - "@budibase/client": "^0.9.140-alpha.11", + "@budibase/bbui": "^0.9.140-alpha.12", + "@budibase/client": "^0.9.140-alpha.12", "@budibase/colorpicker": "1.1.2", - "@budibase/string-templates": "^0.9.140-alpha.11", + "@budibase/string-templates": "^0.9.140-alpha.12", "@sentry/browser": "5.19.1", "@spectrum-css/page": "^3.0.1", "@spectrum-css/vars": "^3.0.1", diff --git a/packages/cli/package.json b/packages/cli/package.json index b71457a71a..f489f4635e 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/cli", - "version": "0.9.140-alpha.11", + "version": "0.9.140-alpha.12", "description": "Budibase CLI, for developers, self hosting and migrations.", "main": "src/index.js", "bin": { diff --git a/packages/client/package.json b/packages/client/package.json index 8d387eb674..44ac4d6f2d 100644 --- a/packages/client/package.json +++ b/packages/client/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/client", - "version": "0.9.140-alpha.11", + "version": "0.9.140-alpha.12", "license": "MPL-2.0", "module": "dist/budibase-client.js", "main": "dist/budibase-client.js", @@ -19,9 +19,9 @@ "dev:builder": "rollup -cw" }, "dependencies": { - "@budibase/bbui": "^0.9.140-alpha.11", + "@budibase/bbui": "^0.9.140-alpha.12", "@budibase/standard-components": "^0.9.139", - "@budibase/string-templates": "^0.9.140-alpha.11", + "@budibase/string-templates": "^0.9.140-alpha.12", "regexparam": "^1.3.0", "shortid": "^2.2.15", "svelte-spa-router": "^3.0.5" diff --git a/packages/server/package.json b/packages/server/package.json index 1e2dc602ae..3b13e4bd28 100644 --- a/packages/server/package.json +++ b/packages/server/package.json @@ -1,7 +1,7 @@ { "name": "@budibase/server", "email": "hi@budibase.com", - "version": "0.9.140-alpha.11", + "version": "0.9.140-alpha.12", "description": "Budibase Web Server", "main": "src/index.js", "repository": { @@ -62,9 +62,9 @@ "author": "Budibase", "license": "AGPL-3.0-or-later", "dependencies": { - "@budibase/auth": "^0.9.140-alpha.11", - "@budibase/client": "^0.9.140-alpha.11", - "@budibase/string-templates": "^0.9.140-alpha.11", + "@budibase/auth": "^0.9.140-alpha.12", + "@budibase/client": "^0.9.140-alpha.12", + "@budibase/string-templates": "^0.9.140-alpha.12", "@elastic/elasticsearch": "7.10.0", "@koa/router": "8.0.0", "@sendgrid/mail": "7.1.1", diff --git a/packages/string-templates/package.json b/packages/string-templates/package.json index 9ea3486427..1e73c0a708 100644 --- a/packages/string-templates/package.json +++ b/packages/string-templates/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/string-templates", - "version": "0.9.140-alpha.11", + "version": "0.9.140-alpha.12", "description": "Handlebars wrapper for Budibase templating.", "main": "src/index.cjs", "module": "dist/bundle.mjs", diff --git a/packages/worker/package.json b/packages/worker/package.json index be9ffabce5..94f3d0b577 100644 --- a/packages/worker/package.json +++ b/packages/worker/package.json @@ -1,7 +1,7 @@ { "name": "@budibase/worker", "email": "hi@budibase.com", - "version": "0.9.140-alpha.11", + "version": "0.9.140-alpha.12", "description": "Budibase background service", "main": "src/index.js", "repository": { @@ -25,8 +25,8 @@ "author": "Budibase", "license": "AGPL-3.0-or-later", "dependencies": { - "@budibase/auth": "^0.9.140-alpha.11", - "@budibase/string-templates": "^0.9.140-alpha.11", + "@budibase/auth": "^0.9.140-alpha.12", + "@budibase/string-templates": "^0.9.140-alpha.12", "@koa/router": "^8.0.0", "@techpass/passport-openidconnect": "^0.3.0", "aws-sdk": "^2.811.0", From 2da28ecc71942670cecb277d8ce5a0a0f46ac908 Mon Sep 17 00:00:00 2001 From: Martin McKeaveney Date: Mon, 27 Sep 2021 16:28:39 +0100 Subject: [PATCH 13/19] prevent SMTP fallback for automations --- .../server/src/automations/steps/sendSmtpEmail.js | 2 +- packages/server/src/utilities/workerRequests.js | 3 ++- .../worker/src/api/controllers/global/email.js | 13 +++++++++++-- packages/worker/src/utilities/email.js | 14 ++++++++------ 4 files changed, 22 insertions(+), 10 deletions(-) diff --git a/packages/server/src/automations/steps/sendSmtpEmail.js b/packages/server/src/automations/steps/sendSmtpEmail.js index 9e4b5a6a3c..07a3059215 100644 --- a/packages/server/src/automations/steps/sendSmtpEmail.js +++ b/packages/server/src/automations/steps/sendSmtpEmail.js @@ -53,7 +53,7 @@ exports.run = async function ({ inputs }) { contents = "

No content

" } try { - let response = await sendSmtpEmail(to, from, subject, contents) + let response = await sendSmtpEmail(to, from, subject, contents, true) return { success: true, response, diff --git a/packages/server/src/utilities/workerRequests.js b/packages/server/src/utilities/workerRequests.js index 377658084f..2ace265ca0 100644 --- a/packages/server/src/utilities/workerRequests.js +++ b/packages/server/src/utilities/workerRequests.js @@ -34,7 +34,7 @@ function request(ctx, request) { exports.request = request // have to pass in the tenant ID as this could be coming from an automation -exports.sendSmtpEmail = async (to, from, subject, contents) => { +exports.sendSmtpEmail = async (to, from, subject, contents, automation) => { // tenant ID will be set in header const response = await fetch( checkSlashesInUrl(env.WORKER_URL + `/api/global/email/send`), @@ -46,6 +46,7 @@ exports.sendSmtpEmail = async (to, from, subject, contents) => { contents, subject, purpose: "custom", + automation, }, }) ) diff --git a/packages/worker/src/api/controllers/global/email.js b/packages/worker/src/api/controllers/global/email.js index 57b78a6d7a..e194a30862 100644 --- a/packages/worker/src/api/controllers/global/email.js +++ b/packages/worker/src/api/controllers/global/email.js @@ -2,8 +2,16 @@ const { sendEmail } = require("../../../utilities/email") const { getGlobalDB } = require("@budibase/auth/tenancy") exports.sendEmail = async ctx => { - let { workspaceId, email, userId, purpose, contents, from, subject } = - ctx.request.body + let { + workspaceId, + email, + userId, + purpose, + contents, + from, + subject, + automation, + } = ctx.request.body let user if (userId) { const db = getGlobalDB() @@ -15,6 +23,7 @@ exports.sendEmail = async ctx => { contents, from, subject, + automation, }) ctx.body = { ...response, diff --git a/packages/worker/src/utilities/email.js b/packages/worker/src/utilities/email.js index 5fd8c05939..14c836952e 100644 --- a/packages/worker/src/utilities/email.js +++ b/packages/worker/src/utilities/email.js @@ -102,9 +102,10 @@ async function buildEmail(purpose, email, context, { user, contents } = {}) { * Utility function for finding most valid SMTP configuration. * @param {object} db The CouchDB database which is to be looked up within. * @param {string|null} workspaceId If using finer grain control of configs a workspace can be used. + * @param {boolean|null} automation Whether or not the configuration is being fetched for an email automation. * @return {Promise} returns the SMTP configuration if it exists */ -async function getSmtpConfiguration(db, workspaceId = null) { +async function getSmtpConfiguration(db, workspaceId = null, automation) { const params = { type: Configs.SMTP, } @@ -116,10 +117,10 @@ async function getSmtpConfiguration(db, workspaceId = null) { if (customConfig) { return customConfig - } - + } + // Use an SMTP fallback configuration from env variables - if (env.SMTP_FALLBACK_ENABLED) { + if (!automation && env.SMTP_FALLBACK_ENABLED) { return { port: env.SMTP_PORT, host: env.SMTP_HOST, @@ -157,16 +158,17 @@ exports.isEmailConfigured = async (workspaceId = null) => { * @param {string|undefined} contents If sending a custom email then can supply contents which will be added to it. * @param {string|undefined} subject A custom subject can be specified if the config one is not desired. * @param {object|undefined} info Pass in a structure of information to be stored alongside the invitation. + * @param {boolean|undefined} disableFallback Prevent email being sent from SMTP fallback to avoid spam. * @return {Promise} returns details about the attempt to send email, e.g. if it is successful; based on * nodemailer response. */ exports.sendEmail = async ( email, purpose, - { workspaceId, user, from, contents, subject, info } = {} + { workspaceId, user, from, contents, subject, info, automation } = {} ) => { const db = getGlobalDB() - let config = (await getSmtpConfiguration(db, workspaceId)) || {} + let config = (await getSmtpConfiguration(db, workspaceId, automation)) || {} if (Object.keys(config).length === 0 && !TEST_MODE) { throw "Unable to find SMTP configuration." } From f6014190c8860cc1529e1be50dad77eaed3337f7 Mon Sep 17 00:00:00 2001 From: Martin McKeaveney Date: Mon, 27 Sep 2021 16:39:08 +0100 Subject: [PATCH 14/19] tests --- packages/server/src/api/routes/view.js | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/packages/server/src/api/routes/view.js b/packages/server/src/api/routes/view.js index 7d390805c6..f096dd4185 100644 --- a/packages/server/src/api/routes/view.js +++ b/packages/server/src/api/routes/view.js @@ -25,9 +25,8 @@ router "/api/views/:viewName", paramResource("viewName"), authorized(BUILDER), - usage, viewController.destroy ) - .post("/api/views", authorized(BUILDER), usage, viewController.save) + .post("/api/views", authorized(BUILDER), viewController.save) module.exports = router From 1e3fca15c5a18b837b9222a72e35c09cb8dad16f Mon Sep 17 00:00:00 2001 From: Martin McKeaveney Date: Mon, 27 Sep 2021 17:35:22 +0100 Subject: [PATCH 15/19] abstract usagedoc creation into auth, create the doc if it doesn't exist --- packages/auth/src/db/constants.js | 1 + packages/auth/src/db/utils.js | 25 +++++++++++++++++++ packages/server/src/api/routes/user.js | 2 -- packages/server/src/api/routes/view.js | 1 - packages/server/src/utilities/usageQuota.js | 18 ++++++++++++- .../src/api/controllers/global/users.js | 23 ++--------------- 6 files changed, 45 insertions(+), 25 deletions(-) diff --git a/packages/auth/src/db/constants.js b/packages/auth/src/db/constants.js index 77643ce4c5..ad4f6c9f66 100644 --- a/packages/auth/src/db/constants.js +++ b/packages/auth/src/db/constants.js @@ -12,6 +12,7 @@ exports.StaticDatabases = { name: "global-info", docs: { tenants: "tenants", + usageQuota: "usage_quota", }, }, } diff --git a/packages/auth/src/db/utils.js b/packages/auth/src/db/utils.js index a1a831523e..09e2ff6314 100644 --- a/packages/auth/src/db/utils.js +++ b/packages/auth/src/db/utils.js @@ -368,8 +368,33 @@ async function getScopedConfig(db, params) { return configDoc && configDoc.config ? configDoc.config : configDoc } +function generateNewUsageQuotaDoc() { + return { + _id: StaticDatabases.PLATFORM_INFO.docs.usageQuota, + quotaReset: Date.now() + 2592000000, + usageQuota: { + automationRuns: 0, + rows: 0, + storage: 0, + apps: 0, + users: 0, + views: 0, + emails: 0, + }, + usageLimits: { + automationRuns: 1000, + rows: 4000, + apps: 4, + storage: 1000, + users: 10, + emails: 50, + }, + } +} + exports.Replication = Replication exports.getScopedConfig = getScopedConfig exports.generateConfigID = generateConfigID exports.getConfigParams = getConfigParams exports.getScopedFullConfig = getScopedFullConfig +exports.generateNewUsageQuotaDoc = generateNewUsageQuotaDoc diff --git a/packages/server/src/api/routes/user.js b/packages/server/src/api/routes/user.js index 465fef82c8..d171870215 100644 --- a/packages/server/src/api/routes/user.js +++ b/packages/server/src/api/routes/user.js @@ -27,13 +27,11 @@ router .post( "/api/users/metadata/self", authorized(PermissionTypes.USER, PermissionLevels.WRITE), - // usage, controller.updateSelfMetadata ) .delete( "/api/users/metadata/:id", authorized(PermissionTypes.USER, PermissionLevels.WRITE), - // usage, controller.destroyMetadata ) diff --git a/packages/server/src/api/routes/view.js b/packages/server/src/api/routes/view.js index f096dd4185..b72fe1ac26 100644 --- a/packages/server/src/api/routes/view.js +++ b/packages/server/src/api/routes/view.js @@ -8,7 +8,6 @@ const { PermissionTypes, PermissionLevels, } = require("@budibase/auth/permissions") -const usage = require("../../middleware/usageQuota") const router = Router() diff --git a/packages/server/src/utilities/usageQuota.js b/packages/server/src/utilities/usageQuota.js index 4e645ec67d..80fddb8303 100644 --- a/packages/server/src/utilities/usageQuota.js +++ b/packages/server/src/utilities/usageQuota.js @@ -1,5 +1,9 @@ const env = require("../environment") const { getGlobalDB } = require("@budibase/auth/tenancy") +const { + StaticDatabases, + generateNewUsageQuotaDoc, +} = require("@budibase/auth/db") function getNewQuotaReset() { return Date.now() + 2592000000 @@ -15,6 +19,18 @@ exports.Properties = { EMAILS: "emails", } +async function getUsageQuotaDoc(db) { + let quota + try { + quota = await db.get(StaticDatabases.PLATFORM_INFO.docs.usageQuota) + } catch (err) { + // doc doesn't exist. Create it + quota = await db.post(generateNewUsageQuotaDoc()) + } + + return quota +} + /** * Given a specified tenantId this will add to the usage object for the specified property. * @param {string} property The property which is to be added to (within the nested usageQuota object). @@ -29,7 +45,7 @@ exports.update = async (property, usage) => { try { const db = getGlobalDB() - const quota = await db.get("usage_quota") + const quota = await getUsageQuotaDoc(db) // Check if the quota needs reset if (Date.now() >= quota.quotaReset) { diff --git a/packages/worker/src/api/controllers/global/users.js b/packages/worker/src/api/controllers/global/users.js index e00fd3ecc0..1d3f38698b 100644 --- a/packages/worker/src/api/controllers/global/users.js +++ b/packages/worker/src/api/controllers/global/users.js @@ -2,6 +2,7 @@ const { generateGlobalUserID, getGlobalUserParams, StaticDatabases, + generateNewUsageQuotaDoc, } = require("@budibase/auth/db") const { hash, getGlobalUserByEmail } = require("@budibase/auth").utils const { UserStatus, EmailTemplatePurpose } = require("../../../constants") @@ -141,27 +142,7 @@ exports.adminUser = async ctx => { // write usage quotas for cloud if (!env.SELF_HOSTED) { - await db.post({ - _id: "usage_quota", - quotaReset: Date.now() + 2592000000, - usageQuota: { - automationRuns: 0, - rows: 0, - storage: 0, - apps: 0, - users: 0, - views: 0, - emails: 0, - }, - usageLimits: { - automationRuns: 1000, - rows: 4000, - apps: 4, - storage: 1000, - users: 10, - emails: 50, - }, - }) + await db.post(generateNewUsageQuotaDoc()) } if (response.rows.some(row => row.doc.admin)) { From b2b13dc719b792c23853ea3f1a8b4c02ca3e435d Mon Sep 17 00:00:00 2001 From: Budibase Staging Release Bot <> Date: Mon, 27 Sep 2021 17:39:48 +0000 Subject: [PATCH 16/19] v0.9.140-alpha.13 --- lerna.json | 2 +- packages/auth/package.json | 2 +- packages/bbui/package.json | 2 +- packages/builder/package.json | 8 ++++---- packages/cli/package.json | 2 +- packages/client/package.json | 6 +++--- packages/server/package.json | 8 ++++---- packages/string-templates/package.json | 2 +- packages/worker/package.json | 6 +++--- 9 files changed, 19 insertions(+), 19 deletions(-) diff --git a/lerna.json b/lerna.json index f3ea43dcc1..faa0bb0841 100644 --- a/lerna.json +++ b/lerna.json @@ -1,5 +1,5 @@ { - "version": "0.9.140-alpha.12", + "version": "0.9.140-alpha.13", "npmClient": "yarn", "packages": [ "packages/*" diff --git a/packages/auth/package.json b/packages/auth/package.json index d42cb7b2c3..6c87de2810 100644 --- a/packages/auth/package.json +++ b/packages/auth/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/auth", - "version": "0.9.140-alpha.12", + "version": "0.9.140-alpha.13", "description": "Authentication middlewares for budibase builder and apps", "main": "src/index.js", "author": "Budibase", diff --git a/packages/bbui/package.json b/packages/bbui/package.json index e9aba805a9..cb54317f6d 100644 --- a/packages/bbui/package.json +++ b/packages/bbui/package.json @@ -1,7 +1,7 @@ { "name": "@budibase/bbui", "description": "A UI solution used in the different Budibase projects.", - "version": "0.9.140-alpha.12", + "version": "0.9.140-alpha.13", "license": "AGPL-3.0", "svelte": "src/index.js", "module": "dist/bbui.es.js", diff --git a/packages/builder/package.json b/packages/builder/package.json index 67ea2da45a..02fdaffc11 100644 --- a/packages/builder/package.json +++ b/packages/builder/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/builder", - "version": "0.9.140-alpha.12", + "version": "0.9.140-alpha.13", "license": "AGPL-3.0", "private": true, "scripts": { @@ -65,10 +65,10 @@ } }, "dependencies": { - "@budibase/bbui": "^0.9.140-alpha.12", - "@budibase/client": "^0.9.140-alpha.12", + "@budibase/bbui": "^0.9.140-alpha.13", + "@budibase/client": "^0.9.140-alpha.13", "@budibase/colorpicker": "1.1.2", - "@budibase/string-templates": "^0.9.140-alpha.12", + "@budibase/string-templates": "^0.9.140-alpha.13", "@sentry/browser": "5.19.1", "@spectrum-css/page": "^3.0.1", "@spectrum-css/vars": "^3.0.1", diff --git a/packages/cli/package.json b/packages/cli/package.json index f489f4635e..9c0f45ea30 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/cli", - "version": "0.9.140-alpha.12", + "version": "0.9.140-alpha.13", "description": "Budibase CLI, for developers, self hosting and migrations.", "main": "src/index.js", "bin": { diff --git a/packages/client/package.json b/packages/client/package.json index 44ac4d6f2d..4974ada5d0 100644 --- a/packages/client/package.json +++ b/packages/client/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/client", - "version": "0.9.140-alpha.12", + "version": "0.9.140-alpha.13", "license": "MPL-2.0", "module": "dist/budibase-client.js", "main": "dist/budibase-client.js", @@ -19,9 +19,9 @@ "dev:builder": "rollup -cw" }, "dependencies": { - "@budibase/bbui": "^0.9.140-alpha.12", + "@budibase/bbui": "^0.9.140-alpha.13", "@budibase/standard-components": "^0.9.139", - "@budibase/string-templates": "^0.9.140-alpha.12", + "@budibase/string-templates": "^0.9.140-alpha.13", "regexparam": "^1.3.0", "shortid": "^2.2.15", "svelte-spa-router": "^3.0.5" diff --git a/packages/server/package.json b/packages/server/package.json index 3b13e4bd28..cb162c3806 100644 --- a/packages/server/package.json +++ b/packages/server/package.json @@ -1,7 +1,7 @@ { "name": "@budibase/server", "email": "hi@budibase.com", - "version": "0.9.140-alpha.12", + "version": "0.9.140-alpha.13", "description": "Budibase Web Server", "main": "src/index.js", "repository": { @@ -62,9 +62,9 @@ "author": "Budibase", "license": "AGPL-3.0-or-later", "dependencies": { - "@budibase/auth": "^0.9.140-alpha.12", - "@budibase/client": "^0.9.140-alpha.12", - "@budibase/string-templates": "^0.9.140-alpha.12", + "@budibase/auth": "^0.9.140-alpha.13", + "@budibase/client": "^0.9.140-alpha.13", + "@budibase/string-templates": "^0.9.140-alpha.13", "@elastic/elasticsearch": "7.10.0", "@koa/router": "8.0.0", "@sendgrid/mail": "7.1.1", diff --git a/packages/string-templates/package.json b/packages/string-templates/package.json index 1e73c0a708..6b37bcf5ab 100644 --- a/packages/string-templates/package.json +++ b/packages/string-templates/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/string-templates", - "version": "0.9.140-alpha.12", + "version": "0.9.140-alpha.13", "description": "Handlebars wrapper for Budibase templating.", "main": "src/index.cjs", "module": "dist/bundle.mjs", diff --git a/packages/worker/package.json b/packages/worker/package.json index 94f3d0b577..97332b1412 100644 --- a/packages/worker/package.json +++ b/packages/worker/package.json @@ -1,7 +1,7 @@ { "name": "@budibase/worker", "email": "hi@budibase.com", - "version": "0.9.140-alpha.12", + "version": "0.9.140-alpha.13", "description": "Budibase background service", "main": "src/index.js", "repository": { @@ -25,8 +25,8 @@ "author": "Budibase", "license": "AGPL-3.0-or-later", "dependencies": { - "@budibase/auth": "^0.9.140-alpha.12", - "@budibase/string-templates": "^0.9.140-alpha.12", + "@budibase/auth": "^0.9.140-alpha.13", + "@budibase/string-templates": "^0.9.140-alpha.13", "@koa/router": "^8.0.0", "@techpass/passport-openidconnect": "^0.3.0", "aws-sdk": "^2.811.0", From ece3010c4e19ca81b4be226e8d156e923a85332a Mon Sep 17 00:00:00 2001 From: Budibase Staging Release Bot <> Date: Mon, 27 Sep 2021 21:39:06 +0000 Subject: [PATCH 17/19] v0.9.143-alpha.0 --- lerna.json | 2 +- packages/auth/package.json | 2 +- packages/bbui/package.json | 2 +- packages/builder/package.json | 8 ++++---- packages/cli/package.json | 2 +- packages/client/package.json | 6 +++--- packages/server/package.json | 8 ++++---- packages/string-templates/package.json | 2 +- packages/worker/package.json | 6 +++--- 9 files changed, 19 insertions(+), 19 deletions(-) diff --git a/lerna.json b/lerna.json index a8cda22a63..01c429c73d 100644 --- a/lerna.json +++ b/lerna.json @@ -1,5 +1,5 @@ { - "version": "0.9.142", + "version": "0.9.143-alpha.0", "npmClient": "yarn", "packages": [ "packages/*" diff --git a/packages/auth/package.json b/packages/auth/package.json index f62e4cb5e6..0457520d91 100644 --- a/packages/auth/package.json +++ b/packages/auth/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/auth", - "version": "0.9.142", + "version": "0.9.143-alpha.0", "description": "Authentication middlewares for budibase builder and apps", "main": "src/index.js", "author": "Budibase", diff --git a/packages/bbui/package.json b/packages/bbui/package.json index ddede26a86..42e4215caa 100644 --- a/packages/bbui/package.json +++ b/packages/bbui/package.json @@ -1,7 +1,7 @@ { "name": "@budibase/bbui", "description": "A UI solution used in the different Budibase projects.", - "version": "0.9.142", + "version": "0.9.143-alpha.0", "license": "AGPL-3.0", "svelte": "src/index.js", "module": "dist/bbui.es.js", diff --git a/packages/builder/package.json b/packages/builder/package.json index 7ab313974b..2c255835ce 100644 --- a/packages/builder/package.json +++ b/packages/builder/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/builder", - "version": "0.9.142", + "version": "0.9.143-alpha.0", "license": "AGPL-3.0", "private": true, "scripts": { @@ -65,10 +65,10 @@ } }, "dependencies": { - "@budibase/bbui": "^0.9.142", - "@budibase/client": "^0.9.142", + "@budibase/bbui": "^0.9.143-alpha.0", + "@budibase/client": "^0.9.143-alpha.0", "@budibase/colorpicker": "1.1.2", - "@budibase/string-templates": "^0.9.142", + "@budibase/string-templates": "^0.9.143-alpha.0", "@sentry/browser": "5.19.1", "@spectrum-css/page": "^3.0.1", "@spectrum-css/vars": "^3.0.1", diff --git a/packages/cli/package.json b/packages/cli/package.json index 0340759e45..29763b9da6 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/cli", - "version": "0.9.142", + "version": "0.9.143-alpha.0", "description": "Budibase CLI, for developers, self hosting and migrations.", "main": "src/index.js", "bin": { diff --git a/packages/client/package.json b/packages/client/package.json index 3d25a1b5d6..f9777629f4 100644 --- a/packages/client/package.json +++ b/packages/client/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/client", - "version": "0.9.142", + "version": "0.9.143-alpha.0", "license": "MPL-2.0", "module": "dist/budibase-client.js", "main": "dist/budibase-client.js", @@ -19,9 +19,9 @@ "dev:builder": "rollup -cw" }, "dependencies": { - "@budibase/bbui": "^0.9.142", + "@budibase/bbui": "^0.9.143-alpha.0", "@budibase/standard-components": "^0.9.139", - "@budibase/string-templates": "^0.9.142", + "@budibase/string-templates": "^0.9.143-alpha.0", "regexparam": "^1.3.0", "shortid": "^2.2.15", "svelte-spa-router": "^3.0.5" diff --git a/packages/server/package.json b/packages/server/package.json index fff341dd57..ad289e8c0d 100644 --- a/packages/server/package.json +++ b/packages/server/package.json @@ -1,7 +1,7 @@ { "name": "@budibase/server", "email": "hi@budibase.com", - "version": "0.9.142", + "version": "0.9.143-alpha.0", "description": "Budibase Web Server", "main": "src/index.js", "repository": { @@ -62,9 +62,9 @@ "author": "Budibase", "license": "AGPL-3.0-or-later", "dependencies": { - "@budibase/auth": "^0.9.142", - "@budibase/client": "^0.9.142", - "@budibase/string-templates": "^0.9.142", + "@budibase/auth": "^0.9.143-alpha.0", + "@budibase/client": "^0.9.143-alpha.0", + "@budibase/string-templates": "^0.9.143-alpha.0", "@elastic/elasticsearch": "7.10.0", "@koa/router": "8.0.0", "@sendgrid/mail": "7.1.1", diff --git a/packages/string-templates/package.json b/packages/string-templates/package.json index 44a3290696..9b3c2487b9 100644 --- a/packages/string-templates/package.json +++ b/packages/string-templates/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/string-templates", - "version": "0.9.142", + "version": "0.9.143-alpha.0", "description": "Handlebars wrapper for Budibase templating.", "main": "src/index.cjs", "module": "dist/bundle.mjs", diff --git a/packages/worker/package.json b/packages/worker/package.json index 83e53e9b45..a1cd5c90e1 100644 --- a/packages/worker/package.json +++ b/packages/worker/package.json @@ -1,7 +1,7 @@ { "name": "@budibase/worker", "email": "hi@budibase.com", - "version": "0.9.142", + "version": "0.9.143-alpha.0", "description": "Budibase background service", "main": "src/index.js", "repository": { @@ -25,8 +25,8 @@ "author": "Budibase", "license": "AGPL-3.0-or-later", "dependencies": { - "@budibase/auth": "^0.9.142", - "@budibase/string-templates": "^0.9.142", + "@budibase/auth": "^0.9.143-alpha.0", + "@budibase/string-templates": "^0.9.143-alpha.0", "@koa/router": "^8.0.0", "@techpass/passport-openidconnect": "^0.3.0", "aws-sdk": "^2.811.0", From 0806e69010d7df91d4684c61e113ec4c07d218b6 Mon Sep 17 00:00:00 2001 From: Rory Powell Date: Tue, 28 Sep 2021 10:15:48 +0100 Subject: [PATCH 18/19] Temp account portal logging --- packages/auth/src/middleware/authenticated.js | 25 +++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/packages/auth/src/middleware/authenticated.js b/packages/auth/src/middleware/authenticated.js index 944f3ee9d9..98773ded44 100644 --- a/packages/auth/src/middleware/authenticated.js +++ b/packages/auth/src/middleware/authenticated.js @@ -10,10 +10,15 @@ function finalise( { authenticated, user, internal, version, publicEndpoint } = {} ) { ctx.publicEndpoint = publicEndpoint || false + console.log("Temp Auth Middleware: public endoint", ctx.publicEndpoint) ctx.isAuthenticated = authenticated || false + console.log("Temp Auth Middleware: isAuthenticated", ctx.isAuthenticated) ctx.user = user + console.log("Temp Auth Middleware: user", ctx.user) ctx.internal = internal || false + console.log("Temp Auth Middleware: internal", ctx.internal) ctx.version = version + console.log("Temp Auth Middleware: version", ctx.version) } /** @@ -27,40 +32,50 @@ module.exports = ( ) => { const noAuthOptions = noAuthPatterns ? buildMatcherRegex(noAuthPatterns) : [] return async (ctx, next) => { + console.log("Temp Auth Middleware: Start auth middleware") let publicEndpoint = false const version = ctx.request.headers[Headers.API_VER] // the path is not authenticated const found = matches(ctx, noAuthOptions) if (found) { + console.log("Temp Auth Middleware: Public endpoint found") publicEndpoint = true } try { + console.log("Temp Auth Middleware: Parsing cookie") // check the actual user is authenticated first const authCookie = getCookie(ctx, Cookies.Auth) let authenticated = false, user = null, internal = false if (authCookie) { + console.log("Temp Auth Middleware: Auth cookie found") let error = null const sessionId = authCookie.sessionId, userId = authCookie.userId + console.log("Temp Auth Middleware: Getting session") const session = await getSession(userId, sessionId) if (!session) { error = "No session found" } else { try { + console.log("Temp Auth Middleware: Getting user") if (opts && opts.populateUser) { + console.log("Temp Auth Middleware: Populate user function found") user = await getUser( userId, session.tenantId, opts.populateUser(ctx) ) } else { + console.log("Temp Auth Middleware: Getting user from DB") user = await getUser(userId, session.tenantId) } delete user.password + console.log("Temp Auth Middleware: User is authenticated") authenticated = true } catch (err) { + console.log("Temp Auth Middleware: Holy shit there was an error") error = err } } @@ -69,6 +84,7 @@ module.exports = ( // remove the cookie as the user does not exist anymore clearCookie(ctx, Cookies.Auth) } else { + console.log("Temp Auth Middleware: No error") // make sure we denote that the session is still in use await updateSessionTTL(session) } @@ -87,14 +103,23 @@ module.exports = ( if (authenticated !== true) { authenticated = false } + console.log("Temp Auth Middleware: Auth status", { + authenticated, + user, + internal, + version, + publicEndpoint, + }) // isAuthenticated is a function, so use a variable to be able to check authed state finalise(ctx, { authenticated, user, internal, version, publicEndpoint }) return next() } catch (err) { + console.log("Temp Auth Middleware: Error:", err) // allow configuring for public access if ((opts && opts.publicAllowed) || publicEndpoint) { finalise(ctx, { authenticated: false, version, publicEndpoint }) } else { + console.log("Temp Auth Middleware: Throwing error status", err.status) ctx.throw(err.status || 403, err) } } From 9e8b14f0fbf6d86229c15071eb294570a026b4c5 Mon Sep 17 00:00:00 2001 From: Budibase Staging Release Bot <> Date: Tue, 28 Sep 2021 09:24:39 +0000 Subject: [PATCH 19/19] v0.9.143-alpha.1 --- lerna.json | 2 +- packages/auth/package.json | 2 +- packages/bbui/package.json | 2 +- packages/builder/package.json | 8 ++++---- packages/cli/package.json | 2 +- packages/client/package.json | 6 +++--- packages/server/package.json | 8 ++++---- packages/string-templates/package.json | 2 +- packages/worker/package.json | 6 +++--- 9 files changed, 19 insertions(+), 19 deletions(-) diff --git a/lerna.json b/lerna.json index 01c429c73d..df7816b085 100644 --- a/lerna.json +++ b/lerna.json @@ -1,5 +1,5 @@ { - "version": "0.9.143-alpha.0", + "version": "0.9.143-alpha.1", "npmClient": "yarn", "packages": [ "packages/*" diff --git a/packages/auth/package.json b/packages/auth/package.json index 0457520d91..98c2c978f8 100644 --- a/packages/auth/package.json +++ b/packages/auth/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/auth", - "version": "0.9.143-alpha.0", + "version": "0.9.143-alpha.1", "description": "Authentication middlewares for budibase builder and apps", "main": "src/index.js", "author": "Budibase", diff --git a/packages/bbui/package.json b/packages/bbui/package.json index 42e4215caa..c25d4dd85e 100644 --- a/packages/bbui/package.json +++ b/packages/bbui/package.json @@ -1,7 +1,7 @@ { "name": "@budibase/bbui", "description": "A UI solution used in the different Budibase projects.", - "version": "0.9.143-alpha.0", + "version": "0.9.143-alpha.1", "license": "AGPL-3.0", "svelte": "src/index.js", "module": "dist/bbui.es.js", diff --git a/packages/builder/package.json b/packages/builder/package.json index 2c255835ce..1846c445aa 100644 --- a/packages/builder/package.json +++ b/packages/builder/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/builder", - "version": "0.9.143-alpha.0", + "version": "0.9.143-alpha.1", "license": "AGPL-3.0", "private": true, "scripts": { @@ -65,10 +65,10 @@ } }, "dependencies": { - "@budibase/bbui": "^0.9.143-alpha.0", - "@budibase/client": "^0.9.143-alpha.0", + "@budibase/bbui": "^0.9.143-alpha.1", + "@budibase/client": "^0.9.143-alpha.1", "@budibase/colorpicker": "1.1.2", - "@budibase/string-templates": "^0.9.143-alpha.0", + "@budibase/string-templates": "^0.9.143-alpha.1", "@sentry/browser": "5.19.1", "@spectrum-css/page": "^3.0.1", "@spectrum-css/vars": "^3.0.1", diff --git a/packages/cli/package.json b/packages/cli/package.json index 29763b9da6..f9d81ed830 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/cli", - "version": "0.9.143-alpha.0", + "version": "0.9.143-alpha.1", "description": "Budibase CLI, for developers, self hosting and migrations.", "main": "src/index.js", "bin": { diff --git a/packages/client/package.json b/packages/client/package.json index f9777629f4..5bd6cdfb71 100644 --- a/packages/client/package.json +++ b/packages/client/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/client", - "version": "0.9.143-alpha.0", + "version": "0.9.143-alpha.1", "license": "MPL-2.0", "module": "dist/budibase-client.js", "main": "dist/budibase-client.js", @@ -19,9 +19,9 @@ "dev:builder": "rollup -cw" }, "dependencies": { - "@budibase/bbui": "^0.9.143-alpha.0", + "@budibase/bbui": "^0.9.143-alpha.1", "@budibase/standard-components": "^0.9.139", - "@budibase/string-templates": "^0.9.143-alpha.0", + "@budibase/string-templates": "^0.9.143-alpha.1", "regexparam": "^1.3.0", "shortid": "^2.2.15", "svelte-spa-router": "^3.0.5" diff --git a/packages/server/package.json b/packages/server/package.json index ad289e8c0d..ba5bdce69b 100644 --- a/packages/server/package.json +++ b/packages/server/package.json @@ -1,7 +1,7 @@ { "name": "@budibase/server", "email": "hi@budibase.com", - "version": "0.9.143-alpha.0", + "version": "0.9.143-alpha.1", "description": "Budibase Web Server", "main": "src/index.js", "repository": { @@ -62,9 +62,9 @@ "author": "Budibase", "license": "AGPL-3.0-or-later", "dependencies": { - "@budibase/auth": "^0.9.143-alpha.0", - "@budibase/client": "^0.9.143-alpha.0", - "@budibase/string-templates": "^0.9.143-alpha.0", + "@budibase/auth": "^0.9.143-alpha.1", + "@budibase/client": "^0.9.143-alpha.1", + "@budibase/string-templates": "^0.9.143-alpha.1", "@elastic/elasticsearch": "7.10.0", "@koa/router": "8.0.0", "@sendgrid/mail": "7.1.1", diff --git a/packages/string-templates/package.json b/packages/string-templates/package.json index 9b3c2487b9..469a3eebb5 100644 --- a/packages/string-templates/package.json +++ b/packages/string-templates/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/string-templates", - "version": "0.9.143-alpha.0", + "version": "0.9.143-alpha.1", "description": "Handlebars wrapper for Budibase templating.", "main": "src/index.cjs", "module": "dist/bundle.mjs", diff --git a/packages/worker/package.json b/packages/worker/package.json index a1cd5c90e1..2952e2bda3 100644 --- a/packages/worker/package.json +++ b/packages/worker/package.json @@ -1,7 +1,7 @@ { "name": "@budibase/worker", "email": "hi@budibase.com", - "version": "0.9.143-alpha.0", + "version": "0.9.143-alpha.1", "description": "Budibase background service", "main": "src/index.js", "repository": { @@ -25,8 +25,8 @@ "author": "Budibase", "license": "AGPL-3.0-or-later", "dependencies": { - "@budibase/auth": "^0.9.143-alpha.0", - "@budibase/string-templates": "^0.9.143-alpha.0", + "@budibase/auth": "^0.9.143-alpha.1", + "@budibase/string-templates": "^0.9.143-alpha.1", "@koa/router": "^8.0.0", "@techpass/passport-openidconnect": "^0.3.0", "aws-sdk": "^2.811.0",