From 515ade6bd369573db7d1c517c775c034a8025c6b Mon Sep 17 00:00:00 2001 From: Rory Powell Date: Thu, 3 Mar 2022 13:37:04 +0000 Subject: [PATCH] Add shared licensing cache --- packages/backend-core/src/cache/user.js | 5 -- packages/backend-core/src/cloud/accounts.js | 3 +- packages/backend-core/src/index.js | 1 + .../backend-core/src/licensing/cache/index.js | 35 +++++++++++ .../backend-core/src/licensing/cache/redis.js | 25 ++++++++ packages/backend-core/src/licensing/index.js | 7 +++ .../src/licensing/middleware/index.js | 17 ++++++ packages/backend-core/src/redis/utils.js | 1 + packages/server/scripts/dev/manage.js | 2 + packages/server/src/api/index.js | 2 + packages/server/src/app.ts | 2 +- packages/server/yarn.lock | 61 +++++++++++-------- .../src/api/controllers/global/users.js | 35 ++++++++--- packages/worker/src/api/index.js | 2 + 14 files changed, 157 insertions(+), 41 deletions(-) create mode 100644 packages/backend-core/src/licensing/cache/index.js create mode 100644 packages/backend-core/src/licensing/cache/redis.js create mode 100644 packages/backend-core/src/licensing/index.js create mode 100644 packages/backend-core/src/licensing/middleware/index.js diff --git a/packages/backend-core/src/cache/user.js b/packages/backend-core/src/cache/user.js index 825b66b99c..60a2d341a8 100644 --- a/packages/backend-core/src/cache/user.js +++ b/packages/backend-core/src/cache/user.js @@ -14,12 +14,7 @@ const populateFromDB = async (userId, tenantId) => { if (!env.SELF_HOSTED && !env.DISABLE_ACCOUNT_PORTAL) { const account = await accounts.getAccount(user.email) - // TODO: Break this out into it's own cache if (account) { - const license = await accounts.getLicense(user.tenantId) - if (license) { - user.license = license - } user.account = account user.accountPortalAccess = true } diff --git a/packages/backend-core/src/cloud/accounts.js b/packages/backend-core/src/cloud/accounts.js index 6f8fd9e837..64b50c2896 100644 --- a/packages/backend-core/src/cloud/accounts.js +++ b/packages/backend-core/src/cloud/accounts.js @@ -42,6 +42,5 @@ exports.getLicense = async tenantId => { throw new Error(`Error getting license for tenant ${tenantId}`) } - const json = await response.json() - return json + return response.json() } diff --git a/packages/backend-core/src/index.js b/packages/backend-core/src/index.js index b0bc524d9b..5b7a0fcd16 100644 --- a/packages/backend-core/src/index.js +++ b/packages/backend-core/src/index.js @@ -15,4 +15,5 @@ module.exports = { auth: require("../auth"), constants: require("../constants"), migrations: require("../migrations"), + licensing: require("./licensing"), } diff --git a/packages/backend-core/src/licensing/cache/index.js b/packages/backend-core/src/licensing/cache/index.js new file mode 100644 index 0000000000..6b11c0cda5 --- /dev/null +++ b/packages/backend-core/src/licensing/cache/index.js @@ -0,0 +1,35 @@ +const redis = require("./redis") +const env = require("../../environment") +const accounts = require("../../cloud/accounts") + +const EXPIRY_SECONDS = 3600 + +const populateLicense = async tenantId => { + if (env.SELF_HOSTED) { + // get license key + } else { + return accounts.getLicense(tenantId) + } +} + +exports.getLicense = async (tenantId, opts = { populateLicense: null }) => { + // try cache + const client = await redis.getClient() + let license = await client.get(tenantId) + if (!license) { + const populate = opts.populateLicense + ? opts.populateLicense + : populateLicense + license = await populate(tenantId) + if (license) { + client.store(tenantId, license, EXPIRY_SECONDS) + } + } + + return license +} + +exports.invalidateLicense = async tenantId => { + const client = await redis.getClient() + await client.delete(tenantId) +} diff --git a/packages/backend-core/src/licensing/cache/redis.js b/packages/backend-core/src/licensing/cache/redis.js new file mode 100644 index 0000000000..a5115d0b45 --- /dev/null +++ b/packages/backend-core/src/licensing/cache/redis.js @@ -0,0 +1,25 @@ +const Redis = require("../../redis") +const utils = require("../../redis/utils") + +let client + +const init = async () => { + client = await new Redis(utils.Databases.LICENSES).init() +} + +const shutdown = async () => { + if (client) { + await client.finish() + } +} + +process.on("exit", async () => { + await shutdown() +}) + +exports.getClient = async () => { + if (!client) { + await init() + } + return client +} diff --git a/packages/backend-core/src/licensing/index.js b/packages/backend-core/src/licensing/index.js new file mode 100644 index 0000000000..cf29c8e51a --- /dev/null +++ b/packages/backend-core/src/licensing/index.js @@ -0,0 +1,7 @@ +const middleware = require("./middleware") +const cache = require("./cache") + +module.exports = { + middleware, + cache, +} diff --git a/packages/backend-core/src/licensing/middleware/index.js b/packages/backend-core/src/licensing/middleware/index.js new file mode 100644 index 0000000000..6d5ba42175 --- /dev/null +++ b/packages/backend-core/src/licensing/middleware/index.js @@ -0,0 +1,17 @@ +const cache = require("../cache") + +const buildLicensingMiddleware = opts => { + return async (ctx, next) => { + if (ctx.user) { + const tenantId = ctx.user.tenantId + const license = await cache.getLicense(tenantId, opts) + if (license) { + ctx.user.license = license + } + } + + return next() + } +} + +module.exports = buildLicensingMiddleware diff --git a/packages/backend-core/src/redis/utils.js b/packages/backend-core/src/redis/utils.js index 3461d2a511..03f02fe8c1 100644 --- a/packages/backend-core/src/redis/utils.js +++ b/packages/backend-core/src/redis/utils.js @@ -17,6 +17,7 @@ exports.Databases = { FLAGS: "flags", APP_METADATA: "appMetadata", QUERY_VARS: "queryVars", + LICENSES: "license", } exports.SEPARATOR = SEPARATOR diff --git a/packages/server/scripts/dev/manage.js b/packages/server/scripts/dev/manage.js index 46a221ca33..a18c795e38 100644 --- a/packages/server/scripts/dev/manage.js +++ b/packages/server/scripts/dev/manage.js @@ -41,6 +41,8 @@ async function init() { REDIS_URL: "localhost:6379", WORKER_URL: "http://localhost:4002", INTERNAL_API_KEY: "budibase", + ACCOUNT_PORTAL_URL: "http://localhost:10001", + ACCOUNT_PORTAL_API_KEY: "budibase", JWT_SECRET: "testsecret", REDIS_PASSWORD: "budibase", MINIO_ACCESS_KEY: "budibase", diff --git a/packages/server/src/api/index.js b/packages/server/src/api/index.js index 4a94063e31..4318a7ea95 100644 --- a/packages/server/src/api/index.js +++ b/packages/server/src/api/index.js @@ -11,6 +11,7 @@ const zlib = require("zlib") const { mainRoutes, staticRoutes } = require("./routes") const pkg = require("../../package.json") const env = require("../environment") +const { licensing } = require("@budibase/backend-core") const router = new Router() @@ -54,6 +55,7 @@ router .use(currentApp) // this middleware will try to use the app ID to determine the tenancy .use(buildAppTenancyMiddleware()) + .use(licensing.middleware()) .use(auditLog) // error handling middleware diff --git a/packages/server/src/app.ts b/packages/server/src/app.ts index a2e5bec873..ae91b4b939 100644 --- a/packages/server/src/app.ts +++ b/packages/server/src/app.ts @@ -20,7 +20,7 @@ const automations = require("./automations/index") const Sentry = require("@sentry/node") const fileSystem = require("./utilities/fileSystem") const bullboard = require("./automations/bullboard") -const redis = require("./utilities/redis") +import redis from "./utilities/redis" import * as migrations from "./migrations" const app = new Koa() diff --git a/packages/server/yarn.lock b/packages/server/yarn.lock index 60cafb1fd9..a45a05d9d3 100644 --- a/packages/server/yarn.lock +++ b/packages/server/yarn.lock @@ -1069,7 +1069,14 @@ svelte-apexcharts "^1.0.2" svelte-flatpickr "^3.1.0" -"@bull-board/api@3.7.0", "@bull-board/api@^3.7.0": +"@bull-board/api@3.9.4": + version "3.9.4" + resolved "https://registry.yarnpkg.com/@bull-board/api/-/api-3.9.4.tgz#984f25e6d5501d97152d81184968ff135757b57a" + integrity sha512-1X1YCqPEID2kKwq+g4aEspZm2j+vUgEYDlqINCLztThBXWbzJhI1vqwktVGJF9DAe98Jl6R84vb7cO/AgjaKMA== + dependencies: + redis-info "^3.0.8" + +"@bull-board/api@^3.7.0": version "3.7.0" resolved "https://registry.yarnpkg.com/@bull-board/api/-/api-3.7.0.tgz#231f687187c0cb34e0b97f463917b6aaeb4ef6af" integrity sha512-BGAqOUqMa7KMsqR+07LhMDVARLBHRekGGxWCIOYx17mMbSev54ausSGQsVaSKvzPbHpp1YbRlh7RzIJUjxsY/A== @@ -2253,6 +2260,7 @@ integrity sha512-8DbSPMSsZH5PWPnGEkAZLYgJEH4ghHJNKF7LB6Wr5R0/v6g+Vs+JoaA7kcvLtHE936xg2WpFPkaoaJgExOmKDw== dependencies: "@types/ioredis" "*" + "@types/redis" "^2.8.0" "@types/connect@*": version "3.4.35" @@ -2292,11 +2300,16 @@ "@types/estree" "*" "@types/json-schema" "*" -"@types/estree@*", "@types/estree@^0.0.50": +"@types/estree@*": version "0.0.50" resolved "https://registry.yarnpkg.com/@types/estree/-/estree-0.0.50.tgz#1e0caa9364d3fccd2931c3ed96fdbeaa5d4cca83" integrity sha512-C6N5s2ZFtuZRj54k2/zyRhNDjJwwcViAM3Nbm8zjBpbqAdZ00mr0CFxvSKeO8Y/e03WVFLpQMdHYVfUd6SB+Hw== +"@types/estree@^0.0.51": + version "0.0.51" + resolved "https://registry.yarnpkg.com/@types/estree/-/estree-0.0.51.tgz#cfd70924a25a3fd32b218e5e420e6897e1ac4f40" + integrity sha512-CuPgU6f3eT/XgKKPqKd/gLZV1Xmvf1a2R5POBOGQa6uv82xpls89HU5zKeVoyR8XzHd1RGNOlQlvUe3CFkjWNQ== + "@types/express-serve-static-core@^4.17.18": version "4.17.28" resolved "https://registry.yarnpkg.com/@types/express-serve-static-core/-/express-serve-static-core-4.17.28.tgz#c47def9f34ec81dc6328d0b1b5303d1ec98d86b8" @@ -2481,13 +2494,6 @@ "@types/node" "*" safe-buffer "*" -"@types/redis@^2.8.0": - version "2.8.32" - resolved "https://registry.yarnpkg.com/@types/redis/-/redis-2.8.32.tgz#1d3430219afbee10f8cfa389dad2571a05ecfb11" - integrity sha512-7jkMKxcGq9p242exlbsVzuJb57KqHRhNl4dHoQu2Y5v9bCAbtIXXH0R3HleSQW4CTOqpHIYUW3t6tpUj4BVQ+w== - dependencies: - "@types/node" "*" - "@types/serve-static@*": version "1.13.10" resolved "https://registry.yarnpkg.com/@types/serve-static/-/serve-static-1.13.10.tgz#f5e0ce8797d2d7cc5ebeda48a52c96c4fa47a8d9" @@ -2837,7 +2843,7 @@ acorn-walk@^7.1.1: resolved "https://registry.yarnpkg.com/acorn-walk/-/acorn-walk-7.2.0.tgz#0de889a601203909b0fbe07b8938dc21d2e967bc" integrity sha512-OPdCF6GsMIP+Az+aWfAAOEt2/+iVDKE7oy6lJ098aoe59oAmK76qV6Gw60SbZ8jHuG2wH058GF4pLFbYamYrVA== -acorn-walk@^8.1.1, acorn-walk@^8.2.0: +acorn-walk@^8.1.1: version "8.2.0" resolved "https://registry.yarnpkg.com/acorn-walk/-/acorn-walk-8.2.0.tgz#741210f2e2426454508853a2f44d0ab83b7f69c1" integrity sha512-k+iyHEuPgSw6SbuDpGQM+06HQUa04DZ3o+F6CSzXMvvI5KMvnaEqXe+YVe555R9nn6GPt404fos4wcgpw12SDA== @@ -2862,7 +2868,7 @@ acorn@^7.1.1: resolved "https://registry.yarnpkg.com/acorn/-/acorn-7.4.1.tgz#feaed255973d2e77555b83dbc08851a6c63520fa" integrity sha512-nQyp0o1/mNdbTO1PO6kHkwSrmgZ0MT/jCCpNiwbUjGoRN4dlBhqJtoQuCnEOKzgTVwg0ZWiCoQy6SxMebQVh8A== -acorn@^8.2.4, acorn@^8.4.1, acorn@^8.7.0: +acorn@^8.2.4, acorn@^8.4.1: version "8.7.0" resolved "https://registry.yarnpkg.com/acorn/-/acorn-8.7.0.tgz#90951fde0f8f09df93549481e5fc141445b791cf" integrity sha512-V/LGr1APy+PXIwKebEWrkZPwoeoF+w1jiOBUmuxuiUIaOHtob8Qc9BTrYo7VuI5fR8tqsy+buA2WFooR5olqvQ== @@ -6004,10 +6010,10 @@ google-auth-library@^7.11.0: jws "^4.0.0" lru-cache "^6.0.0" -google-p12-pem@^3.0.3: - version "3.1.2" - resolved "https://registry.yarnpkg.com/google-p12-pem/-/google-p12-pem-3.1.2.tgz#c3d61c2da8e10843ff830fdb0d2059046238c1d4" - integrity sha512-tjf3IQIt7tWCDsa0ofDQ1qqSCNzahXDxdAGJDbruWqu3eCg5CKLYKN+hi0s6lfvzYZ1GDVr+oDF9OOWlDSdf0A== +google-p12-pem@^3.1.3: + version "3.1.3" + resolved "https://registry.yarnpkg.com/google-p12-pem/-/google-p12-pem-3.1.3.tgz#5497998798ee86c2fc1f4bb1f92b7729baf37537" + integrity sha512-MC0jISvzymxePDVembypNefkAQp+DRP7dBE+zNUPaIjEspIlYg0++OrsNr248V9tPbz6iqtZ7rX1hxWA5B8qBQ== dependencies: node-forge "^1.0.0" @@ -7981,6 +7987,13 @@ keyv@3.0.0: dependencies: json-buffer "3.0.0" +keyv@^3.0.0: + version "3.1.0" + resolved "https://registry.yarnpkg.com/keyv/-/keyv-3.1.0.tgz#ecc228486f69991e49e9476485a5be1e8fc5c4d9" + integrity sha512-9ykJ/46SN/9KPM/sichzQ7OvXyGDYKGTaDlKMGCAlg2UK8KRy4jb0d8sFc+0Tt0YYnThq8X2RZgCg74RPxgcVA== + dependencies: + json-buffer "3.0.0" + kind-of@^3.0.2, kind-of@^3.0.3, kind-of@^3.2.0: version "3.2.2" resolved "https://registry.yarnpkg.com/kind-of/-/kind-of-3.2.2.tgz#31ea21a734bab9bbb0f32466d893aea51e4a3c64" @@ -9016,10 +9029,10 @@ node-fetch@2.6.7, node-fetch@^2.6.0, node-fetch@^2.6.1: dependencies: whatwg-url "^5.0.0" -node-forge@^0.10.0: - version "0.10.0" - resolved "https://registry.yarnpkg.com/node-forge/-/node-forge-0.10.0.tgz#32dea2afb3e9926f02ee5ce8794902691a676bf3" - integrity sha512-PPmu8eEeG9saEUvI97fm4OYxXVB6bFvyNTyiUOBichBpFG8A1Ljw3bY62+5oOjDEMHRnd0Y7HQ+x7uzxOzC6JA== +node-forge@^1.0.0: + version "1.2.1" + resolved "https://registry.yarnpkg.com/node-forge/-/node-forge-1.2.1.tgz#82794919071ef2eb5c509293325cec8afd0fd53c" + integrity sha512-Fcvtbb+zBcZXbTTVwqGA5W+MKBj56UjVRevvchv5XrcyXbmNdesfZL37nlcWOfpgHhgmxApw3tQbTr4CqNmX4w== node-gyp-build@~4.1.0: version "4.1.1" @@ -11931,10 +11944,10 @@ typedarray-to-buffer@^3.1.5: dependencies: is-typedarray "^1.0.0" -typescript@^4.3.5: - version "4.3.5" - resolved "https://registry.yarnpkg.com/typescript/-/typescript-4.3.5.tgz#4d1c37cc16e893973c45a06886b7113234f119f4" - integrity sha512-DqQgihaQ9cUrskJo9kIyW/+g0Vxsk8cDtZ52a3NGh0YNTfpUSArXSohyUGnvbPazEPLu398C0UxmKSOrPumUzA== +typescript@^4.5.5: + version "4.6.2" + resolved "https://registry.yarnpkg.com/typescript/-/typescript-4.6.2.tgz#fe12d2727b708f4eef40f51598b3398baa9611d4" + integrity sha512-HM/hFigTBHZhLXshn9sN37H085+hQGeJHJ/X7LpBWLID/fbc2acUMfU+lGD98X81sKP+pFa9f0DZmCwB9GnbAg== uc.micro@^1.0.1, uc.micro@^1.0.5: version "1.0.6" @@ -12149,7 +12162,7 @@ util.promisify@^1.0.0, util.promisify@^1.0.1: has-symbols "^1.0.1" object.getownpropertydescriptors "^2.1.1" -uuid@3.3.2, uuid@^3.1.0, uuid@^3.3.2: +uuid@3.3.2: version "3.3.2" resolved "https://registry.yarnpkg.com/uuid/-/uuid-3.3.2.tgz#1b4af4955eb3077c501c23872fc6513811587131" integrity sha512-yXJmeNaw3DnnKAOKJE51sL/ZaYfWJRl1pK9dr19YFCu0ObS231AB1/LbqTKRAQ5kw8A90rA6fr4riOUpTZvQZA== diff --git a/packages/worker/src/api/controllers/global/users.js b/packages/worker/src/api/controllers/global/users.js index b2db340f9a..8acce2a481 100644 --- a/packages/worker/src/api/controllers/global/users.js +++ b/packages/worker/src/api/controllers/global/users.js @@ -158,6 +158,29 @@ exports.removeAppRole = async ctx => { } } +/** + * Add the attributes that are session based to the current user. + */ +const addSessionAttributesToUser = ctx => { + ctx.body.account = ctx.user.account + ctx.body.license = ctx.user.license + ctx.body.budibaseAccess = ctx.user.budibaseAccess + ctx.body.accountPortalAccess = ctx.user.accountPortalAccess + ctx.body.csrfToken = ctx.user.csrfToken +} + +/** + * Remove the attributes that are session based from the current user, + * so that stale values are not written to the db + */ +const removeSessionAttributesFromUser = ctx => { + delete ctx.request.body.csrfToken + delete ctx.request.body.account + delete ctx.request.body.accountPortalAccess + delete ctx.request.body.budibaseAccess + delete ctx.request.body.license +} + exports.getSelf = async ctx => { if (!ctx.user) { ctx.throw(403, "User not logged in") @@ -167,13 +190,7 @@ exports.getSelf = async ctx => { } // this will set the body await exports.find(ctx) - - // forward session information not found in db - ctx.body.account = ctx.user.account - ctx.body.license = ctx.user.license - ctx.body.budibaseAccess = ctx.user.budibaseAccess - ctx.body.accountPortalAccess = ctx.user.accountPortalAccess - ctx.body.csrfToken = ctx.user.csrfToken + addSessionAttributesToUser(ctx) } exports.updateSelf = async ctx => { @@ -192,8 +209,8 @@ exports.updateSelf = async ctx => { // don't allow sending up an ID/Rev, always use the existing one delete ctx.request.body._id delete ctx.request.body._rev - // don't allow setting the csrf token - delete ctx.request.body.csrfToken + removeSessionAttributesFromUser(ctx) + const response = await db.put({ ...user, ...ctx.request.body, diff --git a/packages/worker/src/api/index.js b/packages/worker/src/api/index.js index 607d8283f9..2817da685f 100644 --- a/packages/worker/src/api/index.js +++ b/packages/worker/src/api/index.js @@ -8,6 +8,7 @@ const { buildTenancyMiddleware, buildCsrfMiddleware, } = require("@budibase/backend-core/auth") +const { licensing } = require("@budibase/backend-core") const PUBLIC_ENDPOINTS = [ // old deprecated endpoints kept for backwards compat @@ -91,6 +92,7 @@ router .use(buildAuthMiddleware(PUBLIC_ENDPOINTS)) .use(buildTenancyMiddleware(PUBLIC_ENDPOINTS, NO_TENANCY_ENDPOINTS)) .use(buildCsrfMiddleware({ noCsrfPatterns: NO_CSRF_ENDPOINTS })) + .use(licensing.middleware()) // for now no public access is allowed to worker (bar health check) .use((ctx, next) => { if (ctx.publicEndpoint) {