From f13257bebe5fbfc699e535f300a468d26b46366e Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Mon, 15 Nov 2021 17:40:45 +0000 Subject: [PATCH] Updating the getAllApps function to use a cached version of the app metadata, rather than retrieving it individually everytime. Also invalidating the results everytime they are updated (at least in the important locations). --- packages/auth/cache.js | 1 + packages/auth/src/cache/appMetadata.js | 35 +++++++++++++++++++ packages/auth/src/db/index.js | 4 +-- packages/auth/src/db/utils.js | 3 +- packages/auth/src/redis/authRedis.js | 10 +++++- packages/auth/src/redis/utils.js | 1 + .../server/src/api/controllers/application.js | 7 +++- .../src/api/controllers/deploy/index.js | 2 ++ packages/server/src/api/controllers/dev.js | 2 ++ packages/server/src/middleware/builder.js | 2 ++ packages/server/src/utilities/index.js | 1 + 11 files changed, 63 insertions(+), 5 deletions(-) create mode 100644 packages/auth/src/cache/appMetadata.js diff --git a/packages/auth/cache.js b/packages/auth/cache.js index 48563a16f3..02344586a9 100644 --- a/packages/auth/cache.js +++ b/packages/auth/cache.js @@ -1,3 +1,4 @@ module.exports = { user: require("./src/cache/user"), + app: require("./src/cache/appMetadata"), } diff --git a/packages/auth/src/cache/appMetadata.js b/packages/auth/src/cache/appMetadata.js new file mode 100644 index 0000000000..e30a58770f --- /dev/null +++ b/packages/auth/src/cache/appMetadata.js @@ -0,0 +1,35 @@ +const redis = require("../redis/authRedis") +const { getDB } = require("../db") +const { DocumentTypes } = require("../db/constants") + +const EXPIRY_SECONDS = 3600 + +/** + * The default populate app metadata function + */ +const populateFromDB = async appId => { + return getDB(appId, { skip_setup: true }).get(DocumentTypes.APP_METADATA) +} + +/** + * Get the requested app metadata by id. + * Use redis cache to first read the app metadata. + * If not present fallback to loading the app metadata directly and re-caching. + * @param {*} appId the id of the app to get metadata from. + * @returns {object} the app metadata. + */ +exports.getAppMetadata = async appId => { + const client = await redis.getAppClient() + // try cache + let metadata = await client.get(appId) + if (!metadata) { + metadata = await populateFromDB(appId) + client.store(appId, metadata, EXPIRY_SECONDS) + } + return metadata +} + +exports.invalidateAppMetadata = async appId => { + const client = await redis.getAppClient() + await client.delete(appId) +} diff --git a/packages/auth/src/db/index.js b/packages/auth/src/db/index.js index 163364dbf3..94f5513f19 100644 --- a/packages/auth/src/db/index.js +++ b/packages/auth/src/db/index.js @@ -4,8 +4,8 @@ module.exports.setDB = pouch => { Pouch = pouch } -module.exports.getDB = dbName => { - return new Pouch(dbName) +module.exports.getDB = (dbName, opts = {}) => { + return new Pouch(dbName, opts) } module.exports.getCouch = () => { diff --git a/packages/auth/src/db/utils.js b/packages/auth/src/db/utils.js index b956089660..f8f36b17b3 100644 --- a/packages/auth/src/db/utils.js +++ b/packages/auth/src/db/utils.js @@ -6,6 +6,7 @@ const { StaticDatabases, SEPARATOR, DocumentTypes } = require("./constants") const { getTenantId, getTenantIDFromAppID } = require("../tenancy") const fetch = require("node-fetch") const { getCouch } = require("./index") +const { getAppMetadata } = require("../cache/appMetadata") const UNICODE_MAX = "\ufff0" @@ -234,7 +235,7 @@ exports.getAllApps = async (CouchDB, { dev, all, idsOnly } = {}) => { } const appPromises = appDbNames.map(db => // skip setup otherwise databases could be re-created - new CouchDB(db, { skip_setup: true }).get(DocumentTypes.APP_METADATA) + getAppMetadata(db) ) if (appPromises.length === 0) { return [] diff --git a/packages/auth/src/redis/authRedis.js b/packages/auth/src/redis/authRedis.js index decce6763b..ca5c9bae37 100644 --- a/packages/auth/src/redis/authRedis.js +++ b/packages/auth/src/redis/authRedis.js @@ -1,16 +1,18 @@ const Client = require("./index") const utils = require("./utils") -let userClient, sessionClient +let userClient, sessionClient, appClient async function init() { userClient = await new Client(utils.Databases.USER_CACHE).init() sessionClient = await new Client(utils.Databases.SESSIONS).init() + appClient = await new Client(utils.Databases.APP_METADATA).init() } process.on("exit", async () => { if (userClient) await userClient.finish() if (sessionClient) await sessionClient.finish() + if (appClient) await appClient.finish() }) module.exports = { @@ -26,4 +28,10 @@ module.exports = { } return sessionClient }, + getAppClient: async () => { + if (!appClient) { + await init() + } + return appClient + }, } diff --git a/packages/auth/src/redis/utils.js b/packages/auth/src/redis/utils.js index 6befecd9ba..466b117e96 100644 --- a/packages/auth/src/redis/utils.js +++ b/packages/auth/src/redis/utils.js @@ -15,6 +15,7 @@ exports.Databases = { SESSIONS: "session", USER_CACHE: "users", FLAGS: "flags", + APP_METADATA: "appMetadata", } exports.SEPARATOR = SEPARATOR diff --git a/packages/server/src/api/controllers/application.js b/packages/server/src/api/controllers/application.js index 99c16a975c..d38312af2f 100644 --- a/packages/server/src/api/controllers/application.js +++ b/packages/server/src/api/controllers/application.js @@ -45,6 +45,7 @@ const { } = require("../../utilities/fileSystem/clientLibrary") const { getTenantId, isMultiTenant } = require("@budibase/auth/tenancy") const { syncGlobalUsers } = require("./user") +const { app: appCache } = require("@budibase/auth/cache") const URL_REGEX_SLASH = /\/|\\/g @@ -319,6 +320,7 @@ exports.delete = async ctx => { } // make sure the app/role doesn't stick around after the app has been deleted await removeAppFromUserRoles(ctx, ctx.params.appId) + await appCache.invalidateAppMetadata(ctx.params.appId) ctx.status = 200 ctx.body = result @@ -387,7 +389,10 @@ const updateAppPackage = async (ctx, appPackage, appId) => { // Redis, shouldn't ever store it delete newAppPackage.lockedBy - return await db.put(newAppPackage) + const response = await db.put(newAppPackage) + // remove any cached metadata, so that it will be updated + await appCache.invalidateAppMetadata(appId) + return response } const createEmptyAppPackage = async (ctx, app) => { diff --git a/packages/server/src/api/controllers/deploy/index.js b/packages/server/src/api/controllers/deploy/index.js index 13002476fc..c1138f4b03 100644 --- a/packages/server/src/api/controllers/deploy/index.js +++ b/packages/server/src/api/controllers/deploy/index.js @@ -6,6 +6,7 @@ const { disableAllCrons, enableCronTrigger, } = require("../../../automations/utils") +const { app: appCache } = require("@budibase/auth/cache") // the max time we can wait for an invalidation to complete before considering it failed const MAX_PENDING_TIME_MS = 30 * 60000 @@ -103,6 +104,7 @@ async function deployApp(deployment) { appDoc.appId = productionAppId appDoc.instance._id = productionAppId await db.put(appDoc) + await appCache.invalidateAppMetadata(productionAppId) console.log("New app doc written successfully.") await initDeployedApp(productionAppId) console.log("Deployed app initialised, setting deployment to successful") diff --git a/packages/server/src/api/controllers/dev.js b/packages/server/src/api/controllers/dev.js index ed58b8048b..dbea82b06b 100644 --- a/packages/server/src/api/controllers/dev.js +++ b/packages/server/src/api/controllers/dev.js @@ -6,6 +6,7 @@ const { request } = require("../../utilities/workerRequests") const { clearLock } = require("../../utilities/redis") const { Replication } = require("@budibase/auth").db const { DocumentTypes } = require("../../db/utils") +const { app: appCache } = require("@budibase/auth/cache") async function redirect(ctx, method, path = "global") { const { devPath } = ctx.params @@ -106,6 +107,7 @@ exports.revert = async ctx => { appDoc.appId = appId appDoc.instance._id = appId await db.put(appDoc) + await appCache.invalidateAppMetadata(appId) ctx.body = { message: "Reverted changes successfully.", } diff --git a/packages/server/src/middleware/builder.js b/packages/server/src/middleware/builder.js index 8ea49a3b48..427e54287a 100644 --- a/packages/server/src/middleware/builder.js +++ b/packages/server/src/middleware/builder.js @@ -8,6 +8,7 @@ const { const CouchDB = require("../db") const { DocumentTypes } = require("../db/utils") const { PermissionTypes } = require("@budibase/auth/permissions") +const { app: appCache } = require("@budibase/auth/cache") const DEBOUNCE_TIME_SEC = 30 @@ -51,6 +52,7 @@ async function updateAppUpdatedAt(ctx) { const metadata = await db.get(DocumentTypes.APP_METADATA) metadata.updatedAt = new Date().toISOString() await db.put(metadata) + await appCache.invalidateAppMetadata(appId) // set a new debounce record with a short TTL await setDebounce(appId, DEBOUNCE_TIME_SEC) } diff --git a/packages/server/src/utilities/index.js b/packages/server/src/utilities/index.js index c2ebbcd9f1..eacf9708e2 100644 --- a/packages/server/src/utilities/index.js +++ b/packages/server/src/utilities/index.js @@ -48,6 +48,7 @@ exports.objectStoreUrl = () => { * via a specific endpoint (under /api/assets/client). * @param {string} appId In production we need the appId to look up the correct bucket, as the * version of the client lib may differ between apps. + * @param {string} version The version to retrieve. * @return {string} The URL to be inserted into appPackage response or server rendered * app index file. */