Merge pull request #3392 from Budibase/fix/app-list-perf

Caching app metadata to improve application fetch performance

packages/auth/cache.js

@@ -1,3 +1,4 @@
 module.exports = {
   user: require("./src/cache/user"),
+  app: require("./src/cache/appMetadata"),
 }

packages/auth/src/cache/appMetadata.js

@@ -0,0 +1,39 @@
+const redis = require("../redis/authRedis")
+const { getCouch } = require("../db")
+const { DocumentTypes } = require("../db/constants")
+
+const EXPIRY_SECONDS = 3600
+
+/**
+ * The default populate app metadata function
+ */
+const populateFromDB = async (appId, CouchDB = null) => {
+  if (!CouchDB) {
+    CouchDB = getCouch()
+  }
+  const db = new CouchDB(appId, { skip_setup: true })
+  return db.get(DocumentTypes.APP_METADATA)
+}
+
+/**
+ * Get the requested app metadata by id.
+ * Use redis cache to first read the app metadata.
+ * If not present fallback to loading the app metadata directly and re-caching.
+ * @param {*} appId the id of the app to get metadata from.
+ * @returns {object} the app metadata.
+ */
+exports.getAppMetadata = async (appId, CouchDB = null) => {
+  const client = await redis.getAppClient()
+  // try cache
+  let metadata = await client.get(appId)
+  if (!metadata) {
+    metadata = await populateFromDB(appId, CouchDB)
+    client.store(appId, metadata, EXPIRY_SECONDS)
+  }
+  return metadata
+}
+
+exports.invalidateAppMetadata = async appId => {
+  const client = await redis.getAppClient()
+  await client.delete(appId)
+}

packages/auth/src/db/utils.js

@@ -6,6 +6,9 @@ const { StaticDatabases, SEPARATOR, DocumentTypes } = require("./constants")
 const { getTenantId, getTenantIDFromAppID } = require("../tenancy")
 const fetch = require("node-fetch")
 const { getCouch } = require("./index")
+const { getAppMetadata } = require("../cache/appMetadata")
+
+const NO_APP_ERROR = "No app provided"
 
 const UNICODE_MAX = "\ufff0"
 
@@ -45,14 +48,23 @@ function getDocParams(docType, docId = null, otherProps = {}) {
 }
 
 exports.isDevAppID = appId => {
+  if (!appId) {
+    throw NO_APP_ERROR
+  }
   return appId.startsWith(exports.APP_DEV_PREFIX)
 }
 
 exports.isProdAppID = appId => {
+  if (!appId) {
+    throw NO_APP_ERROR
+  }
   return appId.startsWith(exports.APP_PREFIX) && !exports.isDevAppID(appId)
 }
 
 function isDevApp(app) {
+  if (!app) {
+    throw NO_APP_ERROR
+  }
   return exports.isDevAppID(app.appId)
 }
 
@@ -232,16 +244,16 @@ exports.getAllApps = async (CouchDB, { dev, all, idsOnly } = {}) => {
   if (idsOnly) {
     return appDbNames
   }
-  const appPromises = appDbNames.map(db =>
+  const appPromises = appDbNames.map(app =>
     // skip setup otherwise databases could be re-created
-    new CouchDB(db, { skip_setup: true }).get(DocumentTypes.APP_METADATA)
+    getAppMetadata(app, CouchDB)
   )
   if (appPromises.length === 0) {
     return []
   } else {
     const response = await Promise.allSettled(appPromises)
     const apps = response
-      .filter(result => result.status === "fulfilled")
+      .filter(result => result.status === "fulfilled" && result.value != null)
       .map(({ value }) => value)
     if (!all) {
       return apps.filter(app => {

packages/auth/src/redis/authRedis.js

@@ -1,16 +1,18 @@
 const Client = require("./index")
 const utils = require("./utils")
 
-let userClient, sessionClient
+let userClient, sessionClient, appClient
 
 async function init() {
   userClient = await new Client(utils.Databases.USER_CACHE).init()
   sessionClient = await new Client(utils.Databases.SESSIONS).init()
+  appClient = await new Client(utils.Databases.APP_METADATA).init()
 }
 
 process.on("exit", async () => {
   if (userClient) await userClient.finish()
   if (sessionClient) await sessionClient.finish()
+  if (appClient) await appClient.finish()
 })
 
 module.exports = {
@@ -26,4 +28,10 @@ module.exports = {
     }
     return sessionClient
   },
+  getAppClient: async () => {
+    if (!appClient) {
+      await init()
+    }
+    return appClient
+  },
 }

packages/auth/src/redis/utils.js

@@ -15,6 +15,7 @@ exports.Databases = {
   SESSIONS: "session",
   USER_CACHE: "users",
   FLAGS: "flags",
+  APP_METADATA: "appMetadata",
 }
 
 exports.SEPARATOR = SEPARATOR

packages/server/src/api/controllers/application.js

@@ -45,6 +45,7 @@ const {
 } = require("../../utilities/fileSystem/clientLibrary")
 const { getTenantId, isMultiTenant } = require("@budibase/auth/tenancy")
 const { syncGlobalUsers } = require("./user")
+const { app: appCache } = require("@budibase/auth/cache")
 
 const URL_REGEX_SLASH = /\/|\\/g
 
@@ -319,6 +320,7 @@ exports.delete = async ctx => {
   }
   // make sure the app/role doesn't stick around after the app has been deleted
   await removeAppFromUserRoles(ctx, ctx.params.appId)
+  await appCache.invalidateAppMetadata(ctx.params.appId)
 
   ctx.status = 200
   ctx.body = result
@@ -387,7 +389,10 @@ const updateAppPackage = async (ctx, appPackage, appId) => {
   // Redis, shouldn't ever store it
   delete newAppPackage.lockedBy
 
-  return await db.put(newAppPackage)
+  const response = await db.put(newAppPackage)
+  // remove any cached metadata, so that it will be updated
+  await appCache.invalidateAppMetadata(appId)
+  return response
 }
 
 const createEmptyAppPackage = async (ctx, app) => {

packages/server/src/api/controllers/deploy/index.js

@@ -6,6 +6,7 @@ const {
   disableAllCrons,
   enableCronTrigger,
 } = require("../../../automations/utils")
+const { app: appCache } = require("@budibase/auth/cache")
 
 // the max time we can wait for an invalidation to complete before considering it failed
 const MAX_PENDING_TIME_MS = 30 * 60000
@@ -103,6 +104,7 @@ async function deployApp(deployment) {
     appDoc.appId = productionAppId
     appDoc.instance._id = productionAppId
     await db.put(appDoc)
+    await appCache.invalidateAppMetadata(productionAppId)
     console.log("New app doc written successfully.")
     await initDeployedApp(productionAppId)
     console.log("Deployed app initialised, setting deployment to successful")

packages/server/src/api/controllers/dev.js

@@ -6,6 +6,7 @@ const { request } = require("../../utilities/workerRequests")
 const { clearLock } = require("../../utilities/redis")
 const { Replication } = require("@budibase/auth").db
 const { DocumentTypes } = require("../../db/utils")
+const { app: appCache } = require("@budibase/auth/cache")
 
 async function redirect(ctx, method, path = "global") {
   const { devPath } = ctx.params
@@ -106,6 +107,7 @@ exports.revert = async ctx => {
     appDoc.appId = appId
     appDoc.instance._id = appId
     await db.put(appDoc)
+    await appCache.invalidateAppMetadata(appId)
     ctx.body = {
       message: "Reverted changes successfully.",
     }

packages/server/src/middleware/builder.js

@@ -8,6 +8,7 @@ const {
 const CouchDB = require("../db")
 const { DocumentTypes } = require("../db/utils")
 const { PermissionTypes } = require("@budibase/auth/permissions")
+const { app: appCache } = require("@budibase/auth/cache")
 
 const DEBOUNCE_TIME_SEC = 30
 
@@ -51,6 +52,7 @@ async function updateAppUpdatedAt(ctx) {
   const metadata = await db.get(DocumentTypes.APP_METADATA)
   metadata.updatedAt = new Date().toISOString()
   await db.put(metadata)
+  await appCache.invalidateAppMetadata(appId)
   // set a new debounce record with a short TTL
   await setDebounce(appId, DEBOUNCE_TIME_SEC)
 }

packages/server/src/utilities/index.js

@@ -48,6 +48,7 @@ exports.objectStoreUrl = () => {
  * via a specific endpoint (under /api/assets/client).
  * @param {string} appId In production we need the appId to look up the correct bucket, as the
  * version of the client lib may differ between apps.
+ * @param {string} version The version to retrieve.
  * @return {string} The URL to be inserted into appPackage response or server rendered
  * app index file.
  */

packages/worker/src/api/controllers/global/users.js

@@ -43,11 +43,7 @@ exports.save = async ctx => {
 }
 
 const parseBooleanParam = param => {
-  if (param && param === "false") {
-    return false
-  } else {
-    return true
-  }
+  return !(param && param === "false")
 }
 
 exports.adminUser = async ctx => {