Merge pull request #3400 from Budibase/fix/app-cache-invalid

Update for app metadata caching to improve long term performance

packages/auth/src/cache/appMetadata.js

@@ -2,6 +2,9 @@ const redis = require("../redis/authRedis")
 const { getCouch } = require("../db")
 const { DocumentTypes } = require("../db/constants")
 
+const AppState = {
+  INVALID: "invalid",
+}
 const EXPIRY_SECONDS = 3600
 
 /**
@@ -15,11 +18,16 @@ const populateFromDB = async (appId, CouchDB = null) => {
   return db.get(DocumentTypes.APP_METADATA)
 }
 
+const isInvalid = metadata => {
+  return !metadata || metadata.state === AppState.INVALID
+}
+
 /**
  * Get the requested app metadata by id.
  * Use redis cache to first read the app metadata.
  * If not present fallback to loading the app metadata directly and re-caching.
- * @param {*} appId the id of the app to get metadata from.
+ * @param {string} appId the id of the app to get metadata from.
+ * @param {object} CouchDB the database being passed
  * @returns {object} the app metadata.
  */
 exports.getAppMetadata = async (appId, CouchDB = null) => {
@@ -27,13 +35,51 @@ exports.getAppMetadata = async (appId, CouchDB = null) => {
   // try cache
   let metadata = await client.get(appId)
   if (!metadata) {
-    metadata = await populateFromDB(appId, CouchDB)
+    let expiry = EXPIRY_SECONDS
-    client.store(appId, metadata, EXPIRY_SECONDS)
+    try {
+      metadata = await populateFromDB(appId, CouchDB)
+    } catch (err) {
+      // app DB left around, but no metadata, it is invalid
+      if (err && err.status === 404) {
+        metadata = { state: AppState.INVALID }
+        // don't expire the reference to an invalid app, it'll only be
+        // updated if a metadata doc actually gets stored (app is remade/reverted)
+        expiry = undefined
+      } else {
+        throw err
+      }
+    }
+    // needed for cypress/some scenarios where the caching happens
+    // so quickly the requests can get slightly out of sync
+    // might store its invalid just before it stores its valid
+    if (isInvalid(metadata)) {
+      const temp = await client.get(appId)
+      if (temp) {
+        metadata = temp
+      }
+    }
+    await client.store(appId, metadata, expiry)
+  }
+  // we've stored in the cache an object to tell us that it is currently invalid
+  if (isInvalid(metadata)) {
+    throw { status: 404, message: "No app metadata found" }
   }
   return metadata
 }
 
-exports.invalidateAppMetadata = async appId => {
+/**
+ * Invalidate/reset the cached metadata when a change occurs in the db.
+ * @param appId {string} the cache key to bust/update.
+ * @param newMetadata {object|undefined} optional - can simply provide the new metadata to update with.
+ * @return {Promise<void>} will respond with success when cache is updated.
+ */
+exports.invalidateAppMetadata = async (appId, newMetadata = null) => {
+  if (!appId) {
+    throw "Cannot invalidate if no app ID provided."
+  }
   const client = await redis.getAppClient()
   await client.delete(appId)
+  if (newMetadata) {
+    await client.store(appId, newMetadata, EXPIRY_SECONDS)
+  }
 }

packages/auth/src/middleware/authenticated.js

@@ -92,6 +92,10 @@ module.exports = (
       finalise(ctx, { authenticated, user, internal, version, publicEndpoint })
       return next()
     } catch (err) {
+      // invalid token, clear the cookie
+      if (err && err.name === "JsonWebTokenError") {
+        clearCookie(ctx, Cookies.Auth)
+      }
       // allow configuring for public access
       if ((opts && opts.publicAllowed) || publicEndpoint) {
         finalise(ctx, { authenticated: false, version, publicEndpoint })

packages/builder/cypress/setup.js

@@ -17,6 +17,7 @@ process.env.JWT_SECRET = cypressConfig.env.JWT_SECRET
 process.env.COUCH_URL = `leveldb://${tmpdir}/.data/`
 process.env.SELF_HOSTED = 1
 process.env.WORKER_URL = "http://localhost:10002/"
+process.env.APPS_URL = `http://localhost:${MAIN_PORT}/`
 process.env.MINIO_URL = `http://localhost:${MAIN_PORT}/`
 process.env.MINIO_ACCESS_KEY = "budibase"
 process.env.MINIO_SECRET_KEY = "budibase"

packages/server/scripts/likeCypress.ts

@@ -0,0 +1,33 @@
+/******************************************************
+ * This script just makes it easy to re-create        *
+ * a cypress like environment for testing the backend *
+ ******************************************************/
+const path = require("path")
+const tmpdir = path.join(require("os").tmpdir(), ".budibase")
+
+const MAIN_PORT = "10001"
+const WORKER_PORT = "10002"
+
+// @ts-ignore
+process.env.PORT = MAIN_PORT
+process.env.BUDIBASE_API_KEY = "6BE826CB-6B30-4AEC-8777-2E90464633DE"
+process.env.NODE_ENV = "cypress"
+process.env.ENABLE_ANALYTICS = "false"
+process.env.JWT_SECRET = "budibase"
+process.env.COUCH_URL = `leveldb://${tmpdir}/.data/`
+process.env.SELF_HOSTED = "1"
+process.env.WORKER_URL = `http://localhost:${WORKER_PORT}/`
+process.env.MINIO_URL = `http://localhost:${MAIN_PORT}/`
+process.env.MINIO_ACCESS_KEY = "budibase"
+process.env.MINIO_SECRET_KEY = "budibase"
+process.env.COUCH_DB_USER = "budibase"
+process.env.COUCH_DB_PASSWORD = "budibase"
+process.env.INTERNAL_API_KEY = "budibase"
+process.env.ALLOW_DEV_AUTOMATIONS = "1"
+
+// don't make this a variable or top level require
+// it will cause environment module to be loaded prematurely
+const server = require("../src/app")
+process.env.PORT = WORKER_PORT
+const worker = require("../../worker/src/index")
+process.env.PORT = MAIN_PORT

packages/server/src/api/controllers/application.js

@@ -255,6 +255,7 @@ exports.create = async ctx => {
     await createApp(appId)
   }
 
+  await appCache.invalidateAppMetadata(appId, newApplication)
   ctx.status = 200
   ctx.body = newApplication
 }

packages/server/src/api/controllers/dev.js

@@ -25,7 +25,8 @@ async function redirect(ctx, method, path = "global") {
     )
   )
   if (response.status !== 200) {
-    ctx.throw(response.status, response.statusText)
+    const err = await response.text()
+    ctx.throw(400, err)
   }
   const cookie = response.headers.get("set-cookie")
   if (cookie) {

packages/server/src/middleware/builder.js

@@ -51,8 +51,9 @@ async function updateAppUpdatedAt(ctx) {
   const db = new CouchDB(appId)
   const metadata = await db.get(DocumentTypes.APP_METADATA)
   metadata.updatedAt = new Date().toISOString()
-  await db.put(metadata)
+  const response = await db.put(metadata)
-  await appCache.invalidateAppMetadata(appId)
+  metadata._rev = response.rev
+  await appCache.invalidateAppMetadata(appId, metadata)
   // set a new debounce record with a short TTL
   await setDebounce(appId, DEBOUNCE_TIME_SEC)
 }