Merge pull request #12664 from Budibase/fix/budi-7857-redis-issue

Support Redis reconnection after connection interruption

packages/backend-core/src/queue/queue.ts

@@ -47,7 +47,7 @@ export function createQueue<T>(
     cleanupInterval = timers.set(cleanup, CLEANUP_PERIOD_MS)
     // fire off an initial cleanup
     cleanup().catch(err => {
-      console.error(`Unable to cleanup automation queue initially - ${err}`)
+      console.error(`Unable to cleanup ${jobQueue} initially - ${err}`)
     })
   }
   return queue

packages/backend-core/src/redis/redis.ts

@@ -18,6 +18,7 @@ import {
   SelectableDatabase,
   getRedisConnectionDetails,
 } from "./utils"
+import { logAlert } from "../logging"
 import * as timers from "../timers"
 
 const RETRY_PERIOD_MS = 2000
@@ -39,21 +40,16 @@ function pickClient(selectDb: number): any {
   return CLIENTS[selectDb]
 }
 
-function connectionError(
-  selectDb: number,
-  timeout: NodeJS.Timeout,
-  err: Error | string
-) {
+function connectionError(timeout: NodeJS.Timeout, err: Error | string) {
   // manually shut down, ignore errors
   if (CLOSED) {
     return
   }
-  pickClient(selectDb).disconnect()
   CLOSED = true
   // always clear this on error
   clearTimeout(timeout)
   CONNECTED = false
-  console.error("Redis connection failed - " + err)
+  logAlert("Redis connection failed", err)
   setTimeout(() => {
     init()
   }, RETRY_PERIOD_MS)
@@ -79,11 +75,7 @@ function init(selectDb = DEFAULT_SELECT_DB) {
   // start the timer - only allowed 5 seconds to connect
   timeout = setTimeout(() => {
     if (!CONNECTED) {
-      connectionError(
-        selectDb,
-        timeout,
-        "Did not successfully connect in timeout"
-      )
+      connectionError(timeout, "Did not successfully connect in timeout")
     }
   }, STARTUP_TIMEOUT_MS)
 
@@ -106,12 +98,13 @@ function init(selectDb = DEFAULT_SELECT_DB) {
       // allow the process to exit
       return
     }
-    connectionError(selectDb, timeout, err)
+    connectionError(timeout, err)
   })
   client.on("error", (err: Error) => {
-    connectionError(selectDb, timeout, err)
+    connectionError(timeout, err)
   })
   client.on("connect", () => {
+    console.log(`Connected to Redis DB: ${selectDb}`)
     clearTimeout(timeout)
     CONNECTED = true
   })