From ff9d59e1ee8052ce89f55a8d030a72bbc323ff9b Mon Sep 17 00:00:00 2001
From: Sam Rose <hello@samwho.dev>
Date: Tue, 5 Nov 2024 15:35:38 +0000
Subject: [PATCH] Log SQS connection errors, and retry connection a few times
 before giving up.

---
 packages/shared-core/src/helpers/async.ts     |  3 ++
 packages/shared-core/src/helpers/index.ts     |  2 ++
 packages/shared-core/src/helpers/retry.ts     | 28 +++++++++++++++++++
 .../src/api/controllers/system/environment.ts | 16 +++++++----
 4 files changed, 44 insertions(+), 5 deletions(-)
 create mode 100644 packages/shared-core/src/helpers/async.ts
 create mode 100644 packages/shared-core/src/helpers/retry.ts

diff --git a/packages/shared-core/src/helpers/async.ts b/packages/shared-core/src/helpers/async.ts
new file mode 100644
index 0000000000..d9f64ac825
--- /dev/null
+++ b/packages/shared-core/src/helpers/async.ts
@@ -0,0 +1,3 @@
+export async function wait(ms: number) {
+  return new Promise(resolve => setTimeout(resolve, ms))
+}
diff --git a/packages/shared-core/src/helpers/index.ts b/packages/shared-core/src/helpers/index.ts
index 7603a9b88b..0aa378b46a 100644
--- a/packages/shared-core/src/helpers/index.ts
+++ b/packages/shared-core/src/helpers/index.ts
@@ -1,5 +1,7 @@
 export * from "./helpers"
 export * from "./integrations"
+export * from "./async"
+export * from "./retry"
 export * as cron from "./cron"
 export * as schema from "./schema"
 export * as views from "./views"
diff --git a/packages/shared-core/src/helpers/retry.ts b/packages/shared-core/src/helpers/retry.ts
new file mode 100644
index 0000000000..04494da227
--- /dev/null
+++ b/packages/shared-core/src/helpers/retry.ts
@@ -0,0 +1,28 @@
+import { wait } from "./async"
+
+interface RetryOpts {
+  times?: number
+}
+
+export async function retry<T>(
+  fn: () => Promise<T>,
+  opts?: RetryOpts
+): Promise<T> {
+  const { times = 3 } = opts || {}
+  if (times < 1) {
+    throw new Error(`invalid retry count: ${times}`)
+  }
+
+  let lastError: any
+  for (let i = 0; i < times; i++) {
+    const backoff = 1.5 ** i * 1000 * (Math.random() + 0.5)
+    await wait(backoff)
+
+    try {
+      return await fn()
+    } catch (e) {
+      lastError = e
+    }
+  }
+  throw lastError
+}
diff --git a/packages/worker/src/api/controllers/system/environment.ts b/packages/worker/src/api/controllers/system/environment.ts
index b6352ea5e7..1a1308beec 100644
--- a/packages/worker/src/api/controllers/system/environment.ts
+++ b/packages/worker/src/api/controllers/system/environment.ts
@@ -2,6 +2,7 @@ import { Ctx, MaintenanceType, FeatureFlag } from "@budibase/types"
 import env from "../../../environment"
 import { env as coreEnv, db as dbCore, features } from "@budibase/backend-core"
 import nodeFetch from "node-fetch"
+import { retry } from "../../../../../shared-core/src/helpers"
 
 let sqsAvailable: boolean
 async function isSqsAvailable() {
@@ -12,17 +13,22 @@ async function isSqsAvailable() {
   }
 
   try {
-    const couchInfo = dbCore.getCouchInfo()
-    if (!couchInfo.sqlUrl) {
+    const { url } = dbCore.getCouchInfo()
+    if (!url) {
       sqsAvailable = false
       return false
     }
-    await nodeFetch(couchInfo.sqlUrl, {
-      timeout: 1000,
-    })
+    await retry(
+      async () => {
+        await nodeFetch(url, { timeout: 2000 })
+      },
+      { times: 3 }
+    )
+    console.log("connected to SQS")
     sqsAvailable = true
     return true
   } catch (e) {
+    console.warn("failed to connect to SQS", e)
     sqsAvailable = false
     return false
   }