Log SQS connection errors, and retry connection a few times before giving up.

packages/shared-core/src/helpers/async.ts

@@ -0,0 +1,3 @@
+export async function wait(ms: number) {
+  return new Promise(resolve => setTimeout(resolve, ms))
+}

packages/shared-core/src/helpers/index.ts

@@ -1,5 +1,7 @@
 export * from "./helpers"
 export * from "./integrations"
+export * from "./async"
+export * from "./retry"
 export * as cron from "./cron"
 export * as schema from "./schema"
 export * as views from "./views"

packages/shared-core/src/helpers/retry.ts

@@ -0,0 +1,28 @@
+import { wait } from "./async"
+
+interface RetryOpts {
+  times?: number
+}
+
+export async function retry<T>(
+  fn: () => Promise<T>,
+  opts?: RetryOpts
+): Promise<T> {
+  const { times = 3 } = opts || {}
+  if (times < 1) {
+    throw new Error(`invalid retry count: ${times}`)
+  }
+
+  let lastError: any
+  for (let i = 0; i < times; i++) {
+    const backoff = 1.5 ** i * 1000 * (Math.random() + 0.5)
+    await wait(backoff)
+
+    try {
+      return await fn()
+    } catch (e) {
+      lastError = e
+    }
+  }
+  throw lastError
+}

packages/worker/src/api/controllers/system/environment.ts

@@ -2,6 +2,7 @@ import { Ctx, MaintenanceType, FeatureFlag } from "@budibase/types"
 import env from "../../../environment"
 import { env as coreEnv, db as dbCore, features } from "@budibase/backend-core"
 import nodeFetch from "node-fetch"
+import { retry } from "../../../../../shared-core/src/helpers"
 
 let sqsAvailable: boolean
 async function isSqsAvailable() {
@@ -12,17 +13,22 @@ async function isSqsAvailable() {
   }
 
   try {
-    const couchInfo = dbCore.getCouchInfo()
-    if (!couchInfo.sqlUrl) {
+    const { url } = dbCore.getCouchInfo()
+    if (!url) {
       sqsAvailable = false
       return false
     }
-    await nodeFetch(couchInfo.sqlUrl, {
-      timeout: 1000,
-    })
+    await retry(
+      async () => {
+        await nodeFetch(url, { timeout: 2000 })
+      },
+      { times: 3 }
+    )
+    console.log("connected to SQS")
     sqsAvailable = true
     return true
   } catch (e) {
+    console.warn("failed to connect to SQS", e)
     sqsAvailable = false
     return false
   }