diff --git a/packages/shared-core/src/helpers/async.ts b/packages/shared-core/src/helpers/async.ts new file mode 100644 index 0000000000..d9f64ac825 --- /dev/null +++ b/packages/shared-core/src/helpers/async.ts @@ -0,0 +1,3 @@ +export async function wait(ms: number) { + return new Promise(resolve => setTimeout(resolve, ms)) +} diff --git a/packages/shared-core/src/helpers/index.ts b/packages/shared-core/src/helpers/index.ts index 7603a9b88b..0aa378b46a 100644 --- a/packages/shared-core/src/helpers/index.ts +++ b/packages/shared-core/src/helpers/index.ts @@ -1,5 +1,7 @@ export * from "./helpers" export * from "./integrations" +export * from "./async" +export * from "./retry" export * as cron from "./cron" export * as schema from "./schema" export * as views from "./views" diff --git a/packages/shared-core/src/helpers/retry.ts b/packages/shared-core/src/helpers/retry.ts new file mode 100644 index 0000000000..04494da227 --- /dev/null +++ b/packages/shared-core/src/helpers/retry.ts @@ -0,0 +1,28 @@ +import { wait } from "./async" + +interface RetryOpts { + times?: number +} + +export async function retry( + fn: () => Promise, + opts?: RetryOpts +): Promise { + const { times = 3 } = opts || {} + if (times < 1) { + throw new Error(`invalid retry count: ${times}`) + } + + let lastError: any + for (let i = 0; i < times; i++) { + const backoff = 1.5 ** i * 1000 * (Math.random() + 0.5) + await wait(backoff) + + try { + return await fn() + } catch (e) { + lastError = e + } + } + throw lastError +} diff --git a/packages/worker/src/api/controllers/system/environment.ts b/packages/worker/src/api/controllers/system/environment.ts index b6352ea5e7..1a1308beec 100644 --- a/packages/worker/src/api/controllers/system/environment.ts +++ b/packages/worker/src/api/controllers/system/environment.ts @@ -2,6 +2,7 @@ import { Ctx, MaintenanceType, FeatureFlag } from "@budibase/types" import env from "../../../environment" import { env as coreEnv, db as dbCore, features } from "@budibase/backend-core" import nodeFetch from "node-fetch" +import { retry } from "../../../../../shared-core/src/helpers" let sqsAvailable: boolean async function isSqsAvailable() { @@ -12,17 +13,22 @@ async function isSqsAvailable() { } try { - const couchInfo = dbCore.getCouchInfo() - if (!couchInfo.sqlUrl) { + const { url } = dbCore.getCouchInfo() + if (!url) { sqsAvailable = false return false } - await nodeFetch(couchInfo.sqlUrl, { - timeout: 1000, - }) + await retry( + async () => { + await nodeFetch(url, { timeout: 2000 }) + }, + { times: 3 } + ) + console.log("connected to SQS") sqsAvailable = true return true } catch (e) { + console.warn("failed to connect to SQS", e) sqsAvailable = false return false }