From 6b491815a5d8b237c6b142de47c39cac6ab07de8 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Mon, 11 Dec 2023 15:22:02 +0000 Subject: [PATCH 1/8] Add isSqsAvailable key to the environment endpoint. --- packages/backend-core/src/environment.ts | 1 + .../src/api/controllers/system/environment.ts | 20 +++++++++++++++++++ 2 files changed, 21 insertions(+) diff --git a/packages/backend-core/src/environment.ts b/packages/backend-core/src/environment.ts index ed882fe96a..138dbbd9e0 100644 --- a/packages/backend-core/src/environment.ts +++ b/packages/backend-core/src/environment.ts @@ -107,6 +107,7 @@ const environment = { ENCRYPTION_KEY: process.env.ENCRYPTION_KEY, API_ENCRYPTION_KEY: getAPIEncryptionKey(), COUCH_DB_URL: process.env.COUCH_DB_URL || "http://localhost:4005", + COUCH_DB_SQL_URL: process.env.COUCH_DB_SQL_URL || "http://localhost:4984", COUCH_DB_USERNAME: process.env.COUCH_DB_USER, COUCH_DB_PASSWORD: process.env.COUCH_DB_PASSWORD, GOOGLE_CLIENT_ID: process.env.GOOGLE_CLIENT_ID, diff --git a/packages/worker/src/api/controllers/system/environment.ts b/packages/worker/src/api/controllers/system/environment.ts index ade5f241e2..c619d484f7 100644 --- a/packages/worker/src/api/controllers/system/environment.ts +++ b/packages/worker/src/api/controllers/system/environment.ts @@ -1,6 +1,25 @@ import { Ctx } from "@budibase/types" import env from "../../../environment" import { env as coreEnv } from "@budibase/backend-core" +import nodeFetch from "node-fetch" + +let sqsAvailable: boolean +async function isSqsAvailable() { + if (sqsAvailable !== undefined) { + return sqsAvailable + } + + try { + await nodeFetch(coreEnv.COUCH_DB_SQL_URL, { + timeout: 1000, + }) + } catch (e) { + sqsAvailable = false + return false + } + sqsAvailable = true + return true +} export const fetch = async (ctx: Ctx) => { ctx.body = { @@ -11,5 +30,6 @@ export const fetch = async (ctx: Ctx) => { disableAccountPortal: env.DISABLE_ACCOUNT_PORTAL, baseUrl: env.PLATFORM_URL, isDev: env.isDev() && !env.isTest(), + isSqsAvailable: await isSqsAvailable(), } } From 2c3b3d03e1734730d52bea73845c0f19521107cf Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Mon, 11 Dec 2023 16:49:44 +0000 Subject: [PATCH 2/8] Respond to PR comments. --- .../src/api/controllers/system/environment.ts | 7 +++- .../routes/system/tests/environment.spec.ts | 17 +++++++++ .../worker/src/tests/TestConfiguration.ts | 35 +++++++++++++++++++ 3 files changed, 58 insertions(+), 1 deletion(-) diff --git a/packages/worker/src/api/controllers/system/environment.ts b/packages/worker/src/api/controllers/system/environment.ts index c619d484f7..b62da949ad 100644 --- a/packages/worker/src/api/controllers/system/environment.ts +++ b/packages/worker/src/api/controllers/system/environment.ts @@ -30,6 +30,11 @@ export const fetch = async (ctx: Ctx) => { disableAccountPortal: env.DISABLE_ACCOUNT_PORTAL, baseUrl: env.PLATFORM_URL, isDev: env.isDev() && !env.isTest(), - isSqsAvailable: await isSqsAvailable(), + } + + if (env.SELF_HOSTED) { + ctx.body.infrastructure = { + sqs: await isSqsAvailable(), + } } } diff --git a/packages/worker/src/api/routes/system/tests/environment.spec.ts b/packages/worker/src/api/routes/system/tests/environment.spec.ts index 897cc970cc..5dc2e552e1 100644 --- a/packages/worker/src/api/routes/system/tests/environment.spec.ts +++ b/packages/worker/src/api/routes/system/tests/environment.spec.ts @@ -27,5 +27,22 @@ describe("/api/system/environment", () => { offlineMode: false, }) }) + + it("returns the expected environment for self hosters", async () => { + config.withEnv({ SELF_HOSTED: true }, async () => { + const env = await config.api.environment.getEnvironment() + expect(env.body).toEqual({ + cloud: true, + disableAccountPortal: 0, + isDev: false, + multiTenancy: true, + baseUrl: "http://localhost:10000", + offlineMode: false, + infrastructure: { + sqs: false, + }, + }) + }) + }) }) }) diff --git a/packages/worker/src/tests/TestConfiguration.ts b/packages/worker/src/tests/TestConfiguration.ts index c43d1b9d13..41a074ee89 100644 --- a/packages/worker/src/tests/TestConfiguration.ts +++ b/packages/worker/src/tests/TestConfiguration.ts @@ -36,6 +36,7 @@ import { } from "@budibase/types" import API from "./api" import jwt, { Secret } from "jsonwebtoken" +import { cloneDeep } from "lodash" class TestConfiguration { server: any @@ -240,6 +241,40 @@ class TestConfiguration { return { message: "Admin user only endpoint.", status: 403 } } + async withEnv(newEnvVars: Partial, f: () => Promise) { + let cleanup = this.setEnv(newEnvVars) + try { + await f() + } finally { + cleanup() + } + } + + /* + * Sets the environment variables to the given values and returns a function + * that can be called to reset the environment variables to their original values. + */ + setEnv(newEnvVars: Partial): () => void { + const oldEnv = cloneDeep(env) + const oldCoreEnv = cloneDeep(coreEnv) + + let key: keyof typeof newEnvVars + for (key in newEnvVars) { + env._set(key, newEnvVars[key]) + coreEnv._set(key, newEnvVars[key]) + } + + return () => { + for (const [key, value] of Object.entries(oldEnv)) { + env._set(key, value) + } + + for (const [key, value] of Object.entries(oldCoreEnv)) { + coreEnv._set(key, value) + } + } + } + // USERS async createDefaultUser() { From e0243b495ce5bf25f3b3297049b9a133fc940166 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Mon, 11 Dec 2023 17:20:00 +0000 Subject: [PATCH 3/8] Fix tests. --- packages/server/src/tests/utilities/TestConfiguration.ts | 8 +++++++- packages/worker/src/api/controllers/system/environment.ts | 4 ++-- .../src/api/routes/system/tests/environment.spec.ts | 6 ++++-- packages/worker/src/tests/TestConfiguration.ts | 7 +++++++ 4 files changed, 20 insertions(+), 5 deletions(-) diff --git a/packages/server/src/tests/utilities/TestConfiguration.ts b/packages/server/src/tests/utilities/TestConfiguration.ts index afaad64723..cbcb7888aa 100644 --- a/packages/server/src/tests/utilities/TestConfiguration.ts +++ b/packages/server/src/tests/utilities/TestConfiguration.ts @@ -17,7 +17,6 @@ import { basicWebhook, } from "./structures" import { - auth, cache, constants, context, @@ -227,6 +226,13 @@ class TestConfiguration { } return () => { + for (const key in newEnvVars) { + // @ts-ignore + delete env[key] + // @ts-ignore + delete coreEnv[key] + } + for (const [key, value] of Object.entries(oldEnv)) { env._set(key, value) } diff --git a/packages/worker/src/api/controllers/system/environment.ts b/packages/worker/src/api/controllers/system/environment.ts index b62da949ad..bf9270607f 100644 --- a/packages/worker/src/api/controllers/system/environment.ts +++ b/packages/worker/src/api/controllers/system/environment.ts @@ -13,12 +13,12 @@ async function isSqsAvailable() { await nodeFetch(coreEnv.COUCH_DB_SQL_URL, { timeout: 1000, }) + sqsAvailable = true + return true } catch (e) { sqsAvailable = false return false } - sqsAvailable = true - return true } export const fetch = async (ctx: Ctx) => { diff --git a/packages/worker/src/api/routes/system/tests/environment.spec.ts b/packages/worker/src/api/routes/system/tests/environment.spec.ts index 5dc2e552e1..2efbfa07c9 100644 --- a/packages/worker/src/api/routes/system/tests/environment.spec.ts +++ b/packages/worker/src/api/routes/system/tests/environment.spec.ts @@ -1,5 +1,7 @@ import { TestConfiguration } from "../../../../tests" +jest.unmock("node-fetch") + describe("/api/system/environment", () => { const config = new TestConfiguration() @@ -29,10 +31,10 @@ describe("/api/system/environment", () => { }) it("returns the expected environment for self hosters", async () => { - config.withEnv({ SELF_HOSTED: true }, async () => { + await config.withEnv({ SELF_HOSTED: true }, async () => { const env = await config.api.environment.getEnvironment() expect(env.body).toEqual({ - cloud: true, + cloud: false, disableAccountPortal: 0, isDev: false, multiTenancy: true, diff --git a/packages/worker/src/tests/TestConfiguration.ts b/packages/worker/src/tests/TestConfiguration.ts index 41a074ee89..918083cf56 100644 --- a/packages/worker/src/tests/TestConfiguration.ts +++ b/packages/worker/src/tests/TestConfiguration.ts @@ -265,6 +265,13 @@ class TestConfiguration { } return () => { + for (const key in newEnvVars) { + // @ts-ignore + delete env[key] + // @ts-ignore + delete coreEnv[key] + } + for (const [key, value] of Object.entries(oldEnv)) { env._set(key, value) } From 867d68d4298de105a5b361fb3b584eeaa5a28e3b Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Mon, 11 Dec 2023 17:25:38 +0000 Subject: [PATCH 4/8] Cleanup. --- packages/server/src/tests/utilities/TestConfiguration.ts | 7 ------- packages/worker/src/tests/TestConfiguration.ts | 7 ------- 2 files changed, 14 deletions(-) diff --git a/packages/server/src/tests/utilities/TestConfiguration.ts b/packages/server/src/tests/utilities/TestConfiguration.ts index cbcb7888aa..82704220f8 100644 --- a/packages/server/src/tests/utilities/TestConfiguration.ts +++ b/packages/server/src/tests/utilities/TestConfiguration.ts @@ -226,13 +226,6 @@ class TestConfiguration { } return () => { - for (const key in newEnvVars) { - // @ts-ignore - delete env[key] - // @ts-ignore - delete coreEnv[key] - } - for (const [key, value] of Object.entries(oldEnv)) { env._set(key, value) } diff --git a/packages/worker/src/tests/TestConfiguration.ts b/packages/worker/src/tests/TestConfiguration.ts index 918083cf56..41a074ee89 100644 --- a/packages/worker/src/tests/TestConfiguration.ts +++ b/packages/worker/src/tests/TestConfiguration.ts @@ -265,13 +265,6 @@ class TestConfiguration { } return () => { - for (const key in newEnvVars) { - // @ts-ignore - delete env[key] - // @ts-ignore - delete coreEnv[key] - } - for (const [key, value] of Object.entries(oldEnv)) { env._set(key, value) } From 779bd8f230a0a455f28cb8c7c6d61f02d3801be7 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Mon, 11 Dec 2023 17:28:33 +0000 Subject: [PATCH 5/8] Remove core env overriding in worker TestConfiguration. Should be split out in server, too. --- packages/worker/src/tests/TestConfiguration.ts | 6 ------ 1 file changed, 6 deletions(-) diff --git a/packages/worker/src/tests/TestConfiguration.ts b/packages/worker/src/tests/TestConfiguration.ts index 41a074ee89..747b520296 100644 --- a/packages/worker/src/tests/TestConfiguration.ts +++ b/packages/worker/src/tests/TestConfiguration.ts @@ -256,22 +256,16 @@ class TestConfiguration { */ setEnv(newEnvVars: Partial): () => void { const oldEnv = cloneDeep(env) - const oldCoreEnv = cloneDeep(coreEnv) let key: keyof typeof newEnvVars for (key in newEnvVars) { env._set(key, newEnvVars[key]) - coreEnv._set(key, newEnvVars[key]) } return () => { for (const [key, value] of Object.entries(oldEnv)) { env._set(key, value) } - - for (const [key, value] of Object.entries(oldCoreEnv)) { - coreEnv._set(key, value) - } } } From 3a7c30c7f3386f36222b1f68e0a539bc1901ac2d Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Mon, 11 Dec 2023 17:54:02 +0000 Subject: [PATCH 6/8] Seeing if CI is happy if I remove the unmock call. --- packages/worker/src/api/routes/system/tests/environment.spec.ts | 2 -- 1 file changed, 2 deletions(-) diff --git a/packages/worker/src/api/routes/system/tests/environment.spec.ts b/packages/worker/src/api/routes/system/tests/environment.spec.ts index 2efbfa07c9..9fe707d288 100644 --- a/packages/worker/src/api/routes/system/tests/environment.spec.ts +++ b/packages/worker/src/api/routes/system/tests/environment.spec.ts @@ -1,7 +1,5 @@ import { TestConfiguration } from "../../../../tests" -jest.unmock("node-fetch") - describe("/api/system/environment", () => { const config = new TestConfiguration() From b4f0423ca331c31ab95493be11460309b2cc4c8c Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Tue, 12 Dec 2023 14:43:53 +0000 Subject: [PATCH 7/8] Respond to Adri's feedback. --- packages/worker/src/tests/TestConfiguration.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/worker/src/tests/TestConfiguration.ts b/packages/worker/src/tests/TestConfiguration.ts index 747b520296..8e163f0373 100644 --- a/packages/worker/src/tests/TestConfiguration.ts +++ b/packages/worker/src/tests/TestConfiguration.ts @@ -36,7 +36,7 @@ import { } from "@budibase/types" import API from "./api" import jwt, { Secret } from "jsonwebtoken" -import { cloneDeep } from "lodash" +import cloneDeep from "lodash/fp/cloneDeep" class TestConfiguration { server: any From 0a7af05f01874723041930149d2b4e1fb3b2d371 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Tue, 12 Dec 2023 14:46:08 +0000 Subject: [PATCH 8/8] Unmock node-fetch again. --- packages/worker/src/api/routes/system/tests/environment.spec.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/packages/worker/src/api/routes/system/tests/environment.spec.ts b/packages/worker/src/api/routes/system/tests/environment.spec.ts index 9fe707d288..2efbfa07c9 100644 --- a/packages/worker/src/api/routes/system/tests/environment.spec.ts +++ b/packages/worker/src/api/routes/system/tests/environment.spec.ts @@ -1,5 +1,7 @@ import { TestConfiguration } from "../../../../tests" +jest.unmock("node-fetch") + describe("/api/system/environment", () => { const config = new TestConfiguration()