From 69d54b523dd08a33c401e13282fc1f3f22616fdd Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Mon, 8 Jul 2024 14:21:07 +0100 Subject: [PATCH] Finish implementation, fix tests. --- packages/backend-core/src/db/couch/utils.ts | 10 ---------- packages/backend-core/src/db/utils.ts | 18 ++++++++++++++++++ packages/backend-core/src/environment.ts | 2 ++ .../server/src/api/routes/tests/search.spec.ts | 2 +- .../src/api/routes/tests/templates.spec.ts | 4 +++- .../server/src/api/routes/tests/viewV2.spec.ts | 10 ++++++++-- .../server/src/appMigrations/migrations.ts | 2 +- .../migrations/20240604153647_initial_sqs.ts | 3 +-- .../tests/20240604153647_initial_sqs.spec.ts | 4 ++-- packages/server/src/environment.ts | 2 -- .../sdk/app/rows/search/tests/search.spec.ts | 11 +++++++++-- packages/server/src/sdk/app/tables/getters.ts | 3 +-- .../src/tests/utilities/TestConfiguration.ts | 10 +++++----- .../src/api/controllers/system/environment.ts | 2 +- packages/worker/src/api/index.ts | 3 +-- packages/worker/src/environment.ts | 1 - 16 files changed, 53 insertions(+), 34 deletions(-) diff --git a/packages/backend-core/src/db/couch/utils.ts b/packages/backend-core/src/db/couch/utils.ts index 8502bd9ab6..270d953320 100644 --- a/packages/backend-core/src/db/couch/utils.ts +++ b/packages/backend-core/src/db/couch/utils.ts @@ -1,8 +1,6 @@ import { getCouchInfo } from "./connections" import fetch from "node-fetch" import { checkSlashesInUrl } from "../../helpers" -import * as context from "../../context" -import env from "../../environment" export async function directCouchCall( path: string, @@ -55,11 +53,3 @@ export async function directCouchQuery( throw "Cannot connect to CouchDB instance" } } - -export function isSqsEnabledForTenant(): boolean { - const tenantId = context.getTenantId() - return ( - env.SQS_SEARCH_ENABLE !== undefined && - env.SQS_SEARCH_ENABLE_TENANTS.includes(tenantId) - ) -} diff --git a/packages/backend-core/src/db/utils.ts b/packages/backend-core/src/db/utils.ts index 906a95e1db..4362bd2c94 100644 --- a/packages/backend-core/src/db/utils.ts +++ b/packages/backend-core/src/db/utils.ts @@ -206,3 +206,21 @@ export function pagination( nextPage, } } + +export function isSqsEnabledForTenant(): boolean { + const tenantId = getTenantId() + if (!env.SQS_SEARCH_ENABLE) { + return false + } + + // This is to guard against the situation in tests where tests pass because + // we're not actually using SQS, we're using Lucene and the tests pass due to + // parity. + if (env.isTest() && env.SQS_SEARCH_ENABLE_TENANTS.length === 0) { + throw new Error( + "to enable SQS you must specify a list of tenants in the SQS_SEARCH_ENABLE_TENANTS env var" + ) + } + + return env.SQS_SEARCH_ENABLE_TENANTS.includes(tenantId) +} diff --git a/packages/backend-core/src/environment.ts b/packages/backend-core/src/environment.ts index 06256239f1..d3d4ab96ff 100644 --- a/packages/backend-core/src/environment.ts +++ b/packages/backend-core/src/environment.ts @@ -1,5 +1,6 @@ import { existsSync, readFileSync } from "fs" import { ServiceType } from "@budibase/types" +import { SQS } from "aws-sdk" function isTest() { return isJest() @@ -118,6 +119,7 @@ const environment = { SQS_SEARCH_ENABLE: process.env.SQS_SEARCH_ENABLE, SQS_SEARCH_ENABLE_TENANTS: process.env.SQS_SEARCH_ENABLE_TENANTS?.split(",") || [], + SQS_MIGRATION_ENABLE: process.env.SQS_MIGRATION_ENABLE, COUCH_DB_USERNAME: process.env.COUCH_DB_USER, COUCH_DB_PASSWORD: process.env.COUCH_DB_PASSWORD, GOOGLE_CLIENT_ID: process.env.GOOGLE_CLIENT_ID, diff --git a/packages/server/src/api/routes/tests/search.spec.ts b/packages/server/src/api/routes/tests/search.spec.ts index d9036b22fb..a64685d87f 100644 --- a/packages/server/src/api/routes/tests/search.spec.ts +++ b/packages/server/src/api/routes/tests/search.spec.ts @@ -55,7 +55,7 @@ describe.each([ beforeAll(async () => { if (isSqs) { - envCleanup = config.setEnv({ SQS_SEARCH_ENABLE: "true" }) + envCleanup = config.setCoreEnv({ SQS_SEARCH_ENABLE: "true" }) } await config.init() diff --git a/packages/server/src/api/routes/tests/templates.spec.ts b/packages/server/src/api/routes/tests/templates.spec.ts index 680ddb39d7..03f13ba88e 100644 --- a/packages/server/src/api/routes/tests/templates.spec.ts +++ b/packages/server/src/api/routes/tests/templates.spec.ts @@ -2,6 +2,7 @@ import * as setup from "./utilities" import path from "path" import nock from "nock" import { generator } from "@budibase/backend-core/tests" +import { SQS } from "aws-sdk" interface App { background: string @@ -86,9 +87,10 @@ describe("/templates", () => { async source => { const env = { SQS_SEARCH_ENABLE: source === "sqs" ? "true" : "false", + SQS_SEARCH_ENABLE_TENANTS: [config.getTenantId()], } - await config.withEnv(env, async () => { + await config.withCoreEnv(env, async () => { const name = generator.guid().replaceAll("-", "") const url = `/${name}` diff --git a/packages/server/src/api/routes/tests/viewV2.spec.ts b/packages/server/src/api/routes/tests/viewV2.spec.ts index 43a6d39172..fcfeb74236 100644 --- a/packages/server/src/api/routes/tests/viewV2.spec.ts +++ b/packages/server/src/api/routes/tests/viewV2.spec.ts @@ -88,10 +88,16 @@ describe.each([ } beforeAll(async () => { + await config.withCoreEnv( + { SQS_SEARCH_ENABLE: isSqs ? "true" : "false" }, + () => config.init() + ) if (isSqs) { - envCleanup = config.setEnv({ SQS_SEARCH_ENABLE: "true" }) + envCleanup = config.setCoreEnv({ + SQS_SEARCH_ENABLE: "true", + SQS_SEARCH_ENABLE_TENANTS: [config.getTenantId()], + }) } - await config.init() if (dsProvider) { datasource = await config.createDatasource({ diff --git a/packages/server/src/appMigrations/migrations.ts b/packages/server/src/appMigrations/migrations.ts index 7437753ada..247b74ebfc 100644 --- a/packages/server/src/appMigrations/migrations.ts +++ b/packages/server/src/appMigrations/migrations.ts @@ -1,6 +1,6 @@ // This file should never be manually modified, use `yarn add-app-migration` in order to add a new one -import env from "../environment" +import { env } from "@budibase/backend-core" import { AppMigration } from "." import m20240604153647_initial_sqs from "./migrations/20240604153647_initial_sqs" diff --git a/packages/server/src/appMigrations/migrations/20240604153647_initial_sqs.ts b/packages/server/src/appMigrations/migrations/20240604153647_initial_sqs.ts index da435705c1..ec7fba99e9 100644 --- a/packages/server/src/appMigrations/migrations/20240604153647_initial_sqs.ts +++ b/packages/server/src/appMigrations/migrations/20240604153647_initial_sqs.ts @@ -1,8 +1,7 @@ -import { context } from "@budibase/backend-core" +import { context, env } from "@budibase/backend-core" import { allLinkDocs } from "../../db/utils" import LinkDocumentImpl from "../../db/linkedRows/LinkDocument" import sdk from "../../sdk" -import env from "../../environment" const migration = async () => { const linkDocs = await allLinkDocs() diff --git a/packages/server/src/appMigrations/migrations/tests/20240604153647_initial_sqs.spec.ts b/packages/server/src/appMigrations/migrations/tests/20240604153647_initial_sqs.spec.ts index 0a34fb2bb4..4301cb791f 100644 --- a/packages/server/src/appMigrations/migrations/tests/20240604153647_initial_sqs.spec.ts +++ b/packages/server/src/appMigrations/migrations/tests/20240604153647_initial_sqs.spec.ts @@ -69,11 +69,11 @@ function oldLinkDocument(): Omit { type SQSEnvVar = "SQS_MIGRATION_ENABLE" | "SQS_SEARCH_ENABLE" async function sqsDisabled(envVar: SQSEnvVar, cb: () => Promise) { - await config.withEnv({ [envVar]: "" }, cb) + await config.withCoreEnv({ [envVar]: "" }, cb) } async function sqsEnabled(envVar: SQSEnvVar, cb: () => Promise) { - await config.withEnv({ [envVar]: "1" }, cb) + await config.withCoreEnv({ [envVar]: "1" }, cb) } describe.each(["SQS_MIGRATION_ENABLE", "SQS_SEARCH_ENABLE"] as SQSEnvVar[])( diff --git a/packages/server/src/environment.ts b/packages/server/src/environment.ts index b643bd50b0..28673dcaa8 100644 --- a/packages/server/src/environment.ts +++ b/packages/server/src/environment.ts @@ -87,8 +87,6 @@ const environment = { SQL_MAX_ROWS: process.env.SQL_MAX_ROWS, SQL_LOGGING_ENABLE: process.env.SQL_LOGGING_ENABLE, SQL_ALIASING_DISABLE: process.env.SQL_ALIASING_DISABLE, - SQS_SEARCH_ENABLE: process.env.SQS_SEARCH_ENABLE, - SQS_MIGRATION_ENABLE: process.env.SQS_MIGRATION_ENABLE, // flags ALLOW_DEV_AUTOMATIONS: process.env.ALLOW_DEV_AUTOMATIONS, DISABLE_THREADING: process.env.DISABLE_THREADING, diff --git a/packages/server/src/sdk/app/rows/search/tests/search.spec.ts b/packages/server/src/sdk/app/rows/search/tests/search.spec.ts index 2f347475f4..4b80d5bcd3 100644 --- a/packages/server/src/sdk/app/rows/search/tests/search.spec.ts +++ b/packages/server/src/sdk/app/rows/search/tests/search.spec.ts @@ -31,10 +31,17 @@ describe.each([ let rows: Row[] beforeAll(async () => { + await config.withCoreEnv( + { SQS_SEARCH_ENABLE: isSqs ? "true" : "false" }, + () => config.init() + ) + if (isSqs) { - envCleanup = config.setEnv({ SQS_SEARCH_ENABLE: "true" }) + envCleanup = config.setCoreEnv({ + SQS_SEARCH_ENABLE: "true", + SQS_SEARCH_ENABLE_TENANTS: [config.getTenantId()], + }) } - await config.init() if (dsProvider) { datasource = await config.createDatasource({ diff --git a/packages/server/src/sdk/app/tables/getters.ts b/packages/server/src/sdk/app/tables/getters.ts index a12933c966..a775a6d37e 100644 --- a/packages/server/src/sdk/app/tables/getters.ts +++ b/packages/server/src/sdk/app/tables/getters.ts @@ -1,4 +1,4 @@ -import { context, db } from "@budibase/backend-core" +import { context, db, env } from "@budibase/backend-core" import { getTableParams } from "../../../db/utils" import { breakExternalTableId, @@ -15,7 +15,6 @@ import { } from "@budibase/types" import datasources from "../datasources" import sdk from "../../../sdk" -import env from "../../../environment" export function processTable(table: Table): Table { if (!table) { diff --git a/packages/server/src/tests/utilities/TestConfiguration.ts b/packages/server/src/tests/utilities/TestConfiguration.ts index 793bfa8c6a..828b389add 100644 --- a/packages/server/src/tests/utilities/TestConfiguration.ts +++ b/packages/server/src/tests/utilities/TestConfiguration.ts @@ -245,10 +245,10 @@ export default class TestConfiguration { } } - async withEnv(newEnvVars: Partial, f: () => Promise) { + async withEnv(newEnvVars: Partial, f: () => Promise) { let cleanup = this.setEnv(newEnvVars) try { - await f() + return await f() } finally { cleanup() } @@ -273,13 +273,13 @@ export default class TestConfiguration { } } - async withCoreEnv( + async withCoreEnv( newEnvVars: Partial, - f: () => Promise + f: () => Promise ) { let cleanup = this.setCoreEnv(newEnvVars) try { - await f() + return await f() } finally { cleanup() } diff --git a/packages/worker/src/api/controllers/system/environment.ts b/packages/worker/src/api/controllers/system/environment.ts index 4deca5df6e..788b45f25d 100644 --- a/packages/worker/src/api/controllers/system/environment.ts +++ b/packages/worker/src/api/controllers/system/environment.ts @@ -24,7 +24,7 @@ async function isSqsAvailable() { } async function isSqsMissing() { - return env.SQS_SEARCH_ENABLE && !(await isSqsAvailable()) + return coreEnv.SQS_SEARCH_ENABLE && !(await isSqsAvailable()) } export const fetch = async (ctx: Ctx) => { diff --git a/packages/worker/src/api/index.ts b/packages/worker/src/api/index.ts index 08c65b98d4..8ab093a359 100644 --- a/packages/worker/src/api/index.ts +++ b/packages/worker/src/api/index.ts @@ -5,8 +5,7 @@ const compress = require("koa-compress") import zlib from "zlib" import { routes } from "./routes" import { middleware as pro, sdk } from "@budibase/pro" -import { auth, middleware } from "@budibase/backend-core" -import env from "../environment" +import { auth, middleware, env } from "@budibase/backend-core" if (env.SQS_SEARCH_ENABLE) { sdk.auditLogs.useSQLSearch() diff --git a/packages/worker/src/environment.ts b/packages/worker/src/environment.ts index d642d50846..9f7baf9e9b 100644 --- a/packages/worker/src/environment.ts +++ b/packages/worker/src/environment.ts @@ -46,7 +46,6 @@ const environment = { DISABLE_ACCOUNT_PORTAL: process.env.DISABLE_ACCOUNT_PORTAL, SMTP_FALLBACK_ENABLED: process.env.SMTP_FALLBACK_ENABLED, DISABLE_DEVELOPER_LICENSE: process.env.DISABLE_DEVELOPER_LICENSE, - SQS_SEARCH_ENABLE: process.env.SQS_SEARCH_ENABLE, BUDIBASE_ENVIRONMENT: process.env.BUDIBASE_ENVIRONMENT, // smtp SMTP_USER: process.env.SMTP_USER,