From 95b8a4ea10911916b3d8fb1a502aaf293891ecd2 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Thu, 17 Aug 2023 16:39:25 +0100 Subject: [PATCH] Adding feature flagging, the option to only start the automations, or the API, meaning we can split the service if needed. --- packages/server/src/app.ts | 123 ++++-------------- packages/server/src/automations/index.ts | 4 + packages/server/src/automations/utils.ts | 6 +- packages/server/src/environment.ts | 2 + packages/server/src/features.ts | 36 +++++ packages/server/src/koa.ts | 102 +++++++++++++++ packages/server/src/startup.ts | 5 +- .../src/tests/utilities/TestConfiguration.ts | 4 +- packages/worker/src/environment.ts | 2 + packages/worker/src/features.ts | 26 ++++ 10 files changed, 205 insertions(+), 105 deletions(-) create mode 100644 packages/server/src/features.ts create mode 100644 packages/server/src/koa.ts create mode 100644 packages/worker/src/features.ts diff --git a/packages/server/src/app.ts b/packages/server/src/app.ts index d41f908059..6cb122480a 100644 --- a/packages/server/src/app.ts +++ b/packages/server/src/app.ts @@ -1,120 +1,41 @@ +import Sentry from "@sentry/node" + if (process.env.DD_APM_ENABLED) { require("./ddApm") } // need to load environment first import env from "./environment" - -import { ExtendableContext } from "koa" import * as db from "./db" db.init() -import Koa from "koa" -import koaBody from "koa-body" -import http from "http" -import * as api from "./api" -import * as automations from "./automations" -import { Thread } from "./threads" -import * as redis from "./utilities/redis" import { ServiceType } from "@budibase/types" -import { - events, - logging, - middleware, - timers, - env as coreEnv, -} from "@budibase/backend-core" +import { env as coreEnv } from "@budibase/backend-core" coreEnv._set("SERVICE_TYPE", ServiceType.APPS) +import { apiEnabled } from "./features" +import createKoaApp from "./koa" +import Koa from "koa" +import { Server } from "http" import { startup } from "./startup" -const Sentry = require("@sentry/node") -const destroyable = require("server-destroy") -const { userAgent } = require("koa-useragent") -const app = new Koa() +let app: Koa, server: Server -let mbNumber = parseInt(env.HTTP_MB_LIMIT || "10") -if (!mbNumber || isNaN(mbNumber)) { - mbNumber = 10 -} -// set up top level koa middleware -app.use( - koaBody({ - multipart: true, - formLimit: `${mbNumber}mb`, - jsonLimit: `${mbNumber}mb`, - textLimit: `${mbNumber}mb`, - // @ts-ignore - enableTypes: ["json", "form", "text"], - parsedMethods: ["POST", "PUT", "PATCH", "DELETE"], - }) -) - -app.use(middleware.correlation) -app.use(middleware.pino) -app.use(userAgent) - -if (env.isProd()) { - env._set("NODE_ENV", "production") - Sentry.init() - - app.on("error", (err: any, ctx: ExtendableContext) => { - Sentry.withScope(function (scope: any) { - scope.addEventProcessor(function (event: any) { - return Sentry.Handlers.parseRequest(event, ctx.request) - }) - Sentry.captureException(err) - }) - }) -} - -const server = http.createServer(app.callback()) -destroyable(server) - -let shuttingDown = false, - errCode = 0 - -server.on("close", async () => { - // already in process - if (shuttingDown) { - return +async function start() { + if (apiEnabled()) { + const koa = createKoaApp() + app = koa.app + server = koa.server } - shuttingDown = true - console.log("Server Closed") - timers.cleanup() - await automations.shutdown() - await redis.shutdown() - events.shutdown() - await Thread.shutdown() - api.shutdown() - if (!env.isTest()) { - process.exit(errCode) - } -}) - -export default server.listen(env.PORT || 0, async () => { await startup(app, server) -}) - -const shutdown = () => { - server.close() - // @ts-ignore - server.destroy() + if (env.isProd()) { + env._set("NODE_ENV", "production") + Sentry.init() + } } -process.on("uncaughtException", err => { - // @ts-ignore - // don't worry about this error, comes from zlib isn't important - if (err && err["code"] === "ERR_INVALID_CHAR") { - return - } - errCode = -1 - logging.logAlert("Uncaught exception.", err) - shutdown() +start().catch(err => { + console.error(`Failed server startup - ${err.message}`) }) -process.on("SIGTERM", () => { - shutdown() -}) - -process.on("SIGINT", () => { - shutdown() -}) +export function getServer() { + return app +} diff --git a/packages/server/src/automations/index.ts b/packages/server/src/automations/index.ts index 9bbab95a27..4ef3210932 100644 --- a/packages/server/src/automations/index.ts +++ b/packages/server/src/automations/index.ts @@ -2,6 +2,7 @@ import { processEvent } from "./utils" import { automationQueue } from "./bullboard" import { rebootTrigger } from "./triggers" import BullQueue from "bull" +import { automationsEnabled } from "../features" export { automationQueue } from "./bullboard" export { shutdown } from "./bullboard" @@ -12,6 +13,9 @@ export { BUILTIN_ACTION_DEFINITIONS, getActionDefinitions } from "./actions" * This module is built purely to kick off the worker farm and manage the inputs/outputs */ export async function init() { + if (!automationsEnabled()) { + return + } // this promise will not complete const promise = automationQueue.process(async job => { await processEvent(job) diff --git a/packages/server/src/automations/utils.ts b/packages/server/src/automations/utils.ts index 14835820d9..18d2d30f82 100644 --- a/packages/server/src/automations/utils.ts +++ b/packages/server/src/automations/utils.ts @@ -15,9 +15,13 @@ import { WebhookActionType, } from "@budibase/types" import sdk from "../sdk" +import { automationsEnabled } from "../features" const WH_STEP_ID = definitions.WEBHOOK.stepId -const Runner = new Thread(ThreadType.AUTOMATION) +let Runner: Thread +if (automationsEnabled()) { + Runner = new Thread(ThreadType.AUTOMATION) +} function loggingArgs( job: AutomationJob, diff --git a/packages/server/src/environment.ts b/packages/server/src/environment.ts index 2d3b717efd..06fd659911 100644 --- a/packages/server/src/environment.ts +++ b/packages/server/src/environment.ts @@ -38,6 +38,8 @@ function parseIntSafe(number?: string) { } const environment = { + // features + APP_FEATURES: process.env.APP_FEATURES, // important - prefer app port to generic port PORT: process.env.APP_PORT || process.env.PORT, COUCH_DB_URL: process.env.COUCH_DB_URL, diff --git a/packages/server/src/features.ts b/packages/server/src/features.ts new file mode 100644 index 0000000000..0120e48e2a --- /dev/null +++ b/packages/server/src/features.ts @@ -0,0 +1,36 @@ +import env from "./environment" + +enum AppFeature { + API = "api", + AUTOMATIONS = "automations", +} + +const featureList = processFeatureList() + +function processFeatureList() { + const fullList = Object.values(AppFeature) as string[] + let list + if (!env.APP_FEATURES) { + list = fullList + } else { + list = env.APP_FEATURES.split(",") + } + for (let feature of list) { + if (!fullList.includes(feature)) { + throw new Error(`Feature: ${feature} is not an allowed option`) + } + } + return list +} + +export function isFeatureEnabled(feature: AppFeature) { + return featureList.includes(feature) +} + +export function automationsEnabled() { + return featureList.includes(AppFeature.AUTOMATIONS) +} + +export function apiEnabled() { + return featureList.includes(AppFeature.API) +} diff --git a/packages/server/src/koa.ts b/packages/server/src/koa.ts new file mode 100644 index 0000000000..de11bf973a --- /dev/null +++ b/packages/server/src/koa.ts @@ -0,0 +1,102 @@ +import env from "./environment" +import { ExtendableContext } from "koa" +import Koa from "koa" +import koaBody from "koa-body" +import http from "http" +import * as api from "./api" +import * as automations from "./automations" +import { Thread } from "./threads" +import * as redis from "./utilities/redis" +import { events, logging, middleware, timers } from "@budibase/backend-core" +const Sentry = require("@sentry/node") +const destroyable = require("server-destroy") +const { userAgent } = require("koa-useragent") + +export default function createKoaApp() { + const app = new Koa() + + let mbNumber = parseInt(env.HTTP_MB_LIMIT || "10") + if (!mbNumber || isNaN(mbNumber)) { + mbNumber = 10 + } + // set up top level koa middleware + app.use( + koaBody({ + multipart: true, + formLimit: `${mbNumber}mb`, + jsonLimit: `${mbNumber}mb`, + textLimit: `${mbNumber}mb`, + // @ts-ignore + enableTypes: ["json", "form", "text"], + parsedMethods: ["POST", "PUT", "PATCH", "DELETE"], + }) + ) + + app.use(middleware.correlation) + app.use(middleware.pino) + app.use(userAgent) + + if (env.isProd()) { + app.on("error", (err: any, ctx: ExtendableContext) => { + Sentry.withScope(function (scope: any) { + scope.addEventProcessor(function (event: any) { + return Sentry.Handlers.parseRequest(event, ctx.request) + }) + Sentry.captureException(err) + }) + }) + } + + const server = http.createServer(app.callback()) + destroyable(server) + + let shuttingDown = false, + errCode = 0 + + server.on("close", async () => { + // already in process + if (shuttingDown) { + return + } + shuttingDown = true + console.log("Server Closed") + timers.cleanup() + await automations.shutdown() + await redis.shutdown() + events.shutdown() + await Thread.shutdown() + api.shutdown() + if (!env.isTest()) { + process.exit(errCode) + } + }) + + const listener = server.listen(env.PORT || 0) + + const shutdown = () => { + server.close() + // @ts-ignore + server.destroy() + } + + process.on("uncaughtException", err => { + // @ts-ignore + // don't worry about this error, comes from zlib isn't important + if (err && err["code"] === "ERR_INVALID_CHAR") { + return + } + errCode = -1 + logging.logAlert("Uncaught exception.", err) + shutdown() + }) + + process.on("SIGTERM", () => { + shutdown() + }) + + process.on("SIGINT", () => { + shutdown() + }) + + return { app, server: listener } +} diff --git a/packages/server/src/startup.ts b/packages/server/src/startup.ts index 9da26ac2aa..b4a287d2d4 100644 --- a/packages/server/src/startup.ts +++ b/packages/server/src/startup.ts @@ -17,6 +17,7 @@ import * as pro from "@budibase/pro" import * as api from "./api" import sdk from "./sdk" import { initialise as initialiseWebsockets } from "./websockets" +import { automationsEnabled } from "./features" let STARTUP_RAN = false @@ -97,7 +98,9 @@ export async function startup(app?: any, server?: any) { // configure events to use the pro audit log write // can't integrate directly into backend-core due to cyclic issues queuePromises.push(events.processors.init(pro.sdk.auditLogs.write)) - queuePromises.push(automations.init()) + if (automationsEnabled()) { + queuePromises.push(automations.init()) + } queuePromises.push(initPro()) if (app) { // bring routes online as final step once everything ready diff --git a/packages/server/src/tests/utilities/TestConfiguration.ts b/packages/server/src/tests/utilities/TestConfiguration.ts index a93c78d5fc..c8b917f626 100644 --- a/packages/server/src/tests/utilities/TestConfiguration.ts +++ b/packages/server/src/tests/utilities/TestConfiguration.ts @@ -87,7 +87,7 @@ class TestConfiguration { if (openServer) { // use a random port because it doesn't matter env.PORT = "0" - this.server = require("../../app").default + this.server = require("../../app").getServer() // we need the request for logging in, involves cookies, hard to fake this.request = supertest(this.server) this.started = true @@ -178,7 +178,7 @@ class TestConfiguration { if (this.server) { this.server.close() } else { - require("../../app").default.close() + require("../../app").getServer().close() } if (this.allApps) { cleanup(this.allApps.map(app => app.appId)) diff --git a/packages/worker/src/environment.ts b/packages/worker/src/environment.ts index 6ef6dab03c..c357ceb65b 100644 --- a/packages/worker/src/environment.ts +++ b/packages/worker/src/environment.ts @@ -31,6 +31,8 @@ function parseIntSafe(number: any) { } const environment = { + // features + WORKER_FEATURES: process.env.WORKER_FEATURES, // auth MINIO_ACCESS_KEY: process.env.MINIO_ACCESS_KEY, MINIO_SECRET_KEY: process.env.MINIO_SECRET_KEY, diff --git a/packages/worker/src/features.ts b/packages/worker/src/features.ts new file mode 100644 index 0000000000..c9b5af7fad --- /dev/null +++ b/packages/worker/src/features.ts @@ -0,0 +1,26 @@ +import env from "./environment" + +enum WorkerFeature {} + +const featureList: WorkerFeature[] = processFeatureList() + +function processFeatureList() { + const fullList = Object.values(WorkerFeature) as string[] + let list + if (!env.WORKER_FEATURES) { + list = fullList + } else { + list = env.WORKER_FEATURES.split(",") + } + for (let feature of list) { + if (!fullList.includes(feature)) { + throw new Error(`Feature: ${feature} is not an allowed option`) + } + } + // casting ok - confirmed definitely is a list of worker features + return list as unknown as WorkerFeature[] +} + +export function isFeatureEnabled(feature: WorkerFeature) { + return featureList.includes(feature) +}