diff --git a/charts/budibase/templates/app-service-deployment.yaml b/charts/budibase/templates/app-service-deployment.yaml index 278bd1767f..171bf5e6c1 100644 --- a/charts/budibase/templates/app-service-deployment.yaml +++ b/charts/budibase/templates/app-service-deployment.yaml @@ -30,6 +30,7 @@ spec: {{- toYaml .Values.services.apps.templateLabels | indent 8 -}} {{ end }} spec: + terminationGracePeriodSeconds: 60 containers: - env: - name: BUDIBASE_ENVIRONMENT diff --git a/charts/budibase/templates/worker-service-deployment.yaml b/charts/budibase/templates/worker-service-deployment.yaml index 94fdd0b94e..a68d466487 100644 --- a/charts/budibase/templates/worker-service-deployment.yaml +++ b/charts/budibase/templates/worker-service-deployment.yaml @@ -30,6 +30,7 @@ spec: {{- toYaml .Values.services.worker.templateLabels | indent 8 -}} {{ end }} spec: + terminationGracePeriodSeconds: 60 containers: - env: - name: BUDIBASE_ENVIRONMENT diff --git a/packages/server/package.json b/packages/server/package.json index 2915e45069..253db3d5c2 100644 --- a/packages/server/package.json +++ b/packages/server/package.json @@ -135,7 +135,8 @@ "validate.js": "0.13.1", "worker-farm": "1.7.0", "xml2js": "0.6.2", - "zod-validation-error": "^3.4.0" + "zod-validation-error": "^3.4.0", + "http-graceful-shutdown": "^3.1.12" }, "devDependencies": { "@babel/core": "^7.22.5", diff --git a/packages/server/src/koa.ts b/packages/server/src/koa.ts index acae433cc3..c9da38d459 100644 --- a/packages/server/src/koa.ts +++ b/packages/server/src/koa.ts @@ -7,8 +7,8 @@ import * as automations from "./automations" import { Thread } from "./threads" import * as redis from "./utilities/redis" import { events, logging, middleware, timers } from "@budibase/backend-core" -import destroyable from "server-destroy" import { userAgent } from "koa-useragent" +import gracefulShutdown from "http-graceful-shutdown" export default function createKoaApp() { const app = new Koa() @@ -40,55 +40,49 @@ export default function createKoaApp() { app.use(userAgent) const server = http.createServer(app.callback()) - destroyable(server) - let shuttingDown = false, - errCode = 0 - - server.on("close", async () => { - // already in process - if (shuttingDown) { - return - } - shuttingDown = true - console.log("Server Closed") + const shutdown = async () => { + console.log("Server shutting down gracefully...") timers.cleanup() await automations.shutdown() await redis.shutdown() events.shutdown() await Thread.shutdown() api.shutdown() - if (!env.isTest()) { - process.exit(errCode) + } + + gracefulShutdown(server, { + signals: "SIGINT SIGTERM", + timeout: 30000, // in ms + onShutdown: shutdown, + forceExit: !env.isTest, + finally: () => { + console.log("Server shutdown complete") + }, + }) + + process.on("uncaughtException", async err => { + // @ts-ignore + // don't worry about this error, comes from zlib isn't important + if (err?.["code"] === "ERR_INVALID_CHAR") { + logging.logAlert("Uncaught exception.", err) + return + } + await shutdown() + if (!env.isTest) { + process.exit(1) + } + }) + + process.on("unhandledRejection", async reason => { + logging.logAlert("Unhandled Promise Rejection", reason as Error) + await shutdown() + if (!env.isTest) { + process.exit(1) } }) const listener = server.listen(env.PORT || 0) - const shutdown = () => { - server.close() - // @ts-ignore - server.destroy() - } - - process.on("uncaughtException", err => { - // @ts-ignore - // don't worry about this error, comes from zlib isn't important - if (err && err["code"] === "ERR_INVALID_CHAR") { - return - } - errCode = -1 - logging.logAlert("Uncaught exception.", err) - shutdown() - }) - - process.on("SIGTERM", () => { - shutdown() - }) - - process.on("SIGINT", () => { - shutdown() - }) - return { app, server: listener } } diff --git a/packages/worker/package.json b/packages/worker/package.json index 135e0528ac..4ed98fd532 100644 --- a/packages/worker/package.json +++ b/packages/worker/package.json @@ -75,7 +75,8 @@ "pouchdb": "7.3.0", "pouchdb-all-dbs": "1.1.1", "server-destroy": "1.0.1", - "uuid": "^8.3.2" + "uuid": "^8.3.2", + "http-graceful-shutdown": "^3.1.12" }, "devDependencies": { "@jest/types": "^29.6.3", diff --git a/packages/worker/src/index.ts b/packages/worker/src/index.ts index bfb022f213..767de5c7b2 100644 --- a/packages/worker/src/index.ts +++ b/packages/worker/src/index.ts @@ -26,12 +26,12 @@ db.init() import koaBody from "koa-body" import http from "http" import api from "./api" +import gracefulShutdown from "http-graceful-shutdown" const koaSession = require("koa-session") import { userAgent } from "koa-useragent" -import destroyable from "server-destroy" import { initPro } from "./initPro" import { handleScimBody } from "./middleware/handleScimBody" @@ -86,29 +86,40 @@ app.use(auth.passport.session()) app.use(api.routes()) const server = http.createServer(app.callback()) -destroyable(server) -let shuttingDown = false, - errCode = 0 -server.on("close", async () => { - if (shuttingDown) { - return - } - shuttingDown = true - console.log("Server Closed") +const shutdown = async () => { + console.log("Worker service shutting down gracefully...") timers.cleanup() events.shutdown() await redis.clients.shutdown() await queue.shutdown() - if (!env.isTest()) { - process.exit(errCode) +} + +gracefulShutdown(server, { + signals: "SIGINT SIGTERM", + timeout: 30000, + onShutdown: shutdown, + forceExit: !env.isTest, + finally: () => { + console.log("Worker service shutdown complete") + }, +}) + +process.on("uncaughtException", async err => { + logging.logAlert("Uncaught exception.", err) + await shutdown() + if (!env.isTest) { + process.exit(1) } }) -const shutdown = () => { - server.close() - server.destroy() -} +process.on("unhandledRejection", async reason => { + logging.logAlert("Unhandled Promise Rejection", reason as Error) + await shutdown() + if (!env.isTest) { + process.exit(1) + } +}) export default server.listen(parseInt(env.PORT || "4002"), async () => { let startupLog = `Worker running on ${JSON.stringify(server.address())}` @@ -125,17 +136,3 @@ export default server.listen(parseInt(env.PORT || "4002"), async () => { // can't integrate directly into backend-core due to cyclic issues await events.processors.init(proSdk.auditLogs.write) }) - -process.on("uncaughtException", err => { - errCode = -1 - logging.logAlert("Uncaught exception.", err) - shutdown() -}) - -process.on("SIGTERM", () => { - shutdown() -}) - -process.on("SIGINT", () => { - shutdown() -}) diff --git a/yarn.lock b/yarn.lock index e9fe907277..a26d093ae7 100644 --- a/yarn.lock +++ b/yarn.lock @@ -13161,6 +13161,13 @@ http-errors@~1.6.2: setprototypeof "1.1.0" statuses ">= 1.4.0 < 2" +http-graceful-shutdown@^3.1.12: + version "3.1.14" + resolved "https://registry.yarnpkg.com/http-graceful-shutdown/-/http-graceful-shutdown-3.1.14.tgz#a4d48ac5d985da18b4d35c050acd3ef10f02113d" + integrity sha512-aTbGAZDUtRt7gRmU+li7rt5WbJeemULZHLNrycJ1dRBU80Giut6NvzG8h5u1TW1zGHXkPGpEtoEKhPKogIRKdA== + dependencies: + debug "^4.3.4" + http-proxy-agent@^4.0.1: version "4.0.1" resolved "https://registry.yarnpkg.com/http-proxy-agent/-/http-proxy-agent-4.0.1.tgz#8a8c8ef7f5932ccf953c296ca8291b95aa74aa3a"