Merge pull request #16083 from Budibase/ops/graceful-termination

Graceful termination of the worker/app servers

charts/budibase/templates/app-service-deployment.yaml

@@ -30,6 +30,7 @@ spec:
 {{- toYaml .Values.services.apps.templateLabels | indent 8 -}}
 {{ end }}
     spec:
+      terminationGracePeriodSeconds: 60
       containers:
       - env:
         - name: BUDIBASE_ENVIRONMENT

charts/budibase/templates/worker-service-deployment.yaml

@@ -30,6 +30,7 @@ spec:
 {{- toYaml .Values.services.worker.templateLabels | indent 8 -}}
 {{ end }}
     spec:
+      terminationGracePeriodSeconds: 60
       containers:
       - env:
         - name: BUDIBASE_ENVIRONMENT

packages/server/package.json

@@ -135,7 +135,8 @@
     "validate.js": "0.13.1",
     "worker-farm": "1.7.0",
     "xml2js": "0.6.2",
-    "zod-validation-error": "^3.4.0"
+    "zod-validation-error": "^3.4.0",
+    "http-graceful-shutdown": "^3.1.12"
   },
   "devDependencies": {
     "@babel/core": "^7.22.5",

packages/server/src/koa.ts

@@ -7,8 +7,8 @@ import * as automations from "./automations"
 import { Thread } from "./threads"
 import * as redis from "./utilities/redis"
 import { events, logging, middleware, timers } from "@budibase/backend-core"
-import destroyable from "server-destroy"
 import { userAgent } from "koa-useragent"
+import gracefulShutdown from "http-graceful-shutdown"
 
 export default function createKoaApp() {
   const app = new Koa()
@@ -40,55 +40,49 @@ export default function createKoaApp() {
   app.use(userAgent)
 
   const server = http.createServer(app.callback())
-  destroyable(server)
 
-  let shuttingDown = false,
-    errCode = 0
-
-  server.on("close", async () => {
-    // already in process
-    if (shuttingDown) {
-      return
-    }
-    shuttingDown = true
-    console.log("Server Closed")
+  const shutdown = async () => {
+    console.log("Server shutting down gracefully...")
     timers.cleanup()
     await automations.shutdown()
     await redis.shutdown()
     events.shutdown()
     await Thread.shutdown()
     api.shutdown()
-    if (!env.isTest()) {
-      process.exit(errCode)
+  }
+
+  gracefulShutdown(server, {
+    signals: "SIGINT SIGTERM",
+    timeout: 30000, // in ms
+    onShutdown: shutdown,
+    forceExit: !env.isTest,
+    finally: () => {
+      console.log("Server shutdown complete")
+    },
+  })
+
+  process.on("uncaughtException", async err => {
+    // @ts-ignore
+    // don't worry about this error, comes from zlib isn't important
+    if (err?.["code"] === "ERR_INVALID_CHAR") {
+      logging.logAlert("Uncaught exception.", err)
+      return
+    }
+    await shutdown()
+    if (!env.isTest) {
+      process.exit(1)
+    }
+  })
+
+  process.on("unhandledRejection", async reason => {
+    logging.logAlert("Unhandled Promise Rejection", reason as Error)
+    await shutdown()
+    if (!env.isTest) {
+      process.exit(1)
     }
   })
 
   const listener = server.listen(env.PORT || 0)
 
-  const shutdown = () => {
-    server.close()
-    // @ts-ignore
-    server.destroy()
-  }
-
-  process.on("uncaughtException", err => {
-    // @ts-ignore
-    // don't worry about this error, comes from zlib isn't important
-    if (err && err["code"] === "ERR_INVALID_CHAR") {
-      return
-    }
-    errCode = -1
-    logging.logAlert("Uncaught exception.", err)
-    shutdown()
-  })
-
-  process.on("SIGTERM", () => {
-    shutdown()
-  })
-
-  process.on("SIGINT", () => {
-    shutdown()
-  })
-
   return { app, server: listener }
 }

packages/worker/package.json

@@ -75,7 +75,8 @@
     "pouchdb": "7.3.0",
     "pouchdb-all-dbs": "1.1.1",
     "server-destroy": "1.0.1",
-    "uuid": "^8.3.2"
+    "uuid": "^8.3.2",
+    "http-graceful-shutdown": "^3.1.12"
   },
   "devDependencies": {
     "@jest/types": "^29.6.3",

packages/worker/src/index.ts

@@ -26,12 +26,12 @@ db.init()
 import koaBody from "koa-body"
 import http from "http"
 import api from "./api"
+import gracefulShutdown from "http-graceful-shutdown"
 
 const koaSession = require("koa-session")
 
 import { userAgent } from "koa-useragent"
 
-import destroyable from "server-destroy"
 import { initPro } from "./initPro"
 import { handleScimBody } from "./middleware/handleScimBody"
 
@@ -86,29 +86,40 @@ app.use(auth.passport.session())
 app.use(api.routes())
 
 const server = http.createServer(app.callback())
-destroyable(server)
 
-let shuttingDown = false,
-  errCode = 0
-server.on("close", async () => {
-  if (shuttingDown) {
-    return
-  }
-  shuttingDown = true
-  console.log("Server Closed")
+const shutdown = async () => {
+  console.log("Worker service shutting down gracefully...")
   timers.cleanup()
   events.shutdown()
   await redis.clients.shutdown()
   await queue.shutdown()
-  if (!env.isTest()) {
-    process.exit(errCode)
+}
+
+gracefulShutdown(server, {
+  signals: "SIGINT SIGTERM",
+  timeout: 30000,
+  onShutdown: shutdown,
+  forceExit: !env.isTest,
+  finally: () => {
+    console.log("Worker service shutdown complete")
+  },
+})
+
+process.on("uncaughtException", async err => {
+  logging.logAlert("Uncaught exception.", err)
+  await shutdown()
+  if (!env.isTest) {
+    process.exit(1)
   }
 })
 
-const shutdown = () => {
-  server.close()
-  server.destroy()
-}
+process.on("unhandledRejection", async reason => {
+  logging.logAlert("Unhandled Promise Rejection", reason as Error)
+  await shutdown()
+  if (!env.isTest) {
+    process.exit(1)
+  }
+})
 
 export default server.listen(parseInt(env.PORT || "4002"), async () => {
   let startupLog = `Worker running on ${JSON.stringify(server.address())}`
@@ -125,17 +136,3 @@ export default server.listen(parseInt(env.PORT || "4002"), async () => {
   // can't integrate directly into backend-core due to cyclic issues
   await events.processors.init(proSdk.auditLogs.write)
 })
-
-process.on("uncaughtException", err => {
-  errCode = -1
-  logging.logAlert("Uncaught exception.", err)
-  shutdown()
-})
-
-process.on("SIGTERM", () => {
-  shutdown()
-})
-
-process.on("SIGINT", () => {
-  shutdown()
-})

yarn.lock

@@ -13161,6 +13161,13 @@ http-errors@~1.6.2:
     setprototypeof "1.1.0"
     statuses ">= 1.4.0 < 2"
 
+http-graceful-shutdown@^3.1.12:
+  version "3.1.14"
+  resolved "https://registry.yarnpkg.com/http-graceful-shutdown/-/http-graceful-shutdown-3.1.14.tgz#a4d48ac5d985da18b4d35c050acd3ef10f02113d"
+  integrity sha512-aTbGAZDUtRt7gRmU+li7rt5WbJeemULZHLNrycJ1dRBU80Giut6NvzG8h5u1TW1zGHXkPGpEtoEKhPKogIRKdA==
+  dependencies:
+    debug "^4.3.4"
+
 http-proxy-agent@^4.0.1:
   version "4.0.1"
   resolved "https://registry.yarnpkg.com/http-proxy-agent/-/http-proxy-agent-4.0.1.tgz#8a8c8ef7f5932ccf953c296ca8291b95aa74aa3a"