diff --git a/charts/budibase/Chart.yaml b/charts/budibase/Chart.yaml index 694c8c77fe..227a515432 100644 --- a/charts/budibase/Chart.yaml +++ b/charts/budibase/Chart.yaml @@ -11,7 +11,7 @@ sources: - https://github.com/Budibase/budibase - https://budibase.com type: application -version: 0.2.9 +version: 0.2.10 appVersion: 1.0.48 dependencies: - name: couchdb diff --git a/charts/budibase/templates/app-service-deployment.yaml b/charts/budibase/templates/app-service-deployment.yaml index 98a949418c..2e5e923b3e 100644 --- a/charts/budibase/templates/app-service-deployment.yaml +++ b/charts/budibase/templates/app-service-deployment.yaml @@ -78,6 +78,10 @@ spec: value: {{ .Values.services.objectStore.url }} - name: PORT value: {{ .Values.services.apps.port | quote }} + {{ if .Values.services.worker.publicApiRateLimitPerSecond }} + - name: API_REQ_LIMIT_PER_SEC + value: {{ .Values.globals.apps.publicApiRateLimitPerSecond | quote }} + {{ end }} - name: MULTI_TENANCY value: {{ .Values.globals.multiTenancy | quote }} - name: LOG_LEVEL @@ -119,6 +123,12 @@ spec: image: budibase/apps:{{ .Values.globals.appVersion }} imagePullPolicy: Always + livenessProbe: + httpGet: + path: /health + port: {{ .Values.services.apps.port }} + initialDelaySeconds: 5 + periodSeconds: 5 name: bbapps ports: - containerPort: {{ .Values.services.apps.port }} diff --git a/charts/budibase/templates/worker-service-deployment.yaml b/charts/budibase/templates/worker-service-deployment.yaml index 15ff05e214..8a053032d6 100644 --- a/charts/budibase/templates/worker-service-deployment.yaml +++ b/charts/budibase/templates/worker-service-deployment.yaml @@ -119,6 +119,12 @@ spec: value: {{ .Values.globals.google.secret | quote }} image: budibase/worker:{{ .Values.globals.appVersion }} imagePullPolicy: Always + livenessProbe: + httpGet: + path: /health + port: {{ .Values.services.worker.port }} + initialDelaySeconds: 5 + periodSeconds: 5 name: bbworker ports: - containerPort: {{ .Values.services.worker.port }} diff --git a/lerna.json b/lerna.json index 716f366af9..806b5a6cab 100644 --- a/lerna.json +++ b/lerna.json @@ -1,5 +1,5 @@ { - "version": "1.0.189", + "version": "1.0.191", "npmClient": "yarn", "packages": [ "packages/*" diff --git a/packages/backend-core/logging.js b/packages/backend-core/logging.js new file mode 100644 index 0000000000..da40fe3100 --- /dev/null +++ b/packages/backend-core/logging.js @@ -0,0 +1 @@ +module.exports = require("./src/logging") diff --git a/packages/backend-core/package.json b/packages/backend-core/package.json index 453c6e164f..589c676fb5 100644 --- a/packages/backend-core/package.json +++ b/packages/backend-core/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/backend-core", - "version": "1.0.189", + "version": "1.0.191", "description": "Budibase backend core libraries used in server and worker", "main": "src/index.js", "author": "Budibase", diff --git a/packages/backend-core/src/logging.js b/packages/backend-core/src/logging.js new file mode 100644 index 0000000000..425d7f8133 --- /dev/null +++ b/packages/backend-core/src/logging.js @@ -0,0 +1,16 @@ +const NonErrors = ["AccountError"] + +function isSuppressed(e) { + return e && e["suppressAlert"] +} + +module.exports.logAlert = (message, e = null) => { + if (e && NonErrors.includes(e.name) && isSuppressed(e)) { + return + } + let errorJson = "" + if (e) { + errorJson = ": " + JSON.stringify(e, Object.getOwnPropertyNames(e)) + } + console.error(`bb-alert: ${message} ${errorJson}`) +} diff --git a/packages/backend-core/src/redis/index.js b/packages/backend-core/src/redis/index.js index 0ee17265ce..158b5e3841 100644 --- a/packages/backend-core/src/redis/index.js +++ b/packages/backend-core/src/redis/index.js @@ -23,7 +23,7 @@ function connectionError(timeout, err) { if (CLOSED) { return } - CLIENT.end() + CLIENT.disconnect() CLOSED = true // always clear this on error clearTimeout(timeout) diff --git a/packages/bbui/package.json b/packages/bbui/package.json index 78628f9e68..bff9ae4f0f 100644 --- a/packages/bbui/package.json +++ b/packages/bbui/package.json @@ -1,7 +1,7 @@ { "name": "@budibase/bbui", "description": "A UI solution used in the different Budibase projects.", - "version": "1.0.189", + "version": "1.0.191", "license": "MPL-2.0", "svelte": "src/index.js", "module": "dist/bbui.es.js", @@ -38,7 +38,7 @@ ], "dependencies": { "@adobe/spectrum-css-workflow-icons": "^1.2.1", - "@budibase/string-templates": "^1.0.189", + "@budibase/string-templates": "^1.0.191", "@spectrum-css/actionbutton": "^1.0.1", "@spectrum-css/actiongroup": "^1.0.1", "@spectrum-css/avatar": "^3.0.2", diff --git a/packages/builder/package.json b/packages/builder/package.json index 9dcda69a53..8df9541552 100644 --- a/packages/builder/package.json +++ b/packages/builder/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/builder", - "version": "1.0.189", + "version": "1.0.191", "license": "GPL-3.0", "private": true, "scripts": { @@ -69,10 +69,10 @@ } }, "dependencies": { - "@budibase/bbui": "^1.0.189", - "@budibase/client": "^1.0.189", - "@budibase/frontend-core": "^1.0.189", - "@budibase/string-templates": "^1.0.189", + "@budibase/bbui": "^1.0.191", + "@budibase/client": "^1.0.191", + "@budibase/frontend-core": "^1.0.191", + "@budibase/string-templates": "^1.0.191", "@sentry/browser": "5.19.1", "@spectrum-css/page": "^3.0.1", "@spectrum-css/vars": "^3.0.1", diff --git a/packages/cli/package.json b/packages/cli/package.json index a500b8b539..802c3025c5 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/cli", - "version": "1.0.189", + "version": "1.0.191", "description": "Budibase CLI, for developers, self hosting and migrations.", "main": "src/index.js", "bin": { diff --git a/packages/client/package.json b/packages/client/package.json index 31980da192..de9a2aaa21 100644 --- a/packages/client/package.json +++ b/packages/client/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/client", - "version": "1.0.189", + "version": "1.0.191", "license": "MPL-2.0", "module": "dist/budibase-client.js", "main": "dist/budibase-client.js", @@ -19,9 +19,9 @@ "dev:builder": "rollup -cw" }, "dependencies": { - "@budibase/bbui": "^1.0.189", - "@budibase/frontend-core": "^1.0.189", - "@budibase/string-templates": "^1.0.189", + "@budibase/bbui": "^1.0.191", + "@budibase/frontend-core": "^1.0.191", + "@budibase/string-templates": "^1.0.191", "@spectrum-css/button": "^3.0.3", "@spectrum-css/card": "^3.0.3", "@spectrum-css/divider": "^1.0.3", diff --git a/packages/frontend-core/package.json b/packages/frontend-core/package.json index ca4e82c56c..74fc1c9baa 100644 --- a/packages/frontend-core/package.json +++ b/packages/frontend-core/package.json @@ -1,12 +1,12 @@ { "name": "@budibase/frontend-core", - "version": "1.0.189", + "version": "1.0.191", "description": "Budibase frontend core libraries used in builder and client", "author": "Budibase", "license": "MPL-2.0", "svelte": "src/index.js", "dependencies": { - "@budibase/bbui": "^1.0.189", + "@budibase/bbui": "^1.0.191", "lodash": "^4.17.21", "svelte": "^3.46.2" } diff --git a/packages/server/package.json b/packages/server/package.json index 19397b9283..9bdb36d043 100644 --- a/packages/server/package.json +++ b/packages/server/package.json @@ -1,7 +1,7 @@ { "name": "@budibase/server", "email": "hi@budibase.com", - "version": "1.0.189", + "version": "1.0.191", "description": "Budibase Web Server", "main": "src/index.ts", "repository": { @@ -70,10 +70,10 @@ "license": "GPL-3.0", "dependencies": { "@apidevtools/swagger-parser": "^10.0.3", - "@budibase/backend-core": "^1.0.189", - "@budibase/client": "^1.0.189", - "@budibase/pro": "1.0.189", - "@budibase/string-templates": "^1.0.189", + "@budibase/backend-core": "^1.0.191", + "@budibase/client": "^1.0.191", + "@budibase/pro": "1.0.191", + "@budibase/string-templates": "^1.0.191", "@bull-board/api": "^3.7.0", "@bull-board/koa": "^3.7.0", "@elastic/elasticsearch": "7.10.0", diff --git a/packages/server/src/api/index.js b/packages/server/src/api/index.js index 4bf9d9e14d..1aae78cb30 100644 --- a/packages/server/src/api/index.js +++ b/packages/server/src/api/index.js @@ -12,6 +12,7 @@ const { mainRoutes, staticRoutes, publicRoutes } = require("./routes") const pkg = require("../../package.json") const env = require("../environment") const { middleware: pro } = require("@budibase/pro") +const { shutdown } = require("./routes/public") const router = new Router() @@ -90,4 +91,5 @@ router.use(publicRoutes.allowedMethods()) router.use(staticRoutes.routes()) router.use(staticRoutes.allowedMethods()) -module.exports = router +module.exports.router = router +module.exports.shutdown = shutdown diff --git a/packages/server/src/api/routes/public/index.ts b/packages/server/src/api/routes/public/index.ts index 6f1c69560e..ca49a1a7d6 100644 --- a/packages/server/src/api/routes/public/index.ts +++ b/packages/server/src/api/routes/public/index.ts @@ -29,6 +29,7 @@ function getApiLimitPerSecond(): number { return parseInt(env.API_REQ_LIMIT_PER_SEC) } +let rateLimitStore: any = null if (!env.isTest()) { const REDIS_OPTS = getRedisOptions() let options @@ -47,8 +48,9 @@ if (!env.isTest()) { database: 1, } } + rateLimitStore = new Stores.Redis(options) RateLimit.defaultOptions({ - store: new Stores.Redis(options), + store: rateLimitStore, }) } // rate limiting, allows for 2 requests per second @@ -128,3 +130,10 @@ applyRoutes(queryEndpoints, PermissionTypes.QUERY, "queryId") applyRoutes(rowEndpoints, PermissionTypes.TABLE, "tableId", "rowId") export default publicRouter + +export const shutdown = () => { + if (rateLimitStore) { + rateLimitStore.client.disconnect() + rateLimitStore = null + } +} diff --git a/packages/server/src/app.ts b/packages/server/src/app.ts index 8efc383194..f73c90c895 100644 --- a/packages/server/src/app.ts +++ b/packages/server/src/app.ts @@ -14,6 +14,8 @@ const automations = require("./automations/index") const Sentry = require("@sentry/node") const fileSystem = require("./utilities/fileSystem") const bullboard = require("./automations/bullboard") +const { logAlert } = require("@budibase/backend-core/logging") +const { Thread } = require("./threads") import redis from "./utilities/redis" import * as migrations from "./migrations" @@ -49,7 +51,7 @@ app.context.eventEmitter = eventEmitter app.context.auth = {} // api routes -app.use(api.routes()) +app.use(api.router.routes()) if (env.isProd()) { env._set("NODE_ENV", "production") @@ -68,11 +70,24 @@ if (env.isProd()) { const server = http.createServer(app.callback()) destroyable(server) +let shuttingDown = false, + errCode = 0 server.on("close", async () => { - if (env.NODE_ENV !== "jest") { + // already in process + if (shuttingDown) { + return + } + shuttingDown = true + if (!env.isTest()) { console.log("Server Closed") } + await automations.shutdown() await redis.shutdown() + await Thread.shutdown() + api.shutdown() + if (!env.isTest()) { + process.exit(errCode) + } }) module.exports = server.listen(env.PORT || 0, async () => { @@ -90,7 +105,13 @@ const shutdown = () => { } process.on("uncaughtException", err => { - console.error(err) + // @ts-ignore + // don't worry about this error, comes from zlib isn't important + if (err && err["code"] === "ERR_INVALID_CHAR") { + return + } + errCode = -1 + logAlert("Uncaught exception.", err) shutdown() }) @@ -102,7 +123,7 @@ process.on("SIGTERM", () => { // not recommended in a clustered environment if (!env.HTTP_MIGRATIONS) { migrations.migrate().catch(err => { - console.error("Error performing migrations. Exiting.\n", err) + logAlert("Error performing migrations. Exiting.", err) shutdown() }) } diff --git a/packages/server/src/automations/bullboard.js b/packages/server/src/automations/bullboard.js index 32336c4714..cba6594ae7 100644 --- a/packages/server/src/automations/bullboard.js +++ b/packages/server/src/automations/bullboard.js @@ -45,4 +45,12 @@ exports.init = () => { return serverAdapter.registerPlugin() } +exports.shutdown = async () => { + if (automationQueue) { + clearInterval(cleanupInternal) + await automationQueue.close() + automationQueue = null + } +} + exports.queue = automationQueue diff --git a/packages/server/src/automations/index.js b/packages/server/src/automations/index.js index 87f35ce763..e543365183 100644 --- a/packages/server/src/automations/index.js +++ b/packages/server/src/automations/index.js @@ -1,5 +1,5 @@ const { processEvent } = require("./utils") -const { queue } = require("./bullboard") +const { queue, shutdown } = require("./bullboard") /** * This module is built purely to kick off the worker farm and manage the inputs/outputs @@ -14,4 +14,9 @@ exports.init = function () { exports.getQueues = () => { return [queue] } + +exports.shutdown = () => { + return shutdown() +} + exports.queue = queue diff --git a/packages/server/src/threads/index.ts b/packages/server/src/threads/index.ts index c19453cf44..8516b62596 100644 --- a/packages/server/src/threads/index.ts +++ b/packages/server/src/threads/index.ts @@ -28,6 +28,8 @@ export class Thread { workers: any timeoutMs: any + static workerRefs: any[] = [] + constructor(type: any, opts: any = { timeoutMs: null, count: 1 }) { this.type = type this.count = opts.count ? opts.count : 1 @@ -46,6 +48,7 @@ export class Thread { workerOpts.maxCallTime = opts.timeoutMs } this.workers = workerFarm(workerOpts, typeToFile(type)) + Thread.workerRefs.push(this.workers) } } @@ -73,4 +76,23 @@ export class Thread { }) }) } + + static shutdown() { + return new Promise(resolve => { + if (Thread.workerRefs.length === 0) { + resolve() + } + let count = 0 + function complete() { + count++ + if (count >= Thread.workerRefs.length) { + resolve() + } + } + for (let worker of Thread.workerRefs) { + workerFarm.end(worker, complete) + } + Thread.workerRefs = [] + }) + } } diff --git a/packages/server/src/utilities/queue/inMemoryQueue.js b/packages/server/src/utilities/queue/inMemoryQueue.js index aebc0ba919..375092609e 100644 --- a/packages/server/src/utilities/queue/inMemoryQueue.js +++ b/packages/server/src/utilities/queue/inMemoryQueue.js @@ -75,6 +75,13 @@ class InMemoryQueue { this._emitter.emit("message") } + /** + * replicating the close function from bull, which waits for jobs to finish. + */ + async close() { + return [] + } + /** * This removes a cron which has been implemented, this is part of Bull API. * @param {string} cronJobId The cron which is to be removed. diff --git a/packages/string-templates/package.json b/packages/string-templates/package.json index 43d48b0a6e..80b565a6e8 100644 --- a/packages/string-templates/package.json +++ b/packages/string-templates/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/string-templates", - "version": "1.0.189", + "version": "1.0.191", "description": "Handlebars wrapper for Budibase templating.", "main": "src/index.cjs", "module": "dist/bundle.mjs", diff --git a/packages/worker/package.json b/packages/worker/package.json index 698b57a393..d63d7cbe09 100644 --- a/packages/worker/package.json +++ b/packages/worker/package.json @@ -1,7 +1,7 @@ { "name": "@budibase/worker", "email": "hi@budibase.com", - "version": "1.0.189", + "version": "1.0.191", "description": "Budibase background service", "main": "src/index.ts", "repository": { @@ -32,9 +32,9 @@ "author": "Budibase", "license": "GPL-3.0", "dependencies": { - "@budibase/backend-core": "^1.0.189", - "@budibase/pro": "1.0.189", - "@budibase/string-templates": "^1.0.189", + "@budibase/backend-core": "^1.0.191", + "@budibase/pro": "1.0.191", + "@budibase/string-templates": "^1.0.191", "@koa/router": "^8.0.0", "@sentry/node": "6.17.7", "@techpass/passport-openidconnect": "^0.3.0", diff --git a/packages/worker/src/index.ts b/packages/worker/src/index.ts index 1cec2868c6..fb395a7e6a 100644 --- a/packages/worker/src/index.ts +++ b/packages/worker/src/index.ts @@ -12,6 +12,7 @@ const destroyable = require("server-destroy") const koaBody = require("koa-body") const koaSession = require("koa-session") const { passport } = require("@budibase/backend-core/auth") +const { logAlert } = require("@budibase/backend-core/logging") const logger = require("koa-pino-logger") const http = require("http") const api = require("./api") @@ -28,7 +29,6 @@ app.keys = ["secret", "key"] // set up top level koa middleware app.use(koaBody({ multipart: true })) app.use(koaSession(app)) - app.use( logger({ prettyPrint: { @@ -62,25 +62,38 @@ if (env.isProd()) { const server = http.createServer(app.callback()) destroyable(server) +let shuttingDown = false, + errCode = 0 server.on("close", async () => { - if (env.isProd()) { + if (shuttingDown) { + return + } + shuttingDown = true + if (!env.isTest()) { console.log("Server Closed") } await redis.shutdown() + if (!env.isTest()) { + process.exit(errCode) + } }) +const shutdown = () => { + server.close() + server.destroy() +} + module.exports = server.listen(parseInt(env.PORT || 4002), async () => { console.log(`Worker running on ${JSON.stringify(server.address())}`) await redis.init() }) process.on("uncaughtException", err => { - console.error(err) - server.close() - server.destroy() + errCode = -1 + logAlert("Uncaught exception.", err) + shutdown() }) process.on("SIGTERM", () => { - server.close() - server.destroy() + shutdown() })