diff --git a/packages/backend-core/package.json b/packages/backend-core/package.json index aeb1b55f8b..6873f855c5 100644 --- a/packages/backend-core/package.json +++ b/packages/backend-core/package.json @@ -31,6 +31,7 @@ "bcrypt": "5.0.1", "bcryptjs": "2.4.3", "bull": "4.10.1", + "correlation-id": "4.0.0", "dotenv": "16.0.1", "emitter-listener": "1.1.2", "ioredis": "4.28.0", @@ -62,6 +63,7 @@ "@types/ioredis": "4.28.0", "@types/jest": "27.5.1", "@types/koa": "2.13.4", + "@types/koa-pino-logger": "3.0.0", "@types/lodash": "4.14.180", "@types/node": "14.18.20", "@types/node-fetch": "2.6.1", @@ -70,7 +72,7 @@ "@types/semver": "7.3.7", "@types/tar-fs": "2.0.1", "@types/uuid": "8.3.4", - "chance": "1.1.3", + "chance": "1.1.8", "ioredis-mock": "5.8.0", "jest": "28.1.1", "koa": "2.13.4", diff --git a/packages/backend-core/src/cloud/api.ts b/packages/backend-core/src/cloud/api.ts index b601b2e64b..ef9b9d88e6 100644 --- a/packages/backend-core/src/cloud/api.ts +++ b/packages/backend-core/src/cloud/api.ts @@ -1,4 +1,5 @@ import fetch from "node-fetch" +import * as logging from "../logging" export default class API { host: string @@ -22,6 +23,9 @@ export default class API { let json = options.headers["Content-Type"] === "application/json" + // add x-budibase-correlation-id header + logging.correlation.setHeader(options.headers) + const requestOptions = { method: method, body: json ? JSON.stringify(options.body) : options.body, diff --git a/packages/backend-core/src/constants/misc.ts b/packages/backend-core/src/constants/misc.ts index 61b3cea1f6..0bf3df4094 100644 --- a/packages/backend-core/src/constants/misc.ts +++ b/packages/backend-core/src/constants/misc.ts @@ -22,6 +22,7 @@ export enum Header { TENANT_ID = "x-budibase-tenant-id", TOKEN = "x-budibase-token", CSRF_TOKEN = "x-csrf-token", + CORRELATION_ID = "x-budibase-correlation-id", } export enum GlobalRole { diff --git a/packages/backend-core/src/events/processors/LoggingProcessor.ts b/packages/backend-core/src/events/processors/LoggingProcessor.ts index d41a82fbb4..6bb691a83a 100644 --- a/packages/backend-core/src/events/processors/LoggingProcessor.ts +++ b/packages/backend-core/src/events/processors/LoggingProcessor.ts @@ -23,7 +23,7 @@ export default class LoggingProcessor implements EventProcessor { return } let timestampString = getTimestampString(timestamp) - let message = `[audit] [tenant=${identity.tenantId}] [identityType=${identity.type}] [identity=${identity.id}] ${timestampString} ${event} ` + let message = `[audit] [identityType=${identity.type}] ${timestampString} ${event} ` if (env.isDev()) { message = message + `[debug: [properties=${JSON.stringify(properties)}] ]` } diff --git a/packages/backend-core/src/logging.ts b/packages/backend-core/src/logging.ts index beffc98e8d..57295c6595 100644 --- a/packages/backend-core/src/logging.ts +++ b/packages/backend-core/src/logging.ts @@ -1,4 +1,6 @@ +import { Header } from "./constants" import env from "./environment" +const correlator = require("correlation-id") const NonErrors = ["AccountError"] @@ -42,3 +44,14 @@ export function pinoSettings() { }, } } + +const setCorrelationHeader = (headers: any) => { + const correlationId = correlator.getId() + if (correlationId) { + headers[Header.CORRELATION_ID] = correlationId + } +} + +export const correlation = { + setHeader: setCorrelationHeader, +} diff --git a/packages/backend-core/src/middleware/index.ts b/packages/backend-core/src/middleware/index.ts index 91de59d0f8..4986cde64b 100644 --- a/packages/backend-core/src/middleware/index.ts +++ b/packages/backend-core/src/middleware/index.ts @@ -15,4 +15,5 @@ export { default as csrf } from "./csrf" export { default as adminOnly } from "./adminOnly" export { default as builderOrAdmin } from "./builderOrAdmin" export { default as builderOnly } from "./builderOnly" +export { default as logging } from "./logging" export * as joiValidator from "./joi-validator" diff --git a/packages/backend-core/src/middleware/logging.ts b/packages/backend-core/src/middleware/logging.ts new file mode 100644 index 0000000000..d1f2d6566b --- /dev/null +++ b/packages/backend-core/src/middleware/logging.ts @@ -0,0 +1,88 @@ +const correlator = require("correlation-id") +import { Header } from "../constants" +import { v4 as uuid } from "uuid" +import * as context from "../context" + +const debug = console.warn +const trace = console.trace +const log = console.log +const info = console.info +const warn = console.warn +const error = console.error + +const getTenantId = () => { + let tenantId + try { + tenantId = context.getTenantId() + } catch (e: any) { + // do nothing + } + return tenantId +} + +const getAppId = () => { + let appId + try { + appId = context.getAppId() + } catch (e) { + // do nothing + } + return appId +} + +const getIdentityId = () => { + let identityId + try { + const identity = context.getIdentity() + identityId = identity?._id + } catch (e) { + // do nothing + } + return identityId +} + +const print = (fn: any, data: any[]) => { + let message = "" + + const correlationId = correlator.getId() + if (correlationId) { + message = message + `[correlationId=${correlator.getId()}]` + } + + const tenantId = getTenantId() + if (tenantId) { + message = message + ` [tenantId=${tenantId}]` + } + + const appId = getAppId() + if (appId) { + message = message + ` [appId=${appId}]` + } + + const identityId = getIdentityId() + if (identityId) { + message = message + ` [identityId=${identityId}]` + } + + fn(message, data) +} + +const logging = (ctx: any, next: any) => { + // use the provided correlation id header if present + let correlationId = ctx.headers[Header.CORRELATION_ID] + if (!correlationId) { + correlationId = uuid() + } + + return correlator.withId(correlationId, () => { + console.debug = data => print(debug, data) + console.trace = data => print(trace, data) + console.log = data => print(log, data) + console.info = data => print(info, data) + console.warn = data => print(warn, data) + console.error = data => print(error, data) + return next() + }) +} + +export default logging diff --git a/packages/backend-core/src/pino.ts b/packages/backend-core/src/pino.ts new file mode 100644 index 0000000000..415aebff5b --- /dev/null +++ b/packages/backend-core/src/pino.ts @@ -0,0 +1,14 @@ +const env = require("./environment") +import { Options } from "pino-http" +const correlator = require("correlation-id") + +export const pinoSettings = (): Options => ({ + prettyPrint: { + levelFirst: true, + }, + genReqId: correlator.getId, + level: env.LOG_LEVEL || "error", + autoLogging: { + ignore: (req: any) => req.url.includes("/health"), + }, +}) diff --git a/packages/backend-core/tsconfig.build.json b/packages/backend-core/tsconfig.build.json index 25cfb70899..12f8255a7c 100644 --- a/packages/backend-core/tsconfig.build.json +++ b/packages/backend-core/tsconfig.build.json @@ -12,6 +12,7 @@ "declaration": true, "types": [ "node", "jest" ], "outDir": "dist", + "skipLibCheck": true }, "include": [ "**/*.js", diff --git a/packages/backend-core/yarn.lock b/packages/backend-core/yarn.lock index 76d18eba7d..981f9026a3 100644 --- a/packages/backend-core/yarn.lock +++ b/packages/backend-core/yarn.lock @@ -1183,6 +1183,15 @@ dependencies: "@types/koa" "*" +"@types/koa-pino-logger@3.0.0": + version "3.0.0" + resolved "https://registry.yarnpkg.com/@types/koa-pino-logger/-/koa-pino-logger-3.0.0.tgz#275d4b000abc14b1928dc2e9ab476c8296a64b6a" + integrity sha512-sP+12JNX01q+nHpCRqkVIuLjaRemQEfDoFg0evpTnjUEI3jUI2ZrOkhQ5coxn3yVm2tedui/2YhlaPn/XrYNWA== + dependencies: + "@types/koa" "*" + "@types/pino" "*" + "@types/pino-http" "*" + "@types/koa@*", "@types/koa@2.13.4": version "2.13.4" resolved "https://registry.yarnpkg.com/@types/koa/-/koa-2.13.4.tgz#10620b3f24a8027ef5cbae88b393d1b31205726b" @@ -1230,6 +1239,44 @@ resolved "https://registry.yarnpkg.com/@types/node/-/node-14.18.20.tgz#268f028b36eaf51181c3300252f605488c4f0650" integrity sha512-Q8KKwm9YqEmUBRsqJ2GWJDtXltBDxTdC4m5vTdXBolu2PeQh8LX+f6BTwU+OuXPu37fLxoN6gidqBmnky36FXA== +"@types/pino-http@*": + version "5.8.1" + resolved "https://registry.yarnpkg.com/@types/pino-http/-/pino-http-5.8.1.tgz#ebb194750ad2f9245c3028b5d2c4e6d64f685ba9" + integrity sha512-A9MW6VCnx5ii7s+Fs5aFIw+aSZcBCpsZ/atpxamu8tTsvWFacxSf2Hrn1Ohn1jkVRB/LiPGOapRXcFawDBnDnA== + dependencies: + "@types/pino" "6.3" + +"@types/pino-pretty@*": + version "5.0.0" + resolved "https://registry.yarnpkg.com/@types/pino-pretty/-/pino-pretty-5.0.0.tgz#aa7a61cfd553b051764acfa0a49872f7a09a1722" + integrity sha512-N1uzqSzioqz8R3AkDbSJwcfDWeI3YMPNapSQQhnB2ISU4NYgUIcAh+hYT5ygqBM+klX4htpEhXMmoJv3J7GrdA== + dependencies: + pino-pretty "*" + +"@types/pino-std-serializers@*": + version "4.0.0" + resolved "https://registry.yarnpkg.com/@types/pino-std-serializers/-/pino-std-serializers-4.0.0.tgz#1e28b80b554c8222858e99a4e0fc77fd070e10e8" + integrity sha512-gXfUZx2xIBbFYozGms53fT0nvkacx/+62c8iTxrEqH5PkIGAQvDbXg2774VWOycMPbqn5YJBQ3BMsg4Li3dWbg== + dependencies: + pino-std-serializers "*" + +"@types/pino@*": + version "7.0.5" + resolved "https://registry.yarnpkg.com/@types/pino/-/pino-7.0.5.tgz#1c84a81b924a6a9e263dbb581dffdbad7a3c60c4" + integrity sha512-wKoab31pknvILkxAF8ss+v9iNyhw5Iu/0jLtRkUD74cNfOOLJNnqfFKAv0r7wVaTQxRZtWrMpGfShwwBjOcgcg== + dependencies: + pino "*" + +"@types/pino@6.3": + version "6.3.12" + resolved "https://registry.yarnpkg.com/@types/pino/-/pino-6.3.12.tgz#4425db6ced806109c3df957100cba9dfcd73c228" + integrity sha512-dsLRTq8/4UtVSpJgl9aeqHvbh6pzdmjYD3C092SYgLD2TyoCqHpTJk6vp8DvCTGGc7iowZ2MoiYiVUUCcu7muw== + dependencies: + "@types/node" "*" + "@types/pino-pretty" "*" + "@types/pino-std-serializers" "*" + sonic-boom "^2.1.0" + "@types/pouchdb-adapter-cordova-sqlite@*": version "1.0.1" resolved "https://registry.yarnpkg.com/@types/pouchdb-adapter-cordova-sqlite/-/pouchdb-adapter-cordova-sqlite-1.0.1.tgz#49e5ee6df7cc0c23196fcb340f43a560e74eb1d6" @@ -1473,7 +1520,7 @@ abbrev@1: resolved "https://registry.yarnpkg.com/abbrev/-/abbrev-1.1.1.tgz#f8f2c887ad10bf67f634f005b6987fed3179aac8" integrity sha512-nne9/IiQ/hzIhY6pdDnbBtz7DjPTKrY00P/zvPSm5pOFkl6xuGrGnXn/VtTNNfNtAfZ9/1RtehkszU9qcTii0Q== -abort-controller@3.0.0: +abort-controller@3.0.0, abort-controller@^3.0.0: version "3.0.0" resolved "https://registry.yarnpkg.com/abort-controller/-/abort-controller-3.0.0.tgz#eaf54d53b62bae4138e809ca225c8439a6efb392" integrity sha512-h8lQ8tacZYnR3vNQTgibj+tODHI5/+l06Au2Pcriv/Gmet0eaj4TwWH41sO9wnHDiQsEj19q0drzdWdeAHtweg== @@ -1637,6 +1684,11 @@ asynckit@^0.4.0: resolved "https://registry.yarnpkg.com/asynckit/-/asynckit-0.4.0.tgz#c79ed97f7f34cb8f2ba1bc9790bcc366474b4b79" integrity sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q== +atomic-sleep@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/atomic-sleep/-/atomic-sleep-1.0.0.tgz#eb85b77a601fc932cfe432c5acd364a9e2c9075b" + integrity sha512-kNOjDqAh7px0XWNI+4QbzoiR/nTkHAWNud2uvnJquD1/x5a7EQZMJT0AczqK0Qn67oY/TTQ1LbUKajZpp3I9tQ== + aws-cloudfront-sign@2.2.0: version "2.2.0" resolved "https://registry.yarnpkg.com/aws-cloudfront-sign/-/aws-cloudfront-sign-2.2.0.tgz#3910f5a6d0d90fec07f2b4ef8ab07f3eefb5625d" @@ -1829,6 +1881,13 @@ brace-expansion@^1.1.7: balanced-match "^1.0.0" concat-map "0.0.1" +brace-expansion@^2.0.1: + version "2.0.1" + resolved "https://registry.yarnpkg.com/brace-expansion/-/brace-expansion-2.0.1.tgz#1edc459e0f0c548486ecf9fc99f2221364b9a0ae" + integrity sha512-XnAIvQ8eM+kC6aULx6wuQiwVsnzsi9d3WxzV3FpWTGA19F621kwdbsAcFKXgKUHZWsy+mY6iL1sHTxWEFCytDA== + dependencies: + balanced-match "^1.0.0" + braces@^3.0.2, braces@~3.0.2: version "3.0.2" resolved "https://registry.yarnpkg.com/braces/-/braces-3.0.2.tgz#3454e1a462ee8d599e236df336cd9ea4f8afe107" @@ -1903,6 +1962,14 @@ buffer@^5.5.0, buffer@^5.6.0: base64-js "^1.3.1" ieee754 "^1.1.13" +buffer@^6.0.3: + version "6.0.3" + resolved "https://registry.yarnpkg.com/buffer/-/buffer-6.0.3.tgz#2ace578459cc8fbe2a70aaa8f52ee63b6a74c6c6" + integrity sha512-FTiCpNxtwiZZHEZbcbTIcZjERVICn9yq/pDFkTl95/AxzD1naBctN7YO68riM/gLSDY7sdrMby8hofADYuuqOA== + dependencies: + base64-js "^1.3.1" + ieee754 "^1.2.1" + bull@4.10.1: version "4.10.1" resolved "https://registry.yarnpkg.com/bull/-/bull-4.10.1.tgz#f14974b6089358b62b495a2cbf838aadc098e43f" @@ -1994,10 +2061,10 @@ chalk@^4.0.0, chalk@^4.1.0: ansi-styles "^4.1.0" supports-color "^7.1.0" -chance@1.1.3: - version "1.1.3" - resolved "https://registry.yarnpkg.com/chance/-/chance-1.1.3.tgz#414f08634ee479c7a316b569050ea20751b82dd3" - integrity sha512-XeJsdoVAzDb1WRPRuMBesRSiWpW1uNTo5Fd7mYxPJsAfgX71+jfuCOHOdbyBz2uAUZ8TwKcXgWk3DMedFfJkbg== +chance@1.1.8: + version "1.1.8" + resolved "https://registry.yarnpkg.com/chance/-/chance-1.1.8.tgz#5d6c2b78c9170bf6eb9df7acdda04363085be909" + integrity sha512-v7fi5Hj2VbR6dJEGRWLmJBA83LJMS47pkAbmROFxHWd9qmE1esHRZW8Clf1Fhzr3rjxnNZVCjOEv/ivFxeIMtg== char-regex@^1.0.2: version "1.0.2" @@ -2119,6 +2186,11 @@ color-support@^1.1.2: resolved "https://registry.yarnpkg.com/color-support/-/color-support-1.1.3.tgz#93834379a1cc9a0c61f82f52f0d04322251bd5a2" integrity sha512-qiBjkpbMLO/HL68y+lh4q0/O1MZFj2RX6X/KmMa3+gJD3z+WwI1ZzDHysvqHGS3mP6mznPckpXmw1nI9cJjyRg== +colorette@^2.0.7: + version "2.0.19" + resolved "https://registry.yarnpkg.com/colorette/-/colorette-2.0.19.tgz#cdf044f47ad41a0f4b56b3a0d5b4e6e1a2d5a798" + integrity sha512-3tlv/dIP7FWvj3BsbHrGLJ6l/oKh1O3TcgBqMn+yyCagOxc23fyzDS6HypQbgxWbkpDnf52p1LuR4eWDQ/K9WQ== + combined-stream@^1.0.6, combined-stream@^1.0.8, combined-stream@~1.0.6: version "1.0.8" resolved "https://registry.yarnpkg.com/combined-stream/-/combined-stream-1.0.8.tgz#c3d45a8b34fd730631a110a8a2520682b31d5a7f" @@ -2190,6 +2262,13 @@ core-util-is@~1.0.0: resolved "https://registry.yarnpkg.com/core-util-is/-/core-util-is-1.0.3.tgz#a6042d3634c2b27e9328f837b965fac83808db85" integrity sha512-ZQBvi1DcpJ4GDqanjucZ2Hj3wEO5pZDS89BWbkcrvdxksJorwUDDZamX9ldFkp9aw2lmBDLgkObEA4DWNJ9FYQ== +correlation-id@4.0.0: + version "4.0.0" + resolved "https://registry.yarnpkg.com/correlation-id/-/correlation-id-4.0.0.tgz#c1d3038e5f30d7bfeae5728ff96f27a7506bc2c0" + integrity sha512-WvXtJBlovvOBKqTz/YwWP2gm6CXJZJArfGimp9s/ehmhJMPFbmnPMQe3K60Q9idGNixMvKojMjleyDhZEFdHfg== + dependencies: + uuid "^8.3.1" + create-require@^1.1.0: version "1.1.1" resolved "https://registry.yarnpkg.com/create-require/-/create-require-1.1.1.tgz#c1d7e8f1e5f6cfc9ff65f9cd352d37348756c333" @@ -2228,6 +2307,11 @@ dashdash@^1.12.0: dependencies: assert-plus "^1.0.0" +dateformat@^4.6.3: + version "4.6.3" + resolved "https://registry.yarnpkg.com/dateformat/-/dateformat-4.6.3.tgz#556fa6497e5217fedb78821424f8a1c22fa3f4b5" + integrity sha512-2P0p0pFGzHS5EMnhdxQi7aJN+iMheud0UhG4dlE1DLAlvL8JHjJJTX/CSm4JXwV0Ka5nGk3zC5mcb5bUQUxxMA== + debug@4, debug@^4.1.0, debug@^4.1.1, debug@^4.3.1, debug@^4.3.2: version "4.3.4" resolved "https://registry.yarnpkg.com/debug/-/debug-4.3.4.tgz#1319f6579357f2338d3337d2cdd4914bb5dcc865" @@ -2494,6 +2578,11 @@ events@1.1.1: resolved "https://registry.yarnpkg.com/events/-/events-1.1.1.tgz#9ebdb7635ad099c70dcc4c2a1f5004288e8bd924" integrity sha512-kEcvvCBByWXGnZy6JUlgAp2gBIUjfCAV6P6TgT1/aaQKcmuAEC4OZTV1I4EWQLz2gxZw76atuVyvHhTxvi0Flw== +events@^3.3.0: + version "3.3.0" + resolved "https://registry.yarnpkg.com/events/-/events-3.3.0.tgz#31a95ad0a924e2d2c419a813aeb2c4e878ea7400" + integrity sha512-mQw+2fkQbALzQ7V0MY0IqdnXNOeTtP4r0lN9z7AAawCXgqea7bDii20AYrIBrFd/Hx0M2Ocz6S111CaFkUcb0Q== + execa@^5.0.0: version "5.1.1" resolved "https://registry.yarnpkg.com/execa/-/execa-5.1.1.tgz#f80ad9cbf4298f7bd1d4c9555c21e93741c411dd" @@ -2540,6 +2629,11 @@ extsprintf@^1.2.0: resolved "https://registry.yarnpkg.com/extsprintf/-/extsprintf-1.4.1.tgz#8d172c064867f235c0c84a596806d279bf4bcc07" integrity sha512-Wrk35e8ydCKDj/ArClo1VrPVmN8zph5V4AtHwIuHhvMXsKf73UT3BOD+azBIW+3wOJ4FhEH7zyaJCFvChjYvMA== +fast-copy@^3.0.0: + version "3.0.0" + resolved "https://registry.yarnpkg.com/fast-copy/-/fast-copy-3.0.0.tgz#875ebf33b13948ae012b6e51d33da5e6e7571ab8" + integrity sha512-4HzS+9pQ5Yxtv13Lhs1Z1unMXamBdn5nA4bEi1abYpDNSpSp7ODYQ1KPMF6nTatfEzgH6/zPvXKU1zvHiUjWlA== + fast-deep-equal@^3.1.1: version "3.1.3" resolved "https://registry.yarnpkg.com/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz#3a7d56b559d6cbc3eb512325244e619a65c6c525" @@ -2550,6 +2644,16 @@ fast-json-stable-stringify@2.x, fast-json-stable-stringify@^2.0.0: resolved "https://registry.yarnpkg.com/fast-json-stable-stringify/-/fast-json-stable-stringify-2.1.0.tgz#874bf69c6f404c2b5d99c481341399fd55892633" integrity sha512-lhd/wF+Lk98HZoTCtlVraHtfh5XYijIjalXck7saUtuanSDyLMxnHhSXEDJqHxD7msR8D0uCmqlkwjCV8xvwHw== +fast-redact@^3.1.1: + version "3.1.2" + resolved "https://registry.yarnpkg.com/fast-redact/-/fast-redact-3.1.2.tgz#d58e69e9084ce9fa4c1a6fa98a3e1ecf5d7839aa" + integrity sha512-+0em+Iya9fKGfEQGcd62Yv6onjBmmhV1uh86XVfOU8VwAe6kaFdQCWI9s0/Nnugx5Vd9tdbZ7e6gE2tR9dzXdw== + +fast-safe-stringify@^2.1.1: + version "2.1.1" + resolved "https://registry.yarnpkg.com/fast-safe-stringify/-/fast-safe-stringify-2.1.1.tgz#c406a83b6e70d9e35ce3b30a81141df30aeba884" + integrity sha512-W+KJc2dmILlPplD/H4K9l9LcAHAfPtP6BY84uVLXQ6Evcz9Lcg33Y2z1IVblT6xdY54PXYVHEv+0Wpq8Io6zkA== + fb-watchman@^2.0.0: version "2.0.1" resolved "https://registry.yarnpkg.com/fb-watchman/-/fb-watchman-2.0.1.tgz#fc84fb39d2709cf3ff6d743706157bb5708a8a85" @@ -2768,6 +2872,17 @@ glob@^7.1.3, glob@^7.1.4: once "^1.3.0" path-is-absolute "^1.0.0" +glob@^8.0.0: + version "8.0.3" + resolved "https://registry.yarnpkg.com/glob/-/glob-8.0.3.tgz#415c6eb2deed9e502c68fa44a272e6da6eeca42e" + integrity sha512-ull455NHSHI/Y1FqGaaYFaLGkNMMJbavMrEGFXG/PGrg6y7sutWHUHrz6gy6WEBH6akM1M414dWKCNs+IhKdiQ== + dependencies: + fs.realpath "^1.0.0" + inflight "^1.0.4" + inherits "2" + minimatch "^5.0.1" + once "^1.3.0" + global-dirs@^3.0.0: version "3.0.0" resolved "https://registry.yarnpkg.com/global-dirs/-/global-dirs-3.0.0.tgz#70a76fe84ea315ab37b1f5576cbde7d48ef72686" @@ -2854,6 +2969,14 @@ has@^1.0.3: dependencies: function-bind "^1.1.1" +help-me@^4.0.1: + version "4.2.0" + resolved "https://registry.yarnpkg.com/help-me/-/help-me-4.2.0.tgz#50712bfd799ff1854ae1d312c36eafcea85b0563" + integrity sha512-TAOnTB8Tz5Dw8penUuzHVrKNKlCIbwwbHnXraNJxPwf8LRtE2HlM84RYuezMFcwOJmoYOCWVDyJ8TQGxn9PgxA== + dependencies: + glob "^8.0.0" + readable-stream "^3.6.0" + html-escaper@^2.0.0: version "2.0.2" resolved "https://registry.yarnpkg.com/html-escaper/-/html-escaper-2.0.2.tgz#dfd60027da36a36dfcbe236262c00a5822681453" @@ -2917,7 +3040,7 @@ ieee754@1.1.13: resolved "https://registry.yarnpkg.com/ieee754/-/ieee754-1.1.13.tgz#ec168558e95aa181fd87d37f55c32bbcb6708b84" integrity sha512-4vf7I2LYV/HaWerSo3XmlMkp5eZ83i+/CDluXi/IGTs/O1sejBNhTtnxzmRZfvOUqj7lZjqHkeTvpgSFDlWZTg== -ieee754@^1.1.13, ieee754@^1.1.4: +ieee754@^1.1.13, ieee754@^1.1.4, ieee754@^1.2.1: version "1.2.1" resolved "https://registry.yarnpkg.com/ieee754/-/ieee754-1.2.1.tgz#8eb7a10a63fff25d15a57b001586d177d1b0d352" integrity sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA== @@ -3596,6 +3719,11 @@ join-component@^1.1.0: resolved "https://registry.yarnpkg.com/join-component/-/join-component-1.1.0.tgz#b8417b750661a392bee2c2537c68b2a9d4977cd5" integrity sha512-bF7vcQxbODoGK1imE2P9GS9aw4zD0Sd+Hni68IMZLj7zRnquH7dXUmMw9hDI5S/Jzt7q+IyTXN0rSg2GI0IKhQ== +joycon@^3.1.1: + version "3.1.1" + resolved "https://registry.yarnpkg.com/joycon/-/joycon-3.1.1.tgz#bce8596d6ae808f8b68168f5fc69280996894f03" + integrity sha512-34wB/Y7MW7bzjKRjUKTa46I2Z7eV62Rkhva+KkopW7Qvv/OSWBqvkSY7vusOPrNuZcUG3tApvdVgNB8POj3SPw== + js-tokens@^4.0.0: version "4.0.0" resolved "https://registry.yarnpkg.com/js-tokens/-/js-tokens-4.0.0.tgz#19203fb59991df98e3a287050d4647cdeaf32499" @@ -4094,6 +4222,13 @@ minimatch@^3.0.4, minimatch@^3.1.1: dependencies: brace-expansion "^1.1.7" +minimatch@^5.0.1: + version "5.1.2" + resolved "https://registry.yarnpkg.com/minimatch/-/minimatch-5.1.2.tgz#0939d7d6f0898acbd1508abe534d1929368a8fff" + integrity sha512-bNH9mmM9qsJ2X4r2Nat1B//1dJVcn3+iBLa3IgqJ7EbGaDNepL9QSHOxN4ng33s52VMMhhIfgCYDk3C4ZmlDAg== + dependencies: + brace-expansion "^2.0.1" + minimist@^1.2.0: version "1.2.6" resolved "https://registry.yarnpkg.com/minimist/-/minimist-1.2.6.tgz#8637a5b759ea0d6e98702cfb3a9283323c93af44" @@ -4325,6 +4460,11 @@ object-inspect@^1.9.0: resolved "https://registry.yarnpkg.com/object-inspect/-/object-inspect-1.12.2.tgz#c0641f26394532f28ab8d796ab954e43c009a8ea" integrity sha512-z+cPxW0QGUp0mcqcsgQyLVRDoXFQbXOwBaqyF7VIgI4TWNQsDHrBpUQslRmIfAoYWdYzs6UlKJtB2XJpTaNSpQ== +on-exit-leak-free@^2.1.0: + version "2.1.0" + resolved "https://registry.yarnpkg.com/on-exit-leak-free/-/on-exit-leak-free-2.1.0.tgz#5c703c968f7e7f851885f6459bf8a8a57edc9cc4" + integrity sha512-VuCaZZAjReZ3vUwgOB8LxAosIurDiAW0s13rI1YwmaP++jvcxP77AWoQvenZebpCA2m8WC1/EosPYPMjnRAp/w== + on-finished@^2.3.0: version "2.4.1" resolved "https://registry.yarnpkg.com/on-finished/-/on-finished-2.4.1.tgz#58c8c44116e54845ad57f14ab10b03533184ac3f" @@ -4544,6 +4684,56 @@ picomatch@^2.0.4, picomatch@^2.2.1, picomatch@^2.2.3, picomatch@^2.3.1: resolved "https://registry.yarnpkg.com/picomatch/-/picomatch-2.3.1.tgz#3ba3833733646d9d3e4995946c1365a67fb07a42" integrity sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA== +pino-abstract-transport@^1.0.0, pino-abstract-transport@v1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/pino-abstract-transport/-/pino-abstract-transport-1.0.0.tgz#cc0d6955fffcadb91b7b49ef220a6cc111d48bb3" + integrity sha512-c7vo5OpW4wIS42hUVcT5REsL8ZljsUfBjqV/e2sFxmFEFZiq1XLUp5EYLtuDH6PEHq9W1egWqRbnLUP5FuZmOA== + dependencies: + readable-stream "^4.0.0" + split2 "^4.0.0" + +pino-pretty@*: + version "9.1.1" + resolved "https://registry.yarnpkg.com/pino-pretty/-/pino-pretty-9.1.1.tgz#e7d64c1db98266ca428ab56567b844ba780cd0e1" + integrity sha512-iJrnjgR4FWQIXZkUF48oNgoRI9BpyMhaEmihonHeCnZ6F50ZHAS4YGfGBT/ZVNsPmd+hzkIPGzjKdY08+/yAXw== + dependencies: + colorette "^2.0.7" + dateformat "^4.6.3" + fast-copy "^3.0.0" + fast-safe-stringify "^2.1.1" + help-me "^4.0.1" + joycon "^3.1.1" + minimist "^1.2.6" + on-exit-leak-free "^2.1.0" + pino-abstract-transport "^1.0.0" + pump "^3.0.0" + readable-stream "^4.0.0" + secure-json-parse "^2.4.0" + sonic-boom "^3.0.0" + strip-json-comments "^3.1.1" + +pino-std-serializers@*, pino-std-serializers@^6.0.0: + version "6.1.0" + resolved "https://registry.yarnpkg.com/pino-std-serializers/-/pino-std-serializers-6.1.0.tgz#307490fd426eefc95e06067e85d8558603e8e844" + integrity sha512-KO0m2f1HkrPe9S0ldjx7za9BJjeHqBku5Ch8JyxETxT8dEFGz1PwgrHaOQupVYitpzbFSYm7nnljxD8dik2c+g== + +pino@*: + version "8.8.0" + resolved "https://registry.yarnpkg.com/pino/-/pino-8.8.0.tgz#1f0d6695a224aa06afc7ad60f2ccc4772d3b9233" + integrity sha512-cF8iGYeu2ODg2gIwgAHcPrtR63ILJz3f7gkogaHC/TXVVXxZgInmNYiIpDYEwgEkxZti2Se6P2W2DxlBIZe6eQ== + dependencies: + atomic-sleep "^1.0.0" + fast-redact "^3.1.1" + on-exit-leak-free "^2.1.0" + pino-abstract-transport v1.0.0 + pino-std-serializers "^6.0.0" + process-warning "^2.0.0" + quick-format-unescaped "^4.0.3" + real-require "^0.2.0" + safe-stable-stringify "^2.3.1" + sonic-boom "^3.1.0" + thread-stream "^2.0.0" + pirates@^4.0.4: version "4.0.5" resolved "https://registry.yarnpkg.com/pirates/-/pirates-4.0.5.tgz#feec352ea5c3268fb23a37c702ab1699f35a5f3b" @@ -4806,6 +4996,16 @@ process-nextick-args@~2.0.0: resolved "https://registry.yarnpkg.com/process-nextick-args/-/process-nextick-args-2.0.1.tgz#7820d9b16120cc55ca9ae7792680ae7dba6d7fe2" integrity sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag== +process-warning@^2.0.0: + version "2.1.0" + resolved "https://registry.yarnpkg.com/process-warning/-/process-warning-2.1.0.tgz#1e60e3bfe8183033bbc1e702c2da74f099422d1a" + integrity sha512-9C20RLxrZU/rFnxWncDkuF6O999NdIf3E1ws4B0ZeY3sRVPzWBMsYDE2lxjxhiXxg464cQTgKUGm8/i6y2YGXg== + +process@^0.11.10: + version "0.11.10" + resolved "https://registry.yarnpkg.com/process/-/process-0.11.10.tgz#7332300e840161bda3e69a1d1d91a7d4bc16f182" + integrity sha512-cdGef/drWFoydD1JsMzuFf8100nZl+GT+yacc2bEced5f9Rjk4z+WtFUTBu9PhOi9j/jfmBPu0mMEY4wIdAF8A== + prompts@^2.0.1: version "2.4.2" resolved "https://registry.yarnpkg.com/prompts/-/prompts-2.4.2.tgz#7b57e73b3a48029ad10ebd44f74b01722a4cb069" @@ -4881,6 +5081,11 @@ querystringify@^2.1.1: resolved "https://registry.yarnpkg.com/querystringify/-/querystringify-2.2.0.tgz#3345941b4153cb9d082d8eee4cda2016a9aef7f6" integrity sha512-FIqgj2EUvTa7R50u0rGsyTftzjYmv/a3hO345bZNrqabNqjtgiDMgmo4mkUjd+nzU5oF3dClKqFIPUKybUyqoQ== +quick-format-unescaped@^4.0.3: + version "4.0.4" + resolved "https://registry.yarnpkg.com/quick-format-unescaped/-/quick-format-unescaped-4.0.4.tgz#93ef6dd8d3453cbc7970dd614fad4c5954d6b5a7" + integrity sha512-tYC1Q1hgyRuHgloV/YXs2w15unPVh8qfu/qCTfhTYamaw7fyhumKa2yGpdSo87vY32rIclj+4fWYQXUMs9EHvg== + range-parser@^1.2.0: version "1.2.1" resolved "https://registry.yarnpkg.com/range-parser/-/range-parser-1.2.1.tgz#3cf37023d199e1c24d1a55b84800c2f3e6468031" @@ -4925,6 +5130,16 @@ readable-stream@1.1.14, readable-stream@^1.0.27-1: string_decoder "^1.1.1" util-deprecate "^1.0.1" +readable-stream@^4.0.0: + version "4.3.0" + resolved "https://registry.yarnpkg.com/readable-stream/-/readable-stream-4.3.0.tgz#0914d0c72db03b316c9733bb3461d64a3cc50cba" + integrity sha512-MuEnA0lbSi7JS8XM+WNJlWZkHAAdm7gETHdFK//Q/mChGyj2akEFtdLZh32jSdkWGbRwCW9pn6g3LWDdDeZnBQ== + dependencies: + abort-controller "^3.0.0" + buffer "^6.0.3" + events "^3.3.0" + process "^0.11.10" + readable-stream@~0.0.2: version "0.0.4" resolved "https://registry.yarnpkg.com/readable-stream/-/readable-stream-0.0.4.tgz#f32d76e3fb863344a548d79923007173665b3b8d" @@ -4955,6 +5170,11 @@ readline-sync@^1.4.9: resolved "https://registry.yarnpkg.com/readline-sync/-/readline-sync-1.4.10.tgz#41df7fbb4b6312d673011594145705bf56d8873b" integrity sha512-gNva8/6UAe8QYepIQH/jQ2qn91Qj0B9sYjMBBs3QOB8F2CXcKgLxQaJRP76sWVRQt+QU+8fAkCbCvjjMFu7Ycw== +real-require@^0.2.0: + version "0.2.0" + resolved "https://registry.yarnpkg.com/real-require/-/real-require-0.2.0.tgz#209632dea1810be2ae063a6ac084fee7e33fba78" + integrity sha512-57frrGM/OCTLqLOAh0mhVA9VBMHd+9U7Zb2THMGdBUoZVOtGbJzjxsYGDJ3A9AYYCP4hn6y1TVbaOfzWtm5GFg== + redis-commands@1.7.0: version "1.7.0" resolved "https://registry.yarnpkg.com/redis-commands/-/redis-commands-1.7.0.tgz#15a6fea2d58281e27b1cd1acfb4b293e278c3a89" @@ -5089,6 +5309,11 @@ safe-buffer@~5.1.0, safe-buffer@~5.1.1: resolved "https://registry.yarnpkg.com/safe-buffer/-/safe-buffer-5.1.2.tgz#991ec69d296e0313747d59bdfd2b745c35f8828d" integrity sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g== +safe-stable-stringify@^2.3.1: + version "2.4.2" + resolved "https://registry.yarnpkg.com/safe-stable-stringify/-/safe-stable-stringify-2.4.2.tgz#ec7b037768098bf65310d1d64370de0dc02353aa" + integrity sha512-gMxvPJYhP0O9n2pvcfYfIuYgbledAOJFcqRThtPRmjscaipiwcwPPKLytpVzMkG2HAN87Qmo2d4PtGiri1dSLA== + safer-buffer@^2.0.2, safer-buffer@^2.1.0, safer-buffer@~2.1.0: version "2.1.2" resolved "https://registry.yarnpkg.com/safer-buffer/-/safer-buffer-2.1.2.tgz#44fa161b0187b9549dd84bb91802f9bd8385cd6a" @@ -5109,6 +5334,11 @@ sax@>=0.1.1, sax@>=0.6.0: resolved "https://registry.yarnpkg.com/sax/-/sax-1.2.4.tgz#2816234e2378bddc4e5354fab5caa895df7100d9" integrity sha512-NqVDv9TpANUjFm0N8uM5GxL36UgKi9/atZw+x7YFnQ8ckwFGKrl4xX4yWtrey3UJm5nP1kUbnYgLopqWNSRhWw== +secure-json-parse@^2.4.0: + version "2.7.0" + resolved "https://registry.yarnpkg.com/secure-json-parse/-/secure-json-parse-2.7.0.tgz#5a5f9cd6ae47df23dba3151edd06855d47e09862" + integrity sha512-6aU+Rwsezw7VR8/nyvKTx8QpWH9FrcYiXXlqC4z5d5XQBDRqtbfsRjnwGyqbi3gddNtWHuEk9OANUotL26qKUw== + semver-diff@^3.1.1: version "3.1.1" resolved "https://registry.yarnpkg.com/semver-diff/-/semver-diff-3.1.1.tgz#05f77ce59f325e00e2706afd67bb506ddb1ca32b" @@ -5191,6 +5421,20 @@ slash@^3.0.0: resolved "https://registry.yarnpkg.com/slash/-/slash-3.0.0.tgz#6539be870c165adbd5240220dbe361f1bc4d4634" integrity sha512-g9Q1haeby36OSStwb4ntCGGGaKsaVSjQ68fBxoQcutl5fS1vuY18H3wSt3jFyFtrkx+Kz0V1G85A4MyAdDMi2Q== +sonic-boom@^2.1.0: + version "2.8.0" + resolved "https://registry.yarnpkg.com/sonic-boom/-/sonic-boom-2.8.0.tgz#c1def62a77425090e6ad7516aad8eb402e047611" + integrity sha512-kuonw1YOYYNOve5iHdSahXPOK49GqwA+LZhI6Wz/l0rP57iKyXXIHaRagOBHAPmGwJC6od2Z9zgvZ5loSgMlVg== + dependencies: + atomic-sleep "^1.0.0" + +sonic-boom@^3.0.0, sonic-boom@^3.1.0: + version "3.2.1" + resolved "https://registry.yarnpkg.com/sonic-boom/-/sonic-boom-3.2.1.tgz#972ceab831b5840a08a002fa95a672008bda1c38" + integrity sha512-iITeTHxy3B9FGu8aVdiDXUVAcHMF9Ss0cCsAOo2HfCrmVGT3/DT5oYaeu0M/YKZDlKTvChEyPq0zI9Hf33EX6A== + dependencies: + atomic-sleep "^1.0.0" + source-map-support@0.5.13: version "0.5.13" resolved "https://registry.yarnpkg.com/source-map-support/-/source-map-support-0.5.13.tgz#31b24a9c2e73c2de85066c0feb7d44767ed52932" @@ -5221,6 +5465,11 @@ split2@^2.1.0: dependencies: through2 "^2.0.2" +split2@^4.0.0: + version "4.1.0" + resolved "https://registry.yarnpkg.com/split2/-/split2-4.1.0.tgz#101907a24370f85bb782f08adaabe4e281ecf809" + integrity sha512-VBiJxFkxiXRlUIeyMQi8s4hgvKCSjtknJv/LVYbrgALPwf5zSKmEwV9Lst25AkvMDnvxODugjdl6KZgwKM1WYQ== + sprintf-js@^1.1.1: version "1.1.2" resolved "https://registry.yarnpkg.com/sprintf-js/-/sprintf-js-1.1.2.tgz#da1765262bf8c0f571749f2ad6c26300207ae673" @@ -5430,6 +5679,13 @@ test-exclude@^6.0.0: glob "^7.1.4" minimatch "^3.0.4" +thread-stream@^2.0.0: + version "2.2.0" + resolved "https://registry.yarnpkg.com/thread-stream/-/thread-stream-2.2.0.tgz#310c03a253f729094ce5d4638ef5186dfa80a9e8" + integrity sha512-rUkv4/fnb4rqy/gGy7VuqK6wE1+1DOCOWy4RMeaV69ZHMP11tQKZvZSip1yTgrKCMZzEMcCL/bKfHvSfDHx+iQ== + dependencies: + real-require "^0.2.0" + through2@3.0.2: version "3.0.2" resolved "https://registry.yarnpkg.com/through2/-/through2-3.0.2.tgz#99f88931cfc761ec7678b41d5d7336b5b6a07bf4" @@ -5723,7 +5979,7 @@ uuid@8.1.0: resolved "https://registry.yarnpkg.com/uuid/-/uuid-8.1.0.tgz#6f1536eb43249f473abc6bd58ff983da1ca30d8d" integrity sha512-CI18flHDznR0lq54xBycOVmphdCYnQLKn8abKn7PXUiKUGdEd+/l9LWNJmugXel4hXq7S+RMNl34ecyC9TntWg== -uuid@8.3.2, uuid@^8.3.0, uuid@^8.3.2: +uuid@8.3.2, uuid@^8.3.0, uuid@^8.3.1, uuid@^8.3.2: version "8.3.2" resolved "https://registry.yarnpkg.com/uuid/-/uuid-8.3.2.tgz#80d5b5ced271bb9af6c445f21a1a04c606cefbe2" integrity sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg== diff --git a/packages/server/src/app.ts b/packages/server/src/app.ts index 3ec5b20205..177393dee7 100644 --- a/packages/server/src/app.ts +++ b/packages/server/src/app.ts @@ -19,7 +19,7 @@ import * as api from "./api" import * as automations from "./automations" import { Thread } from "./threads" import * as redis from "./utilities/redis" -import { events, logging } from "@budibase/backend-core" +import { events, logging, middleware } from "@budibase/backend-core" import { initialise as initialiseWebsockets } from "./websocket" import { startup } from "./startup" const Sentry = require("@sentry/node") @@ -40,6 +40,8 @@ app.use( }) ) +app.use(middleware.logging) + if (env.isProd()) { env._set("NODE_ENV", "production") Sentry.init() diff --git a/packages/server/src/utilities/workerRequests.ts b/packages/server/src/utilities/workerRequests.ts index b3a61ddd68..e318b12f82 100644 --- a/packages/server/src/utilities/workerRequests.ts +++ b/packages/server/src/utilities/workerRequests.ts @@ -1,7 +1,12 @@ import fetch from "node-fetch" import env from "../environment" import { checkSlashesInUrl } from "./index" -import { db as dbCore, constants, tenancy } from "@budibase/backend-core" +import { + db as dbCore, + constants, + tenancy, + logging, +} from "@budibase/backend-core" import { updateAppRole } from "./global" import { BBContext, User } from "@budibase/types" @@ -27,6 +32,10 @@ export function request(ctx?: BBContext, request?: any) { if (ctx && ctx.headers) { request.headers = ctx.headers } + + // add x-budibase-correlation-id header + logging.correlation.setHeader(request.headers) + return request } diff --git a/packages/worker/src/index.ts b/packages/worker/src/index.ts index 0cb39222c6..dfb96a061c 100644 --- a/packages/worker/src/index.ts +++ b/packages/worker/src/index.ts @@ -14,7 +14,7 @@ import { Event } from "@sentry/types/dist/event" import Application from "koa" import { bootstrap } from "global-agent" import * as db from "./db" -import { auth, logging, events } from "@budibase/backend-core" +import { auth, logging, events, middleware } from "@budibase/backend-core" db.init() import Koa from "koa" import koaBody from "koa-body" @@ -36,6 +36,7 @@ app.keys = ["secret", "key"] // set up top level koa middleware app.use(koaBody({ multipart: true })) app.use(koaSession(app)) +app.use(middleware.logging) app.use(logger(logging.pinoSettings())) // authentication diff --git a/packages/worker/src/utilities/appService.ts b/packages/worker/src/utilities/appService.ts index a0c4314f65..244f10336f 100644 --- a/packages/worker/src/utilities/appService.ts +++ b/packages/worker/src/utilities/appService.ts @@ -1,5 +1,5 @@ import fetch from "node-fetch" -import { constants, tenancy } from "@budibase/backend-core" +import { constants, tenancy, logging } from "@budibase/backend-core" import { checkSlashesInUrl } from "../utilities" import env from "../environment" @@ -17,6 +17,10 @@ async function makeAppRequest(url: string, method: string, body: any) { request.body = JSON.stringify(body) } request.method = method + + // add x-budibase-correlation-id header + logging.correlation.setHeader(request.headers) + return fetch(checkSlashesInUrl(env.APPS_URL + url), request) } diff --git a/tsconfig.json b/tsconfig.json index a0a2cc5905..8382281d99 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -3,7 +3,7 @@ { "path": "./packages/types" }, { "path": "./packages/backend-core" }, { "path": "./packages/server" }, - { "path": "./packages/worker" }, + { "path": "./packages/worker" } ], "files" :[] } \ No newline at end of file