From b30c045663b58c4cae76b7bf805b552f8ebde191 Mon Sep 17 00:00:00 2001 From: Martin McKeaveney Date: Mon, 23 May 2022 16:24:29 +0100 Subject: [PATCH 1/3] enable clustering on server and worker services, better log output on user not found errors --- packages/backend-core/src/auth.js | 2 +- packages/backend-core/src/middleware/passport/local.js | 2 +- packages/builder/src/pages/builder/auth/login.svelte | 2 +- packages/server/Dockerfile | 4 +++- packages/server/docker_run.sh | 5 +++++ packages/server/package.json | 1 + packages/server/pm2.js | 9 +++++++++ packages/worker/Dockerfile | 4 +++- packages/worker/docker_run.sh | 5 +++++ packages/worker/package.json | 1 + packages/worker/pm2.js | 9 +++++++++ packages/worker/src/api/controllers/global/configs.js | 4 +--- packages/worker/src/environment.js | 7 +++++++ 13 files changed, 47 insertions(+), 8 deletions(-) create mode 100755 packages/server/docker_run.sh create mode 100644 packages/server/pm2.js create mode 100755 packages/worker/docker_run.sh create mode 100644 packages/worker/pm2.js diff --git a/packages/backend-core/src/auth.js b/packages/backend-core/src/auth.js index f6d53522d5..b13cd932c6 100644 --- a/packages/backend-core/src/auth.js +++ b/packages/backend-core/src/auth.js @@ -29,7 +29,7 @@ passport.deserializeUser(async (user, done) => { const user = await db.get(user._id) return done(null, user) } catch (err) { - console.error("User not found", err) + console.error(`User not found`, err) return done(null, false, { message: "User not found" }) } }) diff --git a/packages/backend-core/src/middleware/passport/local.js b/packages/backend-core/src/middleware/passport/local.js index 2149bd3e18..716ebc1755 100644 --- a/packages/backend-core/src/middleware/passport/local.js +++ b/packages/backend-core/src/middleware/passport/local.js @@ -30,7 +30,7 @@ exports.authenticate = async function (ctx, email, password, done) { const dbUser = await getGlobalUserByEmail(email) if (dbUser == null) { - return authError(done, "User not found") + return authError(done, `User not found: [${email}]`) } // check that the user is currently inactive, if this is the case throw invalid diff --git a/packages/builder/src/pages/builder/auth/login.svelte b/packages/builder/src/pages/builder/auth/login.svelte index 84af5b6338..c2ec547e38 100644 --- a/packages/builder/src/pages/builder/auth/login.svelte +++ b/packages/builder/src/pages/builder/auth/login.svelte @@ -28,7 +28,7 @@ async function login() { try { await auth.login({ - username, + username: username.trim(), password, }) if ($auth?.user?.forceResetPassword) { diff --git a/packages/server/Dockerfile b/packages/server/Dockerfile index 0ef0fe4244..e5063a88ed 100644 --- a/packages/server/Dockerfile +++ b/packages/server/Dockerfile @@ -16,6 +16,7 @@ ENV BUDIBASE_ENVIRONMENT=PRODUCTION # copy files and install dependencies COPY . ./ RUN yarn +RUN yarn global add pm2 RUN yarn build # Install client for oracle datasource @@ -28,4 +29,5 @@ EXPOSE 4001 # due to this causing yarn to stop installing dev dependencies # which are actually needed to get this environment up and running ENV NODE_ENV=production -CMD ["yarn", "run:docker"] +ENV CLUSTER_MODE=${CLUSTER_MODE} +CMD ["./docker_run.sh"] diff --git a/packages/server/docker_run.sh b/packages/server/docker_run.sh new file mode 100755 index 0000000000..0045fe0c44 --- /dev/null +++ b/packages/server/docker_run.sh @@ -0,0 +1,5 @@ +if [ -z $CLUSTER_MODE ]; then + yarn run:docker +else + yarn run:docker:cluster +fi diff --git a/packages/server/package.json b/packages/server/package.json index f428ad5328..8ebad9923b 100644 --- a/packages/server/package.json +++ b/packages/server/package.json @@ -18,6 +18,7 @@ "build:docker": "yarn run predocker && docker build . -t app-service --label version=$BUDIBASE_RELEASE_VERSION", "build:docs": "node ./scripts/docs/generate.js open", "run:docker": "node dist/index.js", + "run:docker:cluster": "pm2-runtime start pm2.js", "dev:stack:up": "node scripts/dev/manage.js up", "dev:stack:down": "node scripts/dev/manage.js down", "dev:stack:nuke": "node scripts/dev/manage.js nuke", diff --git a/packages/server/pm2.js b/packages/server/pm2.js new file mode 100644 index 0000000000..33829f327c --- /dev/null +++ b/packages/server/pm2.js @@ -0,0 +1,9 @@ +module.exports = { + apps: [ + { + script: "dist/index.js", + instances: "max", + exec_mode: "cluster", + }, + ], +} diff --git a/packages/worker/Dockerfile b/packages/worker/Dockerfile index 614a724ab6..b73efcc4a1 100644 --- a/packages/worker/Dockerfile +++ b/packages/worker/Dockerfile @@ -10,6 +10,7 @@ WORKDIR /app # copy files and install dependencies COPY . ./ RUN yarn +RUN yarn global add pm2 EXPOSE 4001 @@ -17,4 +18,5 @@ EXPOSE 4001 # due to this causing yarn to stop installing dev dependencies # which are actually needed to get this environment up and running ENV NODE_ENV=production -CMD ["yarn", "run:docker"] +ENV CLUSTER_MODE=${CLUSTER_MODE} +CMD ["./docker_run.sh"] diff --git a/packages/worker/docker_run.sh b/packages/worker/docker_run.sh new file mode 100755 index 0000000000..20694e5df0 --- /dev/null +++ b/packages/worker/docker_run.sh @@ -0,0 +1,5 @@ +if [[ -z $CLUSTER_MODE ]]; then + yarn run:docker +else + yarn run:docker:cluster +fi diff --git a/packages/worker/package.json b/packages/worker/package.json index 885f55003e..32b7610d8c 100644 --- a/packages/worker/package.json +++ b/packages/worker/package.json @@ -15,6 +15,7 @@ "build": "rimraf dist/ && tsc", "postbuild": "copyfiles -u 1 src/**/*.hbs dist/", "run:docker": "node dist/index.js", + "run:docker:cluster": "pm2-runtime start pm2.js", "build:docker": "docker build . -t worker-service --label version=$BUDIBASE_RELEASE_VERSION", "dev:stack:init": "node ./scripts/dev/manage.js init", "dev:builder": "npm run dev:stack:init && nodemon", diff --git a/packages/worker/pm2.js b/packages/worker/pm2.js new file mode 100644 index 0000000000..33829f327c --- /dev/null +++ b/packages/worker/pm2.js @@ -0,0 +1,9 @@ +module.exports = { + apps: [ + { + script: "dist/index.js", + instances: "max", + exec_mode: "cluster", + }, + ], +} diff --git a/packages/worker/src/api/controllers/global/configs.js b/packages/worker/src/api/controllers/global/configs.js index 571e102e4c..0301ec354f 100644 --- a/packages/worker/src/api/controllers/global/configs.js +++ b/packages/worker/src/api/controllers/global/configs.js @@ -250,11 +250,9 @@ exports.configChecklist = async function (ctx) { const tenantId = getTenantId() try { - const ONE_MINUTE = 600 - ctx.body = await withCache( `checklist:${tenantId}`, - ONE_MINUTE, + env.CHECKLIST_CACHE_TTL, async () => { let apps = [] if (!env.MULTI_TENANCY || tenantId) { diff --git a/packages/worker/src/environment.js b/packages/worker/src/environment.js index 8ef12e3877..790448d7e8 100644 --- a/packages/worker/src/environment.js +++ b/packages/worker/src/environment.js @@ -20,6 +20,12 @@ if (!LOADED && isDev() && !isTest()) { LOADED = true } +function parseIntSafe(number) { + if (number) { + return parseInt(number) + } +} + module.exports = { NODE_ENV: process.env.NODE_ENV, SELF_HOSTED: !!parseInt(process.env.SELF_HOSTED), @@ -46,6 +52,7 @@ module.exports = { SMTP_FROM_ADDRESS: process.env.SMTP_FROM_ADDRESS, PLATFORM_URL: process.env.PLATFORM_URL, COOKIE_DOMAIN: process.env.COOKIE_DOMAIN, + CHECKLIST_CACHE_TTL: parseIntSafe(process.env.CHECKLIST_CACHE_TTL) || 600, APPS_URL: process.env.APPS_URL, _set(key, value) { process.env[key] = value From 1cc140d50ae9fea5f9d68fede82d9339600d3963 Mon Sep 17 00:00:00 2001 From: Martin McKeaveney Date: Mon, 23 May 2022 16:57:15 +0100 Subject: [PATCH 2/3] making checklist cache length controllable through env var --- .../src/api/controllers/global/configs.js | 103 +++++++++--------- packages/worker/src/environment.js | 4 +- 2 files changed, 56 insertions(+), 51 deletions(-) diff --git a/packages/worker/src/api/controllers/global/configs.js b/packages/worker/src/api/controllers/global/configs.js index 147c5296ce..df3326ef26 100644 --- a/packages/worker/src/api/controllers/global/configs.js +++ b/packages/worker/src/api/controllers/global/configs.js @@ -17,7 +17,6 @@ const { googleCallbackUrl, oidcCallbackUrl } = require("./auth") const { withCache, CacheKeys, - TTL, bustCache, } = require("@budibase/backend-core/cache") @@ -256,58 +255,62 @@ exports.configChecklist = async function (ctx) { const tenantId = getTenantId() try { - ctx.body = await withCache(CacheKeys.CHECKLIST, TTL.ONE_HOUR, async () => { - let apps = [] - if (!env.MULTI_TENANCY || tenantId) { - // Apps exist - apps = await getAllApps({ idsOnly: true, efficient: true }) - } + ctx.body = await withCache( + CacheKeys.CHECKLIST, + env.CHECKLIST_CACHE_TTL, + async () => { + let apps = [] + if (!env.MULTI_TENANCY || tenantId) { + // Apps exist + apps = await getAllApps({ idsOnly: true, efficient: true }) + } - // They have set up SMTP - const smtpConfig = await getScopedFullConfig(db, { - type: Configs.SMTP, - }) - - // They have set up Google Auth - const googleConfig = await getScopedFullConfig(db, { - type: Configs.GOOGLE, - }) - - // They have set up OIDC - const oidcConfig = await getScopedFullConfig(db, { - type: Configs.OIDC, - }) - - // They have set up an global user - const users = await db.allDocs( - getGlobalUserParams(null, { - include_docs: true, - limit: 1, + // They have set up SMTP + const smtpConfig = await getScopedFullConfig(db, { + type: Configs.SMTP, }) - ) - return { - apps: { - checked: apps.length > 0, - label: "Create your first app", - link: "/builder/portal/apps", - }, - smtp: { - checked: !!smtpConfig, - label: "Set up email", - link: "/builder/portal/manage/email", - }, - adminUser: { - checked: users && users.rows.length >= 1, - label: "Create your first user", - link: "/builder/portal/manage/users", - }, - sso: { - checked: !!googleConfig || !!oidcConfig, - label: "Set up single sign-on", - link: "/builder/portal/manage/auth", - }, + + // They have set up Google Auth + const googleConfig = await getScopedFullConfig(db, { + type: Configs.GOOGLE, + }) + + // They have set up OIDC + const oidcConfig = await getScopedFullConfig(db, { + type: Configs.OIDC, + }) + + // They have set up an global user + const users = await db.allDocs( + getGlobalUserParams(null, { + include_docs: true, + limit: 1, + }) + ) + return { + apps: { + checked: apps.length > 0, + label: "Create your first app", + link: "/builder/portal/apps", + }, + smtp: { + checked: !!smtpConfig, + label: "Set up email", + link: "/builder/portal/manage/email", + }, + adminUser: { + checked: users && users.rows.length >= 1, + label: "Create your first user", + link: "/builder/portal/manage/users", + }, + sso: { + checked: !!googleConfig || !!oidcConfig, + label: "Set up single sign-on", + link: "/builder/portal/manage/auth", + }, + } } - }) + ) } catch (err) { ctx.throw(err.status, err) } diff --git a/packages/worker/src/environment.js b/packages/worker/src/environment.js index 790448d7e8..a6a60c0a88 100644 --- a/packages/worker/src/environment.js +++ b/packages/worker/src/environment.js @@ -1,4 +1,5 @@ const { join } = require("path") +const { TTL } = require("@budibase/backend-core/cache") function isDev() { return process.env.NODE_ENV !== "production" @@ -52,7 +53,8 @@ module.exports = { SMTP_FROM_ADDRESS: process.env.SMTP_FROM_ADDRESS, PLATFORM_URL: process.env.PLATFORM_URL, COOKIE_DOMAIN: process.env.COOKIE_DOMAIN, - CHECKLIST_CACHE_TTL: parseIntSafe(process.env.CHECKLIST_CACHE_TTL) || 600, + CHECKLIST_CACHE_TTL: + parseIntSafe(process.env.CHECKLIST_CACHE_TTL) || TTL.ONE_HOUR, APPS_URL: process.env.APPS_URL, _set(key, value) { process.env[key] = value From 73404a6fb39a314baa56dfd927792a4ae00b33ef Mon Sep 17 00:00:00 2001 From: Martin McKeaveney Date: Mon, 23 May 2022 17:38:02 +0100 Subject: [PATCH 3/3] prevent issue with TTL dependency import --- packages/server/yarn.lock | 18 +++++++++--------- packages/worker/src/environment.js | 4 +--- packages/worker/yarn.lock | 18 +++++++++--------- 3 files changed, 19 insertions(+), 21 deletions(-) diff --git a/packages/server/yarn.lock b/packages/server/yarn.lock index 219b5add75..277fda70ca 100644 --- a/packages/server/yarn.lock +++ b/packages/server/yarn.lock @@ -1014,10 +1014,10 @@ resolved "https://registry.yarnpkg.com/@bcoe/v8-coverage/-/v8-coverage-0.2.3.tgz#75a2e8b51cb758a7553d6804a5932d7aace75c39" integrity sha512-0hYQ8SB4Db5zvZB4axdMHGwEaQjkZzFjQiN9LVYvIFB2nSUHW9tYpxWriPrWDASIxiaXax83REcLxuSdnGPZtw== -"@budibase/backend-core@1.0.171": - version "1.0.171" - resolved "https://registry.yarnpkg.com/@budibase/backend-core/-/backend-core-1.0.171.tgz#39fb68fc1a6e233589cf65e8354ff5e5098b4b4e" - integrity sha512-au7nP2x4Oj/5PCU7qd538FDrz/Dhq9nSX6AjR+3+hoU658Gv3xes4RKXgFjf5qE0tdSKTSe1pMntMhwQxshUBQ== +"@budibase/backend-core@1.0.172": + version "1.0.172" + resolved "https://registry.yarnpkg.com/@budibase/backend-core/-/backend-core-1.0.172.tgz#81f825d5e141fbcbad085202509e1f97dfed0dfc" + integrity sha512-JDyR646dhDmQMWqeJxkdD70wyFv26hfNgRl768sgCgN4fG9B8829QsKeNjOquZf+U2diaKbDmKX+f/7gqoA5FQ== dependencies: "@techpass/passport-openidconnect" "^0.3.0" aws-sdk "^2.901.0" @@ -1091,12 +1091,12 @@ svelte-flatpickr "^3.2.3" svelte-portal "^1.0.0" -"@budibase/pro@1.0.171": - version "1.0.171" - resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-1.0.171.tgz#b64de1e3b80407bd004d4cf6e2b247f0b7855c02" - integrity sha512-iTu63sm/DA6AJbVNOnASZ03mPR6GFJf5rLRE672XMF2ftQ2WcjDx9QIuwKF4aPWQqB7LuzX8/isNugNYzgNuVw== +"@budibase/pro@1.0.172": + version "1.0.172" + resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-1.0.172.tgz#c6fe078098c72b8d163cfcf62d5389a9ca937b20" + integrity sha512-hHNobwr+8G99ofi9zp0cGB13iIC0StEV3JdF2a2HXwB4NzMz4b0vJ6VfcB1J+nVJPEe42DWmJViwJkuSX0jGhw== dependencies: - "@budibase/backend-core" "1.0.171" + "@budibase/backend-core" "1.0.172" node-fetch "^2.6.1" "@budibase/standard-components@^0.9.139": diff --git a/packages/worker/src/environment.js b/packages/worker/src/environment.js index a6a60c0a88..4d65d5d834 100644 --- a/packages/worker/src/environment.js +++ b/packages/worker/src/environment.js @@ -1,5 +1,4 @@ const { join } = require("path") -const { TTL } = require("@budibase/backend-core/cache") function isDev() { return process.env.NODE_ENV !== "production" @@ -53,8 +52,7 @@ module.exports = { SMTP_FROM_ADDRESS: process.env.SMTP_FROM_ADDRESS, PLATFORM_URL: process.env.PLATFORM_URL, COOKIE_DOMAIN: process.env.COOKIE_DOMAIN, - CHECKLIST_CACHE_TTL: - parseIntSafe(process.env.CHECKLIST_CACHE_TTL) || TTL.ONE_HOUR, + CHECKLIST_CACHE_TTL: parseIntSafe(process.env.CHECKLIST_CACHE_TTL) || 3600, APPS_URL: process.env.APPS_URL, _set(key, value) { process.env[key] = value diff --git a/packages/worker/yarn.lock b/packages/worker/yarn.lock index 832422f224..57cf9f09af 100644 --- a/packages/worker/yarn.lock +++ b/packages/worker/yarn.lock @@ -293,10 +293,10 @@ resolved "https://registry.yarnpkg.com/@bcoe/v8-coverage/-/v8-coverage-0.2.3.tgz#75a2e8b51cb758a7553d6804a5932d7aace75c39" integrity sha512-0hYQ8SB4Db5zvZB4axdMHGwEaQjkZzFjQiN9LVYvIFB2nSUHW9tYpxWriPrWDASIxiaXax83REcLxuSdnGPZtw== -"@budibase/backend-core@1.0.171": - version "1.0.171" - resolved "https://registry.yarnpkg.com/@budibase/backend-core/-/backend-core-1.0.171.tgz#39fb68fc1a6e233589cf65e8354ff5e5098b4b4e" - integrity sha512-au7nP2x4Oj/5PCU7qd538FDrz/Dhq9nSX6AjR+3+hoU658Gv3xes4RKXgFjf5qE0tdSKTSe1pMntMhwQxshUBQ== +"@budibase/backend-core@1.0.172": + version "1.0.172" + resolved "https://registry.yarnpkg.com/@budibase/backend-core/-/backend-core-1.0.172.tgz#81f825d5e141fbcbad085202509e1f97dfed0dfc" + integrity sha512-JDyR646dhDmQMWqeJxkdD70wyFv26hfNgRl768sgCgN4fG9B8829QsKeNjOquZf+U2diaKbDmKX+f/7gqoA5FQ== dependencies: "@techpass/passport-openidconnect" "^0.3.0" aws-sdk "^2.901.0" @@ -321,12 +321,12 @@ uuid "^8.3.2" zlib "^1.0.5" -"@budibase/pro@1.0.171": - version "1.0.171" - resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-1.0.171.tgz#b64de1e3b80407bd004d4cf6e2b247f0b7855c02" - integrity sha512-iTu63sm/DA6AJbVNOnASZ03mPR6GFJf5rLRE672XMF2ftQ2WcjDx9QIuwKF4aPWQqB7LuzX8/isNugNYzgNuVw== +"@budibase/pro@1.0.172": + version "1.0.172" + resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-1.0.172.tgz#c6fe078098c72b8d163cfcf62d5389a9ca937b20" + integrity sha512-hHNobwr+8G99ofi9zp0cGB13iIC0StEV3JdF2a2HXwB4NzMz4b0vJ6VfcB1J+nVJPEe42DWmJViwJkuSX0jGhw== dependencies: - "@budibase/backend-core" "1.0.171" + "@budibase/backend-core" "1.0.172" node-fetch "^2.6.1" "@cspotcode/source-map-consumer@0.8.0":