MirrorSave
/
budibase
mirror of https://github.com/Budibase/budibase.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #6034 from Budibase/infra/cluster-mode 

enable clustering on server and worker services, better log output on…
This commit is contained in:
Martin McKeaveney 2022-05-23 18:05:02 +01:00 committed by GitHub
parent 8101b6ea06 d18de6c4c9
commit 80d7dcd3cb
15 changed files with 117 additions and 73 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
packages/backend-core/src/auth.js
View File

@ -29,7 +29,7 @@ passport.deserializeUser(async (user, done) => {
const user = await db.get(user._id) const user = await db.get(user._id)
return done(null, user) return done(null, user)
} catch (err) { } catch (err) {
console.error("User not found", err) console.error(`User not found`, err)
return done(null, false, { message: "User not found" }) return done(null, false, { message: "User not found" })
} }
}) })

2
packages/backend-core/src/middleware/passport/local.js
View File

@ -30,7 +30,7 @@ exports.authenticate = async function (ctx, email, password, done) {
const dbUser = await getGlobalUserByEmail(email) const dbUser = await getGlobalUserByEmail(email)
if (dbUser == null) { if (dbUser == null) {
return authError(done, "User not found") return authError(done, `User not found: [${email}]`)
} }
// check that the user is currently inactive, if this is the case throw invalid // check that the user is currently inactive, if this is the case throw invalid

2
packages/builder/src/pages/builder/auth/login.svelte
View File

@ -28,7 +28,7 @@
async function login() { async function login() {
try { try {
await auth.login({ await auth.login({
username, username: username.trim(),
password, password,
}) })
if ($auth?.user?.forceResetPassword) { if ($auth?.user?.forceResetPassword) {

4
packages/server/Dockerfile
View File

@ -16,6 +16,7 @@ ENV BUDIBASE_ENVIRONMENT=PRODUCTION
# copy files and install dependencies # copy files and install dependencies
COPY . ./ COPY . ./
RUN yarn RUN yarn
RUN yarn global add pm2
RUN yarn build RUN yarn build
# Install client for oracle datasource # Install client for oracle datasource
@ -28,4 +29,5 @@ EXPOSE 4001
# due to this causing yarn to stop installing dev dependencies # due to this causing yarn to stop installing dev dependencies
# which are actually needed to get this environment up and running # which are actually needed to get this environment up and running
ENV NODE_ENV=production ENV NODE_ENV=production
CMD ["yarn", "run:docker"] ENV CLUSTER_MODE=${CLUSTER_MODE}
CMD ["./docker_run.sh"]

5
packages/server/docker_run.sh Executable file
View File

@ -0,0 +1,5 @@
if [ -z $CLUSTER_MODE ]; then
yarn run:docker
else
yarn run:docker:cluster
fi

1
packages/server/package.json
View File

@ -18,6 +18,7 @@
"build:docker": "yarn run predocker && docker build . -t app-service --label version=$BUDIBASE_RELEASE_VERSION", "build:docker": "yarn run predocker && docker build . -t app-service --label version=$BUDIBASE_RELEASE_VERSION",
"build:docs": "node ./scripts/docs/generate.js open", "build:docs": "node ./scripts/docs/generate.js open",
"run:docker": "node dist/index.js", "run:docker": "node dist/index.js",
"run:docker:cluster": "pm2-runtime start pm2.js",
"dev:stack:up": "node scripts/dev/manage.js up", "dev:stack:up": "node scripts/dev/manage.js up",
"dev:stack:down": "node scripts/dev/manage.js down", "dev:stack:down": "node scripts/dev/manage.js down",
"dev:stack:nuke": "node scripts/dev/manage.js nuke", "dev:stack:nuke": "node scripts/dev/manage.js nuke",

9
packages/server/pm2.js Normal file
View File

@ -0,0 +1,9 @@
module.exports = {
apps: [
{
script: "dist/index.js",
instances: "max",
exec_mode: "cluster",
},
],
}

18
packages/server/yarn.lock
View File

@ -1014,10 +1014,10 @@
resolved "https://registry.yarnpkg.com/@bcoe/v8-coverage/-/v8-coverage-0.2.3.tgz#75a2e8b51cb758a7553d6804a5932d7aace75c39" resolved "https://registry.yarnpkg.com/@bcoe/v8-coverage/-/v8-coverage-0.2.3.tgz#75a2e8b51cb758a7553d6804a5932d7aace75c39"
integrity sha512-0hYQ8SB4Db5zvZB4axdMHGwEaQjkZzFjQiN9LVYvIFB2nSUHW9tYpxWriPrWDASIxiaXax83REcLxuSdnGPZtw== integrity sha512-0hYQ8SB4Db5zvZB4axdMHGwEaQjkZzFjQiN9LVYvIFB2nSUHW9tYpxWriPrWDASIxiaXax83REcLxuSdnGPZtw==
"@budibase/backend-core@1.0.171": "@budibase/backend-core@1.0.172":
version "1.0.171" version "1.0.172"
resolved "https://registry.yarnpkg.com/@budibase/backend-core/-/backend-core-1.0.171.tgz#39fb68fc1a6e233589cf65e8354ff5e5098b4b4e" resolved "https://registry.yarnpkg.com/@budibase/backend-core/-/backend-core-1.0.172.tgz#81f825d5e141fbcbad085202509e1f97dfed0dfc"
integrity sha512-au7nP2x4Oj/5PCU7qd538FDrz/Dhq9nSX6AjR+3+hoU658Gv3xes4RKXgFjf5qE0tdSKTSe1pMntMhwQxshUBQ== integrity sha512-JDyR646dhDmQMWqeJxkdD70wyFv26hfNgRl768sgCgN4fG9B8829QsKeNjOquZf+U2diaKbDmKX+f/7gqoA5FQ==
dependencies: dependencies:
"@techpass/passport-openidconnect" "^0.3.0" "@techpass/passport-openidconnect" "^0.3.0"
aws-sdk "^2.901.0" aws-sdk "^2.901.0"
@ -1091,12 +1091,12 @@
svelte-flatpickr "^3.2.3" svelte-flatpickr "^3.2.3"
svelte-portal "^1.0.0" svelte-portal "^1.0.0"
"@budibase/pro@1.0.171": "@budibase/pro@1.0.172":
version "1.0.171" version "1.0.172"
resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-1.0.171.tgz#b64de1e3b80407bd004d4cf6e2b247f0b7855c02" resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-1.0.172.tgz#c6fe078098c72b8d163cfcf62d5389a9ca937b20"
integrity sha512-iTu63sm/DA6AJbVNOnASZ03mPR6GFJf5rLRE672XMF2ftQ2WcjDx9QIuwKF4aPWQqB7LuzX8/isNugNYzgNuVw== integrity sha512-hHNobwr+8G99ofi9zp0cGB13iIC0StEV3JdF2a2HXwB4NzMz4b0vJ6VfcB1J+nVJPEe42DWmJViwJkuSX0jGhw==
dependencies: dependencies:
"@budibase/backend-core" "1.0.171" "@budibase/backend-core" "1.0.172"
node-fetch "^2.6.1" node-fetch "^2.6.1"
"@budibase/standard-components@^0.9.139": "@budibase/standard-components@^0.9.139":

4
packages/worker/Dockerfile
View File

@ -10,6 +10,7 @@ WORKDIR /app
# copy files and install dependencies # copy files and install dependencies
COPY . ./ COPY . ./
RUN yarn RUN yarn
RUN yarn global add pm2
EXPOSE 4001 EXPOSE 4001
@ -17,4 +18,5 @@ EXPOSE 4001
# due to this causing yarn to stop installing dev dependencies # due to this causing yarn to stop installing dev dependencies
# which are actually needed to get this environment up and running # which are actually needed to get this environment up and running
ENV NODE_ENV=production ENV NODE_ENV=production
CMD ["yarn", "run:docker"] ENV CLUSTER_MODE=${CLUSTER_MODE}
CMD ["./docker_run.sh"]

5
packages/worker/docker_run.sh Executable file
View File

@ -0,0 +1,5 @@
if [[ -z $CLUSTER_MODE ]]; then
yarn run:docker
else
yarn run:docker:cluster
fi

1
packages/worker/package.json
View File

@ -15,6 +15,7 @@
"build": "rimraf dist/ && tsc", "build": "rimraf dist/ && tsc",
"postbuild": "copyfiles -u 1 src/**/*.hbs dist/", "postbuild": "copyfiles -u 1 src/**/*.hbs dist/",
"run:docker": "node dist/index.js", "run:docker": "node dist/index.js",
"run:docker:cluster": "pm2-runtime start pm2.js",
"build:docker": "docker build . -t worker-service --label version=$BUDIBASE_RELEASE_VERSION", "build:docker": "docker build . -t worker-service --label version=$BUDIBASE_RELEASE_VERSION",
"dev:stack:init": "node ./scripts/dev/manage.js init", "dev:stack:init": "node ./scripts/dev/manage.js init",
"dev:builder": "npm run dev:stack:init && nodemon", "dev:builder": "npm run dev:stack:init && nodemon",

9
packages/worker/pm2.js Normal file
View File

@ -0,0 +1,9 @@
module.exports = {
apps: [
{
script: "dist/index.js",
instances: "max",
exec_mode: "cluster",
},
],
}

9
packages/worker/src/api/controllers/global/configs.js
View File

@ -17,7 +17,6 @@ const { googleCallbackUrl, oidcCallbackUrl } = require("./auth")
const { const {
withCache, withCache,
CacheKeys, CacheKeys,
TTL,
bustCache, bustCache,
} = require("@budibase/backend-core/cache") } = require("@budibase/backend-core/cache")
@ -256,7 +255,10 @@ exports.configChecklist = async function (ctx) {
const tenantId = getTenantId() const tenantId = getTenantId()
try { try {
ctx.body = await withCache(CacheKeys.CHECKLIST, TTL.ONE_HOUR, async () => { ctx.body = await withCache(
CacheKeys.CHECKLIST,
env.CHECKLIST_CACHE_TTL,
async () => {
let apps = [] let apps = []
if (!env.MULTI_TENANCY || tenantId) { if (!env.MULTI_TENANCY || tenantId) {
// Apps exist // Apps exist
@ -307,7 +309,8 @@ exports.configChecklist = async function (ctx) {
link: "/builder/portal/manage/auth", link: "/builder/portal/manage/auth",
}, },
} }
}) }
)
} catch (err) { } catch (err) {
ctx.throw(err.status, err) ctx.throw(err.status, err)
} }

7
packages/worker/src/environment.js
View File

@ -20,6 +20,12 @@ if (!LOADED && isDev() && !isTest()) {
LOADED = true LOADED = true
} }
function parseIntSafe(number) {
if (number) {
return parseInt(number)
}
}
module.exports = { module.exports = {
NODE_ENV: process.env.NODE_ENV, NODE_ENV: process.env.NODE_ENV,
SELF_HOSTED: !!parseInt(process.env.SELF_HOSTED), SELF_HOSTED: !!parseInt(process.env.SELF_HOSTED),
@ -46,6 +52,7 @@ module.exports = {
SMTP_FROM_ADDRESS: process.env.SMTP_FROM_ADDRESS, SMTP_FROM_ADDRESS: process.env.SMTP_FROM_ADDRESS,
PLATFORM_URL: process.env.PLATFORM_URL, PLATFORM_URL: process.env.PLATFORM_URL,
COOKIE_DOMAIN: process.env.COOKIE_DOMAIN, COOKIE_DOMAIN: process.env.COOKIE_DOMAIN,
CHECKLIST_CACHE_TTL: parseIntSafe(process.env.CHECKLIST_CACHE_TTL) || 3600,
APPS_URL: process.env.APPS_URL, APPS_URL: process.env.APPS_URL,
_set(key, value) { _set(key, value) {
process.env[key] = value process.env[key] = value

18
packages/worker/yarn.lock
View File

@ -293,10 +293,10 @@
resolved "https://registry.yarnpkg.com/@bcoe/v8-coverage/-/v8-coverage-0.2.3.tgz#75a2e8b51cb758a7553d6804a5932d7aace75c39" resolved "https://registry.yarnpkg.com/@bcoe/v8-coverage/-/v8-coverage-0.2.3.tgz#75a2e8b51cb758a7553d6804a5932d7aace75c39"
integrity sha512-0hYQ8SB4Db5zvZB4axdMHGwEaQjkZzFjQiN9LVYvIFB2nSUHW9tYpxWriPrWDASIxiaXax83REcLxuSdnGPZtw== integrity sha512-0hYQ8SB4Db5zvZB4axdMHGwEaQjkZzFjQiN9LVYvIFB2nSUHW9tYpxWriPrWDASIxiaXax83REcLxuSdnGPZtw==
"@budibase/backend-core@1.0.171": "@budibase/backend-core@1.0.172":
version "1.0.171" version "1.0.172"
resolved "https://registry.yarnpkg.com/@budibase/backend-core/-/backend-core-1.0.171.tgz#39fb68fc1a6e233589cf65e8354ff5e5098b4b4e" resolved "https://registry.yarnpkg.com/@budibase/backend-core/-/backend-core-1.0.172.tgz#81f825d5e141fbcbad085202509e1f97dfed0dfc"
integrity sha512-au7nP2x4Oj/5PCU7qd538FDrz/Dhq9nSX6AjR+3+hoU658Gv3xes4RKXgFjf5qE0tdSKTSe1pMntMhwQxshUBQ== integrity sha512-JDyR646dhDmQMWqeJxkdD70wyFv26hfNgRl768sgCgN4fG9B8829QsKeNjOquZf+U2diaKbDmKX+f/7gqoA5FQ==
dependencies: dependencies:
"@techpass/passport-openidconnect" "^0.3.0" "@techpass/passport-openidconnect" "^0.3.0"
aws-sdk "^2.901.0" aws-sdk "^2.901.0"
@ -321,12 +321,12 @@
uuid "^8.3.2" uuid "^8.3.2"
zlib "^1.0.5" zlib "^1.0.5"
"@budibase/pro@1.0.171": "@budibase/pro@1.0.172":
version "1.0.171" version "1.0.172"
resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-1.0.171.tgz#b64de1e3b80407bd004d4cf6e2b247f0b7855c02" resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-1.0.172.tgz#c6fe078098c72b8d163cfcf62d5389a9ca937b20"
integrity sha512-iTu63sm/DA6AJbVNOnASZ03mPR6GFJf5rLRE672XMF2ftQ2WcjDx9QIuwKF4aPWQqB7LuzX8/isNugNYzgNuVw== integrity sha512-hHNobwr+8G99ofi9zp0cGB13iIC0StEV3JdF2a2HXwB4NzMz4b0vJ6VfcB1J+nVJPEe42DWmJViwJkuSX0jGhw==
dependencies: dependencies:
"@budibase/backend-core" "1.0.171" "@budibase/backend-core" "1.0.172"
node-fetch "^2.6.1" node-fetch "^2.6.1"
"@cspotcode/source-map-consumer@0.8.0": "@cspotcode/source-map-consumer@0.8.0":