MirrorSave
/
budibase
mirror of https://github.com/Budibase/budibase.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #6034 from Budibase/infra/cluster-mode 

enable clustering on server and worker services, better log output on…
This commit is contained in:
Martin McKeaveney 2022-05-23 18:05:02 +01:00 committed by GitHub
parent 8101b6ea06 d18de6c4c9
commit 80d7dcd3cb
15 changed files with 117 additions and 73 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
packages/backend-core/src/auth.js
View File

@ -29,7 +29,7 @@ passport.deserializeUser(async (user, done) => {
const user = await db.get(user._id)
return done(null, user)
} catch (err) {
console.error("User not found", err)
console.error(`User not found`, err)
return done(null, false, { message: "User not found" })
}
})

2
packages/backend-core/src/middleware/passport/local.js
View File

@ -30,7 +30,7 @@ exports.authenticate = async function (ctx, email, password, done) {
const dbUser = await getGlobalUserByEmail(email)
if (dbUser == null) {
return authError(done, "User not found")
return authError(done, `User not found: [${email}]`)
}
// check that the user is currently inactive, if this is the case throw invalid

2
packages/builder/src/pages/builder/auth/login.svelte
View File

@ -28,7 +28,7 @@
async function login() {
try {
await auth.login({
username,
username: username.trim(),
password,
})
if ($auth?.user?.forceResetPassword) {

4
packages/server/Dockerfile
View File

@ -16,6 +16,7 @@ ENV BUDIBASE_ENVIRONMENT=PRODUCTION
# copy files and install dependencies
COPY . ./
RUN yarn
RUN yarn global add pm2
RUN yarn build
# Install client for oracle datasource
@ -28,4 +29,5 @@ EXPOSE 4001
# due to this causing yarn to stop installing dev dependencies
# which are actually needed to get this environment up and running
ENV NODE_ENV=production
CMD ["yarn", "run:docker"]
ENV CLUSTER_MODE=${CLUSTER_MODE}
CMD ["./docker_run.sh"]

5
packages/server/docker_run.sh Executable file
View File

@ -0,0 +1,5 @@
if [ -z $CLUSTER_MODE ]; then
yarn run:docker
else
yarn run:docker:cluster
fi

1
packages/server/package.json
View File

@ -18,6 +18,7 @@
"build:docker": "yarn run predocker && docker build . -t app-service --label version=$BUDIBASE_RELEASE_VERSION",
"build:docs": "node ./scripts/docs/generate.js open",
"run:docker": "node dist/index.js",
"run:docker:cluster": "pm2-runtime start pm2.js",
"dev:stack:up": "node scripts/dev/manage.js up",
"dev:stack:down": "node scripts/dev/manage.js down",
"dev:stack:nuke": "node scripts/dev/manage.js nuke",

9
packages/server/pm2.js Normal file
View File

@ -0,0 +1,9 @@
module.exports = {
apps: [
{
script: "dist/index.js",
instances: "max",
exec_mode: "cluster",
},
],
}

18
packages/server/yarn.lock
View File

@ -1014,10 +1014,10 @@
resolved "https://registry.yarnpkg.com/@bcoe/v8-coverage/-/v8-coverage-0.2.3.tgz#75a2e8b51cb758a7553d6804a5932d7aace75c39"
integrity sha512-0hYQ8SB4Db5zvZB4axdMHGwEaQjkZzFjQiN9LVYvIFB2nSUHW9tYpxWriPrWDASIxiaXax83REcLxuSdnGPZtw==
"@budibase/backend-core@1.0.171":
version "1.0.171"
resolved "https://registry.yarnpkg.com/@budibase/backend-core/-/backend-core-1.0.171.tgz#39fb68fc1a6e233589cf65e8354ff5e5098b4b4e"
integrity sha512-au7nP2x4Oj/5PCU7qd538FDrz/Dhq9nSX6AjR+3+hoU658Gv3xes4RKXgFjf5qE0tdSKTSe1pMntMhwQxshUBQ==
"@budibase/backend-core@1.0.172":
version "1.0.172"
resolved "https://registry.yarnpkg.com/@budibase/backend-core/-/backend-core-1.0.172.tgz#81f825d5e141fbcbad085202509e1f97dfed0dfc"
integrity sha512-JDyR646dhDmQMWqeJxkdD70wyFv26hfNgRl768sgCgN4fG9B8829QsKeNjOquZf+U2diaKbDmKX+f/7gqoA5FQ==
dependencies:
"@techpass/passport-openidconnect" "^0.3.0"
aws-sdk "^2.901.0"
@ -1091,12 +1091,12 @@
svelte-flatpickr "^3.2.3"
svelte-portal "^1.0.0"
"@budibase/pro@1.0.171":
version "1.0.171"
resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-1.0.171.tgz#b64de1e3b80407bd004d4cf6e2b247f0b7855c02"
integrity sha512-iTu63sm/DA6AJbVNOnASZ03mPR6GFJf5rLRE672XMF2ftQ2WcjDx9QIuwKF4aPWQqB7LuzX8/isNugNYzgNuVw==
"@budibase/pro@1.0.172":
version "1.0.172"
resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-1.0.172.tgz#c6fe078098c72b8d163cfcf62d5389a9ca937b20"
integrity sha512-hHNobwr+8G99ofi9zp0cGB13iIC0StEV3JdF2a2HXwB4NzMz4b0vJ6VfcB1J+nVJPEe42DWmJViwJkuSX0jGhw==
dependencies:
"@budibase/backend-core" "1.0.171"
"@budibase/backend-core" "1.0.172"
node-fetch "^2.6.1"
"@budibase/standard-components@^0.9.139":

4
packages/worker/Dockerfile
View File

@ -10,6 +10,7 @@ WORKDIR /app
# copy files and install dependencies
COPY . ./
RUN yarn
RUN yarn global add pm2
EXPOSE 4001
@ -17,4 +18,5 @@ EXPOSE 4001
# due to this causing yarn to stop installing dev dependencies
# which are actually needed to get this environment up and running
ENV NODE_ENV=production
CMD ["yarn", "run:docker"]
ENV CLUSTER_MODE=${CLUSTER_MODE}
CMD ["./docker_run.sh"]

5
packages/worker/docker_run.sh Executable file
View File

@ -0,0 +1,5 @@
if [[ -z $CLUSTER_MODE ]]; then
yarn run:docker
else
yarn run:docker:cluster
fi

1
packages/worker/package.json
View File

@ -15,6 +15,7 @@
"build": "rimraf dist/ && tsc",
"postbuild": "copyfiles -u 1 src/**/*.hbs dist/",
"run:docker": "node dist/index.js",
"run:docker:cluster": "pm2-runtime start pm2.js",
"build:docker": "docker build . -t worker-service --label version=$BUDIBASE_RELEASE_VERSION",
"dev:stack:init": "node ./scripts/dev/manage.js init",
"dev:builder": "npm run dev:stack:init && nodemon",

9
packages/worker/pm2.js Normal file
View File

@ -0,0 +1,9 @@
module.exports = {
apps: [
{
script: "dist/index.js",
instances: "max",
exec_mode: "cluster",
},
],
}

103
packages/worker/src/api/controllers/global/configs.js
View File

@ -17,7 +17,6 @@ const { googleCallbackUrl, oidcCallbackUrl } = require("./auth")
const {
withCache,
CacheKeys,
TTL,
bustCache,
} = require("@budibase/backend-core/cache")
@ -256,58 +255,62 @@ exports.configChecklist = async function (ctx) {
const tenantId = getTenantId()
try {
ctx.body = await withCache(CacheKeys.CHECKLIST, TTL.ONE_HOUR, async () => {
let apps = []
if (!env.MULTI_TENANCY || tenantId) {
// Apps exist
apps = await getAllApps({ idsOnly: true, efficient: true })
}
ctx.body = await withCache(
CacheKeys.CHECKLIST,
env.CHECKLIST_CACHE_TTL,
async () => {
let apps = []
if (!env.MULTI_TENANCY || tenantId) {
// Apps exist
apps = await getAllApps({ idsOnly: true, efficient: true })
}
// They have set up SMTP
const smtpConfig = await getScopedFullConfig(db, {
type: Configs.SMTP,
})
// They have set up Google Auth
const googleConfig = await getScopedFullConfig(db, {
type: Configs.GOOGLE,
})
// They have set up OIDC
const oidcConfig = await getScopedFullConfig(db, {
type: Configs.OIDC,
})
// They have set up an global user
const users = await db.allDocs(
getGlobalUserParams(null, {
include_docs: true,
limit: 1,
// They have set up SMTP
const smtpConfig = await getScopedFullConfig(db, {
type: Configs.SMTP,
})
)
return {
apps: {
checked: apps.length > 0,
label: "Create your first app",
link: "/builder/portal/apps",
},
smtp: {
checked: !!smtpConfig,
label: "Set up email",
link: "/builder/portal/manage/email",
},
adminUser: {
checked: users && users.rows.length >= 1,
label: "Create your first user",
link: "/builder/portal/manage/users",
},
sso: {
checked: !!googleConfig || !!oidcConfig,
label: "Set up single sign-on",
link: "/builder/portal/manage/auth",
},
// They have set up Google Auth
const googleConfig = await getScopedFullConfig(db, {
type: Configs.GOOGLE,
})
// They have set up OIDC
const oidcConfig = await getScopedFullConfig(db, {
type: Configs.OIDC,
})
// They have set up an global user
const users = await db.allDocs(
getGlobalUserParams(null, {
include_docs: true,
limit: 1,
})
)
return {
apps: {
checked: apps.length > 0,
label: "Create your first app",
link: "/builder/portal/apps",
},
smtp: {
checked: !!smtpConfig,
label: "Set up email",
link: "/builder/portal/manage/email",
},
adminUser: {
checked: users && users.rows.length >= 1,
label: "Create your first user",
link: "/builder/portal/manage/users",
},
sso: {
checked: !!googleConfig || !!oidcConfig,
label: "Set up single sign-on",
link: "/builder/portal/manage/auth",
},
}
}
})
)
} catch (err) {
ctx.throw(err.status, err)
}

7
packages/worker/src/environment.js
View File

@ -20,6 +20,12 @@ if (!LOADED && isDev() && !isTest()) {
LOADED = true
}
function parseIntSafe(number) {
if (number) {
return parseInt(number)
}
}
module.exports = {
NODE_ENV: process.env.NODE_ENV,
SELF_HOSTED: !!parseInt(process.env.SELF_HOSTED),
@ -46,6 +52,7 @@ module.exports = {
SMTP_FROM_ADDRESS: process.env.SMTP_FROM_ADDRESS,
PLATFORM_URL: process.env.PLATFORM_URL,
COOKIE_DOMAIN: process.env.COOKIE_DOMAIN,
CHECKLIST_CACHE_TTL: parseIntSafe(process.env.CHECKLIST_CACHE_TTL) || 3600,
APPS_URL: process.env.APPS_URL,
_set(key, value) {
process.env[key] = value

18
packages/worker/yarn.lock
View File

@ -293,10 +293,10 @@
resolved "https://registry.yarnpkg.com/@bcoe/v8-coverage/-/v8-coverage-0.2.3.tgz#75a2e8b51cb758a7553d6804a5932d7aace75c39"
integrity sha512-0hYQ8SB4Db5zvZB4axdMHGwEaQjkZzFjQiN9LVYvIFB2nSUHW9tYpxWriPrWDASIxiaXax83REcLxuSdnGPZtw==
"@budibase/backend-core@1.0.171":
version "1.0.171"
resolved "https://registry.yarnpkg.com/@budibase/backend-core/-/backend-core-1.0.171.tgz#39fb68fc1a6e233589cf65e8354ff5e5098b4b4e"
integrity sha512-au7nP2x4Oj/5PCU7qd538FDrz/Dhq9nSX6AjR+3+hoU658Gv3xes4RKXgFjf5qE0tdSKTSe1pMntMhwQxshUBQ==
"@budibase/backend-core@1.0.172":
version "1.0.172"
resolved "https://registry.yarnpkg.com/@budibase/backend-core/-/backend-core-1.0.172.tgz#81f825d5e141fbcbad085202509e1f97dfed0dfc"
integrity sha512-JDyR646dhDmQMWqeJxkdD70wyFv26hfNgRl768sgCgN4fG9B8829QsKeNjOquZf+U2diaKbDmKX+f/7gqoA5FQ==
dependencies:
"@techpass/passport-openidconnect" "^0.3.0"
aws-sdk "^2.901.0"
@ -321,12 +321,12 @@
uuid "^8.3.2"
zlib "^1.0.5"
"@budibase/pro@1.0.171":
version "1.0.171"
resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-1.0.171.tgz#b64de1e3b80407bd004d4cf6e2b247f0b7855c02"
integrity sha512-iTu63sm/DA6AJbVNOnASZ03mPR6GFJf5rLRE672XMF2ftQ2WcjDx9QIuwKF4aPWQqB7LuzX8/isNugNYzgNuVw==
"@budibase/pro@1.0.172":
version "1.0.172"
resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-1.0.172.tgz#c6fe078098c72b8d163cfcf62d5389a9ca937b20"
integrity sha512-hHNobwr+8G99ofi9zp0cGB13iIC0StEV3JdF2a2HXwB4NzMz4b0vJ6VfcB1J+nVJPEe42DWmJViwJkuSX0jGhw==
dependencies:
"@budibase/backend-core" "1.0.171"
"@budibase/backend-core" "1.0.172"
node-fetch "^2.6.1"
"@cspotcode/source-map-consumer@0.8.0":