MirrorSave
/
budibase
mirror of https://github.com/Budibase/budibase.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #6034 from Budibase/infra/cluster-mode 

enable clustering on server and worker services, better log output on…
This commit is contained in:
Martin McKeaveney 2022-05-23 18:05:02 +01:00 committed by GitHub
parent d7d1e9a5c3 73404a6fb3
commit bc0ea3e9a7
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
15 changed files with 117 additions and 73 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
packages/backend-core/src/auth.js
View File

@ -29,7 +29,7 @@ passport.deserializeUser(async (user, done) => {
const user = await db.get(user._id)
return done(null, user)
} catch (err) {
console.error("User not found", err)
console.error(`User not found`, err)
return done(null, false, { message: "User not found" })
}
})

2
packages/backend-core/src/middleware/passport/local.js
View File

@ -30,7 +30,7 @@ exports.authenticate = async function (ctx, email, password, done) {
const dbUser = await getGlobalUserByEmail(email)
if (dbUser == null) {
return authError(done, "User not found")
return authError(done, `User not found: [${email}]`)
}
// check that the user is currently inactive, if this is the case throw invalid

2
packages/builder/src/pages/builder/auth/login.svelte
View File

@ -28,7 +28,7 @@
async function login() {
try {
await auth.login({
username,
username: username.trim(),
password,
})
if ($auth?.user?.forceResetPassword) {

4
packages/server/Dockerfile
View File

@ -16,6 +16,7 @@ ENV BUDIBASE_ENVIRONMENT=PRODUCTION
# copy files and install dependencies
COPY . ./
RUN yarn
RUN yarn global add pm2
RUN yarn build
# Install client for oracle datasource
@ -28,4 +29,5 @@ EXPOSE 4001
# due to this causing yarn to stop installing dev dependencies
# which are actually needed to get this environment up and running
ENV NODE_ENV=production
CMD ["yarn", "run:docker"]
ENV CLUSTER_MODE=${CLUSTER_MODE}
CMD ["./docker_run.sh"]

5
packages/server/docker_run.sh Executable file
View File

@ -0,0 +1,5 @@
if [ -z $CLUSTER_MODE ]; then
yarn run:docker
else
yarn run:docker:cluster
fi

1
packages/server/package.json
View File

@ -18,6 +18,7 @@
"build:docker": "yarn run predocker && docker build . -t app-service --label version=$BUDIBASE_RELEASE_VERSION",
"build:docs": "node ./scripts/docs/generate.js open",
"run:docker": "node dist/index.js",
"run:docker:cluster": "pm2-runtime start pm2.js",
"dev:stack:up": "node scripts/dev/manage.js up",
"dev:stack:down": "node scripts/dev/manage.js down",
"dev:stack:nuke": "node scripts/dev/manage.js nuke",

9
packages/server/pm2.js Normal file
View File

@ -0,0 +1,9 @@
module.exports = {
apps: [
{
script: "dist/index.js",
instances: "max",
exec_mode: "cluster",
},
],
}

18
packages/server/yarn.lock
View File

@ -1014,10 +1014,10 @@
resolved "https://registry.yarnpkg.com/@bcoe/v8-coverage/-/v8-coverage-0.2.3.tgz#75a2e8b51cb758a7553d6804a5932d7aace75c39"
integrity sha512-0hYQ8SB4Db5zvZB4axdMHGwEaQjkZzFjQiN9LVYvIFB2nSUHW9tYpxWriPrWDASIxiaXax83REcLxuSdnGPZtw==
"@budibase/backend-core@1.0.171":
version "1.0.171"
resolved "https://registry.yarnpkg.com/@budibase/backend-core/-/backend-core-1.0.171.tgz#39fb68fc1a6e233589cf65e8354ff5e5098b4b4e"
integrity sha512-au7nP2x4Oj/5PCU7qd538FDrz/Dhq9nSX6AjR+3+hoU658Gv3xes4RKXgFjf5qE0tdSKTSe1pMntMhwQxshUBQ==
"@budibase/backend-core@1.0.172":
version "1.0.172"
resolved "https://registry.yarnpkg.com/@budibase/backend-core/-/backend-core-1.0.172.tgz#81f825d5e141fbcbad085202509e1f97dfed0dfc"
integrity sha512-JDyR646dhDmQMWqeJxkdD70wyFv26hfNgRl768sgCgN4fG9B8829QsKeNjOquZf+U2diaKbDmKX+f/7gqoA5FQ==
dependencies:
"@techpass/passport-openidconnect" "^0.3.0"
aws-sdk "^2.901.0"
@ -1091,12 +1091,12 @@
svelte-flatpickr "^3.2.3"
svelte-portal "^1.0.0"
"@budibase/pro@1.0.171":
version "1.0.171"
resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-1.0.171.tgz#b64de1e3b80407bd004d4cf6e2b247f0b7855c02"
integrity sha512-iTu63sm/DA6AJbVNOnASZ03mPR6GFJf5rLRE672XMF2ftQ2WcjDx9QIuwKF4aPWQqB7LuzX8/isNugNYzgNuVw==
"@budibase/pro@1.0.172":
version "1.0.172"
resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-1.0.172.tgz#c6fe078098c72b8d163cfcf62d5389a9ca937b20"
integrity sha512-hHNobwr+8G99ofi9zp0cGB13iIC0StEV3JdF2a2HXwB4NzMz4b0vJ6VfcB1J+nVJPEe42DWmJViwJkuSX0jGhw==
dependencies:
"@budibase/backend-core" "1.0.171"
"@budibase/backend-core" "1.0.172"
node-fetch "^2.6.1"
"@budibase/standard-components@^0.9.139":

4
packages/worker/Dockerfile
View File

@ -10,6 +10,7 @@ WORKDIR /app
# copy files and install dependencies
COPY . ./
RUN yarn
RUN yarn global add pm2
EXPOSE 4001
@ -17,4 +18,5 @@ EXPOSE 4001
# due to this causing yarn to stop installing dev dependencies
# which are actually needed to get this environment up and running
ENV NODE_ENV=production
CMD ["yarn", "run:docker"]
ENV CLUSTER_MODE=${CLUSTER_MODE}
CMD ["./docker_run.sh"]

5
packages/worker/docker_run.sh Executable file
View File

@ -0,0 +1,5 @@
if [[ -z $CLUSTER_MODE ]]; then
yarn run:docker
else
yarn run:docker:cluster
fi

1
packages/worker/package.json
View File

@ -15,6 +15,7 @@
"build": "rimraf dist/ && tsc",
"postbuild": "copyfiles -u 1 src/**/*.hbs dist/",
"run:docker": "node dist/index.js",
"run:docker:cluster": "pm2-runtime start pm2.js",
"build:docker": "docker build . -t worker-service --label version=$BUDIBASE_RELEASE_VERSION",
"dev:stack:init": "node ./scripts/dev/manage.js init",
"dev:builder": "npm run dev:stack:init && nodemon",

9
packages/worker/pm2.js Normal file
View File

@ -0,0 +1,9 @@
module.exports = {
apps: [
{
script: "dist/index.js",
instances: "max",
exec_mode: "cluster",
},
],
}

103
packages/worker/src/api/controllers/global/configs.js
View File

@ -17,7 +17,6 @@ const { googleCallbackUrl, oidcCallbackUrl } = require("./auth")
const {
withCache,
CacheKeys,
TTL,
bustCache,
} = require("@budibase/backend-core/cache")
@ -256,58 +255,62 @@ exports.configChecklist = async function (ctx) {
const tenantId = getTenantId()
try {
ctx.body = await withCache(CacheKeys.CHECKLIST, TTL.ONE_HOUR, async () => {
let apps = []
if (!env.MULTI_TENANCY || tenantId) {
// Apps exist
apps = await getAllApps({ idsOnly: true, efficient: true })
}
ctx.body = await withCache(
CacheKeys.CHECKLIST,
env.CHECKLIST_CACHE_TTL,
async () => {
let apps = []
if (!env.MULTI_TENANCY || tenantId) {
// Apps exist
apps = await getAllApps({ idsOnly: true, efficient: true })
}
// They have set up SMTP
const smtpConfig = await getScopedFullConfig(db, {
type: Configs.SMTP,
})
// They have set up Google Auth
const googleConfig = await getScopedFullConfig(db, {
type: Configs.GOOGLE,
})
// They have set up OIDC
const oidcConfig = await getScopedFullConfig(db, {
type: Configs.OIDC,
})
// They have set up an global user
const users = await db.allDocs(
getGlobalUserParams(null, {
include_docs: true,
limit: 1,
// They have set up SMTP
const smtpConfig = await getScopedFullConfig(db, {
type: Configs.SMTP,
})
)
return {
apps: {
checked: apps.length > 0,
label: "Create your first app",
link: "/builder/portal/apps",
},
smtp: {
checked: !!smtpConfig,
label: "Set up email",
link: "/builder/portal/manage/email",
},
adminUser: {
checked: users && users.rows.length >= 1,
label: "Create your first user",
link: "/builder/portal/manage/users",
},
sso: {
checked: !!googleConfig || !!oidcConfig,
label: "Set up single sign-on",
link: "/builder/portal/manage/auth",
},
// They have set up Google Auth
const googleConfig = await getScopedFullConfig(db, {
type: Configs.GOOGLE,
})
// They have set up OIDC
const oidcConfig = await getScopedFullConfig(db, {
type: Configs.OIDC,
})
// They have set up an global user
const users = await db.allDocs(
getGlobalUserParams(null, {
include_docs: true,
limit: 1,
})
)
return {
apps: {
checked: apps.length > 0,
label: "Create your first app",
link: "/builder/portal/apps",
},
smtp: {
checked: !!smtpConfig,
label: "Set up email",
link: "/builder/portal/manage/email",
},
adminUser: {
checked: users && users.rows.length >= 1,
label: "Create your first user",
link: "/builder/portal/manage/users",
},
sso: {
checked: !!googleConfig || !!oidcConfig,
label: "Set up single sign-on",
link: "/builder/portal/manage/auth",
},
}
}
})
)
} catch (err) {
ctx.throw(err.status, err)
}

7
packages/worker/src/environment.js
View File

@ -20,6 +20,12 @@ if (!LOADED && isDev() && !isTest()) {
LOADED = true
}
function parseIntSafe(number) {
if (number) {
return parseInt(number)
}
}
module.exports = {
NODE_ENV: process.env.NODE_ENV,
SELF_HOSTED: !!parseInt(process.env.SELF_HOSTED),
@ -46,6 +52,7 @@ module.exports = {
SMTP_FROM_ADDRESS: process.env.SMTP_FROM_ADDRESS,
PLATFORM_URL: process.env.PLATFORM_URL,
COOKIE_DOMAIN: process.env.COOKIE_DOMAIN,
CHECKLIST_CACHE_TTL: parseIntSafe(process.env.CHECKLIST_CACHE_TTL) || 3600,
APPS_URL: process.env.APPS_URL,
_set(key, value) {
process.env[key] = value

18
packages/worker/yarn.lock
View File

@ -293,10 +293,10 @@
resolved "https://registry.yarnpkg.com/@bcoe/v8-coverage/-/v8-coverage-0.2.3.tgz#75a2e8b51cb758a7553d6804a5932d7aace75c39"
integrity sha512-0hYQ8SB4Db5zvZB4axdMHGwEaQjkZzFjQiN9LVYvIFB2nSUHW9tYpxWriPrWDASIxiaXax83REcLxuSdnGPZtw==
"@budibase/backend-core@1.0.171":
version "1.0.171"
resolved "https://registry.yarnpkg.com/@budibase/backend-core/-/backend-core-1.0.171.tgz#39fb68fc1a6e233589cf65e8354ff5e5098b4b4e"
integrity sha512-au7nP2x4Oj/5PCU7qd538FDrz/Dhq9nSX6AjR+3+hoU658Gv3xes4RKXgFjf5qE0tdSKTSe1pMntMhwQxshUBQ==
"@budibase/backend-core@1.0.172":
version "1.0.172"
resolved "https://registry.yarnpkg.com/@budibase/backend-core/-/backend-core-1.0.172.tgz#81f825d5e141fbcbad085202509e1f97dfed0dfc"
integrity sha512-JDyR646dhDmQMWqeJxkdD70wyFv26hfNgRl768sgCgN4fG9B8829QsKeNjOquZf+U2diaKbDmKX+f/7gqoA5FQ==
dependencies:
"@techpass/passport-openidconnect" "^0.3.0"
aws-sdk "^2.901.0"
@ -321,12 +321,12 @@
uuid "^8.3.2"
zlib "^1.0.5"
"@budibase/pro@1.0.171":
version "1.0.171"
resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-1.0.171.tgz#b64de1e3b80407bd004d4cf6e2b247f0b7855c02"
integrity sha512-iTu63sm/DA6AJbVNOnASZ03mPR6GFJf5rLRE672XMF2ftQ2WcjDx9QIuwKF4aPWQqB7LuzX8/isNugNYzgNuVw==
"@budibase/pro@1.0.172":
version "1.0.172"
resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-1.0.172.tgz#c6fe078098c72b8d163cfcf62d5389a9ca937b20"
integrity sha512-hHNobwr+8G99ofi9zp0cGB13iIC0StEV3JdF2a2HXwB4NzMz4b0vJ6VfcB1J+nVJPEe42DWmJViwJkuSX0jGhw==
dependencies:
"@budibase/backend-core" "1.0.171"
"@budibase/backend-core" "1.0.172"
node-fetch "^2.6.1"
"@cspotcode/source-map-consumer@0.8.0":