MirrorSave
/
budibase
mirror of https://github.com/Budibase/budibase.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #6021 from Budibase/hotfixes/perf-bg 

Hotfixes/perf bg
This commit is contained in:
Martin McKeaveney 2022-05-22 19:21:50 +01:00 committed by GitHub
parent 617650bf80 e61e118b25
commit fe353b5fce
10 changed files with 196 additions and 53 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

39
packages/backend-core/src/utils.js
View File

@ -197,11 +197,16 @@ exports.getBuildersCount = async () => {
return builders.length
}
exports.saveUser = async (
const DEFAULT_SAVE_USER = {
hashPassword: true,
requirePassword: true,
bulkCreate: false,
}
exports.internalSaveUser = async (
user,
tenantId,
hashPassword = true,
requirePassword = true
{ hashPassword, requirePassword, bulkCreate } = DEFAULT_SAVE_USER
) => {
if (!tenantId) {
throw "No tenancy specified."
@ -213,7 +218,10 @@ exports.saveUser = async (
let { email, password, _id } = user
// make sure another user isn't using the same email
let dbUser
if (email) {
// user can't exist in bulk creation
if (bulkCreate) {
dbUser = null
} else if (email) {
// check budibase users inside the tenant
dbUser = await exports.getGlobalUserByEmail(email)
if (dbUser != null && (dbUser._id !== _id || Array.isArray(dbUser))) {
@ -267,11 +275,17 @@ exports.saveUser = async (
user.status = UserStatus.ACTIVE
}
try {
const response = await db.put({
const putOpts = {
password: hashedPassword,
...user,
})
}
if (bulkCreate) {
return putOpts
}
const response = await db.put(putOpts)
if (env.MULTI_TENANCY) {
await tryAddTenant(tenantId, _id, email)
}
await userCache.invalidateUser(response.id)
return {
_id: response.id,
@ -288,6 +302,19 @@ exports.saveUser = async (
})
}
// maintained for api compat, don't want to change function signature
exports.saveUser = async (
user,
tenantId,
hashPassword = true,
requirePassword = true
) => {
return exports.internalSaveUser(user, tenantId, {
hashPassword,
requirePassword,
})
}
/**
* Logs a user out from budibase. Re-used across account portal and builder.
*/

97
packages/server/scripts/load/users.js Normal file
View File

@ -0,0 +1,97 @@
// get the JWT secret etc
require("../../src/environment")
require("@budibase/backend-core").init()
const {
getProdAppID,
generateGlobalUserID,
} = require("@budibase/backend-core/db")
const { doInTenant, getGlobalDB } = require("@budibase/backend-core/tenancy")
const { internalSaveUser } = require("@budibase/backend-core/utils")
const { publicApiUserFix } = require("../../src/utilities/users")
const { hash } = require("@budibase/backend-core/utils")
const USER_LOAD_NUMBER = 10000
const BATCH_SIZE = 200
const PASSWORD = "test"
const TENANT_ID = "default"
const APP_ID = process.argv[2]
const words = [
"test",
"testing",
"budi",
"mail",
"age",
"risk",
"load",
"uno",
"arm",
"leg",
"pen",
"glass",
"box",
"chicken",
"bottle",
]
if (!APP_ID) {
console.error("Must supply app ID as first CLI option!")
process.exit(-1)
}
const WORD_1 = words[Math.floor(Math.random() * words.length)]
const WORD_2 = words[Math.floor(Math.random() * words.length)]
let HASHED_PASSWORD
function generateUser(count) {
return {
_id: generateGlobalUserID(),
password: HASHED_PASSWORD,
email: `${WORD_1}${count}@${WORD_2}.com`,
roles: {
[getProdAppID(APP_ID)]: "BASIC",
},
status: "active",
forceResetPassword: false,
firstName: "John",
lastName: "Smith",
}
}
async function run() {
HASHED_PASSWORD = await hash(PASSWORD)
return doInTenant(TENANT_ID, async () => {
const db = getGlobalDB()
for (let i = 0; i < USER_LOAD_NUMBER; i += BATCH_SIZE) {
let userSavePromises = []
for (let j = 0; j < BATCH_SIZE; j++) {
// like the public API
const ctx = publicApiUserFix({
request: {
body: generateUser(i + j),
},
})
userSavePromises.push(
internalSaveUser(ctx.request.body, TENANT_ID, {
hashPassword: false,
requirePassword: true,
bulkCreate: true,
})
)
}
const users = await Promise.all(userSavePromises)
await db.bulkDocs(users)
console.log(`${i + BATCH_SIZE} users have been created.`)
}
})
}
run()
.then(() => {
console.log(`Generated ${USER_LOAD_NUMBER} users!`)
})
.catch(err => {
console.error("Failed for reason: ", err)
process.exit(-1)
})

9
packages/server/src/api/controllers/application.ts
View File

@ -439,6 +439,15 @@ export const destroy = async (ctx: any) => {
}
export const sync = async (ctx: any, next: any) => {
if (env.DISABLE_AUTO_PROD_APP_SYNC) {
ctx.status = 200
ctx.body = {
message:
"App sync disabled. You can reenable with the DISABLE_AUTO_PROD_APP_SYNC environment variable.",
}
return next()
}
const appId = ctx.params.appId
if (!isDevAppID(appId)) {
ctx.throw(400, "This action cannot be performed for production apps")

27
packages/server/src/api/controllers/public/users.ts
View File

@ -4,30 +4,9 @@ import {
readGlobalUser,
saveGlobalUser,
} from "../../../utilities/workerRequests"
import { publicApiUserFix } from "../../../utilities/users"
import { search as stringSearch } from "./utils"
const { getProdAppID } = require("@budibase/backend-core/db")
function fixUser(ctx: any) {
if (!ctx.request.body) {
return ctx
}
if (!ctx.request.body._id && ctx.params.userId) {
ctx.request.body._id = ctx.params.userId
}
if (!ctx.request.body.roles) {
ctx.request.body.roles = {}
} else {
const newRoles: { [key: string]: string } = {}
for (let [appId, role] of Object.entries(ctx.request.body.roles)) {
// @ts-ignore
newRoles[getProdAppID(appId)] = role
}
ctx.request.body.roles = newRoles
}
return ctx
}
function getUser(ctx: any, userId?: string) {
if (userId) {
ctx.params = { userId }
@ -45,7 +24,7 @@ export async function search(ctx: any, next: any) {
}
export async function create(ctx: any, next: any) {
const response = await saveGlobalUser(fixUser(ctx))
const response = await saveGlobalUser(publicApiUserFix(ctx))
ctx.body = await getUser(ctx, response._id)
await next()
}
@ -61,7 +40,7 @@ export async function update(ctx: any, next: any) {
...ctx.request.body,
_rev: user._rev,
}
const response = await saveGlobalUser(fixUser(ctx))
const response = await saveGlobalUser(publicApiUserFix(ctx))
ctx.body = await getUser(ctx, response._id)
await next()
}

7
packages/server/src/environment.js
View File

@ -1,3 +1,5 @@
const { join } = require("path")
function isTest() {
return (
process.env.NODE_ENV === "jest" ||
@ -20,7 +22,9 @@ function isCypress() {
let LOADED = false
if (!LOADED && isDev() && !isTest()) {
require("dotenv").config()
require("dotenv").config({
path: join(__dirname, "..", ".env"),
})
LOADED = true
}
@ -57,6 +61,7 @@ module.exports = {
JEST_WORKER_ID: process.env.JEST_WORKER_ID,
BUDIBASE_ENVIRONMENT: process.env.BUDIBASE_ENVIRONMENT,
DISABLE_ACCOUNT_PORTAL: process.env.DISABLE_ACCOUNT_PORTAL,
DISABLE_AUTO_PROD_APP_SYNC: process.env.DISABLE_AUTO_PROD_APP_SYNC,
// minor
SALT_ROUNDS: process.env.SALT_ROUNDS,
LOGGER: process.env.LOGGER,

21
packages/server/src/utilities/users.js
View File

@ -1,6 +1,7 @@
const { InternalTables } = require("../db/utils")
const { getGlobalUser } = require("../utilities/global")
const { getAppDB } = require("@budibase/backend-core/context")
const { getProdAppID } = require("@budibase/backend-core/db")
exports.getFullUser = async (ctx, userId) => {
const global = await getGlobalUser(userId)
@ -22,3 +23,23 @@ exports.getFullUser = async (ctx, userId) => {
_id: userId,
}
}
exports.publicApiUserFix = ctx => {
if (!ctx.request.body) {
return ctx
}
if (!ctx.request.body._id && ctx.params.userId) {
ctx.request.body._id = ctx.params.userId
}
if (!ctx.request.body.roles) {
ctx.request.body.roles = {}
} else {
const newRoles = {}
for (let [appId, role] of Object.entries(ctx.request.body.roles)) {
// @ts-ignore
newRoles[getProdAppID(appId)] = role
}
ctx.request.body.roles = newRoles
}
return ctx
}

18
packages/server/yarn.lock
View File

@ -1014,10 +1014,10 @@
resolved "https://registry.yarnpkg.com/@bcoe/v8-coverage/-/v8-coverage-0.2.3.tgz#75a2e8b51cb758a7553d6804a5932d7aace75c39"
integrity sha512-0hYQ8SB4Db5zvZB4axdMHGwEaQjkZzFjQiN9LVYvIFB2nSUHW9tYpxWriPrWDASIxiaXax83REcLxuSdnGPZtw==
"@budibase/backend-core@1.0.164":
version "1.0.164"
resolved "https://registry.yarnpkg.com/@budibase/backend-core/-/backend-core-1.0.164.tgz#08c111dcebf5c74159a3c18218c7b3a0716de4f6"
integrity sha512-lpMudezndUD1hHBLfT9LDNKCunj8rQNlaJb30/xggdIUvp718u/jVP54hXF26NYxXOTMZ0EvMwCsIS4AucJ1Mg==
"@budibase/backend-core@1.0.167":
version "1.0.167"
resolved "https://registry.yarnpkg.com/@budibase/backend-core/-/backend-core-1.0.167.tgz#6ea4b90f8b8b8ec3cbbe05e39853d44d40938879"
integrity sha512-IG9GZUdjFiqOKbgpZiwGotyT3BttFlChXs7mT8GaOkX7XvlyxxrG/nSI1duglBd6X2iafGESKQU8e6tKKQsxuw==
dependencies:
"@techpass/passport-openidconnect" "^0.3.0"
aws-sdk "^2.901.0"
@ -1091,12 +1091,12 @@
svelte-flatpickr "^3.2.3"
svelte-portal "^1.0.0"
"@budibase/pro@1.0.164":
version "1.0.164"
resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-1.0.164.tgz#780ae38893d0609c87bf51fe96cc2c35bbdb431a"
integrity sha512-PgF7q2vADPPYzet4Wdma+THWuQPrEnN1+TfRly4l0oS9SUxutog3hYn0TlPmPS0AHgrqG/1v65TcEdC4ucX8TA==
"@budibase/pro@1.0.167":
version "1.0.167"
resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-1.0.167.tgz#62fe54b58206eb04151a786a5516351137bd2f84"
integrity sha512-AdWWOub58LMxZoZzXm3jy1ZSVOR1teSH+lwLisdGWLnoVAUV8e46pD7iOyJDM1SKuSyNWeQ1lXj8tcLHVK1+OA==
dependencies:
"@budibase/backend-core" "1.0.164"
"@budibase/backend-core" "1.0.167"
node-fetch "^2.6.1"
"@budibase/standard-components@^0.9.139":

5
packages/worker/src/api/controllers/global/configs.js
View File

@ -271,13 +271,14 @@ exports.configChecklist = async function (ctx) {
const oidcConfig = await getScopedFullConfig(db, {
type: Configs.OIDC,
})
// They have set up an global user
const users = await db.allDocs(
getGlobalUserParams(null, {
include_docs: true,
limit: 1,
})
)
const adminUser = users.rows.some(row => row.doc.admin)
ctx.body = {
apps: {
@ -291,7 +292,7 @@ exports.configChecklist = async function (ctx) {
link: "/builder/portal/manage/email",
},
adminUser: {
checked: adminUser,
checked: users && users.rows.length >= 1,
label: "Create your first user",
link: "/builder/portal/manage/users",
},

6
packages/worker/src/environment.js
View File

@ -1,3 +1,5 @@
const { join } = require("path")
function isDev() {
return process.env.NODE_ENV !== "production"
}
@ -12,7 +14,9 @@ function isTest() {
let LOADED = false
if (!LOADED && isDev() && !isTest()) {
require("dotenv").config()
require("dotenv").config({
path: join(__dirname, "..", ".env"),
})
LOADED = true
}

18
packages/worker/yarn.lock
View File

@ -293,10 +293,10 @@
resolved "https://registry.yarnpkg.com/@bcoe/v8-coverage/-/v8-coverage-0.2.3.tgz#75a2e8b51cb758a7553d6804a5932d7aace75c39"
integrity sha512-0hYQ8SB4Db5zvZB4axdMHGwEaQjkZzFjQiN9LVYvIFB2nSUHW9tYpxWriPrWDASIxiaXax83REcLxuSdnGPZtw==
"@budibase/backend-core@1.0.164":
version "1.0.164"
resolved "https://registry.yarnpkg.com/@budibase/backend-core/-/backend-core-1.0.164.tgz#08c111dcebf5c74159a3c18218c7b3a0716de4f6"
integrity sha512-lpMudezndUD1hHBLfT9LDNKCunj8rQNlaJb30/xggdIUvp718u/jVP54hXF26NYxXOTMZ0EvMwCsIS4AucJ1Mg==
"@budibase/backend-core@1.0.167":
version "1.0.167"
resolved "https://registry.yarnpkg.com/@budibase/backend-core/-/backend-core-1.0.167.tgz#6ea4b90f8b8b8ec3cbbe05e39853d44d40938879"
integrity sha512-IG9GZUdjFiqOKbgpZiwGotyT3BttFlChXs7mT8GaOkX7XvlyxxrG/nSI1duglBd6X2iafGESKQU8e6tKKQsxuw==
dependencies:
"@techpass/passport-openidconnect" "^0.3.0"
aws-sdk "^2.901.0"
@ -321,12 +321,12 @@
uuid "^8.3.2"
zlib "^1.0.5"
"@budibase/pro@1.0.164":
version "1.0.164"
resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-1.0.164.tgz#780ae38893d0609c87bf51fe96cc2c35bbdb431a"
integrity sha512-PgF7q2vADPPYzet4Wdma+THWuQPrEnN1+TfRly4l0oS9SUxutog3hYn0TlPmPS0AHgrqG/1v65TcEdC4ucX8TA==
"@budibase/pro@1.0.167":
version "1.0.167"
resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-1.0.167.tgz#62fe54b58206eb04151a786a5516351137bd2f84"
integrity sha512-AdWWOub58LMxZoZzXm3jy1ZSVOR1teSH+lwLisdGWLnoVAUV8e46pD7iOyJDM1SKuSyNWeQ1lXj8tcLHVK1+OA==
dependencies:
"@budibase/backend-core" "1.0.164"
"@budibase/backend-core" "1.0.167"
node-fetch "^2.6.1"
"@cspotcode/source-map-consumer@0.8.0":