From 82ecaec405d1233c1b804583b0c9363b2baa921a Mon Sep 17 00:00:00 2001 From: Martin McKeaveney Date: Sun, 22 May 2022 16:39:34 +0100 Subject: [PATCH 1/5] lint --- packages/worker/src/api/controllers/global/configs.js | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/packages/worker/src/api/controllers/global/configs.js b/packages/worker/src/api/controllers/global/configs.js index 2562dd4e2e..658789e90a 100644 --- a/packages/worker/src/api/controllers/global/configs.js +++ b/packages/worker/src/api/controllers/global/configs.js @@ -1,7 +1,6 @@ const { generateConfigID, getConfigParams, - getGlobalUserParams, getScopedFullConfig, getAllApps, } = require("@budibase/backend-core/db") @@ -271,13 +270,6 @@ exports.configChecklist = async function (ctx) { const oidcConfig = await getScopedFullConfig(db, { type: Configs.OIDC, }) - // They have set up an global user - const users = await db.allDocs( - getGlobalUserParams(null, { - include_docs: true, - }) - ) - const adminUser = users.rows.some(row => row.doc.admin) ctx.body = { apps: { @@ -291,7 +283,7 @@ exports.configChecklist = async function (ctx) { link: "/builder/portal/manage/email", }, adminUser: { - checked: adminUser, + checked: true, label: "Create your first user", link: "/builder/portal/manage/users", }, From b55635c7ab67a39837ae09ae3195d7aae57d19cc Mon Sep 17 00:00:00 2001 From: Martin McKeaveney Date: Sun, 22 May 2022 17:11:05 +0100 Subject: [PATCH 2/5] disable prod app sync --- packages/server/src/api/controllers/application.ts | 9 +++++++++ packages/server/src/environment.js | 1 + 2 files changed, 10 insertions(+) diff --git a/packages/server/src/api/controllers/application.ts b/packages/server/src/api/controllers/application.ts index aa76dd403c..fbc2250b6b 100644 --- a/packages/server/src/api/controllers/application.ts +++ b/packages/server/src/api/controllers/application.ts @@ -439,6 +439,15 @@ export const destroy = async (ctx: any) => { } export const sync = async (ctx: any, next: any) => { + if (env.DISABLE_AUTO_PROD_APP_SYNC) { + ctx.status = 200 + ctx.body = { + message: + "App sync disabled. You can reenable with the DISABLE_AUTO_PROD_APP_SYNC environment variable.", + } + return next() + } + const appId = ctx.params.appId if (!isDevAppID(appId)) { ctx.throw(400, "This action cannot be performed for production apps") diff --git a/packages/server/src/environment.js b/packages/server/src/environment.js index ff1061dbaf..96f395f153 100644 --- a/packages/server/src/environment.js +++ b/packages/server/src/environment.js @@ -57,6 +57,7 @@ module.exports = { JEST_WORKER_ID: process.env.JEST_WORKER_ID, BUDIBASE_ENVIRONMENT: process.env.BUDIBASE_ENVIRONMENT, DISABLE_ACCOUNT_PORTAL: process.env.DISABLE_ACCOUNT_PORTAL, + DISABLE_AUTO_PROD_APP_SYNC: process.env.DISABLE_AUTO_PROD_APP_SYNC, // minor SALT_ROUNDS: process.env.SALT_ROUNDS, LOGGER: process.env.LOGGER, From 513d901ab416d33e1fbbf5889585a93560074f23 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Sun, 22 May 2022 17:14:24 +0100 Subject: [PATCH 3/5] Adding user load testing script. --- packages/server/scripts/load/users.js | 83 +++++++++++++++++++++++++++ packages/server/yarn.lock | 18 +++--- packages/worker/yarn.lock | 18 +++--- 3 files changed, 101 insertions(+), 18 deletions(-) create mode 100644 packages/server/scripts/load/users.js diff --git a/packages/server/scripts/load/users.js b/packages/server/scripts/load/users.js new file mode 100644 index 0000000000..7c3964b6e7 --- /dev/null +++ b/packages/server/scripts/load/users.js @@ -0,0 +1,83 @@ +const fetch = require("node-fetch") +const { getProdAppID } = require("@budibase/backend-core/db") + +const USER_LOAD_NUMBER = 10000 +const BATCH_SIZE = 25 +const SERVER_URL = "http://localhost:4001" +const PASSWORD = "test" + +const APP_ID = process.argv[2] +const API_KEY = process.argv[3] + +const words = [ + "test", + "testing", + "budi", + "mail", + "age", + "risk", + "load", + "uno", + "arm", + "leg", + "pen", + "glass", + "box", + "chicken", + "bottle", +] + +if (!APP_ID) { + console.error("Must supply app ID as first CLI option!") + process.exit(-1) +} +if (!API_KEY) { + console.error("Must supply API key as second CLI option!") + process.exit(-1) +} + +const WORD_1 = words[Math.floor(Math.random() * words.length)] +const WORD_2 = words[Math.floor(Math.random() * words.length)] + +function generateUser(count) { + return { + password: PASSWORD, + email: `${WORD_1}${count}@${WORD_2}.com`, + roles: { + [getProdAppID(APP_ID)]: "BASIC", + }, + status: "active", + forceResetPassword: false, + firstName: "John", + lastName: "Smith", + } +} + +async function run() { + for (let i = 0; i < USER_LOAD_NUMBER; i += BATCH_SIZE) { + let promises = [] + for (let j = 0; j < BATCH_SIZE; j++) { + promises.push( + fetch(`${SERVER_URL}/api/public/v1/users`, { + method: "POST", + body: JSON.stringify(generateUser(i + j)), + headers: { + "x-budibase-api-key": API_KEY, + "Content-Type": "application/json", + }, + }) + ) + } + await Promise.all(promises) + console.log(`${i + BATCH_SIZE} users have been created.`) + } +} + +run() + .then(() => { + console.log(`Generated ${USER_LOAD_NUMBER} users!`) + }) + .catch(err => { + console.error("Failed for reason: ", err) + process.exit(-1) + }) diff --git a/packages/server/yarn.lock b/packages/server/yarn.lock index 690f4ba0ef..e8a1186adf 100644 --- a/packages/server/yarn.lock +++ b/packages/server/yarn.lock @@ -1014,10 +1014,10 @@ resolved "https://registry.yarnpkg.com/@bcoe/v8-coverage/-/v8-coverage-0.2.3.tgz#75a2e8b51cb758a7553d6804a5932d7aace75c39" integrity sha512-0hYQ8SB4Db5zvZB4axdMHGwEaQjkZzFjQiN9LVYvIFB2nSUHW9tYpxWriPrWDASIxiaXax83REcLxuSdnGPZtw== -"@budibase/backend-core@1.0.164": - version "1.0.164" - resolved "https://registry.yarnpkg.com/@budibase/backend-core/-/backend-core-1.0.164.tgz#08c111dcebf5c74159a3c18218c7b3a0716de4f6" - integrity sha512-lpMudezndUD1hHBLfT9LDNKCunj8rQNlaJb30/xggdIUvp718u/jVP54hXF26NYxXOTMZ0EvMwCsIS4AucJ1Mg== +"@budibase/backend-core@1.0.167": + version "1.0.167" + resolved "https://registry.yarnpkg.com/@budibase/backend-core/-/backend-core-1.0.167.tgz#6ea4b90f8b8b8ec3cbbe05e39853d44d40938879" + integrity sha512-IG9GZUdjFiqOKbgpZiwGotyT3BttFlChXs7mT8GaOkX7XvlyxxrG/nSI1duglBd6X2iafGESKQU8e6tKKQsxuw== dependencies: "@techpass/passport-openidconnect" "^0.3.0" aws-sdk "^2.901.0" @@ -1091,12 +1091,12 @@ svelte-flatpickr "^3.2.3" svelte-portal "^1.0.0" -"@budibase/pro@1.0.164": - version "1.0.164" - resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-1.0.164.tgz#780ae38893d0609c87bf51fe96cc2c35bbdb431a" - integrity sha512-PgF7q2vADPPYzet4Wdma+THWuQPrEnN1+TfRly4l0oS9SUxutog3hYn0TlPmPS0AHgrqG/1v65TcEdC4ucX8TA== +"@budibase/pro@1.0.167": + version "1.0.167" + resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-1.0.167.tgz#62fe54b58206eb04151a786a5516351137bd2f84" + integrity sha512-AdWWOub58LMxZoZzXm3jy1ZSVOR1teSH+lwLisdGWLnoVAUV8e46pD7iOyJDM1SKuSyNWeQ1lXj8tcLHVK1+OA== dependencies: - "@budibase/backend-core" "1.0.164" + "@budibase/backend-core" "1.0.167" node-fetch "^2.6.1" "@budibase/standard-components@^0.9.139": diff --git a/packages/worker/yarn.lock b/packages/worker/yarn.lock index 6ec47f0e09..e52f6e6d9d 100644 --- a/packages/worker/yarn.lock +++ b/packages/worker/yarn.lock @@ -293,10 +293,10 @@ resolved "https://registry.yarnpkg.com/@bcoe/v8-coverage/-/v8-coverage-0.2.3.tgz#75a2e8b51cb758a7553d6804a5932d7aace75c39" integrity sha512-0hYQ8SB4Db5zvZB4axdMHGwEaQjkZzFjQiN9LVYvIFB2nSUHW9tYpxWriPrWDASIxiaXax83REcLxuSdnGPZtw== -"@budibase/backend-core@1.0.164": - version "1.0.164" - resolved "https://registry.yarnpkg.com/@budibase/backend-core/-/backend-core-1.0.164.tgz#08c111dcebf5c74159a3c18218c7b3a0716de4f6" - integrity sha512-lpMudezndUD1hHBLfT9LDNKCunj8rQNlaJb30/xggdIUvp718u/jVP54hXF26NYxXOTMZ0EvMwCsIS4AucJ1Mg== +"@budibase/backend-core@1.0.167": + version "1.0.167" + resolved "https://registry.yarnpkg.com/@budibase/backend-core/-/backend-core-1.0.167.tgz#6ea4b90f8b8b8ec3cbbe05e39853d44d40938879" + integrity sha512-IG9GZUdjFiqOKbgpZiwGotyT3BttFlChXs7mT8GaOkX7XvlyxxrG/nSI1duglBd6X2iafGESKQU8e6tKKQsxuw== dependencies: "@techpass/passport-openidconnect" "^0.3.0" aws-sdk "^2.901.0" @@ -321,12 +321,12 @@ uuid "^8.3.2" zlib "^1.0.5" -"@budibase/pro@1.0.164": - version "1.0.164" - resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-1.0.164.tgz#780ae38893d0609c87bf51fe96cc2c35bbdb431a" - integrity sha512-PgF7q2vADPPYzet4Wdma+THWuQPrEnN1+TfRly4l0oS9SUxutog3hYn0TlPmPS0AHgrqG/1v65TcEdC4ucX8TA== +"@budibase/pro@1.0.167": + version "1.0.167" + resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-1.0.167.tgz#62fe54b58206eb04151a786a5516351137bd2f84" + integrity sha512-AdWWOub58LMxZoZzXm3jy1ZSVOR1teSH+lwLisdGWLnoVAUV8e46pD7iOyJDM1SKuSyNWeQ1lXj8tcLHVK1+OA== dependencies: - "@budibase/backend-core" "1.0.164" + "@budibase/backend-core" "1.0.167" node-fetch "^2.6.1" "@cspotcode/source-map-consumer@0.8.0": From aede23d44e46a51823ec8914bed8c39a6a3bae32 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Sun, 22 May 2022 18:29:02 +0100 Subject: [PATCH 4/5] Improving performance of load script, can generate thousands of users a second. --- packages/backend-core/src/utils.js | 41 +++++++++--- packages/server/scripts/load/users.js | 62 ++++++++++++------- .../src/api/controllers/public/users.ts | 27 +------- packages/server/src/environment.js | 6 +- packages/server/src/utilities/users.js | 21 +++++++ packages/worker/src/environment.js | 6 +- 6 files changed, 106 insertions(+), 57 deletions(-) diff --git a/packages/backend-core/src/utils.js b/packages/backend-core/src/utils.js index 5c922c42ad..e764f35803 100644 --- a/packages/backend-core/src/utils.js +++ b/packages/backend-core/src/utils.js @@ -197,11 +197,16 @@ exports.getBuildersCount = async () => { return builders.length } -exports.saveUser = async ( +const DEFAULT_SAVE_USER = { + hashPassword: true, + requirePassword: true, + bulkCreate: false, +} + +exports.internalSaveUser = async ( user, tenantId, - hashPassword = true, - requirePassword = true + { hashPassword, requirePassword, bulkCreate } = DEFAULT_SAVE_USER ) => { if (!tenantId) { throw "No tenancy specified." @@ -213,7 +218,10 @@ exports.saveUser = async ( let { email, password, _id } = user // make sure another user isn't using the same email let dbUser - if (email) { + // user can't exist in bulk creation + if (bulkCreate) { + dbUser = null + } else if (email) { // check budibase users inside the tenant dbUser = await exports.getGlobalUserByEmail(email) if (dbUser != null && (dbUser._id !== _id || Array.isArray(dbUser))) { @@ -267,11 +275,17 @@ exports.saveUser = async ( user.status = UserStatus.ACTIVE } try { - const response = await db.put({ + const putOpts = { password: hashedPassword, ...user, - }) - await tryAddTenant(tenantId, _id, email) + } + if (bulkCreate) { + return putOpts + } + const response = await db.put(putOpts) + if (env.MULTI_TENANCY) { + await tryAddTenant(tenantId, _id, email) + } await userCache.invalidateUser(response.id) return { _id: response.id, @@ -288,6 +302,19 @@ exports.saveUser = async ( }) } +// maintained for api compat, don't want to change function signature +exports.saveUser = async ( + user, + tenantId, + hashPassword = true, + requirePassword = true +) => { + return exports.internalSaveUser(user, tenantId, { + hashPassword, + requirePassword, + }) +} + /** * Logs a user out from budibase. Re-used across account portal and builder. */ diff --git a/packages/server/scripts/load/users.js b/packages/server/scripts/load/users.js index 7c3964b6e7..3bc8056986 100644 --- a/packages/server/scripts/load/users.js +++ b/packages/server/scripts/load/users.js @@ -1,13 +1,21 @@ -const fetch = require("node-fetch") -const { getProdAppID } = require("@budibase/backend-core/db") +// get the JWT secret etc +require("../../src/environment") +require("@budibase/backend-core").init() +const { + getProdAppID, + generateGlobalUserID, +} = require("@budibase/backend-core/db") +const { doInTenant, getGlobalDB } = require("@budibase/backend-core/tenancy") +const { internalSaveUser } = require("@budibase/backend-core/utils") +const { publicApiUserFix } = require("../../src/utilities/users") +const { hash } = require("@budibase/backend-core/utils") const USER_LOAD_NUMBER = 10000 -const BATCH_SIZE = 25 -const SERVER_URL = "http://localhost:4001" +const BATCH_SIZE = 200 const PASSWORD = "test" +const TENANT_ID = "default" const APP_ID = process.argv[2] -const API_KEY = process.argv[3] const words = [ "test", @@ -31,17 +39,15 @@ if (!APP_ID) { console.error("Must supply app ID as first CLI option!") process.exit(-1) } -if (!API_KEY) { - console.error("Must supply API key as second CLI option!") - process.exit(-1) -} const WORD_1 = words[Math.floor(Math.random() * words.length)] const WORD_2 = words[Math.floor(Math.random() * words.length)] +let HASHED_PASSWORD function generateUser(count) { return { - password: PASSWORD, + _id: generateGlobalUserID(), + password: HASHED_PASSWORD, email: `${WORD_1}${count}@${WORD_2}.com`, roles: { [getProdAppID(APP_ID)]: "BASIC", @@ -54,23 +60,31 @@ function generateUser(count) { } async function run() { - for (let i = 0; i < USER_LOAD_NUMBER; i += BATCH_SIZE) { - let promises = [] - for (let j = 0; j < BATCH_SIZE; j++) { - promises.push( - fetch(`${SERVER_URL}/api/public/v1/users`, { - method: "POST", - body: JSON.stringify(generateUser(i + j)), - headers: { - "x-budibase-api-key": API_KEY, - "Content-Type": "application/json", + HASHED_PASSWORD = await hash(PASSWORD) + return doInTenant(TENANT_ID, async () => { + const db = getGlobalDB() + for (let i = 0; i < USER_LOAD_NUMBER; i += BATCH_SIZE) { + let userSavePromises = [] + for (let j = 0; j < BATCH_SIZE; j++) { + // like the public API + const ctx = publicApiUserFix({ + request: { + body: generateUser(i + j), }, }) - ) + userSavePromises.push( + internalSaveUser(ctx.request.body, TENANT_ID, { + hashPassword: false, + requirePassword: true, + bulkCreate: true, + }) + ) + } + const users = await Promise.all(userSavePromises) + await db.bulkDocs(users) + console.log(`${i + BATCH_SIZE} users have been created.`) } - await Promise.all(promises) - console.log(`${i + BATCH_SIZE} users have been created.`) - } + }) } run() diff --git a/packages/server/src/api/controllers/public/users.ts b/packages/server/src/api/controllers/public/users.ts index f199dcb761..129d2c883f 100644 --- a/packages/server/src/api/controllers/public/users.ts +++ b/packages/server/src/api/controllers/public/users.ts @@ -4,30 +4,9 @@ import { readGlobalUser, saveGlobalUser, } from "../../../utilities/workerRequests" +import { publicApiUserFix } from "../../../utilities/users" import { search as stringSearch } from "./utils" -const { getProdAppID } = require("@budibase/backend-core/db") - -function fixUser(ctx: any) { - if (!ctx.request.body) { - return ctx - } - if (!ctx.request.body._id && ctx.params.userId) { - ctx.request.body._id = ctx.params.userId - } - if (!ctx.request.body.roles) { - ctx.request.body.roles = {} - } else { - const newRoles: { [key: string]: string } = {} - for (let [appId, role] of Object.entries(ctx.request.body.roles)) { - // @ts-ignore - newRoles[getProdAppID(appId)] = role - } - ctx.request.body.roles = newRoles - } - return ctx -} - function getUser(ctx: any, userId?: string) { if (userId) { ctx.params = { userId } @@ -45,7 +24,7 @@ export async function search(ctx: any, next: any) { } export async function create(ctx: any, next: any) { - const response = await saveGlobalUser(fixUser(ctx)) + const response = await saveGlobalUser(publicApiUserFix(ctx)) ctx.body = await getUser(ctx, response._id) await next() } @@ -61,7 +40,7 @@ export async function update(ctx: any, next: any) { ...ctx.request.body, _rev: user._rev, } - const response = await saveGlobalUser(fixUser(ctx)) + const response = await saveGlobalUser(publicApiUserFix(ctx)) ctx.body = await getUser(ctx, response._id) await next() } diff --git a/packages/server/src/environment.js b/packages/server/src/environment.js index 96f395f153..9fa8692298 100644 --- a/packages/server/src/environment.js +++ b/packages/server/src/environment.js @@ -1,3 +1,5 @@ +const { join } = require("path") + function isTest() { return ( process.env.NODE_ENV === "jest" || @@ -20,7 +22,9 @@ function isCypress() { let LOADED = false if (!LOADED && isDev() && !isTest()) { - require("dotenv").config() + require("dotenv").config({ + path: join(__dirname, "..", ".env"), + }) LOADED = true } diff --git a/packages/server/src/utilities/users.js b/packages/server/src/utilities/users.js index b3601986d8..e769441322 100644 --- a/packages/server/src/utilities/users.js +++ b/packages/server/src/utilities/users.js @@ -1,6 +1,7 @@ const { InternalTables } = require("../db/utils") const { getGlobalUser } = require("../utilities/global") const { getAppDB } = require("@budibase/backend-core/context") +const { getProdAppID } = require("@budibase/backend-core/db") exports.getFullUser = async (ctx, userId) => { const global = await getGlobalUser(userId) @@ -22,3 +23,23 @@ exports.getFullUser = async (ctx, userId) => { _id: userId, } } + +exports.publicApiUserFix = ctx => { + if (!ctx.request.body) { + return ctx + } + if (!ctx.request.body._id && ctx.params.userId) { + ctx.request.body._id = ctx.params.userId + } + if (!ctx.request.body.roles) { + ctx.request.body.roles = {} + } else { + const newRoles = {} + for (let [appId, role] of Object.entries(ctx.request.body.roles)) { + // @ts-ignore + newRoles[getProdAppID(appId)] = role + } + ctx.request.body.roles = newRoles + } + return ctx +} diff --git a/packages/worker/src/environment.js b/packages/worker/src/environment.js index c965863a54..8ef12e3877 100644 --- a/packages/worker/src/environment.js +++ b/packages/worker/src/environment.js @@ -1,3 +1,5 @@ +const { join } = require("path") + function isDev() { return process.env.NODE_ENV !== "production" } @@ -12,7 +14,9 @@ function isTest() { let LOADED = false if (!LOADED && isDev() && !isTest()) { - require("dotenv").config() + require("dotenv").config({ + path: join(__dirname, "..", ".env"), + }) LOADED = true } From e61e118b251313ccad7cdc779f7000fea69151c7 Mon Sep 17 00:00:00 2001 From: Martin McKeaveney Date: Sun, 22 May 2022 18:51:13 +0100 Subject: [PATCH 5/5] query optimisation on checklist endpoint --- packages/worker/src/api/controllers/global/configs.js | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/packages/worker/src/api/controllers/global/configs.js b/packages/worker/src/api/controllers/global/configs.js index 658789e90a..3799337bd5 100644 --- a/packages/worker/src/api/controllers/global/configs.js +++ b/packages/worker/src/api/controllers/global/configs.js @@ -1,6 +1,7 @@ const { generateConfigID, getConfigParams, + getGlobalUserParams, getScopedFullConfig, getAllApps, } = require("@budibase/backend-core/db") @@ -271,6 +272,14 @@ exports.configChecklist = async function (ctx) { type: Configs.OIDC, }) + // They have set up an global user + const users = await db.allDocs( + getGlobalUserParams(null, { + include_docs: true, + limit: 1, + }) + ) + ctx.body = { apps: { checked: apps.length > 0, @@ -283,7 +292,7 @@ exports.configChecklist = async function (ctx) { link: "/builder/portal/manage/email", }, adminUser: { - checked: true, + checked: users && users.rows.length >= 1, label: "Create your first user", link: "/builder/portal/manage/users", },