Merge pull request #6021 from Budibase/hotfixes/perf-bg

Hotfixes/perf bg

packages/backend-core/src/utils.js

@@ -197,11 +197,16 @@ exports.getBuildersCount = async () => {
   return builders.length
 }
 
-exports.saveUser = async (
+const DEFAULT_SAVE_USER = {
+  hashPassword: true,
+  requirePassword: true,
+  bulkCreate: false,
+}
+
+exports.internalSaveUser = async (
   user,
   tenantId,
-  hashPassword = true,
+  { hashPassword, requirePassword, bulkCreate } = DEFAULT_SAVE_USER
-  requirePassword = true
 ) => {
   if (!tenantId) {
     throw "No tenancy specified."
@@ -213,7 +218,10 @@ exports.saveUser = async (
     let { email, password, _id } = user
     // make sure another user isn't using the same email
     let dbUser
-    if (email) {
+    // user can't exist in bulk creation
+    if (bulkCreate) {
+      dbUser = null
+    } else if (email) {
       // check budibase users inside the tenant
       dbUser = await exports.getGlobalUserByEmail(email)
       if (dbUser != null && (dbUser._id !== _id || Array.isArray(dbUser))) {
@@ -267,11 +275,17 @@ exports.saveUser = async (
       user.status = UserStatus.ACTIVE
     }
     try {
-      const response = await db.put({
+      const putOpts = {
         password: hashedPassword,
         ...user,
-      })
+      }
+      if (bulkCreate) {
+        return putOpts
+      }
+      const response = await db.put(putOpts)
+      if (env.MULTI_TENANCY) {
         await tryAddTenant(tenantId, _id, email)
+      }
       await userCache.invalidateUser(response.id)
       return {
         _id: response.id,
@@ -288,6 +302,19 @@ exports.saveUser = async (
   })
 }
 
+// maintained for api compat, don't want to change function signature
+exports.saveUser = async (
+  user,
+  tenantId,
+  hashPassword = true,
+  requirePassword = true
+) => {
+  return exports.internalSaveUser(user, tenantId, {
+    hashPassword,
+    requirePassword,
+  })
+}
+
 /**
  * Logs a user out from budibase. Re-used across account portal and builder.
  */

packages/server/scripts/load/users.js

@@ -0,0 +1,97 @@
+// get the JWT secret etc
+require("../../src/environment")
+require("@budibase/backend-core").init()
+const {
+  getProdAppID,
+  generateGlobalUserID,
+} = require("@budibase/backend-core/db")
+const { doInTenant, getGlobalDB } = require("@budibase/backend-core/tenancy")
+const { internalSaveUser } = require("@budibase/backend-core/utils")
+const { publicApiUserFix } = require("../../src/utilities/users")
+const { hash } = require("@budibase/backend-core/utils")
+
+const USER_LOAD_NUMBER = 10000
+const BATCH_SIZE = 200
+const PASSWORD = "test"
+const TENANT_ID = "default"
+
+const APP_ID = process.argv[2]
+
+const words = [
+  "test",
+  "testing",
+  "budi",
+  "mail",
+  "age",
+  "risk",
+  "load",
+  "uno",
+  "arm",
+  "leg",
+  "pen",
+  "glass",
+  "box",
+  "chicken",
+  "bottle",
+]
+
+if (!APP_ID) {
+  console.error("Must supply app ID as first CLI option!")
+  process.exit(-1)
+}
+
+const WORD_1 = words[Math.floor(Math.random() * words.length)]
+const WORD_2 = words[Math.floor(Math.random() * words.length)]
+let HASHED_PASSWORD
+
+function generateUser(count) {
+  return {
+    _id: generateGlobalUserID(),
+    password: HASHED_PASSWORD,
+    email: `${WORD_1}${count}@${WORD_2}.com`,
+    roles: {
+      [getProdAppID(APP_ID)]: "BASIC",
+    },
+    status: "active",
+    forceResetPassword: false,
+    firstName: "John",
+    lastName: "Smith",
+  }
+}
+
+async function run() {
+  HASHED_PASSWORD = await hash(PASSWORD)
+  return doInTenant(TENANT_ID, async () => {
+    const db = getGlobalDB()
+    for (let i = 0; i < USER_LOAD_NUMBER; i += BATCH_SIZE) {
+      let userSavePromises = []
+      for (let j = 0; j < BATCH_SIZE; j++) {
+        // like the public API
+        const ctx = publicApiUserFix({
+          request: {
+            body: generateUser(i + j),
+          },
+        })
+        userSavePromises.push(
+          internalSaveUser(ctx.request.body, TENANT_ID, {
+            hashPassword: false,
+            requirePassword: true,
+            bulkCreate: true,
+          })
+        )
+      }
+      const users = await Promise.all(userSavePromises)
+      await db.bulkDocs(users)
+      console.log(`${i + BATCH_SIZE} users have been created.`)
+    }
+  })
+}
+
+run()
+  .then(() => {
+    console.log(`Generated ${USER_LOAD_NUMBER} users!`)
+  })
+  .catch(err => {
+    console.error("Failed for reason: ", err)
+    process.exit(-1)
+  })

packages/server/src/api/controllers/application.ts

@@ -439,6 +439,15 @@ export const destroy = async (ctx: any) => {
 }
 
 export const sync = async (ctx: any, next: any) => {
+  if (env.DISABLE_AUTO_PROD_APP_SYNC) {
+    ctx.status = 200
+    ctx.body = {
+      message:
+        "App sync disabled. You can reenable with the DISABLE_AUTO_PROD_APP_SYNC environment variable.",
+    }
+    return next()
+  }
+
   const appId = ctx.params.appId
   if (!isDevAppID(appId)) {
     ctx.throw(400, "This action cannot be performed for production apps")

packages/server/src/api/controllers/public/users.ts

@@ -4,30 +4,9 @@ import {
   readGlobalUser,
   saveGlobalUser,
 } from "../../../utilities/workerRequests"
+import { publicApiUserFix } from "../../../utilities/users"
 import { search as stringSearch } from "./utils"
 
-const { getProdAppID } = require("@budibase/backend-core/db")
-
-function fixUser(ctx: any) {
-  if (!ctx.request.body) {
-    return ctx
-  }
-  if (!ctx.request.body._id && ctx.params.userId) {
-    ctx.request.body._id = ctx.params.userId
-  }
-  if (!ctx.request.body.roles) {
-    ctx.request.body.roles = {}
-  } else {
-    const newRoles: { [key: string]: string } = {}
-    for (let [appId, role] of Object.entries(ctx.request.body.roles)) {
-      // @ts-ignore
-      newRoles[getProdAppID(appId)] = role
-    }
-    ctx.request.body.roles = newRoles
-  }
-  return ctx
-}
-
 function getUser(ctx: any, userId?: string) {
   if (userId) {
     ctx.params = { userId }
@@ -45,7 +24,7 @@ export async function search(ctx: any, next: any) {
 }
 
 export async function create(ctx: any, next: any) {
-  const response = await saveGlobalUser(fixUser(ctx))
+  const response = await saveGlobalUser(publicApiUserFix(ctx))
   ctx.body = await getUser(ctx, response._id)
   await next()
 }
@@ -61,7 +40,7 @@ export async function update(ctx: any, next: any) {
     ...ctx.request.body,
     _rev: user._rev,
   }
-  const response = await saveGlobalUser(fixUser(ctx))
+  const response = await saveGlobalUser(publicApiUserFix(ctx))
   ctx.body = await getUser(ctx, response._id)
   await next()
 }

packages/server/src/environment.js

@@ -1,3 +1,5 @@
+const { join } = require("path")
+
 function isTest() {
   return (
     process.env.NODE_ENV === "jest" ||
@@ -20,7 +22,9 @@ function isCypress() {
 
 let LOADED = false
 if (!LOADED && isDev() && !isTest()) {
-  require("dotenv").config()
+  require("dotenv").config({
+    path: join(__dirname, "..", ".env"),
+  })
   LOADED = true
 }
 
@@ -57,6 +61,7 @@ module.exports = {
   JEST_WORKER_ID: process.env.JEST_WORKER_ID,
   BUDIBASE_ENVIRONMENT: process.env.BUDIBASE_ENVIRONMENT,
   DISABLE_ACCOUNT_PORTAL: process.env.DISABLE_ACCOUNT_PORTAL,
+  DISABLE_AUTO_PROD_APP_SYNC: process.env.DISABLE_AUTO_PROD_APP_SYNC,
   // minor
   SALT_ROUNDS: process.env.SALT_ROUNDS,
   LOGGER: process.env.LOGGER,

packages/server/src/utilities/users.js

@@ -1,6 +1,7 @@
 const { InternalTables } = require("../db/utils")
 const { getGlobalUser } = require("../utilities/global")
 const { getAppDB } = require("@budibase/backend-core/context")
+const { getProdAppID } = require("@budibase/backend-core/db")
 
 exports.getFullUser = async (ctx, userId) => {
   const global = await getGlobalUser(userId)
@@ -22,3 +23,23 @@ exports.getFullUser = async (ctx, userId) => {
     _id: userId,
   }
 }
+
+exports.publicApiUserFix = ctx => {
+  if (!ctx.request.body) {
+    return ctx
+  }
+  if (!ctx.request.body._id && ctx.params.userId) {
+    ctx.request.body._id = ctx.params.userId
+  }
+  if (!ctx.request.body.roles) {
+    ctx.request.body.roles = {}
+  } else {
+    const newRoles = {}
+    for (let [appId, role] of Object.entries(ctx.request.body.roles)) {
+      // @ts-ignore
+      newRoles[getProdAppID(appId)] = role
+    }
+    ctx.request.body.roles = newRoles
+  }
+  return ctx
+}

packages/server/yarn.lock

@@ -1014,10 +1014,10 @@
   resolved "https://registry.yarnpkg.com/@bcoe/v8-coverage/-/v8-coverage-0.2.3.tgz#75a2e8b51cb758a7553d6804a5932d7aace75c39"
   integrity sha512-0hYQ8SB4Db5zvZB4axdMHGwEaQjkZzFjQiN9LVYvIFB2nSUHW9tYpxWriPrWDASIxiaXax83REcLxuSdnGPZtw==
 
-"@budibase/backend-core@1.0.164":
+"@budibase/backend-core@1.0.167":
-  version "1.0.164"
+  version "1.0.167"
-  resolved "https://registry.yarnpkg.com/@budibase/backend-core/-/backend-core-1.0.164.tgz#08c111dcebf5c74159a3c18218c7b3a0716de4f6"
+  resolved "https://registry.yarnpkg.com/@budibase/backend-core/-/backend-core-1.0.167.tgz#6ea4b90f8b8b8ec3cbbe05e39853d44d40938879"
-  integrity sha512-lpMudezndUD1hHBLfT9LDNKCunj8rQNlaJb30/xggdIUvp718u/jVP54hXF26NYxXOTMZ0EvMwCsIS4AucJ1Mg==
+  integrity sha512-IG9GZUdjFiqOKbgpZiwGotyT3BttFlChXs7mT8GaOkX7XvlyxxrG/nSI1duglBd6X2iafGESKQU8e6tKKQsxuw==
   dependencies:
     "@techpass/passport-openidconnect" "^0.3.0"
     aws-sdk "^2.901.0"
@@ -1091,12 +1091,12 @@
     svelte-flatpickr "^3.2.3"
     svelte-portal "^1.0.0"
 
-"@budibase/pro@1.0.164":
+"@budibase/pro@1.0.167":
-  version "1.0.164"
+  version "1.0.167"
-  resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-1.0.164.tgz#780ae38893d0609c87bf51fe96cc2c35bbdb431a"
+  resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-1.0.167.tgz#62fe54b58206eb04151a786a5516351137bd2f84"
-  integrity sha512-PgF7q2vADPPYzet4Wdma+THWuQPrEnN1+TfRly4l0oS9SUxutog3hYn0TlPmPS0AHgrqG/1v65TcEdC4ucX8TA==
+  integrity sha512-AdWWOub58LMxZoZzXm3jy1ZSVOR1teSH+lwLisdGWLnoVAUV8e46pD7iOyJDM1SKuSyNWeQ1lXj8tcLHVK1+OA==
   dependencies:
-    "@budibase/backend-core" "1.0.164"
+    "@budibase/backend-core" "1.0.167"
     node-fetch "^2.6.1"
 
 "@budibase/standard-components@^0.9.139":

packages/worker/src/api/controllers/global/configs.js

@@ -271,13 +271,14 @@ exports.configChecklist = async function (ctx) {
     const oidcConfig = await getScopedFullConfig(db, {
       type: Configs.OIDC,
     })
+
     // They have set up an global user
     const users = await db.allDocs(
       getGlobalUserParams(null, {
         include_docs: true,
+        limit: 1,
       })
     )
-    const adminUser = users.rows.some(row => row.doc.admin)
 
     ctx.body = {
       apps: {
@@ -291,7 +292,7 @@ exports.configChecklist = async function (ctx) {
         link: "/builder/portal/manage/email",
       },
       adminUser: {
-        checked: adminUser,
+        checked: users && users.rows.length >= 1,
         label: "Create your first user",
         link: "/builder/portal/manage/users",
       },

packages/worker/src/environment.js

@@ -1,3 +1,5 @@
+const { join } = require("path")
+
 function isDev() {
   return process.env.NODE_ENV !== "production"
 }
@@ -12,7 +14,9 @@ function isTest() {
 
 let LOADED = false
 if (!LOADED && isDev() && !isTest()) {
-  require("dotenv").config()
+  require("dotenv").config({
+    path: join(__dirname, "..", ".env"),
+  })
   LOADED = true
 }
 

packages/worker/yarn.lock

@@ -293,10 +293,10 @@
   resolved "https://registry.yarnpkg.com/@bcoe/v8-coverage/-/v8-coverage-0.2.3.tgz#75a2e8b51cb758a7553d6804a5932d7aace75c39"
   integrity sha512-0hYQ8SB4Db5zvZB4axdMHGwEaQjkZzFjQiN9LVYvIFB2nSUHW9tYpxWriPrWDASIxiaXax83REcLxuSdnGPZtw==
 
-"@budibase/backend-core@1.0.164":
+"@budibase/backend-core@1.0.167":
-  version "1.0.164"
+  version "1.0.167"
-  resolved "https://registry.yarnpkg.com/@budibase/backend-core/-/backend-core-1.0.164.tgz#08c111dcebf5c74159a3c18218c7b3a0716de4f6"
+  resolved "https://registry.yarnpkg.com/@budibase/backend-core/-/backend-core-1.0.167.tgz#6ea4b90f8b8b8ec3cbbe05e39853d44d40938879"
-  integrity sha512-lpMudezndUD1hHBLfT9LDNKCunj8rQNlaJb30/xggdIUvp718u/jVP54hXF26NYxXOTMZ0EvMwCsIS4AucJ1Mg==
+  integrity sha512-IG9GZUdjFiqOKbgpZiwGotyT3BttFlChXs7mT8GaOkX7XvlyxxrG/nSI1duglBd6X2iafGESKQU8e6tKKQsxuw==
   dependencies:
     "@techpass/passport-openidconnect" "^0.3.0"
     aws-sdk "^2.901.0"
@@ -321,12 +321,12 @@
     uuid "^8.3.2"
     zlib "^1.0.5"
 
-"@budibase/pro@1.0.164":
+"@budibase/pro@1.0.167":
-  version "1.0.164"
+  version "1.0.167"
-  resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-1.0.164.tgz#780ae38893d0609c87bf51fe96cc2c35bbdb431a"
+  resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-1.0.167.tgz#62fe54b58206eb04151a786a5516351137bd2f84"
-  integrity sha512-PgF7q2vADPPYzet4Wdma+THWuQPrEnN1+TfRly4l0oS9SUxutog3hYn0TlPmPS0AHgrqG/1v65TcEdC4ucX8TA==
+  integrity sha512-AdWWOub58LMxZoZzXm3jy1ZSVOR1teSH+lwLisdGWLnoVAUV8e46pD7iOyJDM1SKuSyNWeQ1lXj8tcLHVK1+OA==
   dependencies:
-    "@budibase/backend-core" "1.0.164"
+    "@budibase/backend-core" "1.0.167"
     node-fetch "^2.6.1"
 
 "@cspotcode/source-map-consumer@0.8.0":