Merge pull request #3251 from Budibase/fix/user-metadata

Syncing user metadata - allow builder to interact with user metadata in app

hosting/docker-compose.yaml

@@ -45,6 +45,7 @@ services:
       MINIO_ACCESS_KEY: ${MINIO_ACCESS_KEY}
       MINIO_SECRET_KEY: ${MINIO_SECRET_KEY}
       MINIO_URL: http://minio-service:9000
+      APPS_URL: http://app-service:4002
       COUCH_DB_USERNAME: ${COUCH_DB_USER}
       COUCH_DB_PASSWORD: ${COUCH_DB_PASSWORD}
       COUCH_DB_URL: http://${COUCH_DB_USER}:${COUCH_DB_PASSWORD}@couchdb-service:5984

hosting/kubernetes/budibase/templates/worker-service-deployment.yaml

@@ -113,6 +113,8 @@ spec:
           value: {{ .Values.globals.smtp.port | quote }}
         - name: SMTP_FROM_ADDRESS
           value: {{ .Values.globals.smtp.from | quote }}
+        - name: APPS_URL
+          value: http://app-service:{{ .Values.services.apps.port }}
         image: budibase/worker
         imagePullPolicy: Always
         name: bbworker

packages/auth/src/db/utils.js

@@ -152,6 +152,17 @@ exports.getDeployedAppID = appId => {
   return appId
 }
 
+/**
+ * Convert a deployed app ID to a development app ID.
+ */
+exports.getDevelopmentAppID = appId => {
+  if (!appId.startsWith(exports.APP_DEV_PREFIX)) {
+    const id = appId.split(exports.APP_PREFIX)[1]
+    return `${exports.APP_DEV_PREFIX}${id}`
+  }
+  return appId
+}
+
 exports.getCouchUrl = () => {
   if (!env.COUCH_DB_URL) return
 
@@ -248,6 +259,24 @@ exports.getAllApps = async (CouchDB, { dev, all, idsOnly } = {}) => {
   }
 }
 
+/**
+ * Utility function for getAllApps but filters to production apps only.
+ */
+exports.getDeployedAppIDs = async CouchDB => {
+  return (await exports.getAllApps(CouchDB, { idsOnly: true })).filter(
+    id => !exports.isDevAppID(id)
+  )
+}
+
+/**
+ * Utility function for the inverse of above.
+ */
+exports.getDevAppIDs = async CouchDB => {
+  return (await exports.getAllApps(CouchDB, { idsOnly: true })).filter(id =>
+    exports.isDevAppID(id)
+  )
+}
+
 exports.dbExists = async (CouchDB, dbName) => {
   let exists = false
   try {

packages/server/src/api/controllers/application.js

@@ -44,6 +44,7 @@ const {
   revertClientLibrary,
 } = require("../../utilities/fileSystem/clientLibrary")
 const { getTenantId, isMultiTenant } = require("@budibase/auth/tenancy")
+const { syncGlobalUsers } = require("./user")
 
 const URL_REGEX_SLASH = /\/|\\/g
 
@@ -328,6 +329,8 @@ exports.sync = async ctx => {
   if (!isDevAppID(appId)) {
     ctx.throw(400, "This action cannot be performed for production apps")
   }
+
+  // replicate prod to dev
   const prodAppId = getDeployedAppID(appId)
   const replication = new Replication({
     source: prodAppId,
@@ -343,6 +346,10 @@ exports.sync = async ctx => {
   } catch (err) {
     error = err
   }
+
+  // sync the users
+  await syncGlobalUsers(appId)
+
   if (error) {
     ctx.throw(400, error)
   } else {

packages/server/src/api/controllers/auth.js

@@ -14,6 +14,8 @@ exports.fetchSelf = async ctx => {
   }
 
   const user = await getFullUser(ctx, userId)
+  // this shouldn't be returned by the app self
+  delete user.roles
 
   if (appId) {
     const db = new CouchDB(appId)
@@ -36,6 +38,7 @@ exports.fetchSelf = async ctx => {
       // user has a role of some sort, return them
       else if (err.status === 404) {
         const metadata = {
+          ...user,
           _id: userId,
         }
         const dbResp = await db.put(metadata)

packages/server/src/api/controllers/user.js

@@ -4,27 +4,139 @@ const {
   getUserMetadataParams,
 } = require("../../db/utils")
 const { InternalTables } = require("../../db/utils")
-const { getGlobalUsers } = require("../../utilities/global")
+const { getGlobalUsers, getRawGlobalUser } = require("../../utilities/global")
 const { getFullUser } = require("../../utilities/users")
+const { isEqual } = require("lodash")
+const { BUILTIN_ROLE_IDS } = require("@budibase/auth/roles")
+const { getDevelopmentAppID, getDeployedAppIDs } = require("@budibase/auth/db")
+const { doesDatabaseExist } = require("../../utilities")
+const { UserStatus } = require("@budibase/auth/constants")
 
-function removeGlobalProps(user) {
+async function rawMetadata(db) {
-  // make sure to always remove some of the global user props
+  return (
-  delete user.password
+    await db.allDocs(
-  delete user.roles
-  delete user.builder
-  return user
-}
-
-exports.fetchMetadata = async function (ctx) {
-  const database = new CouchDB(ctx.appId)
-  const global = await getGlobalUsers(ctx.appId)
-  const metadata = (
-    await database.allDocs(
       getUserMetadataParams(null, {
         include_docs: true,
       })
     )
   ).rows.map(row => row.doc)
+}
+
+function combineMetadataAndUser(user, metadata) {
+  // skip users with no access
+  if (user.roleId === BUILTIN_ROLE_IDS.PUBLIC) {
+    return null
+  }
+  delete user._rev
+  const metadataId = generateUserMetadataID(user._id)
+  const newDoc = {
+    ...user,
+    _id: metadataId,
+    tableId: InternalTables.USER_METADATA,
+  }
+  const found = Array.isArray(metadata)
+    ? metadata.find(doc => doc._id === metadataId)
+    : metadata
+  // copy rev over for the purposes of equality check
+  if (found) {
+    newDoc._rev = found._rev
+  }
+  if (found == null || !isEqual(newDoc, found)) {
+    return {
+      ...found,
+      ...newDoc,
+    }
+  }
+  return null
+}
+
+exports.syncGlobalUsers = async appId => {
+  // sync user metadata
+  const db = new CouchDB(appId)
+  const [users, metadata] = await Promise.all([
+    getGlobalUsers(appId),
+    rawMetadata(db),
+  ])
+  const toWrite = []
+  for (let user of users) {
+    const combined = await combineMetadataAndUser(user, metadata)
+    if (combined) {
+      toWrite.push(combined)
+    }
+  }
+  await db.bulkDocs(toWrite)
+}
+
+exports.syncUser = async function (ctx) {
+  let deleting = false,
+    user
+  const userId = ctx.params.id
+  try {
+    user = await getRawGlobalUser(userId)
+  } catch (err) {
+    if (err && err.status === 404) {
+      user = {}
+      deleting = true
+    } else {
+      throw err
+    }
+  }
+  const roles = user.roles
+  // remove props which aren't useful to metadata
+  delete user.password
+  delete user.forceResetPassword
+  delete user.roles
+  // run through all production appIDs in the users roles
+  let prodAppIds
+  // if they are a builder then get all production app IDs
+  if ((user.builder && user.builder.global) || deleting) {
+    prodAppIds = await getDeployedAppIDs(CouchDB)
+  } else {
+    prodAppIds = Object.entries(roles)
+      .filter(entry => entry[1] !== BUILTIN_ROLE_IDS.PUBLIC)
+      .map(([appId]) => appId)
+  }
+  for (let prodAppId of prodAppIds) {
+    const devAppId = getDevelopmentAppID(prodAppId)
+    for (let appId of [prodAppId, devAppId]) {
+      if (!(await doesDatabaseExist(appId))) {
+        continue
+      }
+      const db = new CouchDB(appId)
+      const metadataId = generateUserMetadataID(userId)
+      let metadata
+      try {
+        metadata = await db.get(metadataId)
+      } catch (err) {
+        if (deleting) {
+          continue
+        }
+        metadata = {
+          tableId: InternalTables.USER_METADATA,
+        }
+      }
+      let combined
+      if (deleting) {
+        combined = {
+          ...metadata,
+          status: UserStatus.INACTIVE,
+          metadata: BUILTIN_ROLE_IDS.PUBLIC,
+        }
+      } else {
+        combined = combineMetadataAndUser(user, metadata)
+      }
+      await db.put(combined)
+    }
+  }
+  ctx.body = {
+    message: "User synced.",
+  }
+}
+
+exports.fetchMetadata = async function (ctx) {
+  const database = new CouchDB(ctx.appId)
+  const global = await getGlobalUsers(ctx.appId)
+  const metadata = await rawMetadata(database)
   const users = []
   for (let user of global) {
     // find the metadata that matches up to the global ID
@@ -52,7 +164,9 @@ exports.updateSelfMetadata = async function (ctx) {
 exports.updateMetadata = async function (ctx) {
   const appId = ctx.appId
   const db = new CouchDB(appId)
-  const user = removeGlobalProps(ctx.request.body)
+  const user = ctx.request.body
+  // this isn't applicable to the user
+  delete user.roles
   const metadata = {
     tableId: InternalTables.USER_METADATA,
     ...user,

packages/server/src/api/routes/user.js

@@ -34,5 +34,10 @@ router
     authorized(PermissionTypes.USER, PermissionLevels.WRITE),
     controller.destroyMetadata
   )
+  .post(
+    "/api/users/metadata/sync/:id",
+    authorized(PermissionTypes.USER, PermissionLevels.WRITE),
+    controller.syncUser
+  )
 
 module.exports = router

packages/server/src/middleware/authorized.js

@@ -15,7 +15,8 @@ module.exports =
   (permType, permLevel = null) =>
   async (ctx, next) => {
     // webhooks don't need authentication, each webhook unique
-    if (isWebhookEndpoint(ctx)) {
+    // also internal requests (between services) don't need authorized
+    if (isWebhookEndpoint(ctx) || ctx.internal) {
       return next()
     }
 

packages/server/src/utilities/global.js

@@ -46,9 +46,13 @@ exports.getCachedSelf = async (ctx, appId) => {
   return processUser(appId, user)
 }
 
-exports.getGlobalUser = async (appId, userId) => {
+exports.getRawGlobalUser = async userId => {
   const db = getGlobalDB()
-  let user = await db.get(getGlobalIDFromUserMetadataID(userId))
+  return db.get(getGlobalIDFromUserMetadataID(userId))
+}
+
+exports.getGlobalUser = async (appId, userId) => {
+  let user = await exports.getRawGlobalUser(userId)
   return processUser(appId, user)
 }
 
@@ -73,6 +77,7 @@ exports.getGlobalUsers = async (appId = null, users = null) => {
     .filter(user => user != null)
     .map(user => {
       delete user.password
+      delete user.forceResetPassword
       return user
     })
   if (!appId) {

packages/server/src/utilities/index.js

@@ -134,3 +134,13 @@ exports.stringToReadStream = string => {
     },
   })
 }
+
+exports.doesDatabaseExist = async dbName => {
+  try {
+    const db = new CouchDB(dbName, { skip_setup: true })
+    const info = await db.info()
+    return info && !info.error
+  } catch (err) {
+    return false
+  }
+}

packages/worker/scripts/dev/manage.js

@@ -25,6 +25,7 @@ async function init() {
       ACCOUNT_PORTAL_URL: "http://localhost:10001",
       ACCOUNT_PORTAL_API_KEY: "budibase",
       PLATFORM_URL: "http://localhost:10000",
+      APPS_URL: "http://localhost:4001",
     }
     let envFile = ""
     Object.keys(envFileJson).forEach(key => {

packages/worker/src/api/controllers/global/users.js

@@ -19,6 +19,7 @@ const {
 } = require("@budibase/auth/tenancy")
 const { removeUserFromInfoDB } = require("@budibase/auth/deprovision")
 const env = require("../../../environment")
+const { syncUserInApps } = require("../../../utilities/appService")
 
 async function allUsers() {
   const db = getGlobalDB()
@@ -32,7 +33,10 @@ async function allUsers() {
 
 exports.save = async ctx => {
   try {
-    ctx.body = await saveUser(ctx.request.body, getTenantId())
+    const user = await saveUser(ctx.request.body, getTenantId())
+    // let server know to sync user
+    await syncUserInApps(user._id)
+    ctx.body = user
   } catch (err) {
     ctx.throw(err.status || 400, err)
   }
@@ -129,6 +133,8 @@ exports.destroy = async ctx => {
   await db.remove(dbUser._id, dbUser._rev)
   await userCache.invalidateUser(dbUser._id)
   await invalidateSessions(dbUser._id)
+  // let server know to sync user
+  await syncUserInApps(dbUser._id)
   ctx.body = {
     message: `User ${ctx.params.id} deleted.`,
   }

packages/worker/src/environment.js

@@ -42,6 +42,7 @@ module.exports = {
   SMTP_FROM_ADDRESS: process.env.SMTP_FROM_ADDRESS,
   PLATFORM_URL: process.env.PLATFORM_URL,
   COOKIE_DOMAIN: process.env.COOKIE_DOMAIN,
+  APPS_URL: process.env.APPS_URL,
   _set(key, value) {
     process.env[key] = value
     module.exports[key] = value
@@ -53,6 +54,13 @@ module.exports = {
   },
 }
 
+// if some var haven't been set, define them
+if (!module.exports.APPS_URL) {
+  module.exports.APPS_URL = isDev()
+    ? "http://localhost:4001"
+    : "http://app-service:4002"
+}
+
 // clean up any environment variable edge cases
 for (let [key, value] of Object.entries(module.exports)) {
   // handle the edge case of "0" to disable an environment variable

packages/worker/src/utilities/appService.js

@@ -0,0 +1,33 @@
+const fetch = require("node-fetch")
+const { Headers } = require("@budibase/auth/constants")
+const { getTenantId, isTenantIdSet } = require("@budibase/auth/tenancy")
+const { checkSlashesInUrl } = require("../utilities")
+const env = require("../environment")
+
+async function makeAppRequest(url, method, body) {
+  if (env.isTest()) {
+    return
+  }
+  const request = { headers: {} }
+  request.headers[Headers.API_KEY] = env.INTERNAL_API_KEY
+  if (isTenantIdSet()) {
+    request.headers[Headers.TENANT_ID] = getTenantId()
+  }
+  if (body) {
+    request.headers["Content-Type"] = "application/json"
+    request.body = JSON.stringify(body)
+  }
+  request.method = method
+  return fetch(checkSlashesInUrl(env.APPS_URL + url), request)
+}
+
+exports.syncUserInApps = async userId => {
+  const response = await makeAppRequest(
+    `/api/users/metadata/sync/${userId}`,
+    "POST",
+    {}
+  )
+  if (response && response.status !== 200) {
+    throw "Unable to sync user."
+  }
+}