Merge pull request #1668 from Budibase/fix/user-relationships

Fixing some issues around users and export/import

packages/auth/src/db/utils.js

@@ -169,7 +169,8 @@ exports.getAllApps = async ({ CouchDB, dev, all } = {}) => {
     dbName.startsWith(exports.APP_PREFIX)
   )
   const appPromises = appDbNames.map(db =>
-    new CouchDB(db).get(DocumentTypes.APP_METADATA)
+    // skip setup otherwise databases could be re-created
+    new CouchDB(db, { skip_setup: true }).get(DocumentTypes.APP_METADATA)
   )
   if (appPromises.length === 0) {
     return []
@@ -194,6 +195,21 @@ exports.getAllApps = async ({ CouchDB, dev, all } = {}) => {
   }
 }
 
+exports.dbExists = async (CouchDB, dbName) => {
+  let exists = false
+  try {
+    const db = CouchDB(dbName, { skip_setup: true })
+    // check if database exists
+    const info = await db.info()
+    if (info && !info.error) {
+      exists = true
+    }
+  } catch (err) {
+    exists = false
+  }
+  return exists
+}
+
 /**
  * Generates a new configuration ID.
  * @returns {string} The new configuration ID which the config doc can be stored under.

packages/server/scripts/exportAppTemplate.js

@@ -5,6 +5,10 @@ const { join } = require("path")
 const CouchDB = require("../src/db")
 // load environment
 const env = require("../src/environment")
+const {
+  USER_METDATA_PREFIX,
+  LINK_USER_METADATA_PREFIX,
+} = require("../src/db/utils")
 
 // Script to export a chosen budibase app into a package
 // Usage: ./scripts/exportAppTemplate.js export --name=Funky --appId=appId
@@ -44,7 +48,13 @@ yargs
       // perform couch dump
 
       const instanceDb = new CouchDB(appId)
-      await instanceDb.dump(writeStream, {})
+      await instanceDb.dump(writeStream, {
+        filter: doc =>
+          !(
+            doc._id.includes(USER_METDATA_PREFIX) ||
+            doc.includes(LINK_USER_METADATA_PREFIX)
+          ),
+      })
       console.log(`Template ${name} exported to ${exportPath}`)
     }
   )

packages/server/src/api/controllers/application.js

@@ -99,18 +99,11 @@ async function createInstance(template) {
   // replicate the template data to the instance DB
   // this is currently very hard to test, downloading and importing template files
   /* istanbul ignore next */
-  let _rev
   if (template && template.useTemplate === "true") {
     const { ok } = await db.load(await getTemplateStream(template))
     if (!ok) {
       throw "Error loading database dump from template."
     }
-    try {
-      const response = await db.get(DocumentTypes.APP_METADATA)
-      _rev = response._rev
-    } catch (err) {
-      _rev = null
-    }
   } else {
     // create the users table
     await db.put(USERS_TABLE_SCHEMA)
@@ -121,7 +114,7 @@ async function createInstance(template) {
   await createRoutingView(appId)
   await createAllSearchIndex(appId)
 
-  return { _id: appId, _rev }
+  return { _id: appId }
 }
 
 exports.fetch = async function (ctx) {
@@ -188,11 +181,21 @@ exports.create = async function (ctx) {
     instanceConfig.file = ctx.request.files.templateFile
   }
   const instance = await createInstance(instanceConfig)
+  const appId = instance._id
 
   const url = await getAppUrlIfNotInUse(ctx)
-  const appId = instance._id
+  const db = new CouchDB(appId)
+  let _rev
+  try {
+    // if template there will be an existing doc
+    const existing = await db.get(DocumentTypes.APP_METADATA)
+    _rev = existing._rev
+  } catch (err) {
+    // nothing to do
+  }
   const newApplication = {
     _id: DocumentTypes.APP_METADATA,
+    _rev,
     appId: instance._id,
     type: "app",
     version: packageJson.version,
@@ -204,11 +207,7 @@ exports.create = async function (ctx) {
     updatedAt: new Date().toISOString(),
     createdAt: new Date().toISOString(),
   }
-  if (instance._rev) {
+  await db.put(newApplication, { force: true })
-    newApplication._rev = instance._rev
-  }
-  const instanceDb = new CouchDB(appId)
-  await instanceDb.put(newApplication)
 
   await createEmptyAppPackage(ctx, newApplication)
   /* istanbul ignore next */

packages/server/src/api/controllers/user.js

@@ -4,10 +4,8 @@ const {
   getUserMetadataParams,
 } = require("../../db/utils")
 const { InternalTables } = require("../../db/utils")
-const {
+const { addAppRoleToUser } = require("../../utilities/workerRequests")
-  getGlobalUsers,
+const { getGlobalUsers } = require("../../utilities/global")
-  addAppRoleToUser,
-} = require("../../utilities/workerRequests")
 const { getFullUser } = require("../../utilities/users")
 
 function removeGlobalProps(user) {
@@ -20,7 +18,7 @@ function removeGlobalProps(user) {
 
 exports.fetchMetadata = async function (ctx) {
   const database = new CouchDB(ctx.appId)
-  const global = await getGlobalUsers(ctx, ctx.appId)
+  const global = await getGlobalUsers(ctx.appId)
   const metadata = (
     await database.allDocs(
       getUserMetadataParams(null, {

packages/server/src/api/routes/tests/routing.spec.js

@@ -26,11 +26,6 @@ describe("/routing", () => {
 
   describe("fetch", () => {
     it("returns the correct routing for basic user", async () => {
-      workerRequests.getGlobalUsers.mockImplementationOnce((ctx, appId) => {
-        return {
-          roleId: BUILTIN_ROLE_IDS.BASIC,
-        }
-      })
       const res = await request
         .get(`/api/routing/client`)
         .set(await config.roleHeaders({
@@ -52,13 +47,6 @@ describe("/routing", () => {
     })
 
     it("returns the correct routing for power user", async () => {
-      workerRequests.getGlobalUsers.mockImplementationOnce((ctx, appId) => {
-        return {
-          roles: {
-            [appId]: BUILTIN_ROLE_IDS.POWER,
-          }
-        }
-      })
       const res = await request
         .get(`/api/routing/client`)
         .set(await config.roleHeaders({

packages/server/src/api/routes/tests/user.spec.js

@@ -1,7 +1,6 @@
 const { BUILTIN_ROLE_IDS } = require("@budibase/auth/roles")
 const { checkPermissionsEndpoint } = require("./utilities/TestFunctions")
 const setup = require("./utilities")
-const workerRequests = require("../../../utilities/workerRequests")
 
 jest.mock("../../../utilities/workerRequests", () => ({
   getGlobalUsers: jest.fn(() => {
@@ -25,30 +24,18 @@ describe("/users", () => {
   })
 
   describe("fetch", () => {
-    beforeEach(() => {
-      workerRequests.getGlobalUsers.mockImplementationOnce(() => ([
-          {
-            _id: "us_uuid1",
-          },
-          {
-            _id: "us_uuid2",
-          }
-        ]
-      ))
-    })
-
     it("returns a list of users from an instance db", async () => {
-      await config.createUser("brenda@brenda.com", "brendas_password")
+      await config.createUser("uuidx")
-      await config.createUser("pam@pam.com", "pam_password")
+      await config.createUser("uuidy")
       const res = await request
         .get(`/api/users/metadata`)
         .set(config.defaultHeaders())
         .expect("Content-Type", /json/)
         .expect(200)
 
-      expect(res.body.length).toBe(2)
+      expect(res.body.length).toBe(3)
-      expect(res.body.find(u => u._id === `ro_ta_users_us_uuid1`)).toBeDefined()
+      expect(res.body.find(u => u._id === `ro_ta_users_us_uuidx`)).toBeDefined()
-      expect(res.body.find(u => u._id === `ro_ta_users_us_uuid2`)).toBeDefined()
+      expect(res.body.find(u => u._id === `ro_ta_users_us_uuidy`)).toBeDefined()
     })
 
     it("should apply authorization to endpoint", async () => {
@@ -65,9 +52,6 @@ describe("/users", () => {
   })
 
   describe("update", () => {
-    beforeEach(() => {
-    })
-
     it("should be able to update the user", async () => {
       const user = await config.createUser()
       user.roleId = BUILTIN_ROLE_IDS.BASIC
@@ -94,14 +78,6 @@ describe("/users", () => {
   })
 
   describe("find", () => {
-    beforeEach(() => {
-      jest.resetAllMocks()
-      workerRequests.getGlobalUsers.mockImplementationOnce(() => ({
-        _id: "us_uuid1",
-        roleId: BUILTIN_ROLE_IDS.POWER,
-      }))
-    })
-
     it("should be able to find the user", async () => {
       const user = await config.createUser()
       const res = await request
@@ -110,7 +86,7 @@ describe("/users", () => {
         .expect(200)
         .expect("Content-Type", /json/)
       expect(res.body._id).toEqual(user._id)
-      expect(res.body.roleId).toEqual(BUILTIN_ROLE_IDS.POWER)
+      expect(res.body.roleId).toEqual(BUILTIN_ROLE_IDS.ADMIN)
       expect(res.body.tableId).toBeDefined()
     })
   })

packages/server/src/db/linkedRows/index.js

@@ -11,7 +11,9 @@ const {
 const { flatten } = require("lodash")
 const CouchDB = require("../../db")
 const { FieldTypes } = require("../../constants")
-const { getMultiIDParams } = require("../../db/utils")
+const { getMultiIDParams, USER_METDATA_PREFIX } = require("../../db/utils")
+const { partition } = require("lodash")
+const { getGlobalUsers } = require("../../utilities/global")
 
 /**
  * This functionality makes sure that when rows with links are created, updated or deleted they are processed
@@ -57,6 +59,31 @@ async function getLinksForRows(appId, rows) {
   )
 }
 
+async function getFullLinkedDocs(appId, links) {
+  // create DBs
+  const db = new CouchDB(appId)
+  const linkedRowIds = links.map(link => link.id)
+  let linked = (await db.allDocs(getMultiIDParams(linkedRowIds))).rows.map(
+    row => row.doc
+  )
+  // need to handle users as specific cases
+  let [users, other] = partition(linked, linkRow =>
+    linkRow._id.startsWith(USER_METDATA_PREFIX)
+  )
+  const globalUsers = await getGlobalUsers(appId, users)
+  users = users.map(user => {
+    const globalUser = globalUsers.find(
+      globalUser => globalUser && user._id.includes(globalUser._id)
+    )
+    return {
+      ...globalUser,
+      // doing user second overwrites the id and rev (always metadata)
+      ...user,
+    }
+  })
+  return [...other, ...users]
+}
+
 /**
  * Update link documents for a row or table - this is to be called by the API controller when a change is occurring.
  * @param {string} eventType states what type of change which is occurring, means this can be expanded upon in the
@@ -154,14 +181,13 @@ exports.attachFullLinkedDocs = async (appId, table, rows) => {
   if (linkedTableIds.length === 0) {
     return rows
   }
+  // create DBs
   const db = new CouchDB(appId)
+  // get all the links
   const links = (await getLinksForRows(appId, rows)).filter(link =>
     rows.some(row => row._id === link.thisId)
   )
-  const linkedRowIds = links.map(link => link.id)
+  let linked = await getFullLinkedDocs(appId, links)
-  const linked = (await db.allDocs(getMultiIDParams(linkedRowIds))).rows.map(
-    row => row.doc
-  )
   const linkedTables = []
   for (let row of rows) {
     for (let link of links.filter(link => link.thisId === row._id)) {

packages/server/src/db/utils.js

@@ -6,17 +6,11 @@ const {
   APP_DEV_PREFIX,
   APP_PREFIX,
   SEPARATOR,
+  StaticDatabases,
 } = require("@budibase/auth/db")
 
 const UNICODE_MAX = "\ufff0"
 
-const StaticDatabases = {
-  BUILDER: {
-    name: "builder-db",
-    baseDoc: "builder-doc",
-  },
-}
-
 const AppStatus = {
   DEV: "development",
   ALL: "all",
@@ -54,9 +48,18 @@ const SearchIndexes = {
   ROWS: "rows",
 }
 
+exports.StaticDatabases = {
+  BUILDER: {
+    name: "builder-db",
+    baseDoc: "builder-doc",
+  },
+  ...StaticDatabases,
+}
+
 exports.APP_PREFIX = APP_PREFIX
 exports.APP_DEV_PREFIX = APP_DEV_PREFIX
-exports.StaticDatabases = StaticDatabases
+exports.USER_METDATA_PREFIX = `${DocumentTypes.ROW}${SEPARATOR}${InternalTables.USER_METADATA}${SEPARATOR}`
+exports.LINK_USER_METADATA_PREFIX = `${DocumentTypes.LINK}${SEPARATOR}${InternalTables.USER_METADATA}${SEPARATOR}`
 exports.ViewNames = ViewNames
 exports.InternalTables = InternalTables
 exports.DocumentTypes = DocumentTypes

packages/server/src/middleware/currentapp.js

@@ -1,9 +1,16 @@
-const { getAppId, setCookie, getCookie } = require("@budibase/auth").utils
+const {
+  getAppId,
+  setCookie,
+  getCookie,
+  clearCookie,
+} = require("@budibase/auth").utils
 const { Cookies } = require("@budibase/auth").constants
 const { getRole } = require("@budibase/auth/roles")
 const { getGlobalSelf } = require("../utilities/workerRequests")
 const { BUILTIN_ROLE_IDS } = require("@budibase/auth/roles")
 const { generateUserMetadataID } = require("../db/utils")
+const { dbExists } = require("@budibase/auth/db")
+const CouchDB = require("../db")
 
 module.exports = async (ctx, next) => {
   // try to get the appID from the request
@@ -13,6 +20,15 @@ module.exports = async (ctx, next) => {
   if (!appCookie && !requestAppId) {
     return next()
   }
+  // check the app exists referenced in cookie
+  if (appCookie) {
+    const appId = appCookie.appId
+    const exists = await dbExists(CouchDB, appId)
+    if (!exists) {
+      clearCookie(ctx, Cookies.CurrentApp)
+      return next()
+    }
+  }
 
   let updateCookie = false,
     appId,

packages/server/src/tests/utilities/TestConfiguration.js

@@ -306,8 +306,8 @@ class TestConfiguration {
     return await this._req(config, null, controllers.layout.save)
   }
 
-  async createUser() {
+  async createUser(id = null) {
-    const globalId = `us_${Math.random()}`
+    const globalId = !id ? `us_${Math.random()}` : `us_${id}`
     const resp = await this.globalUser(globalId)
     return {
       ...resp,

packages/server/src/utilities/fileSystem/index.js

@@ -18,6 +18,10 @@ const download = require("download")
 const env = require("../../environment")
 const { homedir } = require("os")
 const fetch = require("node-fetch")
+const {
+  USER_METDATA_PREFIX,
+  LINK_USER_METADATA_PREFIX,
+} = require("../../db/utils")
 
 const DEFAULT_AUTOMATION_BUCKET =
   "https://prod-budi-automations.s3-eu-west-1.amazonaws.com"
@@ -117,7 +121,14 @@ exports.performBackup = async (appId, backupName) => {
   const writeStream = fs.createWriteStream(path)
   // perform couch dump
   const instanceDb = new CouchDB(appId)
-  await instanceDb.dump(writeStream, {})
+  await instanceDb.dump(writeStream, {
+    // filter out anything that has a user metadata structure in its ID
+    filter: doc =>
+      !(
+        doc._id.includes(USER_METDATA_PREFIX) ||
+        doc.includes(LINK_USER_METADATA_PREFIX)
+      ),
+  })
   // write the file to the object store
   await streamUpload(
     ObjectStoreBuckets.BACKUPS,

packages/server/src/utilities/global.js

@@ -0,0 +1,64 @@
+const CouchDB = require("../db")
+const {
+  getMultiIDParams,
+  getGlobalIDFromUserMetadataID,
+  StaticDatabases,
+} = require("../db/utils")
+const { BUILTIN_ROLE_IDS } = require("@budibase/auth/roles")
+const { getDeployedAppID } = require("@budibase/auth/db")
+const { getGlobalUserParams } = require("@budibase/auth/db")
+
+exports.updateAppRole = (appId, user) => {
+  if (!user.roles) {
+    return user
+  }
+  if (user.builder && user.builder.global) {
+    user.roleId = BUILTIN_ROLE_IDS.ADMIN
+  } else {
+    // always use the deployed app
+    user.roleId = user.roles[getDeployedAppID(appId)]
+    if (!user.roleId) {
+      user.roleId = BUILTIN_ROLE_IDS.PUBLIC
+    }
+  }
+  delete user.roles
+  return user
+}
+
+exports.getGlobalUser = async (appId, userId) => {
+  const db = CouchDB(StaticDatabases.GLOBAL.name)
+  let user = await db.get(getGlobalIDFromUserMetadataID(userId))
+  if (user) {
+    delete user.password
+  }
+  return exports.updateAppRole(appId, user)
+}
+
+exports.getGlobalUsers = async (appId = null, users = null) => {
+  const db = CouchDB(StaticDatabases.GLOBAL.name)
+  let globalUsers
+  if (users) {
+    const globalIds = users.map(user => getGlobalIDFromUserMetadataID(user._id))
+    globalUsers = (await db.allDocs(getMultiIDParams(globalIds))).rows.map(
+      row => row.doc
+    )
+  } else {
+    globalUsers = (
+      await db.allDocs(
+        getGlobalUserParams(null, {
+          include_docs: true,
+        })
+      )
+    ).rows.map(row => row.doc)
+  }
+  globalUsers = globalUsers
+    .filter(user => user != null)
+    .map(user => {
+      delete user.password
+      return user
+    })
+  if (!appId) {
+    return globalUsers
+  }
+  return globalUsers.map(user => exports.updateAppRole(appId, user))
+}

packages/server/src/utilities/users.js

@@ -1,13 +1,9 @@
 const CouchDB = require("../db")
-const { getGlobalIDFromUserMetadataID, InternalTables } = require("../db/utils")
+const { InternalTables } = require("../db/utils")
-const { getGlobalUsers } = require("../utilities/workerRequests")
+const { getGlobalUser } = require("../utilities/global")
 
 exports.getFullUser = async (ctx, userId) => {
-  const global = await getGlobalUsers(
+  const global = await getGlobalUser(ctx.appId, userId)
-    ctx,
-    ctx.appId,
-    getGlobalIDFromUserMetadataID(userId)
-  )
   let metadata
   try {
     // this will throw an error if the db doesn't exist, or there is no appId

packages/server/src/utilities/workerRequests.js

@@ -1,26 +1,8 @@
 const fetch = require("node-fetch")
 const env = require("../environment")
 const { checkSlashesInUrl } = require("./index")
-const { BUILTIN_ROLE_IDS } = require("@budibase/auth/roles")
 const { getDeployedAppID } = require("@budibase/auth/db")
-const { getGlobalIDFromUserMetadataID } = require("../db/utils")
+const { updateAppRole, getGlobalUser } = require("./global")
-
-function getAppRole(appId, user) {
-  if (!user.roles) {
-    return user
-  }
-  if (user.builder && user.builder.global) {
-    user.roleId = BUILTIN_ROLE_IDS.ADMIN
-  } else {
-    // always use the deployed app
-    user.roleId = user.roles[getDeployedAppID(appId)]
-    if (!user.roleId) {
-      user.roleId = BUILTIN_ROLE_IDS.PUBLIC
-    }
-  }
-  delete user.roles
-  return user
-}
 
 function request(ctx, request, noApiKey) {
   if (!request.headers) {
@@ -90,27 +72,6 @@ exports.getDeployedApps = async ctx => {
   }
 }
 
-exports.getGlobalUsers = async (ctx, appId = null, globalId = null) => {
-  const endpoint = globalId
-    ? `/api/admin/users/${globalId}`
-    : `/api/admin/users`
-  const reqCfg = { method: "GET" }
-  const response = await fetch(
-    checkSlashesInUrl(env.WORKER_URL + endpoint),
-    request(ctx, reqCfg)
-  )
-  let users = await response.json()
-  if (!appId) {
-    return users
-  }
-  if (Array.isArray(users)) {
-    users = users.map(user => getAppRole(appId, user))
-  } else {
-    users = getAppRole(appId, users)
-  }
-  return users
-}
-
 exports.getGlobalSelf = async (ctx, appId = null) => {
   const endpoint = `/api/admin/users/self`
   const response = await fetch(
@@ -123,7 +84,7 @@ exports.getGlobalSelf = async (ctx, appId = null) => {
   }
   let json = await response.json()
   if (appId) {
-    json = getAppRole(appId, json)
+    json = updateAppRole(appId, json)
   }
   return json
 }
@@ -136,8 +97,7 @@ exports.addAppRoleToUser = async (ctx, appId, roleId, userId = null) => {
     user = await exports.getGlobalSelf(ctx)
     endpoint = `/api/admin/users/self`
   } else {
-    userId = getGlobalIDFromUserMetadataID(userId)
+    user = await getGlobalUser(appId, userId)
-    user = await exports.getGlobalUsers(ctx, appId, userId)
     body._id = userId
     endpoint = `/api/admin/users`
   }

packages/worker/src/api/controllers/app.js

@@ -1,19 +1,12 @@
-const fetch = require("node-fetch")
 const { DocumentTypes } = require("@budibase/auth").db
 const CouchDB = require("../../db")
-const env = require("../../environment")
 
 const APP_PREFIX = "app_"
 const URL_REGEX_SLASH = /\/|\\/g
 
 exports.getApps = async ctx => {
-  let allDbs
   // allDbs call of CouchDB is very inaccurate in production
-  if (env.COUCH_DB_URL) {
+  const allDbs = await CouchDB.allDbs()
-    allDbs = await (await fetch(`${env.COUCH_DB_URL}/_all_dbs`)).json()
-  } else {
-    allDbs = await CouchDB.allDbs()
-  }
   const appDbNames = allDbs.filter(dbName => dbName.startsWith(APP_PREFIX))
   const appPromises = appDbNames.map(db =>
     new CouchDB(db).get(DocumentTypes.APP_METADATA)