Merge pull request #7591 from Budibase/fix/platform-email-casing

Fix/platform email casing

packages/backend-core/src/db/constants.ts

@@ -19,6 +19,7 @@ export enum ViewName {
   ROUTING = "screen_routes",
   AUTOMATION_LOGS = "automation_logs",
   ACCOUNT_BY_EMAIL = "account_by_email",
+  PLATFORM_USERS_LOWERCASE = "platform_users_lowercase",
 }
 
 export const DeprecatedViews = {

packages/backend-core/src/db/utils.ts

@@ -2,7 +2,8 @@ import { newid } from "../hashing"
 import { DEFAULT_TENANT_ID, Configs } from "../constants"
 import env from "../environment"
 import { SEPARATOR, DocumentType, UNICODE_MAX, ViewName } from "./constants"
-import { getTenantId, getGlobalDBName, getGlobalDB } from "../tenancy"
+import { getTenantId, getGlobalDB } from "../context"
+import { getGlobalDBName } from "../tenancy/utils"
 import fetch from "node-fetch"
 import { doWithDB, allDbs } from "./index"
 import { getCouchInfo } from "./pouch"

packages/backend-core/src/db/views.js

@@ -1,203 +0,0 @@
-const {
-  DocumentType,
-  ViewName,
-  DeprecatedViews,
-  SEPARATOR,
-} = require("./utils")
-const { getGlobalDB } = require("../tenancy")
-const { StaticDatabases } = require("./constants")
-const { doWithDB } = require("./")
-
-const DESIGN_DB = "_design/database"
-
-function DesignDoc() {
-  return {
-    _id: DESIGN_DB,
-    // view collation information, read before writing any complex views:
-    // https://docs.couchdb.org/en/master/ddocs/views/collation.html#collation-specification
-    views: {},
-  }
-}
-
-async function removeDeprecated(db, viewName) {
-  if (!DeprecatedViews[viewName]) {
-    return
-  }
-  try {
-    const designDoc = await db.get(DESIGN_DB)
-    for (let deprecatedNames of DeprecatedViews[viewName]) {
-      delete designDoc.views[deprecatedNames]
-    }
-    await db.put(designDoc)
-  } catch (err) {
-    // doesn't exist, ignore
-  }
-}
-
-exports.createNewUserEmailView = async () => {
-  const db = getGlobalDB()
-  let designDoc
-  try {
-    designDoc = await db.get(DESIGN_DB)
-  } catch (err) {
-    // no design doc, make one
-    designDoc = DesignDoc()
-  }
-  const view = {
-    // if using variables in a map function need to inject them before use
-    map: `function(doc) {
-      if (doc._id.startsWith("${DocumentType.USER}${SEPARATOR}")) {
-        emit(doc.email.toLowerCase(), doc._id)
-      }
-    }`,
-  }
-  designDoc.views = {
-    ...designDoc.views,
-    [ViewName.USER_BY_EMAIL]: view,
-  }
-  await db.put(designDoc)
-}
-
-exports.createAccountEmailView = async () => {
-  await doWithDB(StaticDatabases.PLATFORM_INFO.name, async db => {
-    let designDoc
-    try {
-      designDoc = await db.get(DESIGN_DB)
-    } catch (err) {
-      // no design doc, make one
-      designDoc = DesignDoc()
-    }
-    const view = {
-      // if using variables in a map function need to inject them before use
-      map: `function(doc) {
-      if (doc._id.startsWith("${DocumentType.ACCOUNT_METADATA}${SEPARATOR}")) {
-        emit(doc.email.toLowerCase(), doc._id)
-      }
-    }`,
-    }
-    designDoc.views = {
-      ...designDoc.views,
-      [ViewName.ACCOUNT_BY_EMAIL]: view,
-    }
-    await db.put(designDoc)
-  })
-}
-
-exports.createUserAppView = async () => {
-  const db = getGlobalDB()
-  let designDoc
-  try {
-    designDoc = await db.get("_design/database")
-  } catch (err) {
-    // no design doc, make one
-    designDoc = DesignDoc()
-  }
-  const view = {
-    // if using variables in a map function need to inject them before use
-    map: `function(doc) {
-      if (doc._id.startsWith("${DocumentType.USER}${SEPARATOR}") && doc.roles) {
-        for (let prodAppId of Object.keys(doc.roles)) {
-          let emitted = prodAppId + "${SEPARATOR}" + doc._id
-          emit(emitted, null)
-        }
-      }
-    }`,
-  }
-  designDoc.views = {
-    ...designDoc.views,
-    [ViewName.USER_BY_APP]: view,
-  }
-  await db.put(designDoc)
-}
-
-exports.createApiKeyView = async () => {
-  const db = getGlobalDB()
-  let designDoc
-  try {
-    designDoc = await db.get("_design/database")
-  } catch (err) {
-    designDoc = DesignDoc()
-  }
-  const view = {
-    map: `function(doc) {
-      if (doc._id.startsWith("${DocumentType.DEV_INFO}") && doc.apiKey) {
-        emit(doc.apiKey, doc.userId)
-      }
-    }`,
-  }
-  designDoc.views = {
-    ...designDoc.views,
-    [ViewName.BY_API_KEY]: view,
-  }
-  await db.put(designDoc)
-}
-
-exports.createUserBuildersView = async () => {
-  const db = getGlobalDB()
-  let designDoc
-  try {
-    designDoc = await db.get("_design/database")
-  } catch (err) {
-    // no design doc, make one
-    designDoc = DesignDoc()
-  }
-  const view = {
-    map: `function(doc) {
-      if (doc.builder && doc.builder.global === true) {
-        emit(doc._id, doc._id)
-      }
-    }`,
-  }
-  designDoc.views = {
-    ...designDoc.views,
-    [ViewName.USER_BY_BUILDERS]: view,
-  }
-  await db.put(designDoc)
-}
-
-exports.queryView = async (viewName, params, db, CreateFuncByName) => {
-  try {
-    let response = (await db.query(`database/${viewName}`, params)).rows
-    response = response.map(resp =>
-      params.include_docs ? resp.doc : resp.value
-    )
-    if (params.arrayResponse) {
-      return response
-    } else {
-      return response.length <= 1 ? response[0] : response
-    }
-  } catch (err) {
-    if (err != null && err.name === "not_found") {
-      const createFunc = CreateFuncByName[viewName]
-      await removeDeprecated(db, viewName)
-      await createFunc()
-      return exports.queryView(viewName, params, db, CreateFuncByName)
-    } else {
-      throw err
-    }
-  }
-}
-
-exports.queryPlatformView = async (viewName, params) => {
-  const CreateFuncByName = {
-    [ViewName.ACCOUNT_BY_EMAIL]: exports.createAccountEmailView,
-  }
-
-  return doWithDB(StaticDatabases.PLATFORM_INFO.name, async db => {
-    return exports.queryView(viewName, params, db, CreateFuncByName)
-  })
-}
-
-exports.queryGlobalView = async (viewName, params, db = null) => {
-  const CreateFuncByName = {
-    [ViewName.USER_BY_EMAIL]: exports.createNewUserEmailView,
-    [ViewName.BY_API_KEY]: exports.createApiKeyView,
-    [ViewName.USER_BY_BUILDERS]: exports.createUserBuildersView,
-    [ViewName.USER_BY_APP]: exports.createUserAppView,
-  }
-  // can pass DB in if working with something specific
-  if (!db) {
-    db = getGlobalDB()
-  }
-  return exports.queryView(viewName, params, db, CreateFuncByName)
-}

packages/backend-core/src/db/views.ts

@@ -0,0 +1,260 @@
+import { DocumentType, ViewName, DeprecatedViews, SEPARATOR } from "./utils"
+import { getGlobalDB } from "../context"
+import { StaticDatabases } from "./constants"
+import { doWithDB } from "./"
+
+const DESIGN_DB = "_design/database"
+
+function DesignDoc() {
+  return {
+    _id: DESIGN_DB,
+    // view collation information, read before writing any complex views:
+    // https://docs.couchdb.org/en/master/ddocs/views/collation.html#collation-specification
+    views: {},
+  }
+}
+
+interface DesignDocument {
+  views: any
+}
+
+async function removeDeprecated(db: PouchDB.Database, viewName: ViewName) {
+  // @ts-ignore
+  if (!DeprecatedViews[viewName]) {
+    return
+  }
+  try {
+    const designDoc = await db.get<DesignDocument>(DESIGN_DB)
+    // @ts-ignore
+    for (let deprecatedNames of DeprecatedViews[viewName]) {
+      delete designDoc.views[deprecatedNames]
+    }
+    await db.put(designDoc)
+  } catch (err) {
+    // doesn't exist, ignore
+  }
+}
+
+export const createNewUserEmailView = async () => {
+  const db = getGlobalDB()
+  let designDoc
+  try {
+    designDoc = await db.get(DESIGN_DB)
+  } catch (err) {
+    // no design doc, make one
+    designDoc = DesignDoc()
+  }
+  const view = {
+    // if using variables in a map function need to inject them before use
+    map: `function(doc) {
+      if (doc._id.startsWith("${DocumentType.USER}${SEPARATOR}")) {
+        emit(doc.email.toLowerCase(), doc._id)
+      }
+    }`,
+  }
+  designDoc.views = {
+    ...designDoc.views,
+    [ViewName.USER_BY_EMAIL]: view,
+  }
+  await db.put(designDoc)
+}
+
+export const createAccountEmailView = async () => {
+  await doWithDB(
+    StaticDatabases.PLATFORM_INFO.name,
+    async (db: PouchDB.Database) => {
+      let designDoc
+      try {
+        designDoc = await db.get<DesignDocument>(DESIGN_DB)
+      } catch (err) {
+        // no design doc, make one
+        designDoc = DesignDoc()
+      }
+      const view = {
+        // if using variables in a map function need to inject them before use
+        map: `function(doc) {
+      if (doc._id.startsWith("${DocumentType.ACCOUNT_METADATA}${SEPARATOR}")) {
+        emit(doc.email.toLowerCase(), doc._id)
+      }
+    }`,
+      }
+      designDoc.views = {
+        ...designDoc.views,
+        [ViewName.ACCOUNT_BY_EMAIL]: view,
+      }
+      await db.put(designDoc)
+    }
+  )
+}
+
+export const createUserAppView = async () => {
+  const db = getGlobalDB() as PouchDB.Database
+  let designDoc
+  try {
+    designDoc = await db.get<DesignDocument>("_design/database")
+  } catch (err) {
+    // no design doc, make one
+    designDoc = DesignDoc()
+  }
+  const view = {
+    // if using variables in a map function need to inject them before use
+    map: `function(doc) {
+      if (doc._id.startsWith("${DocumentType.USER}${SEPARATOR}") && doc.roles) {
+        for (let prodAppId of Object.keys(doc.roles)) {
+          let emitted = prodAppId + "${SEPARATOR}" + doc._id
+          emit(emitted, null)
+        }
+      }
+    }`,
+  }
+  designDoc.views = {
+    ...designDoc.views,
+    [ViewName.USER_BY_APP]: view,
+  }
+  await db.put(designDoc)
+}
+
+export const createApiKeyView = async () => {
+  const db = getGlobalDB()
+  let designDoc
+  try {
+    designDoc = await db.get("_design/database")
+  } catch (err) {
+    designDoc = DesignDoc()
+  }
+  const view = {
+    map: `function(doc) {
+      if (doc._id.startsWith("${DocumentType.DEV_INFO}") && doc.apiKey) {
+        emit(doc.apiKey, doc.userId)
+      }
+    }`,
+  }
+  designDoc.views = {
+    ...designDoc.views,
+    [ViewName.BY_API_KEY]: view,
+  }
+  await db.put(designDoc)
+}
+
+export const createUserBuildersView = async () => {
+  const db = getGlobalDB()
+  let designDoc
+  try {
+    designDoc = await db.get("_design/database")
+  } catch (err) {
+    // no design doc, make one
+    designDoc = DesignDoc()
+  }
+  const view = {
+    map: `function(doc) {
+      if (doc.builder && doc.builder.global === true) {
+        emit(doc._id, doc._id)
+      }
+    }`,
+  }
+  designDoc.views = {
+    ...designDoc.views,
+    [ViewName.USER_BY_BUILDERS]: view,
+  }
+  await db.put(designDoc)
+}
+
+export const createPlatformUserView = async () => {
+  await doWithDB(
+    StaticDatabases.PLATFORM_INFO.name,
+    async (db: PouchDB.Database) => {
+      let designDoc
+      try {
+        designDoc = await db.get<DesignDocument>(DESIGN_DB)
+      } catch (err) {
+        // no design doc, make one
+        designDoc = DesignDoc()
+      }
+      const view = {
+        // if using variables in a map function need to inject them before use
+        map: `function(doc) {
+        if (doc.tenantId) {
+          emit(doc._id.toLowerCase(), doc._id)
+        }
+      }`,
+      }
+      designDoc.views = {
+        ...designDoc.views,
+        [ViewName.PLATFORM_USERS_LOWERCASE]: view,
+      }
+      await db.put(designDoc)
+    }
+  )
+}
+
+export interface QueryViewOptions {
+  arrayResponse?: boolean
+}
+
+export const queryView = async <T>(
+  viewName: ViewName,
+  params: PouchDB.Query.Options<T, T>,
+  db: PouchDB.Database,
+  CreateFuncByName: any,
+  opts?: QueryViewOptions
+): Promise<T[] | T | undefined> => {
+  try {
+    let response = await db.query<T, T>(`database/${viewName}`, params)
+    const rows = response.rows
+    const docs = rows.map((resp: any) =>
+      params.include_docs ? resp.doc : resp.value
+    )
+
+    if (opts?.arrayResponse) {
+      return docs
+    } else {
+      return docs.length <= 1 ? docs[0] : docs
+    }
+  } catch (err: any) {
+    if (err != null && err.name === "not_found") {
+      const createFunc = CreateFuncByName[viewName]
+      await removeDeprecated(db, viewName)
+      await createFunc()
+      return queryView(viewName, params, db, CreateFuncByName, opts)
+    } else {
+      throw err
+    }
+  }
+}
+
+export const queryPlatformView = async <T>(
+  viewName: ViewName,
+  params: PouchDB.Query.Options<T, T>,
+  opts?: QueryViewOptions
+): Promise<T[] | T | undefined> => {
+  const CreateFuncByName = {
+    [ViewName.ACCOUNT_BY_EMAIL]: createAccountEmailView,
+    [ViewName.PLATFORM_USERS_LOWERCASE]: createPlatformUserView,
+  }
+
+  return doWithDB(
+    StaticDatabases.PLATFORM_INFO.name,
+    async (db: PouchDB.Database) => {
+      return queryView(viewName, params, db, CreateFuncByName, opts)
+    }
+  )
+}
+
+export const queryGlobalView = async <T>(
+  viewName: ViewName,
+  params: PouchDB.Query.Options<T, T>,
+  db?: PouchDB.Database,
+  opts?: QueryViewOptions
+): Promise<T[] | T | undefined> => {
+  const CreateFuncByName = {
+    [ViewName.USER_BY_EMAIL]: createNewUserEmailView,
+    [ViewName.BY_API_KEY]: createApiKeyView,
+    [ViewName.USER_BY_BUILDERS]: createUserBuildersView,
+    [ViewName.USER_BY_APP]: createUserAppView,
+  }
+  // can pass DB in if working with something specific
+  if (!db) {
+    db = getGlobalDB() as PouchDB.Database
+  }
+  return queryView(viewName, params, db, CreateFuncByName, opts)
+}

packages/backend-core/src/tenancy/index.ts

@@ -1,9 +1,11 @@
 import * as context from "../context"
 import * as tenancy from "./tenancy"
+import * as utils from "./utils"
 
 const pkg = {
   ...context,
   ...tenancy,
+  ...utils,
 }
 
 export = pkg

packages/backend-core/src/tenancy/tenancy.ts

@@ -1,6 +1,7 @@
 import { doWithDB } from "../db"
-import { StaticDatabases } from "../db/constants"
+import { queryPlatformView } from "../db/views"
-import { baseGlobalDBName } from "./utils"
+import { StaticDatabases, ViewName } from "../db/constants"
+import { getGlobalDBName } from "./utils"
 import {
   getTenantId,
   DEFAULT_TENANT_ID,
@@ -8,6 +9,7 @@ import {
   getTenantIDFromAppID,
 } from "../context"
 import env from "../environment"
+import { PlatformUser, PlatformUserByEmail } from "@budibase/types"
 
 const TENANT_DOC = StaticDatabases.PLATFORM_INFO.docs.tenants
 const PLATFORM_INFO_DB = StaticDatabases.PLATFORM_INFO.name
@@ -87,15 +89,6 @@ export const tryAddTenant = async (
   })
 }
 
-export const getGlobalDBName = (tenantId?: string) => {
-  // tenant ID can be set externally, for example user API where
-  // new tenants are being created, this may be the case
-  if (!tenantId) {
-    tenantId = getTenantId()
-  }
-  return baseGlobalDBName(tenantId)
-}
-
 export const doWithGlobalDB = (tenantId: string, cb: any) => {
   return doWithDB(getGlobalDBName(tenantId), cb)
 }
@@ -116,14 +109,16 @@ export const lookupTenantId = async (userId: string) => {
 }
 
 // lookup, could be email or userId, either will return a doc
-export const getTenantUser = async (identifier: string) => {
+export const getTenantUser = async (
-  return doWithDB(PLATFORM_INFO_DB, async (db: any) => {
+  identifier: string
-    try {
+): Promise<PlatformUser | null> => {
-      return await db.get(identifier)
+  // use the view here and allow to find anyone regardless of casing
-    } catch (err) {
+  // Use lowercase to ensure email login is case insensitive
-      return null
+  const response = queryPlatformView(ViewName.PLATFORM_USERS_LOWERCASE, {
-    }
+    keys: [identifier.toLowerCase()],
-  })
+    include_docs: true,
+  }) as Promise<PlatformUser>
+  return response
 }
 
 export const isUserInAppTenant = (appId: string, user: any) => {

packages/backend-core/src/tenancy/utils.js

@@ -1,12 +0,0 @@
-const { DEFAULT_TENANT_ID } = require("../constants")
-const { StaticDatabases, SEPARATOR } = require("../db/constants")
-
-exports.baseGlobalDBName = tenantId => {
-  let dbName
-  if (!tenantId || tenantId === DEFAULT_TENANT_ID) {
-    dbName = StaticDatabases.GLOBAL.name
-  } else {
-    dbName = `${tenantId}${SEPARATOR}${StaticDatabases.GLOBAL.name}`
-  }
-  return dbName
-}

packages/backend-core/src/tenancy/utils.ts

@@ -0,0 +1,22 @@
+import { DEFAULT_TENANT_ID } from "../constants"
+import { StaticDatabases, SEPARATOR } from "../db/constants"
+import { getTenantId } from "../context"
+
+export const getGlobalDBName = (tenantId?: string) => {
+  // tenant ID can be set externally, for example user API where
+  // new tenants are being created, this may be the case
+  if (!tenantId) {
+    tenantId = getTenantId()
+  }
+  return baseGlobalDBName(tenantId)
+}
+
+export const baseGlobalDBName = (tenantId: string | undefined | null) => {
+  let dbName
+  if (!tenantId || tenantId === DEFAULT_TENANT_ID) {
+    dbName = StaticDatabases.GLOBAL.name
+  } else {
+    dbName = `${tenantId}${SEPARATOR}${StaticDatabases.GLOBAL.name}`
+  }
+  return dbName
+}

packages/types/src/documents/platform/users.ts

@@ -7,3 +7,12 @@ export interface PlatformUserByEmail extends Document {
   tenantId: string
   userId: string
 }
+
+/**
+ * doc id is userId
+ */
+export interface PlatformUserById extends Document {
+  tenantId: string
+}
+
+export type PlatformUser = PlatformUserByEmail | PlatformUserById

packages/types/src/sdk/migrations.ts

@@ -47,6 +47,7 @@ export enum MigrationName {
   EVENT_GLOBAL_BACKFILL = "event_global_backfill",
   EVENT_INSTALLATION_BACKFILL = "event_installation_backfill",
   GLOBAL_INFO_SYNC_USERS = "global_info_sync_users",
+  PLATFORM_USERS_EMAIL_CASING = "platform_users_email_casing",
 }
 
 export interface MigrationDefinition {

packages/worker/package.json

@@ -23,6 +23,7 @@
     "dev:stack:init": "node ./scripts/dev/manage.js init",
     "dev:builder": "npm run dev:stack:init && nodemon",
     "test": "jest --runInBand",
+    "test:watch": "jest --watch",
     "env:multi:enable": "node scripts/multiTenancy.js enable",
     "env:multi:disable": "node scripts/multiTenancy.js disable",
     "env:selfhost:enable": "node scripts/selfhost.js enable",

packages/worker/src/api/routes/global/tests/users.spec.ts

@@ -242,6 +242,26 @@ describe("/api/global/users", () => {
       expect(response.body.message).toBe(`Unavailable`)
       expect(events.user.created).toBeCalledTimes(0)
     })
+
+    it("should not be able to create a user with the same email and different casing", async () => {
+      const user = structures.users.user()
+      await api.users.saveUser(user)
+
+      user.email = user.email.toUpperCase()
+      await api.users.saveUser(user, 400)
+
+      expect(events.user.created).toBeCalledTimes(1)
+    })
+
+    it("should not be able to bulk create a user with the same email and different casing", async () => {
+      const user = structures.users.user()
+      await api.users.saveUser(user)
+
+      user.email = user.email.toUpperCase()
+      await api.users.bulkCreateUsers([user])
+
+      expect(events.user.created).toBeCalledTimes(1)
+    })
   })
 
   describe("update", () => {

packages/worker/src/sdk/users/users.ts

@@ -278,39 +278,61 @@ export const addTenant = async (
 }
 
 const getExistingTenantUsers = async (emails: string[]): Promise<User[]> => {
-  return dbUtils.queryGlobalView(ViewName.USER_BY_EMAIL, {
+  const lcEmails = emails.map(email => email.toLowerCase())
-    keys: emails,
+  const params = {
+    keys: lcEmails,
     include_docs: true,
+  }
+
+  const opts = {
     arrayResponse: true,
-  })
+  }
+
+  return dbUtils.queryGlobalView(
+    ViewName.USER_BY_EMAIL,
+    params,
+    undefined,
+    opts
+  ) as Promise<User[]>
 }
 
 const getExistingPlatformUsers = async (
   emails: string[]
 ): Promise<PlatformUserByEmail[]> => {
-  return dbUtils.doWithDB(
+  const lcEmails = emails.map(email => email.toLowerCase())
-    StaticDatabases.PLATFORM_INFO.name,
+  const params = {
-    async (infoDb: any) => {
+    keys: lcEmails,
-      const response: AllDocsResponse<PlatformUserByEmail> =
-        await infoDb.allDocs({
-          keys: emails,
     include_docs: true,
-        })
-      return response.rows
-        .filter(row => row.doc && (row.error !== "not_found") !== null)
-        .map((row: any) => row.doc)
   }
-  )
+
+  const opts = {
+    arrayResponse: true,
+  }
+  return dbUtils.queryPlatformView(
+    ViewName.PLATFORM_USERS_LOWERCASE,
+    params,
+    opts
+  ) as Promise<PlatformUserByEmail[]>
 }
 
 const getExistingAccounts = async (
   emails: string[]
 ): Promise<AccountMetadata[]> => {
-  return dbUtils.queryPlatformView(ViewName.ACCOUNT_BY_EMAIL, {
+  const lcEmails = emails.map(email => email.toLowerCase())
-    keys: emails,
+  const params = {
+    keys: lcEmails,
     include_docs: true,
+  }
+
+  const opts = {
     arrayResponse: true,
-  })
+  }
+
+  return dbUtils.queryPlatformView(
+    ViewName.ACCOUNT_BY_EMAIL,
+    params,
+    opts
+  ) as Promise<AccountMetadata[]>
 }
 
 /**
@@ -332,7 +354,7 @@ const searchExistingEmails = async (emails: string[]) => {
   const existingAccounts = await getExistingAccounts(emails)
   matchedEmails.push(...existingAccounts.map(account => account.email))
 
-  return [...new Set(matchedEmails)]
+  return [...new Set(matchedEmails.map(email => email.toLowerCase()))]
 }
 
 export const bulkCreate = async (
@@ -351,8 +373,10 @@ export const bulkCreate = async (
 
   for (const newUser of newUsersRequested) {
     if (
-      newUsers.find((x: any) => x.email === newUser.email) ||
+      newUsers.find(
-      existingEmails.includes(newUser.email)
+        (x: User) => x.email.toLowerCase() === newUser.email.toLowerCase()
+      ) ||
+      existingEmails.includes(newUser.email.toLowerCase())
     ) {
       unsuccessful.push({
         email: newUser.email,