Merge branch 'develop' into fix/BUDI-7258

lerna.json

@@ -1,5 +1,5 @@
 {
-  "version": "2.10.9-alpha.1",
+  "version": "2.10.9-alpha.5",
   "npmClient": "yarn",
   "packages": [
     "packages/*"

packages/backend-core/src/cache/tests/user.spec.ts

@@ -0,0 +1,145 @@
+import { User } from "@budibase/types"
+import { generator, structures } from "../../../tests"
+import { DBTestConfiguration } from "../../../tests/extra"
+import { getUsers } from "../user"
+import { getGlobalDB } from "../../context"
+import _ from "lodash"
+
+import * as redis from "../../redis/init"
+import { UserDB } from "../../users"
+
+const config = new DBTestConfiguration()
+
+describe("user cache", () => {
+  describe("getUsers", () => {
+    const users: User[] = []
+    beforeAll(async () => {
+      const userCount = 10
+      const userIds = generator.arrayOf(() => generator.guid(), {
+        min: userCount,
+        max: userCount,
+      })
+
+      await config.doInTenant(async () => {
+        const db = getGlobalDB()
+        for (const userId of userIds) {
+          const user = structures.users.user({ _id: userId })
+          await db.put(user)
+          users.push(user)
+        }
+      })
+    })
+
+    beforeEach(async () => {
+      jest.clearAllMocks()
+
+      const redisClient = await redis.getUserClient()
+      await redisClient.clear()
+    })
+
+    it("when no user is in cache, all of them are retrieved from db", async () => {
+      const usersToRequest = _.sampleSize(users, 5)
+
+      const userIdsToRequest = usersToRequest.map(x => x._id!)
+
+      jest.spyOn(UserDB, "bulkGet")
+
+      const results = await config.doInTenant(() => getUsers(userIdsToRequest))
+
+      expect(results.users).toHaveLength(5)
+      expect(results).toEqual({
+        users: usersToRequest.map(u => ({
+          ...u,
+          budibaseAccess: true,
+          _rev: expect.any(String),
+        })),
+      })
+
+      expect(UserDB.bulkGet).toBeCalledTimes(1)
+      expect(UserDB.bulkGet).toBeCalledWith(userIdsToRequest)
+    })
+
+    it("on a second all, all of them are retrieved from cache", async () => {
+      const usersToRequest = _.sampleSize(users, 5)
+
+      const userIdsToRequest = usersToRequest.map(x => x._id!)
+
+      jest.spyOn(UserDB, "bulkGet")
+
+      await config.doInTenant(() => getUsers(userIdsToRequest))
+      const resultsFromCache = await config.doInTenant(() =>
+        getUsers(userIdsToRequest)
+      )
+
+      expect(resultsFromCache.users).toHaveLength(5)
+      expect(resultsFromCache).toEqual({
+        users: expect.arrayContaining(
+          usersToRequest.map(u => ({
+            ...u,
+            budibaseAccess: true,
+            _rev: expect.any(String),
+          }))
+        ),
+      })
+
+      expect(UserDB.bulkGet).toBeCalledTimes(1)
+    })
+
+    it("when some users are cached, only the missing ones are retrieved from db", async () => {
+      const usersToRequest = _.sampleSize(users, 5)
+
+      const userIdsToRequest = usersToRequest.map(x => x._id!)
+
+      jest.spyOn(UserDB, "bulkGet")
+
+      await config.doInTenant(() =>
+        getUsers([userIdsToRequest[0], userIdsToRequest[3]])
+      )
+      ;(UserDB.bulkGet as jest.Mock).mockClear()
+
+      const results = await config.doInTenant(() => getUsers(userIdsToRequest))
+
+      expect(results.users).toHaveLength(5)
+      expect(results).toEqual({
+        users: expect.arrayContaining(
+          usersToRequest.map(u => ({
+            ...u,
+            budibaseAccess: true,
+            _rev: expect.any(String),
+          }))
+        ),
+      })
+
+      expect(UserDB.bulkGet).toBeCalledTimes(1)
+      expect(UserDB.bulkGet).toBeCalledWith([
+        userIdsToRequest[1],
+        userIdsToRequest[2],
+        userIdsToRequest[4],
+      ])
+    })
+
+    it("requesting existing and unexisting ids will return found ones", async () => {
+      const usersToRequest = _.sampleSize(users, 3)
+      const missingIds = [generator.guid(), generator.guid()]
+
+      const userIdsToRequest = _.shuffle([
+        ...missingIds,
+        ...usersToRequest.map(x => x._id!),
+      ])
+
+      const results = await config.doInTenant(() => getUsers(userIdsToRequest))
+
+      expect(results.users).toHaveLength(3)
+      expect(results).toEqual({
+        users: expect.arrayContaining(
+          usersToRequest.map(u => ({
+            ...u,
+            budibaseAccess: true,
+            _rev: expect.any(String),
+          }))
+        ),
+        notFoundIds: expect.arrayContaining(missingIds),
+      })
+    })
+  })
+})

packages/backend-core/src/cache/user.ts

@@ -6,6 +6,7 @@ import env from "../environment"
 import * as accounts from "../accounts"
 import { UserDB } from "../users"
 import { sdk } from "@budibase/shared-core"
+import { User } from "@budibase/types"
 
 const EXPIRY_SECONDS = 3600
 
@@ -27,6 +28,35 @@ async function populateFromDB(userId: string, tenantId: string) {
   return user
 }
 
+async function populateUsersFromDB(
+  userIds: string[]
+): Promise<{ users: User[]; notFoundIds?: string[] }> {
+  const getUsersResponse = await UserDB.bulkGet(userIds)
+
+  // Handle missed user ids
+  const notFoundIds = userIds.filter((uid, i) => !getUsersResponse[i])
+
+  const users = getUsersResponse.filter(x => x)
+
+  await Promise.all(
+    users.map(async (user: any) => {
+      user.budibaseAccess = true
+      if (!env.SELF_HOSTED && !env.DISABLE_ACCOUNT_PORTAL) {
+        const account = await accounts.getAccount(user.email)
+        if (account) {
+          user.account = account
+          user.accountPortalAccess = true
+        }
+      }
+    })
+  )
+
+  if (notFoundIds.length) {
+    return { users, notFoundIds }
+  }
+  return { users }
+}
+
 /**
  * Get the requested user by id.
  * Use redis cache to first read the user.
@@ -77,6 +107,36 @@ export async function getUser(
   return user
 }
 
+/**
+ * Get the requested users by id.
+ * Use redis cache to first read the users.
+ * If not present fallback to loading the users directly and re-caching.
+ * @param {*} userIds the ids of the user to get
+ * @param {*} tenantId the tenant of the users to get
+ * @returns
+ */
+export async function getUsers(
+  userIds: string[]
+): Promise<{ users: User[]; notFoundIds?: string[] }> {
+  const client = await redis.getUserClient()
+  // try cache
+  let usersFromCache = await client.bulkGet(userIds)
+  const missingUsersFromCache = userIds.filter(uid => !usersFromCache[uid])
+  const users = Object.values(usersFromCache)
+  let notFoundIds
+
+  if (missingUsersFromCache.length) {
+    const usersFromDb = await populateUsersFromDB(missingUsersFromCache)
+
+    notFoundIds = usersFromDb.notFoundIds
+    for (const userToCache of usersFromDb.users) {
+      await client.store(userToCache._id!, userToCache, EXPIRY_SECONDS)
+    }
+    users.push(...usersFromDb.users)
+  }
+  return { users, notFoundIds: notFoundIds }
+}
+
 export async function invalidateUser(userId: string) {
   const client = await redis.getUserClient()
   await client.delete(userId)

packages/backend-core/src/middleware/passport/sso/tests/sso.spec.ts

@@ -102,6 +102,7 @@ describe("sso", () => {
 
           // modified external id to match user format
           ssoUser._id = "us_" + details.userId
+          delete ssoUser.userId
 
           // new sso user won't have a password
           delete ssoUser.password

packages/backend-core/src/redis/redis.ts

@@ -250,7 +250,7 @@ class RedisWrapper {
     const prefixedKeys = keys.map(key => addDbPrefix(db, key))
     let response = await this.getClient().mget(prefixedKeys)
     if (Array.isArray(response)) {
-      let final: any = {}
+      let final: Record<string, any> = {}
       let count = 0
       for (let result of response) {
         if (result) {

packages/backend-core/tests/core/utilities/structures/shared.ts

@@ -1,19 +0,0 @@
-import { User } from "@budibase/types"
-import { generator } from "./generator"
-import { uuid } from "./common"
-
-export const newEmail = () => {
-  return `${uuid()}@test.com`
-}
-
-export const user = (userProps?: any): User => {
-  return {
-    email: newEmail(),
-    password: "test",
-    roles: { app_test: "admin" },
-    firstName: generator.first(),
-    lastName: generator.last(),
-    pictureUrl: "http://test.com",
-    ...userProps,
-  }
-}

packages/backend-core/tests/core/utilities/structures/sso.ts

@@ -13,8 +13,7 @@ import {
 } from "@budibase/types"
 import { generator } from "./generator"
 import { email, uuid } from "./common"
-import * as shared from "./shared"
-import { user } from "./shared"
+import * as users from "./users"
 import sample from "lodash/sample"
 
 export function OAuth(): OAuth2 {
@@ -26,7 +25,7 @@ export function OAuth(): OAuth2 {
 
 export function authDetails(userDoc?: User): SSOAuthDetails {
   if (!userDoc) {
-    userDoc = user()
+    userDoc = users.user()
   }
 
   const userId = userDoc._id || uuid()
@@ -52,7 +51,7 @@ export function providerType(): SSOProviderType {
 
 export function ssoProfile(user?: User): SSOProfile {
   if (!user) {
-    user = shared.user()
+    user = users.user()
   }
   return {
     id: user._id!,

packages/backend-core/tests/core/utilities/structures/users.ts

@@ -4,11 +4,33 @@ import {
   BuilderUser,
   SSOAuthDetails,
   SSOUser,
+  User,
 } from "@budibase/types"
-import { user } from "./shared"
 import { authDetails } from "./sso"
+import { uuid } from "./common"
+import { generator } from "./generator"
+import { tenant } from "."
+import { generateGlobalUserID } from "../../../../src/docIds"
 
-export { user, newEmail } from "./shared"
+export const newEmail = () => {
+  return `${uuid()}@test.com`
+}
+
+export const user = (userProps?: Partial<Omit<User, "userId">>): User => {
+  const userId = userProps?._id || generateGlobalUserID()
+  return {
+    _id: userId,
+    userId,
+    email: newEmail(),
+    password: "test",
+    roles: { app_test: "admin" },
+    firstName: generator.first(),
+    lastName: generator.last(),
+    pictureUrl: "http://test.com",
+    tenantId: tenant.id(),
+    ...userProps,
+  }
+}
 
 export const adminUser = (userProps?: any): AdminUser => {
   return {

packages/bbui/package.json

@@ -82,7 +82,7 @@
     "@spectrum-css/typography": "3.0.1",
     "@spectrum-css/underlay": "2.0.9",
     "@spectrum-css/vars": "3.0.1",
-    "dayjs": "^1.10.4",
+    "dayjs": "^1.10.8",
     "easymde": "^2.16.1",
     "svelte-flatpickr": "3.2.3",
     "svelte-portal": "^1.0.0",

packages/bbui/src/Tooltip/AbsTooltip.svelte

@@ -126,8 +126,9 @@
     transition: top 130ms ease-out, left 130ms ease-out;
   }
   .spectrum-Tooltip-label {
-    text-overflow: ellipsis;
-    white-space: nowrap;
+    display: -webkit-box;
+    -webkit-line-clamp: 3;
+    -webkit-box-orient: vertical;
     overflow: hidden;
     font-size: 12px;
     font-weight: 600;

packages/builder/package.json

@@ -69,7 +69,7 @@
     "@spectrum-css/page": "^3.0.1",
     "@spectrum-css/vars": "^3.0.1",
     "codemirror": "^5.59.0",
-    "dayjs": "^1.11.2",
+    "dayjs": "^1.10.8",
     "downloadjs": "1.4.7",
     "fast-json-patch": "^3.1.1",
     "lodash": "4.17.21",

packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte

@@ -502,7 +502,7 @@
     </div>
     {#if datasource?.source !== "ORACLE" && datasource?.source !== "SQL_SERVER"}
       <div>
-        <div>
+        <div class="row">
           <Label>Time zones</Label>
           <AbsTooltip
             position="top"
@@ -690,18 +690,19 @@
     display: flex;
     align-items: center;
   }
-
   .tooltip-alignment {
     display: flex;
     align-items: center;
     gap: var(--spacing-xs);
   }
-
   .label-length {
     flex-basis: 40%;
   }
-
   .input-length {
     flex-grow: 1;
   }
+  .row {
+    gap: 8px;
+    display: flex;
+  }
 </style>

packages/client/package.json

@@ -33,7 +33,7 @@
     "@spectrum-css/typography": "^3.0.2",
     "@spectrum-css/vars": "^3.0.1",
     "apexcharts": "^3.22.1",
-    "dayjs": "^1.10.5",
+    "dayjs": "^1.10.8",
     "downloadjs": "1.4.7",
     "html5-qrcode": "^2.2.1",
     "leaflet": "^1.7.1",

packages/frontend-core/package.json

@@ -8,7 +8,7 @@
   "dependencies": {
     "@budibase/bbui": "0.0.0",
     "@budibase/shared-core": "0.0.0",
-    "dayjs": "^1.11.7",
+    "dayjs": "^1.10.8",
     "lodash": "^4.17.21",
     "socket.io-client": "^4.6.1",
     "svelte": "^3.46.2"

packages/frontend-core/src/components/grid/cells/DateCell.svelte

@@ -1,5 +1,5 @@
 <script>
-  import { dayjs } from "dayjs"
+  import dayjs from "dayjs"
   import { CoreDatePicker, Icon } from "@budibase/bbui"
   import { onMount } from "svelte"
 

packages/server/src/integrations/mysql.ts

@@ -158,6 +158,12 @@ class MySQLIntegration extends Sql implements DatasourcePlus {
     ) {
       config.ssl.rejectUnauthorized = config.rejectUnauthorized
     }
+    // The MySQL library we use doesn't directly document the parameters that can be passed in the ssl
+    // object, it instead points to an older library that it says it is mostly API compatible with, that
+    // older library actually documents what parameters can be passed in the ssl object.
+    // https://github.com/sidorares/node-mysql2#api-and-configuration
+    // https://github.com/mysqljs/mysql#ssl-options
+
     // @ts-ignore
     delete config.rejectUnauthorized
     this.config = {

packages/string-templates/package.json

@@ -26,7 +26,7 @@
   },
   "dependencies": {
     "@budibase/handlebars-helpers": "^0.11.9",
-    "dayjs": "^1.10.4",
+    "dayjs": "^1.10.8",
     "handlebars": "^4.7.6",
     "handlebars-utils": "^1.0.6",
     "lodash": "^4.17.20",

yarn.lock

@@ -6269,14 +6269,6 @@
     "@types/tedious" "*"
     tarn "^3.0.1"
 
-"@types/node-fetch@2.6.1":
-  version "2.6.1"
-  resolved "https://registry.yarnpkg.com/@types/node-fetch/-/node-fetch-2.6.1.tgz#8f127c50481db65886800ef496f20bbf15518975"
-  integrity sha512-oMqjURCaxoSIsHSr1E47QHzbmzNR5rK8McHuNb11BOM9cHcIK3Avy0s/b2JlXHoQGTYS3NsvWzV1M0iK7l0wbA==
-  dependencies:
-    "@types/node" "*"
-    form-data "^3.0.0"
-
 "@types/node-fetch@2.6.4":
   version "2.6.4"
   resolved "https://registry.yarnpkg.com/@types/node-fetch/-/node-fetch-2.6.4.tgz#1bc3a26de814f6bf466b25aeb1473fa1afe6a660"
@@ -6298,11 +6290,6 @@
   resolved "https://registry.yarnpkg.com/@types/node/-/node-18.11.18.tgz#8dfb97f0da23c2293e554c5a50d61ef134d7697f"
   integrity sha512-DHQpWGjyQKSHj3ebjFI/wRKcqQcdR+MoFBygntYOZytCqNfkd2ZC4ARDJ2DQqhjH5p85Nnd3jhUJIXrszFX/JA==
 
-"@types/node@14.18.20":
-  version "14.18.20"
-  resolved "https://registry.yarnpkg.com/@types/node/-/node-14.18.20.tgz#268f028b36eaf51181c3300252f605488c4f0650"
-  integrity sha512-Q8KKwm9YqEmUBRsqJ2GWJDtXltBDxTdC4m5vTdXBolu2PeQh8LX+f6BTwU+OuXPu37fLxoN6gidqBmnky36FXA==
-
 "@types/node@16.9.1":
   version "16.9.1"
   resolved "https://registry.yarnpkg.com/@types/node/-/node-16.9.1.tgz#0611b37db4246c937feef529ddcc018cf8e35708"
@@ -9792,11 +9779,16 @@ dateformat@^4.5.1, dateformat@^4.6.3:
   resolved "https://registry.yarnpkg.com/dateformat/-/dateformat-4.6.3.tgz#556fa6497e5217fedb78821424f8a1c22fa3f4b5"
   integrity sha512-2P0p0pFGzHS5EMnhdxQi7aJN+iMheud0UhG4dlE1DLAlvL8JHjJJTX/CSm4JXwV0Ka5nGk3zC5mcb5bUQUxxMA==
 
-dayjs@^1.10.4, dayjs@^1.10.5, dayjs@^1.11.2, dayjs@^1.11.7:
+dayjs@^1.10.4, dayjs@^1.10.5:
   version "1.11.7"
   resolved "https://registry.yarnpkg.com/dayjs/-/dayjs-1.11.7.tgz#4b296922642f70999544d1144a2c25730fce63e2"
   integrity sha512-+Yw9U6YO5TQohxLcIkrXBeY73WP3ejHWVvx8XCk3gxvQDCTEmS48ZrSZCKciI7Bhl/uCMyxYtE9UqRILmFphkQ==
 
+dayjs@^1.10.8:
+  version "1.11.9"
+  resolved "https://registry.yarnpkg.com/dayjs/-/dayjs-1.11.9.tgz#9ca491933fadd0a60a2c19f6c237c03517d71d1a"
+  integrity sha512-QvzAURSbQ0pKdIye2txOzNaHmxtUBXerpY0FJsFXUMKbIZeFm5ht1LS/jFsrncjnmtv8HsG0W2g6c0zUjZWmpA==
+
 dd-trace@3.13.2:
   version "3.13.2"
   resolved "https://registry.yarnpkg.com/dd-trace/-/dd-trace-3.13.2.tgz#95b1ec480ab9ac406e1da7591a8c6f678d3799fd"