Merge branch 'master' into fix/role-permission-update

.github/workflows/budibase_ci.yml

@@ -117,9 +117,9 @@ jobs:
       - name: Test
         run: |
           if ${{ env.ONLY_AFFECTED_TASKS }}; then
-            yarn test --ignore=@budibase/worker --ignore=@budibase/server --since=${{ env.NX_BASE_BRANCH }}
+            yarn test --ignore=@budibase/worker --ignore=@budibase/server --ignore @budibase/account-portal-server --since=${{ env.NX_BASE_BRANCH }}
           else
-            yarn test --ignore=@budibase/worker --ignore=@budibase/server
+            yarn test --ignore=@budibase/worker --ignore=@budibase/server --ignore @budibase/account-portal-server
           fi
 
   test-worker:

lerna.json

@@ -1,6 +1,6 @@
 {
   "$schema": "node_modules/lerna/schemas/lerna-schema.json",
-  "version": "2.32.0",
+  "version": "2.32.1",
   "npmClient": "yarn",
   "packages": [
     "packages/*",

packages/backend-core/src/cache/user.ts

@@ -63,14 +63,25 @@ async function populateUsersFromDB(
  * If not present fallback to loading the user directly and re-caching.
  * @param userId the id of the user to get
  * @param tenantId the tenant of the user to get
+ * @param email the email of the user to populate from account if needed
  * @param populateUser function to provide the user for re-caching. default to couch db
  * @returns
  */
-export async function getUser(
+export async function getUser({
-  userId: string,
+  userId,
-  tenantId?: string,
+  tenantId,
-  populateUser?: (userId: string, tenantId: string) => Promise<User>
+  email,
-) {
+  populateUser,
+}: {
+  userId: string
+  email?: string
+  tenantId?: string
+  populateUser?: (
+    userId: string,
+    tenantId: string,
+    email?: string
+  ) => Promise<User>
+}) {
   if (!populateUser) {
     populateUser = populateFromDB
   }
@@ -85,7 +96,7 @@ export async function getUser(
   // try cache
   let user: User = await client.get(userId)
   if (!user) {
-    user = await populateUser(userId, tenantId)
+    user = await populateUser(userId, tenantId, email)
     await client.store(userId, user, EXPIRY_SECONDS)
   }
   if (user && !user.tenantId && tenantId) {

packages/backend-core/src/middleware/authenticated.ts

@@ -43,7 +43,11 @@ function finalise(ctx: any, opts: FinaliseOpts = {}) {
 
 async function checkApiKey(
   apiKey: string,
-  populateUser?: (userId: string, tenantId: string) => Promise<User>
+  populateUser?: (
+    userId: string,
+    tenantId: string,
+    email?: string
+  ) => Promise<User>
 ) {
   // check both the primary and the fallback internal api keys
   // this allows for rotation
@@ -70,7 +74,11 @@ async function checkApiKey(
     if (userId) {
       return {
         valid: true,
-        user: await getUser(userId, tenantId, populateUser),
+        user: await getUser({
+          userId,
+          tenantId,
+          populateUser,
+        }),
       }
     } else {
       throw new InvalidAPIKeyError()
@@ -123,13 +131,18 @@ export default function (
           // getting session handles error checking (if session exists etc)
           session = await getSession(userId, sessionId)
           if (opts && opts.populateUser) {
-            user = await getUser(
+            user = await getUser({
               userId,
-              session.tenantId,
+              tenantId: session.tenantId,
-              opts.populateUser(ctx)
+              email: session.email,
-            )
+              populateUser: opts.populateUser(ctx),
+            })
           } else {
-            user = await getUser(userId, session.tenantId)
+            user = await getUser({
+              userId,
+              tenantId: session.tenantId,
+              email: session.email,
+            })
           }
           // @ts-ignore
           user.csrfToken = session.csrfToken
@@ -148,7 +161,11 @@ export default function (
       }
       // this is an internal request, no user made it
       if (!authenticated && apiKey) {
-        const populateUser = opts.populateUser ? opts.populateUser(ctx) : null
+        const populateUser: (
+          userId: string,
+          tenantId: string,
+          email?: string
+        ) => Promise<User> = opts.populateUser ? opts.populateUser(ctx) : null
         const { valid, user: foundUser } = await checkApiKey(
           apiKey,
           populateUser

packages/server/Dockerfile

@@ -1,4 +1,4 @@
-FROM node:20-slim
+FROM node:20-alpine
 
 LABEL com.centurylinklabs.watchtower.lifecycle.pre-check="scripts/watchtower-hooks/pre-check.sh"
 LABEL com.centurylinklabs.watchtower.lifecycle.pre-update="scripts/watchtower-hooks/pre-update.sh"
@@ -15,37 +15,35 @@ ENV POSTHOG_TOKEN=phc_bIjZL7oh2GEUd2vqvTBH8WvrX0fWTFQMs6H5KQxiUxU
 ENV ACCOUNT_PORTAL_URL=https://account.budibase.app
 ENV TOP_LEVEL_PATH=/
 
-# handle node-gyp
+# handle node-gyp and install postgres client for pg_dump utils
-RUN apt-get update \
+RUN apk add --no-cache \
-    && apt-get install -y --no-install-recommends g++ make python3 jq
+    g++ \
-RUN yarn global add pm2
+    make \
+    python3 \
+    jq \
+    bash \
+    postgresql-client \
+    git
 
-# Install postgres client for pg_dump utils
+RUN yarn global add pm2
-RUN apt update && apt upgrade -y \
-    && apt install software-properties-common apt-transport-https curl gpg -y \
-    && curl -fsSl https://www.postgresql.org/media/keys/ACCC4CF8.asc | gpg --dearmor | tee /usr/share/keyrings/postgresql.gpg > /dev/null \
-    && echo deb [arch=amd64,arm64,ppc64el signed-by=/usr/share/keyrings/postgresql.gpg] http://apt.postgresql.org/pub/repos/apt/ $(lsb_release -cs)-pgdg main | tee /etc/apt/sources.list.d/postgresql.list \
-    && apt update -y \
-    && apt install postgresql-client-15 -y \
-    && apt remove software-properties-common apt-transport-https curl gpg -y
 
 WORKDIR /
 
 COPY scripts/removeWorkspaceDependencies.sh scripts/removeWorkspaceDependencies.sh
 RUN chmod +x ./scripts/removeWorkspaceDependencies.sh
 
-
 WORKDIR /app
 COPY packages/server/package.json .
 COPY packages/server/dist/yarn.lock .
 
 COPY scripts/removeWorkspaceDependencies.sh scripts/removeWorkspaceDependencies.sh
 RUN chmod +x ./scripts/removeWorkspaceDependencies.sh
-RUN ./scripts/removeWorkspaceDependencies.sh  package.json
+RUN ./scripts/removeWorkspaceDependencies.sh package.json
 
+# Install yarn packages with caching
 RUN --mount=type=cache,target=/root/.yarn YARN_CACHE_FOLDER=/root/.yarn yarn install --production=true --network-timeout 1000000 \
-    # Remove unneeded data from file system to reduce image size
+    && yarn cache clean \
-    && yarn cache clean && apt-get remove -y --purge --auto-remove g++ make python jq \
+    && apk del g++ make python3 jq \
     && rm -rf /tmp/* /root/.node-gyp /usr/local/lib/node_modules/npm/node_modules/node-gyp
 
 COPY packages/server/dist/ dist/
@@ -69,7 +67,7 @@ EXPOSE 4001
 # due to this causing yarn to stop installing dev dependencies
 # which are actually needed to get this environment up and running
 ENV NODE_ENV=production
-# this is required for isolated-vm to work on Node 20+
+# This is required for isolated-vm to work on Node 20+
 ENV NODE_OPTIONS="--no-node-snapshot"
 ENV CLUSTER_MODE=${CLUSTER_MODE}
 ENV TOP_LEVEL_PATH=/app

packages/server/src/tests/utilities/TestConfiguration.ts

@@ -333,6 +333,7 @@ export default class TestConfiguration {
       sessionId: this.sessionIdForUser(_id),
       tenantId: this.getTenantId(),
       csrfToken: this.csrfToken,
+      email,
     })
     const resp = await db.put(user)
     await cache.user.invalidateUser(_id)
@@ -396,16 +397,17 @@ export default class TestConfiguration {
       }
       // make sure the user exists in the global DB
       if (roleId !== roles.BUILTIN_ROLE_IDS.PUBLIC) {
-        await this.globalUser({
+        const user = await this.globalUser({
           _id: userId,
           builder: { global: builder },
           roles: { [appId]: roleId || roles.BUILTIN_ROLE_IDS.BASIC },
         })
+        await sessions.createASession(userId, {
+          sessionId: this.sessionIdForUser(userId),
+          tenantId: this.getTenantId(),
+          email: user.email,
+        })
       }
-      await sessions.createASession(userId, {
-        sessionId: this.sessionIdForUser(userId),
-        tenantId: this.getTenantId(),
-      })
       // have to fake this
       const authObj = {
         userId,

packages/server/src/threads/query.ts

@@ -247,7 +247,9 @@ class QueryRunner {
     if (!resp.err) {
       const globalUserId = getGlobalIDFromUserMetadataID(_id)
       await auth.updateUserOAuth(globalUserId, resp)
-      this.ctx.user = await cache.user.getUser(globalUserId)
+      this.ctx.user = await cache.user.getUser({
+        userId: globalUserId,
+      })
     } else {
       // In this event the user may have oAuth issues that
       // could require re-authenticating with their provider.

packages/server/src/utilities/global.ts

@@ -77,7 +77,9 @@ export async function getCachedSelf(
 ): Promise<ContextUser> {
   // this has to be tenant aware, can't depend on the context to find it out
   // running some middlewares before the tenancy causes context to break
-  const user = await cache.user.getUser(ctx.user?._id!)
+  const user = await cache.user.getUser({
+    userId: ctx.user?._id!,
+  })
   return processUser(user, { appId })
 }
 

packages/server/src/utilities/rowProcessor/bbReferenceProcessor.ts

@@ -35,7 +35,9 @@ export async function processInputBBReference(
       }
 
       try {
-        await cache.user.getUser(id)
+        await cache.user.getUser({
+          userId: id,
+        })
         return id
       } catch (e: any) {
         if (e.statusCode === 404) {
@@ -125,7 +127,9 @@ export async function processOutputBBReference(
     case BBReferenceFieldSubType.USER: {
       let user
       try {
-        user = await cache.user.getUser(value as string)
+        user = await cache.user.getUser({
+          userId: value as string,
+        })
       } catch (err: any) {
         if (err.statusCode !== 404) {
           throw err

packages/server/src/utilities/rowProcessor/index.ts

@@ -110,7 +110,9 @@ async function processDefaultValues(table: Table, row: Row) {
 
   const identity = context.getIdentity()
   if (identity?._id && identity.type === IdentityType.USER) {
-    const user = await cache.user.getUser(identity._id)
+    const user = await cache.user.getUser({
+      userId: identity._id,
+    })
     delete user.password
 
     ctx["Current User"] = user

packages/server/src/utilities/rowProcessor/tests/bbReferenceProcessor.spec.ts

@@ -74,7 +74,9 @@ describe("bbReferenceProcessor", () => {
 
         expect(result).toEqual(userId)
         expect(cacheGetUserSpy).toHaveBeenCalledTimes(1)
-        expect(cacheGetUserSpy).toHaveBeenCalledWith(userId)
+        expect(cacheGetUserSpy).toHaveBeenCalledWith({
+          userId,
+        })
       })
 
       it("throws an error given an invalid id", async () => {
@@ -98,7 +100,9 @@ describe("bbReferenceProcessor", () => {
 
         expect(result).toEqual(userId)
         expect(cacheGetUserSpy).toHaveBeenCalledTimes(1)
-        expect(cacheGetUserSpy).toHaveBeenCalledWith(userId)
+        expect(cacheGetUserSpy).toHaveBeenCalledWith({
+          userId,
+        })
       })
 
       it("empty strings will return null", async () => {
@@ -243,7 +247,9 @@ describe("bbReferenceProcessor", () => {
           lastName: user.lastName,
         })
         expect(cacheGetUserSpy).toHaveBeenCalledTimes(1)
-        expect(cacheGetUserSpy).toHaveBeenCalledWith(userId)
+        expect(cacheGetUserSpy).toHaveBeenCalledWith({
+          userId,
+        })
       })
 
       it("returns undefined given an unexisting user", async () => {
@@ -255,7 +261,9 @@ describe("bbReferenceProcessor", () => {
 
         expect(result).toBeUndefined()
         expect(cacheGetUserSpy).toHaveBeenCalledTimes(1)
-        expect(cacheGetUserSpy).toHaveBeenCalledWith(userId)
+        expect(cacheGetUserSpy).toHaveBeenCalledWith({
+          userId,
+        })
       })
     })
   })

packages/types/src/sdk/auth.ts

@@ -10,6 +10,7 @@ export interface AuthToken {
 export interface CreateSession {
   sessionId: string
   tenantId: string
+  email: string
   csrfToken?: string
   hosting?: Hosting
 }

packages/worker/src/sdk/auth/auth.ts

@@ -19,12 +19,17 @@ import { EmailTemplatePurpose } from "../../constants"
 export async function loginUser(user: User) {
   const sessionId = coreUtils.newid()
   const tenantId = tenancy.getTenantId()
-  await sessions.createASession(user._id!, { sessionId, tenantId })
+  await sessions.createASession(user._id!, {
+    sessionId,
+    tenantId,
+    email: user.email,
+  })
   return jwt.sign(
     {
       userId: user._id,
       sessionId,
       tenantId,
+      email: user.email,
     },
     coreEnv.JWT_SECRET!
   )

packages/worker/src/tests/TestConfiguration.ts

@@ -170,19 +170,26 @@ class TestConfiguration {
   async _createSession({
     userId,
     tenantId,
+    email,
   }: {
     userId: string
     tenantId: string
+    email: string
   }) {
     await sessions.createASession(userId!, {
       sessionId: "sessionid",
-      tenantId: tenantId,
+      tenantId,
       csrfToken: CSRF_TOKEN,
+      email,
     })
   }
 
   async createSession(user: User) {
-    return this._createSession({ userId: user._id!, tenantId: user.tenantId })
+    return this._createSession({
+      userId: user._id!,
+      tenantId: user.tenantId,
+      email: user.email,
+    })
   }
 
   cookieHeader(cookies: any) {