Merge remote-tracking branch 'origin/develop' into feat/relationship-configuration

lerna.json

@@ -1,5 +1,5 @@
 {
-  "version": "2.10.12-alpha.26",
+  "version": "2.10.16-alpha.7",
   "npmClient": "yarn",
   "packages": [
     "packages/*"

packages/backend-core/src/constants/db.ts

@@ -18,7 +18,7 @@ export enum ViewName {
   ROUTING = "screen_routes",
   AUTOMATION_LOGS = "automation_logs",
   ACCOUNT_BY_EMAIL = "account_by_email",
-  PLATFORM_USERS_LOWERCASE = "platform_users_lowercase",
+  PLATFORM_USERS_LOWERCASE = "platform_users_lowercase_2",
   USER_BY_GROUP = "user_by_group",
   APP_BACKUP_BY_TRIGGER = "by_trigger",
 }

packages/backend-core/src/db/views.ts

@@ -190,6 +190,10 @@ export const createPlatformUserView = async () => {
     if (doc.tenantId) {
       emit(doc._id.toLowerCase(), doc._id)
     }
+
+    if (doc.ssoId) {
+      emit(doc.ssoId, doc._id)
+    }
   }`
   await createPlatformView(viewJs, ViewName.PLATFORM_USERS_LOWERCASE)
 }

packages/backend-core/src/platform/users.ts

@@ -5,6 +5,7 @@ import {
   PlatformUser,
   PlatformUserByEmail,
   PlatformUserById,
+  PlatformUserBySsoId,
   User,
 } from "@budibase/types"
 
@@ -45,6 +46,20 @@ function newUserEmailDoc(
   }
 }
 
+function newUserSsoIdDoc(
+  ssoId: string,
+  email: string,
+  userId: string,
+  tenantId: string
+): PlatformUserBySsoId {
+  return {
+    _id: ssoId,
+    userId,
+    email,
+    tenantId,
+  }
+}
+
 /**
  * Add a new user id or email doc if it doesn't exist.
  */
@@ -64,11 +79,24 @@ async function addUserDoc(emailOrId: string, newDocFn: () => PlatformUser) {
   }
 }
 
-export async function addUser(tenantId: string, userId: string, email: string) {
-  await Promise.all([
+export async function addUser(
+  tenantId: string,
+  userId: string,
+  email: string,
+  ssoId?: string
+) {
+  const promises = [
     addUserDoc(userId, () => newUserIdDoc(userId, tenantId)),
     addUserDoc(email, () => newUserEmailDoc(userId, email, tenantId)),
-  ])
+  ]
+
+  if (ssoId) {
+    promises.push(
+      addUserDoc(ssoId, () => newUserSsoIdDoc(ssoId, email, userId, tenantId))
+    )
+  }
+
+  await Promise.all(promises)
 }
 
 // DELETE

packages/backend-core/src/security/permissions.ts

@@ -1,8 +1,9 @@
-import { PermissionType, PermissionLevel } from "@budibase/types"
-export { PermissionType, PermissionLevel } from "@budibase/types"
+import { PermissionLevel, PermissionType } from "@budibase/types"
 import flatten from "lodash/flatten"
 import cloneDeep from "lodash/fp/cloneDeep"
 
+export { PermissionType, PermissionLevel } from "@budibase/types"
+
 export type RoleHierarchy = {
   permissionId: string
 }[]
@@ -78,6 +79,7 @@ export const BUILTIN_PERMISSIONS = {
     permissions: [
       new Permission(PermissionType.QUERY, PermissionLevel.READ),
       new Permission(PermissionType.TABLE, PermissionLevel.READ),
+      new Permission(PermissionType.APP, PermissionLevel.READ),
     ],
   },
   WRITE: {
@@ -88,6 +90,7 @@ export const BUILTIN_PERMISSIONS = {
       new Permission(PermissionType.TABLE, PermissionLevel.WRITE),
       new Permission(PermissionType.AUTOMATION, PermissionLevel.EXECUTE),
       new Permission(PermissionType.LEGACY_VIEW, PermissionLevel.READ),
+      new Permission(PermissionType.APP, PermissionLevel.READ),
     ],
   },
   POWER: {
@@ -99,6 +102,7 @@ export const BUILTIN_PERMISSIONS = {
       new Permission(PermissionType.AUTOMATION, PermissionLevel.EXECUTE),
       new Permission(PermissionType.WEBHOOK, PermissionLevel.READ),
       new Permission(PermissionType.LEGACY_VIEW, PermissionLevel.READ),
+      new Permission(PermissionType.APP, PermissionLevel.READ),
     ],
   },
   ADMIN: {
@@ -111,6 +115,7 @@ export const BUILTIN_PERMISSIONS = {
       new Permission(PermissionType.WEBHOOK, PermissionLevel.READ),
       new Permission(PermissionType.QUERY, PermissionLevel.ADMIN),
       new Permission(PermissionType.LEGACY_VIEW, PermissionLevel.READ),
+      new Permission(PermissionType.APP, PermissionLevel.READ),
     ],
   },
 }

packages/backend-core/src/security/roles.ts

@@ -215,21 +215,23 @@ async function getAllUserRoles(userRoleId?: string): Promise<RoleDoc[]> {
   return roles
 }
 
+export async function getUserRoleIdHierarchy(
+  userRoleId?: string
+): Promise<string[]> {
+  const roles = await getUserRoleHierarchy(userRoleId)
+  return roles.map(role => role._id!)
+}
+
 /**
  * Returns an ordered array of the user's inherited role IDs, this can be used
  * to determine if a user can access something that requires a specific role.
  * @param {string} userRoleId The user's role ID, this can be found in their access token.
- * @param {object} opts Various options, such as whether to only retrieve the IDs (default true).
- * @returns {Promise<string[]|object[]>} returns an ordered array of the roles, with the first being their
+ * @returns {Promise<object[]>} returns an ordered array of the roles, with the first being their
  * highest level of access and the last being the lowest level.
  */
-export async function getUserRoleHierarchy(
-  userRoleId?: string,
-  opts = { idOnly: true }
-) {
+export async function getUserRoleHierarchy(userRoleId?: string) {
   // special case, if they don't have a role then they are a public user
-  const roles = await getAllUserRoles(userRoleId)
-  return opts.idOnly ? roles.map(role => role._id) : roles
+  return getAllUserRoles(userRoleId)
 }
 
 // this function checks that the provided permissions are in an array format
@@ -249,6 +251,11 @@ export function checkForRoleResourceArray(
   return rolePerms
 }
 
+export async function getAllRoleIds(appId?: string) {
+  const roles = await getAllRoles(appId)
+  return roles.map(role => role._id)
+}
+
 /**
  * Given an app ID this will retrieve all of the roles that are currently within that app.
  * @return {Promise<object[]>} An array of the role objects that were found.
@@ -332,9 +339,7 @@ export class AccessController {
     }
     let roleIds = userRoleId ? this.userHierarchies[userRoleId] : null
     if (!roleIds && userRoleId) {
-      roleIds = (await getUserRoleHierarchy(userRoleId, {
-        idOnly: true,
-      })) as string[]
+      roleIds = await getUserRoleIdHierarchy(userRoleId)
       this.userHierarchies[userRoleId] = roleIds
     }
 

packages/backend-core/src/users/db.ts

@@ -278,7 +278,12 @@ export class UserDB {
         builtUser._rev = response.rev
 
         await eventHelpers.handleSaveEvents(builtUser, dbUser)
-        await platform.users.addUser(tenantId, builtUser._id!, builtUser.email)
+        await platform.users.addUser(
+          tenantId,
+          builtUser._id!,
+          builtUser.email,
+          builtUser.ssoId
+        )
         await cache.user.invalidateUser(response.id)
 
         await Promise.all(groupPromises)

packages/backend-core/tests/core/utilities/structures/accounts.ts

@@ -1,4 +1,4 @@
-import { generator, uuid, quotas } from "."
+import { generator, quotas, uuid } from "."
 import { generateGlobalUserID } from "../../../../src/docIds"
 import {
   Account,
@@ -6,10 +6,11 @@ import {
   AccountSSOProviderType,
   AuthType,
   CloudAccount,
-  Hosting,
-  SSOAccount,
   CreateAccount,
   CreatePassswordAccount,
+  CreateVerifiableSSOAccount,
+  Hosting,
+  SSOAccount,
 } from "@budibase/types"
 import sample from "lodash/sample"
 
@@ -68,6 +69,23 @@ export function ssoAccount(account: Account = cloudAccount()): SSOAccount {
   }
 }
 
+export function verifiableSsoAccount(
+  account: Account = cloudAccount()
+): SSOAccount {
+  return {
+    ...account,
+    authType: AuthType.SSO,
+    oauth2: {
+      accessToken: generator.string(),
+      refreshToken: generator.string(),
+    },
+    pictureUrl: generator.url(),
+    provider: AccountSSOProvider.MICROSOFT,
+    providerType: AccountSSOProviderType.MICROSOFT,
+    thirdPartyProfile: { id: "abc123" },
+  }
+}
+
 export const cloudCreateAccount: CreatePassswordAccount = {
   email: "cloud@budibase.com",
   tenantId: "cloud",
@@ -91,6 +109,19 @@ export const cloudSSOCreateAccount: CreateAccount = {
   profession: "Software Engineer",
 }
 
+export const cloudVerifiableSSOCreateAccount: CreateVerifiableSSOAccount = {
+  email: "cloud-sso@budibase.com",
+  tenantId: "cloud-sso",
+  hosting: Hosting.CLOUD,
+  authType: AuthType.SSO,
+  tenantName: "cloudsso",
+  name: "Budi Armstrong",
+  size: "10+",
+  profession: "Software Engineer",
+  provider: AccountSSOProvider.MICROSOFT,
+  thirdPartyProfile: { id: "abc123" },
+}
+
 export const selfCreateAccount: CreatePassswordAccount = {
   email: "self@budibase.com",
   tenantId: "self",

packages/builder/README.md

@@ -1,3 +1,5 @@
+test
+
 ## Get Started 
 
 `yarn install`

packages/builder/src/builderStore/store/screenTemplates/rowListScreen.js

@@ -33,6 +33,8 @@ const generateTableBlock = datasource => {
       showTitleButton: true,
       titleButtonText: "Create row",
       titleButtonClickBehaviour: "new",
+      sidePanelSaveLabel: "Save",
+      sidePanelDeleteLabel: "Delete",
     })
     .instanceName(`${datasource.label} - Table block`)
   return tableBlock

packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte

@@ -33,7 +33,7 @@
   import { getBindings } from "components/backend/DataTable/formula"
   import JSONSchemaModal from "./JSONSchemaModal.svelte"
   import { ValidColumnNameRegex } from "@budibase/shared-core"
-  import { FieldSubtype, FieldType } from "@budibase/types"
+  import { FieldType } from "@budibase/types"
   import RelationshipSelector from "components/common/RelationshipSelector.svelte"
 
   const AUTO_TYPE = "auto"
@@ -43,11 +43,7 @@
   const NUMBER_TYPE = FIELDS.NUMBER.type
   const JSON_TYPE = FIELDS.JSON.type
   const DATE_TYPE = FIELDS.DATETIME.type
-  const BB_REFERENCE_TYPE = FieldType.BB_REFERENCE
-  const BB_USER_REFERENCE_TYPE = composeType(
-    BB_REFERENCE_TYPE,
-    FieldSubtype.USER
-  )
+  const USER_REFRENCE_TYPE = FIELDS.BB_REFERENCE_USER.compositeType
 
   const dispatch = createEventDispatcher()
   const PROHIBITED_COLUMN_NAMES = ["type", "_id", "_rev", "tableId"]
@@ -84,33 +80,6 @@
   let relationshipOpts1 = Object.values(PrettyRelationshipDefinitions)
   let relationshipOpts2 = Object.values(PrettyRelationshipDefinitions)
 
-  const bbRefTypeMapping = {}
-
-  function composeType(fieldType, subtype) {
-    return `${fieldType}_${subtype}`
-  }
-
-  // Handling fields with subtypes
-  fieldDefinitions = Object.entries(fieldDefinitions).reduce(
-    (p, [key, field]) => {
-      if (field.type === BB_REFERENCE_TYPE) {
-        const composedType = composeType(field.type, field.subtype)
-        p[key] = {
-          ...field,
-          type: composedType,
-        }
-        bbRefTypeMapping[composedType] = {
-          type: field.type,
-          subtype: field.subtype,
-        }
-      } else {
-        p[key] = field
-      }
-      return p
-    },
-    {}
-  )
-
   $: if (primaryDisplay) {
     editableColumn.constraints.presence = { allowEmpty: false }
   }
@@ -170,12 +139,8 @@
         $tables.selected.primaryDisplay == null ||
         $tables.selected.primaryDisplay === editableColumn.name
 
-      const mapped = Object.entries(bbRefTypeMapping).find(
-        ([_, v]) => v.type === field.type && v.subtype === field.subtype
-      )
-      if (mapped) {
-        editableColumn.type = mapped[0]
-        delete editableColumn.subtype
+      if (editableColumn.type === FieldType.BB_REFERENCE) {
+        editableColumn.type = `${editableColumn.type}_${editableColumn.subtype}`
       }
       // Here we are setting the relationship values based on the editableColumn
       // This part of the code is used when viewing an existing field hence the check
@@ -212,8 +177,6 @@
 
   $: initialiseField(field, savingColumn)
 
-  $: isBBReference = !!bbRefTypeMapping[editableColumn.type]
-
   $: checkConstraints(editableColumn)
   $: required = !!editableColumn?.constraints?.presence || primaryDisplay
   $: uneditable =
@@ -286,11 +249,12 @@
 
     let saveColumn = cloneDeep(editableColumn)
 
-    if (bbRefTypeMapping[saveColumn.type]) {
-      saveColumn = {
-        ...saveColumn,
-        ...bbRefTypeMapping[saveColumn.type],
-      }
+    // Handle types on composite types
+    const definition = fieldDefinitions[saveColumn.type.toUpperCase()]
+    if (definition && saveColumn.type === definition.compositeType) {
+      saveColumn.type = definition.type
+      saveColumn.subtype = definition.subtype
+      delete saveColumn.compositeType
     }
 
     if (saveColumn.type === AUTO_TYPE) {
@@ -373,7 +337,7 @@
       editableColumn.relationshipType = RelationshipType.MANY_TO_MANY
     } else if (editableColumn.type === FORMULA_TYPE) {
       editableColumn.formulaType = "dynamic"
-    } else if (editableColumn.type === BB_USER_REFERENCE_TYPE) {
+    } else if (editableColumn.type === USER_REFRENCE_TYPE) {
       editableColumn.relationshipType = RelationshipType.ONE_TO_MANY
     }
   }
@@ -431,14 +395,12 @@
         FIELDS.BOOLEAN,
         FIELDS.FORMULA,
         FIELDS.BIGINT,
+        FIELDS.BB_REFERENCE_USER,
       ]
       // no-sql or a spreadsheet
       if (!external || table.sql) {
         fields = [...fields, FIELDS.LINK, FIELDS.ARRAY]
       }
-      if (fieldDefinitions.USER) {
-        fields.push(fieldDefinitions.USER)
-      }
       return fields
     }
   }
@@ -447,8 +409,9 @@
     if (!fieldToCheck) {
       return
     }
+
     // most types need this, just make sure its always present
-    if (fieldToCheck && !fieldToCheck.constraints) {
+    if (!fieldToCheck.constraints) {
       fieldToCheck.constraints = {}
     }
     // some string types may have been built by server, may not always have constraints
@@ -528,7 +491,7 @@
     on:change={handleTypeChange}
     options={allowedTypes}
     getOptionLabel={field => field.name}
-    getOptionValue={field => field.type}
+    getOptionValue={field => field.compositeType || field.type}
     getOptionIcon={field => field.icon}
     isOptionEnabled={option => {
       if (option.type == AUTO_TYPE) {
@@ -694,7 +657,7 @@
     <Button primary text on:click={openJsonSchemaEditor}
       >Open schema editor</Button
     >
-  {:else if isBBReference}
+  {:else if editableColumn.type === USER_REFRENCE_TYPE}
     <Toggle
       value={editableColumn.relationshipType === RelationshipType.MANY_TO_MANY}
       on:change={e =>

packages/builder/src/components/integration/QueryViewer.svelte

@@ -337,11 +337,12 @@
     padding: 8px 10px 8px 16px;
     display: flex;
     border-bottom: 2px solid transparent;
-    transition: border-bottom 130ms ease-out;
+    transition: border-bottom 130ms ease-out, background 130ms ease-out;
   }
 
   .header.scrolling {
     border-bottom: var(--border-light);
+    background: var(--background);
   }
 
   .body {

packages/builder/src/constants/backend/index.js

@@ -120,10 +120,11 @@ export const FIELDS = {
       presence: false,
     },
   },
-  USER: {
+  BB_REFERENCE_USER: {
     name: "User",
     type: "bb_reference",
     subtype: "user",
+    compositeType: "bb_reference_user", // Used for working with the subtype on CreateEditColumn as is it was a primary type
     icon: "User",
   },
 }

packages/builder/src/pages/builder/portal/apps/_layout.svelte

@@ -46,7 +46,9 @@
 
 {#if loaded}
   <div class="page">
+    {#if $apps.length > 0}
       <PortalSideBar />
+    {/if}
     <slot />
   </div>
 {/if}

packages/client/src/components/app/Layout.svelte

@@ -4,15 +4,14 @@
   import { Heading, Icon, clickOutside } from "@budibase/bbui"
   import { FieldTypes } from "constants"
   import active from "svelte-spa-router/active"
-  import { RoleUtils } from "@budibase/frontend-core"
 
   const sdk = getContext("sdk")
   const {
     routeStore,
+    roleStore,
     styleable,
     linkable,
     builderStore,
-    currentRole,
     sidePanelStore,
   } = sdk
   const component = getContext("component")
@@ -61,7 +60,7 @@
   })
   setContext("layout", store)
 
-  $: validLinks = getValidLinks(links, $currentRole)
+  $: validLinks = getValidLinks(links, $roleStore)
   $: typeClass = NavigationClasses[navigation] || NavigationClasses.None
   $: navWidthClass = WidthClasses[navWidth || width] || WidthClasses.Large
   $: pageWidthClass = WidthClasses[pageWidth || width] || WidthClasses.Large
@@ -99,14 +98,12 @@
     }
   }
 
-  const getValidLinks = (allLinks, role) => {
+  const getValidLinks = (allLinks, userRoleHierarchy) => {
     // Strip links missing required info
     let validLinks = (allLinks || []).filter(link => link.text && link.url)
-
     // Filter to only links allowed by the current role
-    const priority = RoleUtils.getRolePriority(role)
     return validLinks.filter(link => {
-      return !link.roleId || RoleUtils.getRolePriority(link.roleId) <= priority
+      return userRoleHierarchy?.find(roleId => roleId === link.roleId)
     })
   }
 

packages/client/src/components/app/Section.svelte

@@ -47,28 +47,29 @@
 <style>
   div {
     display: grid;
-    grid-gap: 16px;
+    --gap: 16px;
+    grid-gap: var(--gap);
   }
   .mainSidebar {
-    grid-template-columns: 3fr 1fr;
+    grid-template-columns:
+      calc((100% - var(--gap)) / 4 * 3) /* 75% */
+      calc((100% - var(--gap)) / 4); /* 25% */
   }
   .sidebarMain {
-    grid-template-columns: 1fr 3fr;
-  }
-  .oneColumn {
-    grid-template-columns: 1fr;
-  }
-  .twoColumns {
-    grid-template-columns: 1fr 1fr;
-  }
-  .threeColumns {
-    grid-template-columns: 1fr 1fr 1fr;
+    grid-template-columns:
+      calc((100% - var(--gap)) / 4) /* 25% */
+      calc((100% - var(--gap)) / 4 * 3); /* 75% */
   }
+  .oneColumn,
   .columns-1 {
     grid-template-columns: 1fr;
   }
+  .twoColumns,
   .columns-2 {
-    grid-template-columns: 1fr 1fr;
+    grid-template-columns: repeat(2, calc((100% - var(--gap)) / 2));
+  }
+  .threeColumns {
+    grid-template-columns: repeat(3, calc((100% - var(--gap)) / 3));
   }
   .placeholder {
     border: 2px dashed var(--spectrum-global-color-gray-600);

packages/client/src/components/app/blocks/TableBlock.svelte

@@ -45,8 +45,21 @@
   let enrichedSearchColumns
   let schemaLoaded = false
 
+  $: deleteLabel = setDeleteLabel(sidePanelDeleteLabel, sidePanelShowDelete)
+
+  const setDeleteLabel = sidePanelDeleteLabel => {
     // Accommodate old config to ensure delete button does not reappear
-  $: deleteLabel = sidePanelShowDelete === false ? "" : sidePanelDeleteLabel
+    let labelText = sidePanelShowDelete === false ? "" : sidePanelDeleteLabel
+
+    // Empty text is considered hidden.
+    if (labelText?.trim() === "") {
+      return ""
+    }
+
+    // Default to "Delete" if the value is unset
+    return labelText || "Delete"
+  }
+
   $: isDSPlus = dataSource?.type === "table" || dataSource?.type === "viewV2"
   $: fetchSchema(dataSource)
   $: enrichSearchColumns(searchColumns, schema).then(
@@ -249,7 +262,7 @@
             props={{
               dataSource,
               saveButtonLabel: sidePanelSaveLabel || "Save", //always show
-              deleteButtonLabel: deleteLabel, //respect config
+              deleteButtonLabel: deleteLabel,
               actionType: "Update",
               rowId: `{{ ${safe("state")}.${safe(stateKey)} }}`,
               fields: sidePanelFields || normalFields,

packages/client/src/components/devtools/DevToolsHeader.svelte

@@ -1,32 +1,39 @@
 <script>
   import { Heading, Select, ActionButton } from "@budibase/bbui"
-  import { devToolsStore, appStore } from "../../stores"
-  import { getContext } from "svelte"
+  import { devToolsStore, appStore, roleStore } from "../../stores"
+  import { getContext, onMount } from "svelte"
 
   const context = getContext("context")
+  const SELF_ROLE = "self"
 
-  $: previewOptions = [
-    {
+  let staticRoleList
+
+  $: previewOptions = buildRoleList(staticRoleList)
+
+  function buildRoleList(roleIds) {
+    const list = []
+    list.push({
       label: "View as yourself",
-      value: "self",
-    },
-    {
-      label: "View as public user",
-      value: "PUBLIC",
-    },
-    {
-      label: "View as basic user",
-      value: "BASIC",
-    },
-    {
-      label: "View as power user",
-      value: "POWER",
-    },
-    {
-      label: "View as admin user",
-      value: "ADMIN",
-    },
-  ]
+      value: SELF_ROLE,
+    })
+    if (!roleIds) {
+      return list
+    }
+    for (let roleId of roleIds) {
+      list.push({
+        label: `View as ${roleId.toLowerCase()} user`,
+        value: roleId,
+      })
+    }
+    devToolsStore.actions.changeRole(SELF_ROLE)
+    return list
+  }
+
+  onMount(async () => {
+    // make sure correct before starting
+    await devToolsStore.actions.changeRole(SELF_ROLE)
+    staticRoleList = await roleStore.actions.fetchAccessibleRoles()
+  })
 </script>
 
 <div class="dev-preview-header" class:mobile={$context.device.mobile}>
@@ -34,7 +41,7 @@
   <Select
     quiet
     options={previewOptions}
-    value={$devToolsStore.role || "self"}
+    value={$devToolsStore.role || SELF_ROLE}
     placeholder={null}
     autoWidth
     on:change={e => devToolsStore.actions.changeRole(e.detail)}

packages/client/src/sdk.js

@@ -13,6 +13,7 @@ import {
   sidePanelStore,
   dndIsDragging,
   confirmationStore,
+  roleStore,
 } from "stores"
 import { styleable } from "utils/styleable"
 import { linkable } from "utils/linkable"
@@ -39,6 +40,7 @@ export default {
   dndIsDragging,
   currentRole,
   confirmationStore,
+  roleStore,
   styleable,
   linkable,
   getAction,

packages/client/src/stores/index.js

@@ -11,12 +11,13 @@ export { stateStore } from "./state"
 export { themeStore } from "./theme"
 export { devToolsStore } from "./devTools"
 export { componentStore } from "./components"
-export { uploadStore } from "./uploads.js"
-export { rowSelectionStore } from "./rowSelection.js"
-export { blockStore } from "./blocks.js"
+export { uploadStore } from "./uploads"
+export { rowSelectionStore } from "./rowSelection"
+export { blockStore } from "./blocks"
 export { environmentStore } from "./environment"
-export { eventStore } from "./events.js"
-export { orgStore } from "./org.js"
+export { eventStore } from "./events"
+export { orgStore } from "./org"
+export { roleStore } from "./roles"
 export {
   dndStore,
   dndIndex,
@@ -25,7 +26,7 @@ export {
   dndIsNewComponent,
   dndIsDragging,
 } from "./dnd"
-export { sidePanelStore } from "./sidePanel.js"
+export { sidePanelStore } from "./sidePanel"
 
 // Context stores are layered and duplicated, so it is not a singleton
 export { createContextStore } from "./context"

packages/client/src/stores/roles.js

@@ -0,0 +1,24 @@
+import { API } from "api"
+import { writable } from "svelte/store"
+import { currentRole } from "./derived"
+
+const createRoleStore = () => {
+  const store = writable([])
+
+  // Fetches the user object if someone is logged in and has reloaded the page
+  const fetchAccessibleRoles = async () => {
+    const accessible = await API.getAccessibleRoles()
+    // Use the app self if present, otherwise fallback to the global self
+    store.set(accessible || [])
+    return accessible
+  }
+
+  return {
+    subscribe: store.subscribe,
+    actions: { fetchAccessibleRoles },
+  }
+}
+
+export const roleStore = createRoleStore()
+
+currentRole.subscribe(roleStore.actions.fetchAccessibleRoles)

packages/frontend-core/src/api/roles.js

@@ -38,4 +38,13 @@ export const buildRoleEndpoints = API => ({
       url: `/api/global/roles/${appId}`,
     })
   },
+
+  /**
+   * For the logged in user and current app - retrieves accessible roles.
+   */
+  getAccessibleRoles: async () => {
+    return await API.get({
+      url: `/api/roles/accessible`,
+    })
+  },
 })

packages/server/specs/openapi.json

@@ -1567,8 +1567,7 @@
           }
         },
         "required": [
-          "email",
-          "roles"
+          "email"
         ]
       },
       "userOutput": {
@@ -1639,7 +1638,6 @@
             },
             "required": [
               "email",
-              "roles",
               "_id"
             ]
           }
@@ -1718,7 +1716,6 @@
               },
               "required": [
                 "email",
-                "roles",
                 "_id"
               ]
             }

packages/server/specs/openapi.yaml

@@ -1337,7 +1337,6 @@ components:
               role ID, e.g. ADMIN.
       required:
         - email
-        - roles
     userOutput:
       type: object
       properties:
@@ -1398,7 +1397,6 @@ components:
               type: string
           required:
             - email
-            - roles
             - _id
       required:
         - data
@@ -1464,7 +1462,6 @@ components:
                 type: string
             required:
               - email
-              - roles
               - _id
       required:
         - data

packages/server/specs/resources/user.ts

@@ -92,7 +92,7 @@ const userSchema = object(
       },
     },
   },
-  { required: ["email", "roles"] }
+  { required: ["email"] }
 )
 
 const userOutputSchema = {

packages/server/src/api/controllers/public/mapping/users.ts

@@ -15,10 +15,15 @@ function user(body: any): User {
   }
 }
 
-function mapUser(ctx: any): { data: User } {
-  return {
+function mapUser(ctx: any) {
+  const body: { data: User; message?: string } = {
     data: user(ctx.body),
   }
+  if (ctx.extra?.message) {
+    body.message = ctx.extra.message
+    delete ctx.extra
+  }
+  return body
 }
 
 function mapUsers(ctx: any): { data: User[] } {

packages/server/src/api/controllers/public/users.ts

@@ -10,6 +10,32 @@ import { search as stringSearch } from "./utils"
 import { UserCtx, User } from "@budibase/types"
 import { Next } from "koa"
 import { sdk } from "@budibase/pro"
+import { isEqual, cloneDeep } from "lodash"
+
+function rolesRemoved(base: User, ctx: UserCtx) {
+  return (
+    !isEqual(base.builder, ctx.request.body.builder) ||
+    !isEqual(base.admin, ctx.request.body.admin) ||
+    !isEqual(base.roles, ctx.request.body.roles)
+  )
+}
+
+const NO_ROLES_MSG =
+  "Roles/admin/builder can only be set on business/enterprise licenses - input ignored."
+
+async function createUpdateResponse(ctx: UserCtx, user?: User) {
+  const base = cloneDeep(ctx.request.body)
+  ctx = await sdk.publicApi.users.roleCheck(ctx, user)
+  // check the ctx before any updates to it
+  const removed = rolesRemoved(base, ctx)
+  ctx = publicApiUserFix(ctx)
+  const response = await saveGlobalUser(ctx)
+  ctx.body = await getUser(ctx, response._id)
+  if (removed) {
+    ctx.extra = { message: NO_ROLES_MSG }
+  }
+  return ctx
+}
 
 function isLoggedInUser(ctx: UserCtx, user: User) {
   const loggedInId = ctx.user?._id
@@ -35,9 +61,7 @@ export async function search(ctx: UserCtx, next: Next) {
 }
 
 export async function create(ctx: UserCtx, next: Next) {
-  ctx = publicApiUserFix(await sdk.publicApi.users.roleCheck(ctx))
-  const response = await saveGlobalUser(ctx)
-  ctx.body = await getUser(ctx, response._id)
+  await createUpdateResponse(ctx)
   await next()
 }
 
@@ -52,9 +76,7 @@ export async function update(ctx: UserCtx, next: Next) {
     ...ctx.request.body,
     _rev: user._rev,
   }
-  ctx = publicApiUserFix(await sdk.publicApi.users.roleCheck(ctx, user))
-  const response = await saveGlobalUser(ctx)
-  ctx.body = await getUser(ctx, response._id)
+  await createUpdateResponse(ctx, user)
   await next()
 }
 

packages/server/src/api/controllers/role.ts

@@ -1,6 +1,7 @@
-import { roles, context, events, db as dbCore } from "@budibase/backend-core"
+import { context, db as dbCore, events, roles } from "@budibase/backend-core"
 import { getUserMetadataParams, InternalTables } from "../../db/utils"
-import { UserCtx, Database, UserRoles, Role } from "@budibase/types"
+import { Database, Role, UserCtx, UserRoles } from "@budibase/types"
+import { sdk as sharedSdk } from "@budibase/shared-core"
 import sdk from "../../sdk"
 
 const UpdateRolesOptions = {
@@ -94,7 +95,6 @@ export async function save(ctx: UserCtx) {
   )
   role._rev = result.rev
   ctx.body = role
-  ctx.message = `Role '${role.name}' created successfully.`
 }
 
 export async function destroy(ctx: UserCtx) {
@@ -131,3 +131,16 @@ export async function destroy(ctx: UserCtx) {
   ctx.message = `Role ${ctx.params.roleId} deleted successfully`
   ctx.status = 200
 }
+
+export async function accessible(ctx: UserCtx) {
+  let roleId = ctx.user?.roleId
+  if (!roleId) {
+    roleId = roles.BUILTIN_ROLE_IDS.PUBLIC
+  }
+  if (ctx.user && sharedSdk.users.isAdminOrBuilder(ctx.user)) {
+    const appId = context.getAppId()
+    ctx.body = await roles.getAllRoleIds(appId)
+  } else {
+    ctx.body = await roles.getUserRoleIdHierarchy(roleId!)
+  }
+}

packages/server/src/api/controllers/routing.ts

@@ -63,9 +63,7 @@ export async function fetch(ctx: UserCtx) {
 export async function clientFetch(ctx: UserCtx) {
   const routing = await getRoutingStructure()
   let roleId = ctx.user?.role?._id
-  const roleIds = (await roles.getUserRoleHierarchy(roleId, {
-    idOnly: true,
-  })) as string[]
+  const roleIds = await roles.getUserRoleIdHierarchy(roleId)
   for (let topLevel of Object.values(routing.routes) as any) {
     for (let subpathKey of Object.keys(topLevel.subpaths)) {
       let found = false

packages/server/src/api/controllers/row/ExternalRequest.ts

@@ -269,13 +269,25 @@ function isEditableColumn(column: FieldSchema) {
   return !(isExternalAutoColumn || isFormula)
 }
 
-export class ExternalRequest {
-  private operation: Operation
-  private tableId: string
+export type ExternalRequestReturnType<T> = T extends Operation.READ
+  ?
+      | Row[]
+      | {
+          row: Row
+          table: Table
+        }
+  : {
+      row: Row
+      table: Table
+    }
+
+export class ExternalRequest<T extends Operation> {
+  private readonly operation: T
+  private readonly tableId: string
   private datasource?: Datasource
   private tables: { [key: string]: Table } = {}
 
-  constructor(operation: Operation, tableId: string, datasource?: Datasource) {
+  constructor(operation: T, tableId: string, datasource?: Datasource) {
     this.operation = operation
     this.tableId = tableId
     this.datasource = datasource
@@ -739,7 +751,7 @@ export class ExternalRequest {
     return fields
   }
 
-  async run(config: RunConfig) {
+  async run(config: RunConfig): Promise<ExternalRequestReturnType<T>> {
     const { operation, tableId } = this
     let { datasourceId, tableName } = breakExternalTableId(tableId)
     if (!tableName) {
@@ -830,8 +842,11 @@ export class ExternalRequest {
     }
     const output = this.outputProcessing(response, table, relationships)
     // if reading it'll just be an array of rows, return whole thing
-    return operation === Operation.READ && Array.isArray(response)
+    const result = (
+      operation === Operation.READ && Array.isArray(response)
         ? output
         : { row: output[0], table }
+    ) as ExternalRequestReturnType<T>
+    return result
   }
 }

packages/server/src/api/controllers/row/external.ts

@@ -1,9 +1,13 @@
-import { FieldTypes, NoEmptyFilterStrings } from "../../../constants"
+import { FieldTypes } from "../../../constants"
 import {
   breakExternalTableId,
   breakRowIdField,
 } from "../../../integrations/utils"
-import { ExternalRequest, RunConfig } from "./ExternalRequest"
+import {
+  ExternalRequest,
+  ExternalRequestReturnType,
+  RunConfig,
+} from "./ExternalRequest"
 import {
   Datasource,
   IncludeRelationship,
@@ -18,14 +22,17 @@ import {
 import sdk from "../../../sdk"
 import * as utils from "./utils"
 import { dataFilters } from "@budibase/shared-core"
-import { inputProcessing } from "../../../utilities/rowProcessor"
+import {
+  inputProcessing,
+  outputProcessing,
+} from "../../../utilities/rowProcessor"
 import { cloneDeep, isEqual } from "lodash"
 
-export async function handleRequest(
-  operation: Operation,
+export async function handleRequest<T extends Operation>(
+  operation: T,
   tableId: string,
   opts?: RunConfig
-) {
+): Promise<ExternalRequestReturnType<T>> {
   // make sure the filters are cleaned up, no empty strings for equals, fuzzy or string
   if (opts && opts.filters) {
     opts.filters = sdk.rows.removeEmptyFilters(opts.filters)
@@ -34,7 +41,7 @@ export async function handleRequest(
     !dataFilters.hasFilters(opts?.filters) &&
     opts?.filters?.onEmptyFilter === EmptyFilterOption.RETURN_NONE
   ) {
-    return []
+    return [] as any
   }
 
   return new ExternalRequest(operation, tableId, opts?.datasource).run(
@@ -46,21 +53,26 @@ export async function patch(ctx: UserCtx<PatchRowRequest, PatchRowResponse>) {
   const tableId = utils.getTableId(ctx)
   const { _id, ...rowData } = ctx.request.body
 
+  const table = await sdk.tables.getTable(tableId)
+  const { row: dataToUpdate } = await inputProcessing(
+    ctx.user?._id,
+    cloneDeep(table),
+    rowData
+  )
+
   const validateResult = await sdk.rows.utils.validate({
-    row: rowData,
+    row: dataToUpdate,
     tableId,
   })
   if (!validateResult.valid) {
     throw { validation: validateResult.errors }
   }
+
   const response = await handleRequest(Operation.UPDATE, tableId, {
     id: breakRowIdField(_id),
-    row: rowData,
+    row: dataToUpdate,
   })
-  const row = await sdk.rows.external.getRow(tableId, _id, {
-    relationships: true,
-  })
-  const table = await sdk.tables.getTable(tableId)
+  const row = await outputProcessing(table, response.row)
   return {
     ...response,
     row,
@@ -71,13 +83,6 @@ export async function patch(ctx: UserCtx<PatchRowRequest, PatchRowResponse>) {
 export async function save(ctx: UserCtx) {
   const inputs = ctx.request.body
   const tableId = utils.getTableId(ctx)
-  const validateResult = await sdk.rows.utils.validate({
-    row: inputs,
-    tableId,
-  })
-  if (!validateResult.valid) {
-    throw { validation: validateResult.errors }
-  }
 
   const table = await sdk.tables.getTable(tableId)
   const { table: updatedTable, row } = await inputProcessing(
@@ -86,6 +91,14 @@ export async function save(ctx: UserCtx) {
     inputs
   )
 
+  const validateResult = await sdk.rows.utils.validate({
+    row,
+    tableId,
+  })
+  if (!validateResult.valid) {
+    throw { validation: validateResult.errors }
+  }
+
   const response = await handleRequest(Operation.CREATE, tableId, {
     row,
   })
@@ -103,7 +116,7 @@ export async function save(ctx: UserCtx) {
     })
     return {
       ...response,
-      row,
+      row: await outputProcessing(table, row),
     }
   } else {
     return response
@@ -121,7 +134,12 @@ export async function find(ctx: UserCtx): Promise<Row> {
     ctx.throw(404)
   }
 
-  return row
+  const table = await sdk.tables.getTable(tableId)
+  // Preserving links, as the outputProcessing does not support external rows yet and we don't need it in this use case
+  return await outputProcessing(table, row, {
+    squash: false,
+    preserveLinks: true,
+  })
 }
 
 export async function destroy(ctx: UserCtx) {

packages/server/src/api/controllers/static/index.ts

@@ -107,6 +107,11 @@ export const serveApp = async function (ctx: any) {
   //Public Settings
   const { config } = await configs.getSettingsConfigDoc()
   const branding = await pro.branding.getBrandingConfig(config)
+  // incase running direct from TS
+  let appHbsPath = join(__dirname, "app.hbs")
+  if (!fs.existsSync(appHbsPath)) {
+    appHbsPath = join(__dirname, "templates", "app.hbs")
+  }
 
   let db
   try {
@@ -138,7 +143,7 @@ export const serveApp = async function (ctx: any) {
             ? objectStore.getGlobalFileUrl("settings", "logoUrl")
             : "",
       })
-      const appHbs = loadHandlebarsFile(`${__dirname}/app.hbs`)
+      const appHbs = loadHandlebarsFile(appHbsPath)
       ctx.body = await processString(appHbs, {
         head,
         body: html,
@@ -166,7 +171,7 @@ export const serveApp = async function (ctx: any) {
             : "",
       })
 
-      const appHbs = loadHandlebarsFile(`${__dirname}/app.hbs`)
+      const appHbs = loadHandlebarsFile(appHbsPath)
       ctx.body = await processString(appHbs, {
         head,
         body: html,
@@ -193,8 +198,13 @@ export const serveBuilderPreview = async function (ctx: any) {
 }
 
 export const serveClientLibrary = async function (ctx: any) {
+  let rootPath = join(NODE_MODULES_PATH, "@budibase", "client", "dist")
+  // incase running from TS directly
+  if (env.isDev() && !fs.existsSync(rootPath)) {
+    rootPath = join(require.resolve("@budibase/client"), "..")
+  }
   return send(ctx, "budibase-client.js", {
-    root: join(NODE_MODULES_PATH, "@budibase", "client", "dist"),
+    root: rootPath,
   })
 }
 

packages/server/src/api/routes/public/tests/users.spec.ts

@@ -68,6 +68,7 @@ describe("no user role update in free", () => {
     })
     expect(res.status).toBe(200)
     expect(res.body.data.roles["app_a"]).toBeUndefined()
+    expect(res.body.message).toBeDefined()
   })
 
   it("should not allow 'admin' to be updated", async () => {
@@ -77,6 +78,7 @@ describe("no user role update in free", () => {
     })
     expect(res.status).toBe(200)
     expect(res.body.data.admin).toBeUndefined()
+    expect(res.body.message).toBeDefined()
   })
 
   it("should not allow 'builder' to be updated", async () => {
@@ -86,6 +88,7 @@ describe("no user role update in free", () => {
     })
     expect(res.status).toBe(200)
     expect(res.body.data.builder).toBeUndefined()
+    expect(res.body.message).toBeDefined()
   })
 })
 
@@ -102,6 +105,7 @@ describe("no user role update in business", () => {
     })
     expect(res.status).toBe(200)
     expect(res.body.data.roles["app_a"]).toBe("BASIC")
+    expect(res.body.message).toBeUndefined()
   })
 
   it("should allow 'admin' to be updated", async () => {
@@ -112,6 +116,7 @@ describe("no user role update in business", () => {
     })
     expect(res.status).toBe(200)
     expect(res.body.data.admin.global).toBe(true)
+    expect(res.body.message).toBeUndefined()
   })
 
   it("should allow 'builder' to be updated", async () => {
@@ -122,5 +127,6 @@ describe("no user role update in business", () => {
     })
     expect(res.status).toBe(200)
     expect(res.body.data.builder.global).toBe(true)
+    expect(res.body.message).toBeUndefined()
   })
 })

packages/server/src/api/routes/role.ts

@@ -7,6 +7,9 @@ import { roleValidator } from "./utils/validators"
 const router: Router = new Router()
 
 router
+  // retrieve a list of the roles a user can access
+  // needs to be public for public screens
+  .get("/api/roles/accessible", controller.accessible)
   .post(
     "/api/roles",
     authorized(permissions.BUILDER),

packages/server/src/api/routes/tests/role.spec.js

@@ -15,7 +15,7 @@ describe("/roles", () => {
     await config.init()
   })
 
-  const createRole = async (role) => {
+  const createRole = async role => {
     if (!role) {
       role = basicRole()
     }
@@ -33,9 +33,6 @@ describe("/roles", () => {
       const role = basicRole()
       const res = await createRole(role)
 
-      expect(res.res.statusMessage).toEqual(
-        `Role '${role.name}' created successfully.`
-      )
       expect(res.body._id).toBeDefined()
       expect(res.body._rev).toBeDefined()
       expect(events.role.updated).not.toBeCalled()
@@ -51,9 +48,6 @@ describe("/roles", () => {
       jest.clearAllMocks()
       res = await createRole(res.body)
 
-      expect(res.res.statusMessage).toEqual(
-        `Role '${role.name}' created successfully.`
-      )
       expect(res.body._id).toBeDefined()
       expect(res.body._rev).toBeDefined()
       expect(events.role.created).not.toBeCalled()
@@ -99,7 +93,11 @@ describe("/roles", () => {
 
     it("should be able to get the role with a permission added", async () => {
       const table = await config.createTable()
-      await config.api.permission.set({ roleId: BUILTIN_ROLE_IDS.POWER, resourceId: table._id, level: PermissionLevel.READ })
+      await config.api.permission.set({
+        roleId: BUILTIN_ROLE_IDS.POWER,
+        resourceId: table._id,
+        level: PermissionLevel.READ,
+      })
       const res = await request
         .get(`/api/roles`)
         .set(config.defaultHeaders())
@@ -131,4 +129,34 @@ describe("/roles", () => {
       expect(events.role.deleted).toBeCalledWith(customRole)
     })
   })
+
+  describe("accessible", () => {
+    it("should be able to fetch accessible roles (with builder)", async () => {
+      const res = await request
+        .get("/api/roles/accessible")
+        .set(config.defaultHeaders())
+        .expect(200)
+      expect(res.body.length).toBe(5)
+      expect(typeof res.body[0]).toBe("string")
+    })
+
+    it("should be able to fetch accessible roles (basic user)", async () => {
+      const res = await request
+        .get("/api/roles/accessible")
+        .set(await config.basicRoleHeaders())
+        .expect(200)
+      expect(res.body.length).toBe(2)
+      expect(res.body[0]).toBe("BASIC")
+      expect(res.body[1]).toBe("PUBLIC")
+    })
+
+    it("should be able to fetch accessible roles (no user)", async () => {
+      const res = await request
+        .get("/api/roles/accessible")
+        .set(config.publicHeaders())
+        .expect(200)
+      expect(res.body.length).toBe(1)
+      expect(res.body[0]).toBe("PUBLIC")
+    })
+  })
 })

packages/server/src/api/routes/tests/row.spec.ts

@@ -7,6 +7,7 @@ import { context, InternalTable, roles, tenancy } from "@budibase/backend-core"
 import { quotas } from "@budibase/pro"
 import {
   FieldType,
+  FieldTypeSubtypes,
   MonthlyQuotaName,
   PermissionLevel,
   QuotaUsageType,
@@ -17,6 +18,7 @@ import {
   SortType,
   StaticQuotaName,
   Table,
+  User,
 } from "@budibase/types"
 import {
   expectAnyExternalColsAttributes,
@@ -25,6 +27,7 @@ import {
   mocks,
   structures,
 } from "@budibase/backend-core/tests"
+import _ from "lodash"
 
 const timestamp = new Date("2023-01-26T11:48:57.597Z").toISOString()
 tk.freeze(timestamp)
@@ -34,7 +37,7 @@ const { basicRow } = setup.structures
 describe.each([
   ["internal", undefined],
   ["postgres", databaseTestProviders.postgres],
-])("/rows (%s)", (_, dsProvider) => {
+])("/rows (%s)", (__, dsProvider) => {
   const isInternal = !dsProvider
 
   const request = setup.getRequest()
@@ -1511,4 +1514,393 @@ describe.each([
       })
     })
   })
+
+  describe("bb reference fields", () => {
+    let tableId: string
+    let users: User[]
+
+    beforeAll(async () => {
+      const tableConfig = generateTableConfig()
+
+      if (config.datasource) {
+        tableConfig.sourceId = config.datasource._id
+        if (config.datasource.plus) {
+          tableConfig.type = "external"
+        }
+      }
+      const table = await config.api.table.create({
+        ...tableConfig,
+        schema: {
+          ...tableConfig.schema,
+          user: {
+            name: "user",
+            type: FieldType.BB_REFERENCE,
+            subtype: FieldTypeSubtypes.BB_REFERENCE.USER,
+            relationshipType: RelationshipType.ONE_TO_MANY,
+          },
+          users: {
+            name: "users",
+            type: FieldType.BB_REFERENCE,
+            subtype: FieldTypeSubtypes.BB_REFERENCE.USER,
+            relationshipType: RelationshipType.MANY_TO_MANY,
+          },
+        },
+      })
+      tableId = table._id!
+
+      users = [
+        await config.createUser(),
+        await config.createUser(),
+        await config.createUser(),
+        await config.createUser(),
+      ]
+    })
+
+    it("can save a row when BB reference fields are empty", async () => {
+      const rowData = {
+        ...basicRow(tableId),
+        name: generator.name(),
+        description: generator.name(),
+      }
+      const row = await config.api.row.save(tableId, rowData)
+
+      expect(row).toEqual({
+        name: rowData.name,
+        description: rowData.description,
+        tableId,
+        _id: expect.any(String),
+        _rev: expect.any(String),
+        id: isInternal ? undefined : expect.any(Number),
+        type: isInternal ? "row" : undefined,
+      })
+    })
+
+    it("can save a row with a single BB reference field", async () => {
+      const user = _.sample(users)!
+      const rowData = {
+        ...basicRow(tableId),
+        name: generator.name(),
+        description: generator.name(),
+        user: user,
+      }
+      const row = await config.api.row.save(tableId, rowData)
+
+      expect(row).toEqual({
+        name: rowData.name,
+        description: rowData.description,
+        tableId,
+        user: [
+          {
+            _id: user._id,
+            email: user.email,
+            firstName: user.firstName,
+            lastName: user.lastName,
+            primaryDisplay: user.email,
+          },
+        ],
+        _id: expect.any(String),
+        _rev: expect.any(String),
+        id: isInternal ? undefined : expect.any(Number),
+        type: isInternal ? "row" : undefined,
+      })
+    })
+
+    it("can save a row with a multiple BB reference field", async () => {
+      const selectedUsers = _.sampleSize(users, 2)
+      const rowData = {
+        ...basicRow(tableId),
+        name: generator.name(),
+        description: generator.name(),
+        users: selectedUsers,
+      }
+      const row = await config.api.row.save(tableId, rowData)
+
+      expect(row).toEqual({
+        name: rowData.name,
+        description: rowData.description,
+        tableId,
+        users: selectedUsers.map(u => ({
+          _id: u._id,
+          email: u.email,
+          firstName: u.firstName,
+          lastName: u.lastName,
+          primaryDisplay: u.email,
+        })),
+        _id: expect.any(String),
+        _rev: expect.any(String),
+        id: isInternal ? undefined : expect.any(Number),
+        type: isInternal ? "row" : undefined,
+      })
+    })
+
+    it("can retrieve rows with no populated BB references", async () => {
+      const rowData = {
+        ...basicRow(tableId),
+        name: generator.name(),
+        description: generator.name(),
+      }
+      const row = await config.api.row.save(tableId, rowData)
+
+      const { body: retrieved } = await config.api.row.get(tableId, row._id!)
+      expect(retrieved).toEqual({
+        name: rowData.name,
+        description: rowData.description,
+        tableId,
+        user: undefined,
+        users: undefined,
+        _id: row._id,
+        _rev: expect.any(String),
+        id: isInternal ? undefined : expect.any(Number),
+        ...defaultRowFields,
+      })
+    })
+
+    it("can retrieve rows with populated BB references", async () => {
+      const [user1, user2] = _.sampleSize(users, 2)
+
+      const rowData = {
+        ...basicRow(tableId),
+        name: generator.name(),
+        description: generator.name(),
+        users: [user1, user2],
+        user: [user1],
+      }
+      const row = await config.api.row.save(tableId, rowData)
+
+      const { body: retrieved } = await config.api.row.get(tableId, row._id!)
+      expect(retrieved).toEqual({
+        name: rowData.name,
+        description: rowData.description,
+        tableId,
+        user: [user1].map(u => ({
+          _id: u._id,
+          email: u.email,
+          firstName: u.firstName,
+          lastName: u.lastName,
+          primaryDisplay: u.email,
+        })),
+        users: [user1, user2].map(u => ({
+          _id: u._id,
+          email: u.email,
+          firstName: u.firstName,
+          lastName: u.lastName,
+          primaryDisplay: u.email,
+        })),
+        _id: row._id,
+        _rev: expect.any(String),
+        id: isInternal ? undefined : expect.any(Number),
+        ...defaultRowFields,
+      })
+    })
+
+    it("can update an existing populated row", async () => {
+      const [user1, user2, user3] = _.sampleSize(users, 3)
+
+      const rowData = {
+        ...basicRow(tableId),
+        name: generator.name(),
+        description: generator.name(),
+        users: [user1, user2],
+      }
+      const row = await config.api.row.save(tableId, rowData)
+
+      const updatedRow = await config.api.row.save(tableId, {
+        ...row,
+        user: [user3],
+        users: [user3, user2],
+      })
+      expect(updatedRow).toEqual({
+        name: rowData.name,
+        description: rowData.description,
+        tableId,
+        user: [
+          {
+            _id: user3._id,
+            email: user3.email,
+            firstName: user3.firstName,
+            lastName: user3.lastName,
+            primaryDisplay: user3.email,
+          },
+        ],
+        users: [user3, user2].map(u => ({
+          _id: u._id,
+          email: u.email,
+          firstName: u.firstName,
+          lastName: u.lastName,
+          primaryDisplay: u.email,
+        })),
+        _id: row._id,
+        _rev: expect.any(String),
+        id: isInternal ? undefined : expect.any(Number),
+        type: isInternal ? "row" : undefined,
+      })
+    })
+
+    it("can wipe an existing populated BB references in row", async () => {
+      const [user1, user2] = _.sampleSize(users, 2)
+
+      const rowData = {
+        ...basicRow(tableId),
+        name: generator.name(),
+        description: generator.name(),
+        users: [user1, user2],
+      }
+      const row = await config.api.row.save(tableId, rowData)
+
+      const updatedRow = await config.api.row.save(tableId, {
+        ...row,
+        user: null,
+        users: null,
+      })
+      expect(updatedRow).toEqual({
+        name: rowData.name,
+        description: rowData.description,
+        tableId,
+        user: isInternal ? null : undefined,
+        users: isInternal ? null : undefined,
+        _id: row._id,
+        _rev: expect.any(String),
+        id: isInternal ? undefined : expect.any(Number),
+        type: isInternal ? "row" : undefined,
+      })
+    })
+
+    it("fetch all will populate the BB references", async () => {
+      const [user1, user2, user3] = _.sampleSize(users, 3)
+
+      const rows: {
+        name: string
+        description: string
+        user?: User[]
+        users?: User[]
+        tableId: string
+      }[] = [
+        {
+          ...basicRow(tableId),
+          name: generator.name(),
+          description: generator.name(),
+          users: [user1, user2],
+        },
+        {
+          ...basicRow(tableId),
+          name: generator.name(),
+          description: generator.name(),
+          user: [user1],
+          users: [user1, user3],
+        },
+        {
+          ...basicRow(tableId),
+          name: generator.name(),
+          description: generator.name(),
+          users: [user3],
+        },
+      ]
+
+      await config.api.row.save(tableId, rows[0])
+      await config.api.row.save(tableId, rows[1])
+      await config.api.row.save(tableId, rows[2])
+
+      const res = await config.api.row.fetch(tableId)
+
+      expect(res).toEqual(
+        expect.arrayContaining(
+          rows.map(r => ({
+            name: r.name,
+            description: r.description,
+            tableId,
+            user: r.user?.map(u => ({
+              _id: u._id,
+              email: u.email,
+              firstName: u.firstName,
+              lastName: u.lastName,
+              primaryDisplay: u.email,
+            })),
+            users: r.users?.map(u => ({
+              _id: u._id,
+              email: u.email,
+              firstName: u.firstName,
+              lastName: u.lastName,
+              primaryDisplay: u.email,
+            })),
+            _id: expect.any(String),
+            _rev: expect.any(String),
+            id: isInternal ? undefined : expect.any(Number),
+            ...defaultRowFields,
+          }))
+        )
+      )
+    })
+
+    it("search all will populate the BB references", async () => {
+      const [user1, user2, user3] = _.sampleSize(users, 3)
+
+      const rows: {
+        name: string
+        description: string
+        user?: User[]
+        users?: User[]
+        tableId: string
+      }[] = [
+        {
+          ...basicRow(tableId),
+          name: generator.name(),
+          description: generator.name(),
+          users: [user1, user2],
+        },
+        {
+          ...basicRow(tableId),
+          name: generator.name(),
+          description: generator.name(),
+          user: [user1],
+          users: [user1, user3],
+        },
+        {
+          ...basicRow(tableId),
+          name: generator.name(),
+          description: generator.name(),
+          users: [user3],
+        },
+      ]
+
+      await config.api.row.save(tableId, rows[0])
+      await config.api.row.save(tableId, rows[1])
+      await config.api.row.save(tableId, rows[2])
+
+      const res = await config.api.row.search(tableId)
+
+      expect(res).toEqual({
+        rows: expect.arrayContaining(
+          rows.map(r => ({
+            name: r.name,
+            description: r.description,
+            tableId,
+            user: r.user?.map(u => ({
+              _id: u._id,
+              email: u.email,
+              firstName: u.firstName,
+              lastName: u.lastName,
+              primaryDisplay: u.email,
+            })),
+            users: r.users?.map(u => ({
+              _id: u._id,
+              email: u.email,
+              firstName: u.firstName,
+              lastName: u.lastName,
+              primaryDisplay: u.email,
+            })),
+            _id: expect.any(String),
+            _rev: expect.any(String),
+            id: isInternal ? undefined : expect.any(Number),
+            ...defaultRowFields,
+          }))
+        ),
+        ...(isInternal
+          ? {}
+          : {
+              hasNextPage: false,
+              bookmark: null,
+            }),
+      })
+    })
+  })
 })

packages/server/src/definitions/openapi.ts

@@ -613,7 +613,7 @@ export interface components {
         global?: boolean;
       };
       /** @description Contains the roles of the user per app (assuming they are not a builder user). This field can only be set on a business or enterprise license. */
-      roles: { [key: string]: string };
+      roles?: { [key: string]: string };
     };
     userOutput: {
       data: {
@@ -643,7 +643,7 @@ export interface components {
           global?: boolean;
         };
         /** @description Contains the roles of the user per app (assuming they are not a builder user). This field can only be set on a business or enterprise license. */
-        roles: { [key: string]: string };
+        roles?: { [key: string]: string };
         /** @description The ID of the user. */
         _id: string;
       };
@@ -676,7 +676,7 @@ export interface components {
           global?: boolean;
         };
         /** @description Contains the roles of the user per app (assuming they are not a builder user). This field can only be set on a business or enterprise license. */
-        roles: { [key: string]: string };
+        roles?: { [key: string]: string };
         /** @description The ID of the user. */
         _id: string;
       }[];

packages/server/src/integrations/googlesheets.ts

@@ -1,6 +1,5 @@
 import {
   ConnectionInfo,
-  Datasource,
   DatasourceFeature,
   DatasourceFieldType,
   DatasourcePlus,
@@ -23,7 +22,6 @@ import fetch from "node-fetch"
 import { cache, configs, context, HTTPError } from "@budibase/backend-core"
 import { dataFilters, utils } from "@budibase/shared-core"
 import { GOOGLE_SHEETS_PRIMARY_KEY } from "../constants"
-import sdk from "../sdk"
 
 interface GoogleSheetsConfig {
   spreadsheetId: string
@@ -56,6 +54,7 @@ const ALLOWED_TYPES = [
   FieldType.OPTIONS,
   FieldType.BOOLEAN,
   FieldType.BARCODEQR,
+  FieldType.BB_REFERENCE,
 ]
 
 const SCHEMA: Integration = {
@@ -213,7 +212,7 @@ class GoogleSheetsIntegration implements DatasourcePlus {
       await setupCreationAuth(this.config)
 
       // Initialise oAuth client
-      let googleConfig = await configs.getGoogleDatasourceConfig()
+      const googleConfig = await configs.getGoogleDatasourceConfig()
       if (!googleConfig) {
         throw new HTTPError("Google config not found", 400)
       }
@@ -323,14 +322,14 @@ class GoogleSheetsIntegration implements DatasourcePlus {
       case Operation.UPDATE:
         return this.update({
           // exclude the header row and zero index
-          rowIndex: json.extra?.idFilter?.equal?.rowNumber - 2,
+          rowIndex: json.extra?.idFilter?.equal?.rowNumber,
           sheet,
           row: json.body,
         })
       case Operation.DELETE:
         return this.delete({
           // exclude the header row and zero index
-          rowIndex: json.extra?.idFilter?.equal?.rowNumber - 2,
+          rowIndex: json.extra?.idFilter?.equal?.rowNumber,
           sheet,
         })
       case Operation.CREATE_TABLE:
@@ -541,17 +540,30 @@ class GoogleSheetsIntegration implements DatasourcePlus {
     }
   }
 
+  private async getRowByIndex(sheetTitle: string, rowIndex: number) {
+    const sheet = this.client.sheetsByTitle[sheetTitle]
+    const rows = await sheet.getRows()
+    // We substract 2, as the SDK is skipping the header automatically and Google Spreadsheets is base 1
+    const row = rows[rowIndex - 2]
+    return { sheet, row }
+  }
+
   async update(query: { sheet: string; rowIndex: number; row: any }) {
     try {
       await this.connect()
-      const sheet = this.client.sheetsByTitle[query.sheet]
-      const rows = await sheet.getRows()
-      const row = rows[query.rowIndex]
+      const { sheet, row } = await this.getRowByIndex(
+        query.sheet,
+        query.rowIndex
+      )
       if (row) {
         const updateValues =
           typeof query.row === "string" ? JSON.parse(query.row) : query.row
         for (let key in updateValues) {
           row[key] = updateValues[key]
+
+          if (row[key] === null) {
+            row[key] = ""
+          }
         }
         await row.save()
         return [
@@ -568,9 +580,7 @@ class GoogleSheetsIntegration implements DatasourcePlus {
 
   async delete(query: { sheet: string; rowIndex: number }) {
     await this.connect()
-    const sheet = this.client.sheetsByTitle[query.sheet]
-    const rows = await sheet.getRows()
-    const row = rows[query.rowIndex]
+    const { row } = await this.getRowByIndex(query.sheet, query.rowIndex)
     if (row) {
       await row.delete()
       return [

packages/server/src/middleware/authorized.ts

@@ -55,9 +55,7 @@ const checkAuthorizedResource = async (
 ) => {
   // get the user's roles
   const roleId = ctx.roleId || roles.BUILTIN_ROLE_IDS.PUBLIC
-  const userRoles = (await roles.getUserRoleHierarchy(roleId, {
-    idOnly: false,
-  })) as Role[]
+  const userRoles = await roles.getUserRoleHierarchy(roleId)
   const permError = "User does not have permission"
   // check if the user has the required role
   if (resourceRoles.length > 0) {

packages/server/src/sdk/app/rows/search/external.ts

@@ -17,6 +17,7 @@ import { utils } from "@budibase/shared-core"
 import { ExportRowsParams, ExportRowsResult } from "../search"
 import { HTTPError, db } from "@budibase/backend-core"
 import pick from "lodash/pick"
+import { outputProcessing } from "../../../../utilities/rowProcessor"
 
 export async function search(options: SearchParams) {
   const { tableId } = options
@@ -75,6 +76,9 @@ export async function search(options: SearchParams) {
       rows = rows.map((r: any) => pick(r, fields))
     }
 
+    const table = await sdk.tables.getTable(tableId)
+    rows = await outputProcessing(table, rows)
+
     // need wrapper object for bookmarks etc when paginating
     return { rows, hasNextPage, bookmark: bookmark && bookmark + 1 }
   } catch (err: any) {
@@ -166,9 +170,11 @@ export async function exportRows(
 }
 
 export async function fetch(tableId: string) {
-  return handleRequest(Operation.READ, tableId, {
+  const response = await handleRequest(Operation.READ, tableId, {
     includeSqlRelationships: IncludeRelationship.INCLUDE,
   })
+  const table = await sdk.tables.getTable(tableId)
+  return await outputProcessing(table, response)
 }
 
 export async function fetchView(viewName: string) {

packages/server/src/sdk/tests/rows/row.spec.ts

@@ -7,9 +7,14 @@ import { HTTPError } from "@budibase/backend-core"
 import { Operation } from "@budibase/types"
 
 const mockDatasourcesGet = jest.fn()
+const mockTableGet = jest.fn()
 sdk.datasources.get = mockDatasourcesGet
+sdk.tables.getTable = mockTableGet
 
 jest.mock("../../../api/controllers/row/ExternalRequest")
+jest.mock("../../../utilities/rowProcessor", () => ({
+  outputProcessing: jest.fn((_, rows) => rows),
+}))
 
 jest.mock("../../../api/controllers/view/exporters", () => ({
   ...jest.requireActual("../../../api/controllers/view/exporters"),

packages/server/src/tests/utilities/TestConfiguration.ts

@@ -425,6 +425,15 @@ class TestConfiguration {
     return headers
   }
 
+  async basicRoleHeaders() {
+    return await this.roleHeaders({
+      email: this.defaultUserValues.email,
+      builder: false,
+      prodApp: true,
+      roleId: roles.BUILTIN_ROLE_IDS.BASIC,
+    })
+  }
+
   async roleHeaders({
     email = this.defaultUserValues.email,
     roleId = roles.BUILTIN_ROLE_IDS.ADMIN,

packages/server/src/tests/utilities/api/row.ts

@@ -44,12 +44,12 @@ export class RowAPI extends TestAPI {
   }
 
   save = async (
-    sourceId: string,
+    tableId: string,
     row: SaveRowRequest,
     { expectStatus } = { expectStatus: 200 }
   ): Promise<Row> => {
     const resp = await this.request
-      .post(`/api/${sourceId}/rows`)
+      .post(`/api/${tableId}/rows`)
       .send(row)
       .set(this.config.defaultHeaders())
       .expect("Content-Type", /json/)
@@ -122,4 +122,16 @@ export class RowAPI extends TestAPI {
       .expect(expectStatus)
     return request
   }
+
+  search = async (
+    sourceId: string,
+    { expectStatus } = { expectStatus: 200 }
+  ): Promise<Row[]> => {
+    const request = this.request
+      .post(`/api/${sourceId}/search`)
+      .set(this.config.defaultHeaders())
+      .expect(expectStatus)
+
+    return (await request).body
+  }
 }

packages/server/src/utilities/centralPath.ts

@@ -8,7 +8,7 @@ import path from "path"
  * @param args Any number of string arguments to add to a path
  * @returns {string} The final path ready to use
  */
-export function join(...args: any) {
+export function join(...args: string[]) {
   return path.join(...args)
 }
 
@@ -17,6 +17,6 @@ export function join(...args: any) {
  * @param args Any number of string arguments to add to a path
  * @returns {string} The final path ready to use
  */
-export function resolve(...args: any) {
+export function resolve(...args: string[]) {
   return path.resolve(...args)
 }

packages/server/src/utilities/rowProcessor/bbReferenceProcessor.ts

@@ -6,7 +6,7 @@ import { InvalidBBRefError } from "./errors"
 export async function processInputBBReferences(
   value: string | string[] | { _id: string } | { _id: string }[],
   subtype: FieldSubtype
-): Promise<string | undefined> {
+): Promise<string | null> {
   const referenceIds: string[] = []
 
   if (Array.isArray(value)) {
@@ -39,7 +39,7 @@ export async function processInputBBReferences(
       throw utils.unreachable(subtype)
   }
 
-  return referenceIds.join(",") || undefined
+  return referenceIds.join(",") || null
 }
 
 export async function processOutputBBReferences(

packages/server/src/utilities/rowProcessor/index.ts

@@ -200,7 +200,10 @@ export async function inputProcessing(
 export async function outputProcessing<T extends Row[] | Row>(
   table: Table,
   rows: T,
-  opts = { squash: true }
+  opts: { squash?: boolean; preserveLinks?: boolean } = {
+    squash: true,
+    preserveLinks: false,
+  }
 ): Promise<T> {
   let safeRows: Row[]
   let wasArray = true
@@ -211,7 +214,9 @@ export async function outputProcessing<T extends Row[] | Row>(
     safeRows = rows
   }
   // attach any linked row information
-  let enriched = await linkRows.attachFullLinkedDocs(table, safeRows)
+  let enriched = !opts.preserveLinks
+    ? await linkRows.attachFullLinkedDocs(table, safeRows)
+    : safeRows
 
   // process formulas
   enriched = processFormulas(table, enriched, { dynamic: true }) as Row[]

packages/server/src/utilities/rowProcessor/tests/bbReferenceProcessor.spec.ts

@@ -139,20 +139,20 @@ describe("bbReferenceProcessor", () => {
         expect(cacheGetUsersSpy).toBeCalledWith(userIds)
       })
 
-      it("empty strings will return undefined", async () => {
+      it("empty strings will return null", async () => {
         const result = await config.doInTenant(() =>
           processInputBBReferences("", FieldSubtype.USER)
         )
 
-        expect(result).toEqual(undefined)
+        expect(result).toEqual(null)
       })
 
-      it("empty arrays will return undefined", async () => {
+      it("empty arrays will return null", async () => {
         const result = await config.doInTenant(() =>
           processInputBBReferences([], FieldSubtype.USER)
         )
 
-        expect(result).toEqual(undefined)
+        expect(result).toEqual(null)
       })
     })
   })

packages/server/src/utilities/users.ts

@@ -36,5 +36,8 @@ export function publicApiUserFix(ctx: UserCtx) {
   if (!ctx.request.body._id && ctx.params.userId) {
     ctx.request.body._id = ctx.params.userId
   }
+  if (!ctx.request.body.roles) {
+    ctx.request.body.roles = {}
+  }
   return ctx
 }

packages/types/src/api/account/accounts.ts

@@ -1,4 +1,4 @@
-import { Account } from "../../documents"
+import { Account, AccountSSOProvider } from "../../documents"
 import { Hosting } from "../../sdk"
 
 export interface CreateAccountRequest {
@@ -11,6 +11,8 @@ export interface CreateAccountRequest {
   tenantName?: string
   name?: string
   password: string
+  provider?: AccountSSOProvider
+  thirdPartyProfile: object
 }
 
 export interface SearchAccountsRequest {

packages/types/src/api/web/user.ts

@@ -61,6 +61,7 @@ export interface CreateAdminUserRequest {
   email: string
   password: string
   tenantId: string
+  ssoId?: string
 }
 
 export interface CreateAdminUserResponse {

packages/types/src/documents/account/account.ts

@@ -20,6 +20,11 @@ export interface CreatePassswordAccount extends CreateAccount {
   password: string
 }
 
+export interface CreateVerifiableSSOAccount extends CreateAccount {
+  provider?: AccountSSOProvider
+  thirdPartyProfile?: any
+}
+
 export const isCreatePasswordAccount = (
   account: CreateAccount
 ): account is CreatePassswordAccount => account.authType === AuthType.PASSWORD
@@ -50,6 +55,8 @@ export interface Account extends CreateAccount {
   licenseKeyActivatedAt?: number
   licenseRequestedAt?: number
   licenseOverrides?: LicenseOverrides
+  provider?: AccountSSOProvider
+  providerType?: AccountSSOProviderType
   quotaUsage?: QuotaUsage
   offlineLicenseToken?: string
 }
@@ -87,6 +94,13 @@ export enum AccountSSOProvider {
   MICROSOFT = "microsoft",
 }
 
+const verifiableSSOProviders: AccountSSOProvider[] = [
+  AccountSSOProvider.MICROSOFT,
+]
+export function isVerifiableSSOProvider(provider: AccountSSOProvider): boolean {
+  return verifiableSSOProviders.includes(provider)
+}
+
 export interface AccountSSO {
   provider: AccountSSOProvider
   providerType: AccountSSOProviderType

packages/types/src/documents/global/user.ts

@@ -55,6 +55,7 @@ export interface User extends Document {
   userGroups?: string[]
   onboardedAt?: string
   scimInfo?: { isSync: true } & Record<string, any>
+  ssoId?: string
 }
 
 export enum UserStatus {

packages/types/src/documents/platform/users.ts

@@ -15,4 +15,16 @@ export interface PlatformUserById extends Document {
   tenantId: string
 }
 
-export type PlatformUser = PlatformUserByEmail | PlatformUserById
+/**
+ * doc id is a unique SSO provider ID for the user
+ */
+export interface PlatformUserBySsoId extends Document {
+  tenantId: string
+  userId: string
+  email: string
+}
+
+export type PlatformUser =
+  | PlatformUserByEmail
+  | PlatformUserById
+  | PlatformUserBySsoId

packages/worker/src/api/controllers/global/users.ts

@@ -95,7 +95,7 @@ const parseBooleanParam = (param: any) => {
 export const adminUser = async (
   ctx: Ctx<CreateAdminUserRequest, CreateAdminUserResponse>
 ) => {
-  const { email, password, tenantId } = ctx.request.body
+  const { email, password, tenantId, ssoId } = ctx.request.body
 
   if (await platform.tenants.exists(tenantId)) {
     ctx.throw(403, "Organisation already exists.")
@@ -136,6 +136,7 @@ export const adminUser = async (
         global: true,
       },
       tenantId,
+      ssoId,
     }
     try {
       // always bust checklist beforehand, if an error occurs but can proceed, don't get

packages/worker/src/api/routes/global/users.ts

@@ -14,6 +14,7 @@ function buildAdminInitValidation() {
       email: Joi.string().required(),
       password: Joi.string(),
       tenantId: Joi.string().required(),
+      ssoId: Joi.string(),
     })
       .required()
       .unknown(false)

specs/openapi.json

@@ -1 +0,0 @@
-../packages/server/specs/openapi.json

specs/openapi.yaml

@@ -1 +0,0 @@
-../packages/server/specs/openapi.yaml