Merge branch 'master' into cleanup-isolates

lerna.json

@@ -1,5 +1,5 @@
 {
-  "version": "2.21.4",
+  "version": "2.21.8",
   "npmClient": "yarn",
   "packages": [
     "packages/*",

packages/backend-core/src/cache/docWritethrough.ts

@@ -10,6 +10,7 @@ interface ProcessDocMessage {
 }
 
 const PERSIST_MAX_ATTEMPTS = 100
+let processor: DocWritethroughProcessor | undefined
 
 export const docWritethroughProcessorQueue = createQueue<ProcessDocMessage>(
   JobQueue.DOC_WRITETHROUGH_QUEUE,
@@ -61,8 +62,6 @@ class DocWritethroughProcessor {
   }
 }
 
-export const processor = new DocWritethroughProcessor().init()
-
 export class DocWritethrough {
   private db: Database
   private _docId: string
@@ -84,3 +83,15 @@ export class DocWritethrough {
     })
   }
 }
+
+export function init(): DocWritethroughProcessor {
+  processor = new DocWritethroughProcessor().init()
+  return processor
+}
+
+export function getProcessor(): DocWritethroughProcessor {
+  if (!processor) {
+    return init()
+  }
+  return processor
+}

packages/backend-core/src/cache/tests/docWritethrough.spec.ts

@@ -7,6 +7,7 @@ import { getDB } from "../../db"
 import {
   DocWritethrough,
   docWritethroughProcessorQueue,
+  init,
 } from "../docWritethrough"
 
 import InMemoryQueue from "../../queue/inMemoryQueue"
@@ -19,6 +20,10 @@ async function waitForQueueCompletion() {
 }
 
 describe("docWritethrough", () => {
+  beforeAll(() => {
+    init()
+  })
+
   const config = new DBTestConfiguration()
 
   const db = getDB(structures.db.id())

packages/bbui/src/Icon/IconAvatar.svelte

@@ -1,22 +1,41 @@
 <script>
   import Icon from "./Icon.svelte"
 
+  import Tooltip from "../Tooltip/Tooltip.svelte"
+  import { fade } from "svelte/transition"
+
   export let icon
   export let background
   export let color
   export let size = "M"
+  export let tooltip
+
+  let showTooltip = false
 </script>
 
+<!-- svelte-ignore a11y-no-static-element-interactions -->
+<!-- svelte-ignore a11y-click-events-have-key-events -->
 <div
   class="icon size--{size}"
   style="background: {background || `transparent`};"
   class:filled={!!background}
+  on:mouseover={() => (showTooltip = true)}
+  on:mouseleave={() => (showTooltip = false)}
+  on:focus={() => (showTooltip = true)}
+  on:blur={() => (showTooltip = false)}
+  on:click={() => (showTooltip = false)}
 >
   <Icon name={icon} color={background ? "white" : color} />
+  {#if tooltip && showTooltip}
+    <div class="tooltip" in:fade={{ duration: 130, delay: 250 }}>
+      <Tooltip textWrapping direction="right" text={tooltip} />
+    </div>
+  {/if}
 </div>
 
 <style>
   .icon {
+    position: relative;
     width: 28px;
     height: 28px;
     flex: 0 0 28px;
@@ -32,6 +51,15 @@
     width: 16px;
     height: 16px;
   }
+  .icon.size--XS {
+    width: 18px;
+    height: 18px;
+    flex: 0 0 18px;
+  }
+  .icon.size--XS :global(.spectrum-Icon) {
+    width: 10px;
+    height: 10px;
+  }
   .icon.size--S {
     width: 22px;
     height: 22px;
@@ -58,4 +86,14 @@
     width: 22px;
     height: 22px;
   }
+
+  .tooltip {
+    position: absolute;
+    pointer-events: none;
+    left: calc(50% + 8px);
+    bottom: calc(-50% + 6px);
+    /* transform: translateY(-50%); */
+    text-align: center;
+    z-index: 1;
+  }
 </style>

packages/bbui/src/Table/Table.svelte

@@ -470,6 +470,7 @@
     --table-border: 1px solid var(--spectrum-alias-border-color-mid);
     --cell-padding: var(--spectrum-global-dimension-size-250);
     overflow: auto;
+    display: contents;
   }
   .wrapper--quiet {
     --table-bg: var(--spectrum-alias-background-color-transparent);

packages/builder/src/components/integration/QueryViewer.svelte

@@ -39,7 +39,7 @@
   let integration
   let schemaType
 
-  let autoSchema = {}
+  let schema = {}
   let nestedSchemaFields = {}
   let rows = []
   let keys = {}
@@ -52,6 +52,8 @@
     schemaType = integration.query[query.queryVerb].type
 
     newQuery = cloneDeep(query)
+    // init schema from the query if one already exists
+    schema = newQuery.schema
     // Set the location where the query code will be written to an empty string so that it doesn't
     // get changed from undefined -> "" by the input, breaking our unsaved changes checks
     newQuery.fields[schemaType] ??= ""
@@ -86,12 +88,7 @@
 
       nestedSchemaFields = response.nestedSchemaFields
 
-      if (Object.keys(newQuery.schema).length === 0) {
+      schema = response.schema
-        // Assign this to a variable instead of directly to the newQuery.schema so that a user
-        // can change the table they're querying and have the schema update until they first
-        // edit it
-        autoSchema = response.schema
-      }
       rows = response.rows
 
       notifications.success("Query executed successfully")
@@ -118,10 +115,7 @@
       loading = true
       const response = await queries.save(newQuery.datasourceId, {
         ...newQuery,
-        schema:
+        schema,
-          Object.keys(newQuery.schema).length === 0
-            ? autoSchema
-            : newQuery.schema,
         nestedSchemaFields,
       })
 
@@ -320,12 +314,10 @@
     <QueryViewerSidePanel
       onClose={() => (showSidePanel = false)}
       onSchemaChange={newSchema => {
-        newQuery.schema = newSchema
+        schema = newSchema
       }}
       {rows}
-      schema={Object.keys(newQuery.schema).length === 0
+      {schema}
-        ? autoSchema
-        : newQuery.schema}
     />
   </div>
 </div>

packages/builder/src/pages/builder/portal/users/_components/ActiveDirectoryInfo.svelte

@@ -0,0 +1,23 @@
+<script>
+  import { IconAvatar } from "@budibase/bbui"
+
+  export let text = ""
+  export let iconSize = "S"
+</script>
+
+<div class="scim-banner">
+  <IconAvatar
+    icon="Sync"
+    size={iconSize}
+    background={"var(--spectrum-global-color-gray-500)"}
+    tooltip="Synced from your AD"
+  />
+  {text}
+</div>
+
+<style>
+  .scim-banner {
+    display: flex;
+    gap: var(--spacing-s);
+  }
+</style>

packages/builder/src/pages/builder/portal/users/_components/SCIMBanner.svelte

@@ -1,15 +0,0 @@
-<script>
-  import { Icon } from "@budibase/bbui"
-</script>
-
-<div class="scim-banner">
-  <Icon name="Info" size="S" />
-  Users are synced from your AD
-</div>
-
-<style>
-  .scim-banner {
-    display: flex;
-    gap: var(--spacing-s);
-  }
-</style>

packages/builder/src/pages/builder/portal/users/groups/[groupId].svelte

@@ -13,7 +13,7 @@
   import ConfirmDialog from "components/common/ConfirmDialog.svelte"
   import { Breadcrumb, Breadcrumbs } from "components/portal/page"
   import { roles } from "stores/builder"
-  import { apps, auth, features, groups } from "stores/portal"
+  import { apps, auth, groups } from "stores/portal"
   import { onMount, setContext } from "svelte"
   import AppNameTableRenderer from "../users/_components/AppNameTableRenderer.svelte"
   import AppRoleTableRenderer from "../users/_components/AppRoleTableRenderer.svelte"
@@ -47,9 +47,10 @@
   let loaded = false
   let editModal, deleteModal
 
-  $: scimEnabled = $features.isScimEnabled
-  $: readonly = !sdk.users.isAdmin($auth.user) || scimEnabled
   $: group = $groups.find(x => x._id === groupId)
+  $: isScimGroup = group?.scimInfo?.isSync
+  $: isAdmin = sdk.users.isAdmin($auth.user)
+  $: readonly = !isAdmin || isScimGroup
   $: groupApps = $apps
     .filter(app =>
       groups.actions
@@ -119,23 +120,31 @@
     <div class="header">
       <GroupIcon {group} size="L" />
       <Heading>{group?.name}</Heading>
-      {#if !readonly}
+      <ActionMenu align="right">
-        <ActionMenu align="right">
+        <span slot="control">
-          <span slot="control">
+          <Icon hoverable name="More" />
-            <Icon hoverable name="More" />
+        </span>
-          </span>
+        <MenuItem
-          <MenuItem icon="Refresh" on:click={() => editModal.show()}>
+          icon="Refresh"
-            Edit
+          on:click={() => editModal.show()}
-          </MenuItem>
+          disabled={!isAdmin}
-          <MenuItem icon="Delete" on:click={() => deleteModal.show()}>
+        >
+          Edit
+        </MenuItem>
+        <div title={isScimGroup && "Group synced from your AD"}>
+          <MenuItem
+            icon="Delete"
+            on:click={() => deleteModal.show()}
+            disabled={readonly}
+          >
             Delete
           </MenuItem>
-        </ActionMenu>
+        </div>
-      {/if}
+      </ActionMenu>
     </div>
 
     <Layout noPadding gap="S">
-      <GroupUsers {groupId} />
+      <GroupUsers {groupId} {readonly} {isScimGroup} />
     </Layout>
 
     <Layout noPadding gap="S">

packages/builder/src/pages/builder/portal/users/groups/_components/CreateEditGroupModal.svelte

@@ -11,6 +11,8 @@
   export let group
   export let saveGroup
 
+  let readonlyTitle = group?.scimInfo?.isSync
+
   let nameError
 </script>
 
@@ -26,7 +28,12 @@
   title={group?._rev ? "Edit group" : "Create group"}
   confirmText="Save"
 >
-  <Input bind:value={group.name} label="Name" error={nameError} />
+  <Input
+    bind:value={group.name}
+    label="Name"
+    error={nameError}
+    disabled={readonlyTitle}
+  />
   <div class="modal-format">
     <div class="modal-inner">
       <Body size="XS">Icon</Body>

packages/builder/src/pages/builder/portal/users/groups/_components/EditUserPicker.svelte

@@ -2,8 +2,7 @@
   import { Button, Popover, notifications } from "@budibase/bbui"
   import UserGroupPicker from "components/settings/UserGroupPicker.svelte"
   import { createPaginationStore } from "helpers/pagination"
-  import { auth, groups, users } from "stores/portal"
+  import { groups, users } from "stores/portal"
-  import { sdk } from "@budibase/shared-core"
 
   export let groupId
   export let onUsersUpdated
@@ -14,7 +13,6 @@
   let prevSearch = undefined
   let pageInfo = createPaginationStore()
 
-  $: readonly = !sdk.users.isAdmin($auth.user)
   $: page = $pageInfo.page
   $: searchUsers(page, searchTerm)
   $: group = $groups.find(x => x._id === groupId)
@@ -43,7 +41,7 @@
 </script>
 
 <div bind:this={popoverAnchor}>
-  <Button disabled={readonly} on:click={popover.show()} cta>Add user</Button>
+  <Button on:click={popover.show()} cta>Add user</Button>
 </div>
 <Popover align="right" bind:this={popover} anchor={popoverAnchor}>
   <UserGroupPicker

packages/builder/src/pages/builder/portal/users/groups/_components/GroupIcon.svelte

@@ -5,4 +5,12 @@
   export let size = "M"
 </script>
 
-<IconAvatar icon={group?.icon} background={group?.color} {size} />
+<div class="icon-group">
+  <IconAvatar icon={group?.icon} background={group?.color} {size} />
+</div>
+
+<style>
+  .icon-group {
+    position: relative;
+  }
+</style>

packages/builder/src/pages/builder/portal/users/groups/_components/GroupNameTableRenderer.svelte

@@ -1,5 +1,6 @@
 <script>
   import GroupIcon from "./GroupIcon.svelte"
+  import ActiveDirectoryInfo from "../../_components/ActiveDirectoryInfo.svelte"
 
   export let value
   export let row
@@ -14,13 +15,16 @@
   {:else}
     <div class="text">-</div>
   {/if}
+  {#if row.scimInfo?.isSync}
+    <ActiveDirectoryInfo iconSize="XS" />
+  {/if}
 </div>
 
 <style>
   .align {
     display: flex;
     align-items: center;
-    overflow: hidden;
+    overflow: visible;
     gap: var(--spacing-m);
     max-width: var(--max-cell-width);
     flex: 1 1 auto;

packages/builder/src/pages/builder/portal/users/groups/_components/GroupUsers.svelte

@@ -5,13 +5,15 @@
   import { fetchData } from "@budibase/frontend-core"
   import { goto } from "@roxi/routify"
   import { API } from "api"
-  import { auth, features, groups } from "stores/portal"
+  import { groups } from "stores/portal"
   import { setContext } from "svelte"
-  import ScimBanner from "../../_components/SCIMBanner.svelte"
+
   import RemoveUserTableRenderer from "../_components/RemoveUserTableRenderer.svelte"
-  import { sdk } from "@budibase/shared-core"
+  import ActiveDirectoryInfo from "../../_components/ActiveDirectoryInfo.svelte"
 
   export let groupId
+  export let readonly
+  export let isScimGroup
 
   let emailSearch
   let fetchGroupUsers
@@ -49,9 +51,6 @@
     },
   ]
 
-  $: scimEnabled = $features.isScimEnabled
-  $: readonly = !sdk.users.isAdmin($auth.user) || scimEnabled
-
   const removeUser = async id => {
     await groups.actions.removeUser(groupId, id)
     fetchGroupUsers.refresh()
@@ -63,10 +62,10 @@
 </script>
 
 <div class="header">
-  {#if !scimEnabled}
+  {#if isScimGroup}
+    <ActiveDirectoryInfo text="Users synced from your AD" />
+  {:else if !readonly}
     <EditUserPicker {groupId} onUsersUpdated={fetchGroupUsers.getInitialData} />
-  {:else}
-    <ScimBanner />
   {/if}
 
   <div class="controls-right">

packages/builder/src/pages/builder/portal/users/groups/index.svelte

@@ -13,7 +13,7 @@
     Search,
     notifications,
   } from "@budibase/bbui"
-  import { groups, auth, licensing, admin, features } from "stores/portal"
+  import { groups, auth, licensing, admin } from "stores/portal"
   import { onMount } from "svelte"
   import CreateEditGroupModal from "./_components/CreateEditGroupModal.svelte"
   import { cloneDeep } from "lodash/fp"
@@ -21,7 +21,6 @@
   import UsersTableRenderer from "./_components/UsersTableRenderer.svelte"
   import GroupNameTableRenderer from "./_components/GroupNameTableRenderer.svelte"
   import { goto } from "@roxi/routify"
-  import ScimBanner from "../_components/SCIMBanner.svelte"
   import { sdk } from "@budibase/shared-core"
 
   const DefaultGroup = {
@@ -110,14 +109,10 @@
   <div class="controls">
     <ButtonGroup>
       {#if $licensing.groupsEnabled}
-        {#if !$features.isScimEnabled}
+        <!--Show the group create button-->
-          <!--Show the group create button-->
+        <Button disabled={readonly} cta on:click={showCreateGroupModal}>
-          <Button disabled={readonly} cta on:click={showCreateGroupModal}>
+          Add group
-            Add group
+        </Button>
-          </Button>
-        {:else}
-          <ScimBanner />
-        {/if}
       {:else}
         <Button
           primary

packages/builder/src/pages/builder/portal/users/users/[userId].svelte

@@ -18,7 +18,7 @@
     Table,
   } from "@budibase/bbui"
   import { onMount, setContext } from "svelte"
-  import { users, auth, groups, apps, licensing, features } from "stores/portal"
+  import { users, auth, groups, apps, licensing } from "stores/portal"
   import { roles } from "stores/builder"
   import ForceResetPasswordModal from "./_components/ForceResetPasswordModal.svelte"
   import UserGroupPicker from "components/settings/UserGroupPicker.svelte"
@@ -30,8 +30,8 @@
   import GroupNameTableRenderer from "../groups/_components/GroupNameTableRenderer.svelte"
   import AppNameTableRenderer from "./_components/AppNameTableRenderer.svelte"
   import AppRoleTableRenderer from "./_components/AppRoleTableRenderer.svelte"
-  import ScimBanner from "../_components/SCIMBanner.svelte"
   import { sdk } from "@budibase/shared-core"
+  import ActiveDirectoryInfo from "../_components/ActiveDirectoryInfo.svelte"
 
   export let userId
 
@@ -39,9 +39,10 @@
     name: {
       width: "1fr",
     },
-    ...(readonly
+    ...(!isAdmin
       ? {}
-      : {
+      : // Add
+        {
           _id: {
             displayName: "",
             width: "auto",
@@ -87,12 +88,15 @@
   let user
   let loaded = false
 
-  $: scimEnabled = $features.isScimEnabled
+  $: internalGroups = $groups?.filter(g => !g?.scimInfo?.isSync)
+
   $: isSSO = !!user?.provider
-  $: readonly = !sdk.users.isAdmin($auth.user) || scimEnabled
+  $: isAdmin = sdk.users.isAdmin($auth.user)
+  $: isScim = user?.scimInfo?.isSync
+  $: readonly = !isAdmin || isScim
   $: privileged = sdk.users.isAdminOrGlobalBuilder(user)
   $: nameLabel = getNameLabel(user)
-  $: filteredGroups = getFilteredGroups($groups, searchTerm)
+  $: filteredGroups = getFilteredGroups(internalGroups, searchTerm)
   $: availableApps = getAvailableApps($apps, privileged, user?.roles)
   $: userGroups = $groups.filter(x => {
     return x.users?.find(y => {
@@ -274,8 +278,8 @@
     <Layout noPadding gap="S">
       <div class="details-title">
         <Heading size="S">Details</Heading>
-        {#if scimEnabled}
+        {#if user?.scimInfo?.isSync}
-          <ScimBanner />
+          <ActiveDirectoryInfo text="User synced from your AD" />
         {/if}
       </div>
       <div class="fields">
@@ -321,23 +325,23 @@
       <Layout gap="S" noPadding>
         <div class="tableTitle">
           <Heading size="S">Groups</Heading>
-          <div bind:this={popoverAnchor}>
+          {#if internalGroups?.length && isAdmin}
-            <Button disabled={readonly} on:click={popover.show()} secondary>
+            <div bind:this={popoverAnchor}>
-              Add to group
+              <Button on:click={popover.show()} secondary>Add to group</Button>
-            </Button>
+            </div>
-          </div>
+            <Popover align="right" bind:this={popover} anchor={popoverAnchor}>
-          <Popover align="right" bind:this={popover} anchor={popoverAnchor}>
+              <UserGroupPicker
-            <UserGroupPicker
+                labelKey="name"
-              labelKey="name"
+                bind:searchTerm
-              bind:searchTerm
+                list={filteredGroups}
-              list={filteredGroups}
+                selected={user.userGroups}
-              selected={user.userGroups}
+                on:select={e => addGroup(e.detail)}
-              on:select={e => addGroup(e.detail)}
+                on:deselect={e => removeGroup(e.detail)}
-              on:deselect={e => removeGroup(e.detail)}
+                iconComponent={GroupIcon}
-              iconComponent={GroupIcon}
+                extractIconProps={item => ({ group: item, size: "S" })}
-              extractIconProps={item => ({ group: item, size: "S" })}
+              />
-            />
+            </Popover>
-          </Popover>
+          {/if}
         </div>
         <Table
           schema={groupSchema}

packages/builder/src/pages/builder/portal/users/users/_components/AddUserModal.svelte

@@ -33,6 +33,8 @@
   $: reached = licensing.usersLimitReached(userCount)
   $: exceeded = licensing.usersLimitExceeded(userCount)
 
+  $: internalGroups = $groups?.filter(g => !g?.scimInfo?.isSync)
+
   function removeInput(idx) {
     userData = userData.filter((e, i) => i !== idx)
   }
@@ -133,12 +135,12 @@
     {/if}
   </Layout>
 
-  {#if $licensing.groupsEnabled}
+  {#if $licensing.groupsEnabled && internalGroups?.length}
     <Multiselect
       bind:value={userGroups}
       placeholder="No groups"
       label="Groups"
-      options={$groups}
+      options={internalGroups}
       getOptionLabel={option => option.name}
       getOptionValue={option => option._id}
     />

packages/builder/src/pages/builder/portal/users/users/_components/EmailTableRenderer.svelte

@@ -0,0 +1,11 @@
+<script>
+  import ActiveDirectoryInfo from "../../_components/ActiveDirectoryInfo.svelte"
+
+  export let value
+  export let row
+</script>
+
+{value}
+{#if row.scimInfo?.isSync}
+  <ActiveDirectoryInfo iconSize="XS" />
+{/if}

packages/builder/src/pages/builder/portal/users/users/_components/ImportUsersModal.svelte

@@ -34,6 +34,8 @@
     label: `${option.label} - ${option.subtitle}`,
   }))
 
+  $: internalGroups = $groups?.filter(g => !g?.scimInfo?.isSync)
+
   const validEmails = userEmails => {
     if ($admin.cloud && userEmails.length > MAX_USERS_UPLOAD_LIMIT) {
       notifications.error(
@@ -106,12 +108,12 @@
   {/if}
   <RadioGroup bind:value={usersRole} options={roleOptions} />
 
-  {#if $licensing.groupsEnabled}
+  {#if $licensing.groupsEnabled && internalGroups?.length}
     <Multiselect
       bind:value={userGroups}
       placeholder="No groups"
       label="Groups"
-      options={$groups}
+      options={internalGroups}
       getOptionLabel={option => option.name}
       getOptionValue={option => option._id}
     />

packages/builder/src/pages/builder/portal/users/users/_components/RemoveGroupTableRenderer.svelte

@@ -5,6 +5,7 @@
   import { sdk } from "@budibase/shared-core"
 
   export let value
+  export let row
 
   const groupContext = getContext("groups")
 
@@ -12,12 +13,11 @@
     e.stopPropagation()
     groupContext.removeGroup(value)
   }
+
+  $: disabled = !sdk.users.isAdmin($auth.user) || row?.scimInfo?.isSync
+  $: tooltip = row?.scimInfo?.isSync && "This group is managed via your AD"
 </script>
 
-<ActionButton
+<ActionButton {disabled} size="S" on:click={onClick} {tooltip}
-  disabled={!sdk.users.isAdmin($auth.user)}
+  >Remove</ActionButton
-  size="S"
-  on:click={onClick}
 >
-  Remove
-</ActionButton>

packages/builder/src/pages/builder/portal/users/users/index.svelte

@@ -19,7 +19,6 @@
     auth,
     licensing,
     organisation,
-    features,
     admin,
   } from "stores/portal"
   import { onMount } from "svelte"
@@ -28,6 +27,7 @@
   import GroupsTableRenderer from "./_components/GroupsTableRenderer.svelte"
   import AppsTableRenderer from "./_components/AppsTableRenderer.svelte"
   import RoleTableRenderer from "./_components/RoleTableRenderer.svelte"
+  import EmailTableRenderer from "./_components/EmailTableRenderer.svelte"
   import { goto } from "@roxi/routify"
   import OnboardingTypeModal from "./_components/OnboardingTypeModal.svelte"
   import PasswordModal from "./_components/PasswordModal.svelte"
@@ -37,7 +37,6 @@
   import { Constants, Utils, fetchData } from "@budibase/frontend-core"
   import { API } from "api"
   import { OnboardingType } from "constants"
-  import ScimBanner from "../_components/SCIMBanner.svelte"
   import { sdk } from "@budibase/shared-core"
 
   const fetch = fetchData({
@@ -63,6 +62,7 @@
   let selectedRows = []
   let bulkSaveResponse
   let customRenderers = [
+    { column: "email", component: EmailTableRenderer },
     { column: "userGroups", component: GroupsTableRenderer },
     { column: "apps", component: AppsTableRenderer },
     { column: "role", component: RoleTableRenderer },
@@ -73,7 +73,7 @@
   let parsedInvites = []
 
   $: isOwner = $auth.accountPortalAccess && $admin.cloud
-  $: readonly = !sdk.users.isAdmin($auth.user) || $features.isScimEnabled
+  $: readonly = !sdk.users.isAdmin($auth.user)
   $: debouncedUpdateFetch(searchEmail)
   $: schema = {
     email: {
@@ -247,19 +247,25 @@
     }
   }
 
-  const deleteRows = async () => {
+  const deleteUsers = async () => {
     try {
       let ids = selectedRows.map(user => user._id)
       if (ids.includes(get(auth).user._id)) {
         notifications.error("You cannot delete yourself")
         return
       }
+
+      if (selectedRows.some(u => u.scimInfo?.isSync)) {
+        notifications.error("You cannot remove users created via your AD")
+        return
+      }
+
       await users.bulkDelete(ids)
       notifications.success(`Successfully deleted ${selectedRows.length} rows`)
       selectedRows = []
       await fetch.refresh()
     } catch (error) {
-      notifications.error("Error deleting rows")
+      notifications.error("Error deleting users")
     }
   }
 
@@ -320,8 +326,6 @@
           Import
         </Button>
       </div>
-    {:else}
-      <ScimBanner />
     {/if}
     <div class="controls-right">
       <Search bind:value={searchEmail} placeholder="Search" />
@@ -330,7 +334,7 @@
           item="user"
           on:updaterows
           {selectedRows}
-          {deleteRows}
+          deleteRows={deleteUsers}
         />
       {/if}
     </div>
@@ -346,6 +350,7 @@
     {customRenderers}
     loading={!$fetch.loaded || !groupsLoaded}
   />
+
   <div class="pagination">
     <Pagination
       page={$fetch.pageNumber + 1}
@@ -355,6 +360,7 @@
       goToNextPage={fetch.nextPage}
     />
   </div>
+
   <Table
     schema={pendingSchema}
     data={parsedInvites}
@@ -402,6 +408,7 @@
     display: flex;
     flex-direction: row;
     justify-content: flex-end;
+    margin-left: auto;
   }
   .controls {
     display: flex;

packages/builder/src/stores/portal/groups.js

@@ -35,7 +35,10 @@ export function createGroupsStore() {
     get: getGroup,
 
     save: async group => {
-      const response = await API.saveGroup(group)
+      const { ...dataToSave } = group
+      delete dataToSave.scimInfo
+      delete dataToSave.userGroups
+      const response = await API.saveGroup(dataToSave)
       group._id = response._id
       group._rev = response._rev
       updateStore(group)

packages/client/manifest.json

@@ -1137,6 +1137,12 @@
         "key": "color",
         "showInBar": true
       },
+      {
+        "type": "color",
+        "label": "Text Color",
+        "key": "textColor",
+        "showInBar": true
+      },
       {
         "type": "boolean",
         "label": "Allow delete",

packages/client/src/components/app/Tag.svelte

@@ -6,6 +6,7 @@
   export let onClick
   export let text = ""
   export let color
+  export let textColor
   export let closable = false
   export let size = "M"
 
@@ -14,7 +15,7 @@
 
   // Add color styles to main styles object, otherwise the styleable helper
   // overrides the color when it's passed as inline style.
-  $: styles = enrichStyles($component.styles, color)
+  $: styles = enrichStyles($component.styles, color, textColor)
   $: componentText = getComponentText(text, $builderStore, $component)
 
   const getComponentText = (text, builderState, componentState) => {
@@ -24,7 +25,7 @@
     return text || componentState.name || "Placeholder text"
   }
 
-  const enrichStyles = (styles, color) => {
+  const enrichStyles = (styles, color, textColor) => {
     if (!color) {
       return styles
     }
@@ -34,7 +35,7 @@
         ...styles?.normal,
         "background-color": color,
         "border-color": color,
-        color: "white",
+        color: textColor || "white",
         "--spectrum-clearbutton-medium-icon-color": "white",
       },
     }

packages/pro

@@ -1 +1 @@
-Subproject commit 80a4d8ff998895fc298ee510158a82ce7daebc67
+Subproject commit c4c98ae70f2e936009250893898ecf11f4ddf2c3

packages/server/src/api/controllers/public/utils.ts

@@ -1,11 +1,12 @@
 import { context } from "@budibase/backend-core"
 import { isExternalTableID } from "../../../integrations/utils"
 import { APP_PREFIX, DocumentType } from "../../../db/utils"
+import { Row } from "@budibase/types"
 
 export async function addRev(
   body: { _id?: string; _rev?: string },
   tableId?: string
-) {
+): Promise<Row> {
   if (!body._id || (tableId && isExternalTableID(tableId))) {
     return body
   }

packages/server/src/api/controllers/row/external.ts

@@ -128,7 +128,10 @@ export async function bulkDestroy(ctx: UserCtx) {
     )
   }
   const responses = await Promise.all(promises)
-  return { response: { ok: true }, rows: responses.map(resp => resp.row) }
+  const finalRows = responses
+    .map(resp => resp.row)
+    .filter(row => row && row._id)
+  return { response: { ok: true }, rows: finalRows }
 }
 
 export async function fetchEnrichedRow(ctx: UserCtx) {

packages/server/src/api/controllers/row/index.ts

@@ -131,7 +131,10 @@ async function processDeleteRowsRequest(ctx: UserCtx<DeleteRowRequest>) {
       : fixRow(processedRow, ctx.params)
   })
 
-  return await Promise.all(processedRows)
+  const responses = await Promise.allSettled(processedRows)
+  return responses
+    .filter(resp => resp.status === "fulfilled")
+    .map(resp => (resp as PromiseFulfilledResult<Row>).value)
 }
 
 async function deleteRows(ctx: UserCtx<DeleteRowRequest>) {

packages/server/src/api/routes/tests/row.spec.ts

@@ -636,6 +636,17 @@ describe.each([
       expect(res[0]._id).toEqual(createdRow._id)
       await assertRowUsage(rowUsage - 1)
     })
+
+    it("should be able to bulk delete rows, including a row that doesn't exist", async () => {
+      const createdRow = await config.createRow()
+
+      const res = await config.api.row.bulkDelete(table._id!, {
+        rows: [createdRow, { _id: "2" }],
+      })
+
+      expect(res[0]._id).toEqual(createdRow._id)
+      expect(res.length).toEqual(1)
+    })
   })
 
   describe("validate", () => {

packages/server/src/startup.ts

@@ -7,6 +7,7 @@ import {
   logging,
   tenancy,
   users,
+  cache,
 } from "@budibase/backend-core"
 import fs from "fs"
 import { watch } from "./watch"
@@ -74,6 +75,7 @@ export async function startup(app?: Koa, server?: Server) {
   eventEmitter.emitPort(env.PORT)
   fileSystem.init()
   await redis.init()
+  cache.docWritethrough.init()
   eventInit()
   if (app && server) {
     initialiseWebsockets(app, server)

packages/server/src/threads/query.ts

@@ -15,7 +15,7 @@ import { context, cache, auth } from "@budibase/backend-core"
 import { getGlobalIDFromUserMetadataID } from "../db/utils"
 import sdk from "../sdk"
 import { cloneDeep } from "lodash/fp"
-import { Datasource, Query, SourceName } from "@budibase/types"
+import { Datasource, Query, SourceName, Row } from "@budibase/types"
 
 import { isSQL } from "../integrations/utils"
 import { interpolateSQL } from "../integrations/queries/sql"
@@ -115,7 +115,7 @@ class QueryRunner {
     }
 
     let output = threadUtils.formatResponse(await integration[queryVerb](query))
-    let rows = output,
+    let rows = output as Row[],
       info = undefined,
       extra = undefined,
       pagination = undefined
@@ -170,7 +170,12 @@ class QueryRunner {
     }
 
     // get all the potential fields in the schema
-    let keys = rows.flatMap(Object.keys)
+    const keysSet: Set<string> = new Set()
+    rows.forEach(row => {
+      const keys = Object.keys(row)
+      keys.forEach(key => keysSet.add(key))
+    })
+    const keys: string[] = [...keysSet]
 
     if (integration.end) {
       integration.end()

packages/types/package.json

@@ -23,6 +23,6 @@
     "typescript": "5.2.2"
   },
   "dependencies": {
-    "scim-patch": "^0.7.0"
+    "scim-patch": "^0.8.1"
   }
 }

packages/worker/src/api/routes/global/tests/groups.spec.ts

@@ -104,17 +104,79 @@ describe("/api/global/groups", () => {
       expect(events.group.permissionsEdited).not.toBeCalled()
     })
 
-    describe("destroy", () => {
+    describe("scim", () => {
-      it("should be able to delete a basic group", async () => {
+      async function createScimGroup() {
-        const group = structures.groups.UserGroup()
+        mocks.licenses.useScimIntegration()
-        let oldGroup = await config.api.groups.saveGroup(group)
+        await config.setSCIMConfig(true)
-        await config.api.groups.deleteGroup(
-          oldGroup.body._id,
-          oldGroup.body._rev
-        )
 
-        expect(events.group.deleted).toBeCalledTimes(1)
+        const scimGroup = await config.api.scimGroupsAPI.post({
+          body: structures.scim.createGroupRequest({
+            displayName: generator.word(),
+          }),
+        })
+
+        const { body: group } = await config.api.groups.find(scimGroup.id)
+
+        expect(group).toBeDefined()
+        return group
+      }
+
+      it("update will not allow sending SCIM fields", async () => {
+        const group = await createScimGroup()
+
+        const updatedGroup: UserGroup = {
+          ...group,
+          name: generator.word(),
+        }
+        await config.api.groups.saveGroup(updatedGroup, {
+          expect: {
+            message: 'Invalid body - "scimInfo" is not allowed',
+            status: 400,
+          },
+        })
+
+        expect(events.group.updated).not.toBeCalled()
       })
+
+      it("update will not amend the SCIM fields", async () => {
+        const group: UserGroup = await createScimGroup()
+
+        const updatedGroup: UserGroup = {
+          ...group,
+          name: generator.word(),
+          scimInfo: undefined,
+        }
+
+        await config.api.groups.saveGroup(updatedGroup, {
+          expect: 200,
+        })
+
+        expect(events.group.updated).toBeCalledTimes(1)
+        expect(
+          (
+            await config.api.groups.find(group._id!, {
+              expect: 200,
+            })
+          ).body
+        ).toEqual(
+          expect.objectContaining({
+            ...group,
+            name: updatedGroup.name,
+            scimInfo: group.scimInfo,
+            _rev: expect.any(String),
+          })
+        )
+      })
+    })
+  })
+
+  describe("destroy", () => {
+    it("should be able to delete a basic group", async () => {
+      const group = structures.groups.UserGroup()
+      let oldGroup = await config.api.groups.saveGroup(group)
+      await config.api.groups.deleteGroup(oldGroup.body._id, oldGroup.body._rev)
+
+      expect(events.group.deleted).toBeCalledTimes(1)
     })
   })
 
@@ -147,7 +209,7 @@ describe("/api/global/groups", () => {
 
         await Promise.all(
           Array.from({ length: 30 }).map(async (_, i) => {
-            const email = `user${i}@example.com`
+            const email = `user${i}+${generator.guid()}@example.com`
             const user = await config.api.users.saveUser({
               ...structures.users.user(),
               email,
@@ -257,12 +319,16 @@ describe("/api/global/groups", () => {
       })
     })
 
-    it("update should return 200", async () => {
+    it("update should return forbidden", async () => {
       await config.withUser(builder, async () => {
-        await config.api.groups.updateGroupUsers(group._id!, {
+        await config.api.groups.updateGroupUsers(
-          add: [builder._id!],
+          group._id!,
-          remove: [],
+          {
-        })
+            add: [builder._id!],
+            remove: [],
+          },
+          { expect: 403 }
+        )
       })
     })
   })

packages/worker/src/api/routes/global/tests/scim.spec.ts

@@ -2,6 +2,7 @@ import tk from "timekeeper"
 import _ from "lodash"
 import { generator, mocks, structures } from "@budibase/backend-core/tests"
 import {
+  CloudAccount,
   ScimCreateUserRequest,
   ScimGroupResponse,
   ScimUpdateRequest,
@@ -10,25 +11,20 @@ import {
 import { TestConfiguration } from "../../../../tests"
 import { events } from "@budibase/backend-core"
 
-// this test can 409 - retries reduce issues with this
-jest.retryTimes(2, { logErrorsBeforeRetry: true })
 jest.setTimeout(30000)
 
 describe("scim", () => {
-  beforeAll(async () => {
+  async function setup() {
-    tk.freeze(mocks.date.MOCK_DATE)
-    mocks.licenses.useScimIntegration()
-
-    await config.setSCIMConfig(true)
-  })
-
-  beforeEach(async () => {
     jest.resetAllMocks()
     tk.freeze(mocks.date.MOCK_DATE)
     mocks.licenses.useScimIntegration()
+    mocks.licenses.useGroups()
 
     await config.setSCIMConfig(true)
-  })
+  }
+
+  beforeAll(setup)
+  beforeEach(setup)
 
   const config = new TestConfiguration()
 
@@ -367,13 +363,77 @@ describe("scim", () => {
         })
       })
 
-      it("creating an existing user name returns a conflict", async () => {
+      it("creating an external user that conflicts an internal one syncs the existing user", async () => {
-        const body = structures.scim.createUserRequest()
+        const { body: internalUser } = await config.api.users.saveUser(
+          structures.users.user()
+        )
 
-        await postScimUser({ body })
+        const scimUserData = {
+          externalId: structures.uuid(),
+          email: internalUser.email,
+          firstName: structures.generator.first(),
+          lastName: structures.generator.last(),
+          username: structures.generator.name(),
+        }
+        const scimUserRequest = structures.scim.createUserRequest(scimUserData)
 
-        const res = await postScimUser({ body }, { expect: 409 })
+        const res = await postScimUser(
-        expect((res as any).message).toBe("Email already in use")
+          { body: scimUserRequest },
+          { expect: 200 }
+        )
+
+        const expectedScimUser: ScimUserResponse = {
+          schemas: ["urn:ietf:params:scim:schemas:core:2.0:User"],
+          id: internalUser._id!,
+          externalId: scimUserRequest.externalId,
+          meta: {
+            resourceType: "User",
+            // @ts-ignore
+            created: mocks.date.MOCK_DATE.toISOString(),
+            // @ts-ignore
+            lastModified: mocks.date.MOCK_DATE.toISOString(),
+          },
+          userName: scimUserData.username,
+          name: {
+            formatted: `${scimUserData.firstName} ${scimUserData.lastName}`,
+            familyName: scimUserData.lastName,
+            givenName: scimUserData.firstName,
+          },
+          active: true,
+          emails: [
+            {
+              value: internalUser.email,
+              type: "work",
+              primary: true,
+            },
+          ],
+        }
+
+        expect(res).toEqual(expectedScimUser)
+      })
+
+      it("a user cannot be SCIM synchronised with another SCIM user", async () => {
+        const { body: internalUser } = await config.api.users.saveUser(
+          structures.users.user()
+        )
+
+        await postScimUser(
+          {
+            body: structures.scim.createUserRequest({
+              email: internalUser.email,
+            }),
+          },
+          { expect: 200 }
+        )
+
+        await postScimUser(
+          {
+            body: structures.scim.createUserRequest({
+              email: internalUser.email,
+            }),
+          },
+          { expect: 409 }
+        )
       })
     })
 
@@ -545,6 +605,25 @@ describe("scim", () => {
 
         expect(events.user.deleted).toBeCalledTimes(1)
       })
+
+      it("an account holder cannot be removed even when synched", async () => {
+        const account: CloudAccount = {
+          ...structures.accounts.account(),
+          budibaseUserId: user.id,
+          email: user.emails![0].value,
+        }
+        mocks.accounts.getAccount.mockResolvedValue(account)
+
+        await deleteScimUser(user.id, {
+          expect: {
+            message: "Account holder cannot be deleted",
+            status: 400,
+            error: { code: "http" },
+          },
+        })
+
+        await config.api.scimUsersAPI.find(user.id, { expect: 200 })
+      })
     })
   })
 
@@ -656,7 +735,6 @@ describe("scim", () => {
         })
 
         it("can fetch groups even if internal groups exist", async () => {
-          mocks.licenses.useGroups()
           await config.api.groups.saveGroup(structures.userGroups.userGroup())
           await config.api.groups.saveGroup(structures.userGroups.userGroup())
 
@@ -722,6 +800,43 @@ describe("scim", () => {
           )
         })
       })
+
+      it("creating an external group that conflicts an internal one syncs the existing group", async () => {
+        const groupToSave = structures.userGroups.userGroup()
+        const { body: internalGroup } = await config.api.groups.saveGroup(
+          groupToSave
+        )
+
+        const scimGroupData = {
+          externalId: structures.uuid(),
+          displayName: groupToSave.name,
+        }
+
+        const res = await postScimGroup(
+          { body: structures.scim.createGroupRequest(scimGroupData) },
+          { expect: 200 }
+        )
+
+        expect(res).toEqual(
+          expect.objectContaining({
+            id: internalGroup._id!,
+            externalId: scimGroupData.externalId,
+            displayName: scimGroupData.displayName,
+          })
+        )
+      })
+
+      it("a group cannot be SCIM synchronised with another SCIM group", async () => {
+        const groupToSave = structures.userGroups.userGroup()
+        await config.api.groups.saveGroup(groupToSave)
+
+        const createGroupRequest = structures.scim.createGroupRequest({
+          displayName: groupToSave.name,
+        })
+        await postScimGroup({ body: createGroupRequest }, { expect: 200 })
+
+        await postScimGroup({ body: createGroupRequest }, { expect: 409 })
+      })
     })
 
     describe("GET /api/global/scim/v2/groups/:id", () => {

packages/worker/src/index.ts

@@ -17,6 +17,7 @@ import {
   env as coreEnv,
   timers,
   redis,
+  cache,
 } from "@budibase/backend-core"
 
 db.init()
@@ -90,6 +91,7 @@ export default server.listen(parseInt(env.PORT || "4002"), async () => {
   console.log(`Worker running on ${JSON.stringify(server.address())}`)
   await initPro()
   await redis.clients.init()
+  cache.docWritethrough.init()
   // configure events to use the pro audit log write
   // can't integrate directly into backend-core due to cyclic issues
   await events.processors.init(proSdk.auditLogs.write)

packages/worker/src/tests/api/groups.ts

@@ -7,7 +7,10 @@ export class GroupsAPI extends TestAPI {
     super(config)
   }
 
-  saveGroup = (group: UserGroup, { expect } = { expect: 200 }) => {
+  saveGroup = (
+    group: UserGroup,
+    { expect }: { expect: number | object } = { expect: 200 }
+  ) => {
     return this.request
       .post(`/api/global/groups`)
       .send(group)
@@ -44,14 +47,15 @@ export class GroupsAPI extends TestAPI {
 
   updateGroupUsers = (
     id: string,
-    body: { add: string[]; remove: string[] }
+    body: { add: string[]; remove: string[] },
+    { expect } = { expect: 200 }
   ) => {
     return this.request
       .post(`/api/global/groups/${id}/users`)
       .send(body)
       .set(this.config.defaultHeaders())
       .expect("Content-Type", /json/)
-      .expect(200)
+      .expect(expect)
   }
 
   fetch = ({ expect } = { expect: 200 }) => {
@@ -61,4 +65,12 @@ export class GroupsAPI extends TestAPI {
       .expect("Content-Type", /json/)
       .expect(expect)
   }
+
+  find = (id: string, { expect } = { expect: 200 }) => {
+    return this.request
+      .get(`/api/global/groups/${id}`)
+      .set(this.config.defaultHeaders())
+      .expect("Content-Type", /json/)
+      .expect(expect)
+  }
 }

packages/worker/src/tests/api/scim/shared.ts

@@ -1,13 +1,17 @@
 import TestConfiguration from "../../TestConfiguration"
 import { TestAPI } from "../base"
 
-const defaultConfig = {
+const defaultConfig: RequestSettings = {
   expect: 200,
   setHeaders: true,
   skipContentTypeCheck: false,
 }
 
-export type RequestSettings = typeof defaultConfig
+export type RequestSettings = {
+  expect: number | object
+  setHeaders: boolean
+  skipContentTypeCheck: boolean
+}
 
 export abstract class ScimTestAPI extends TestAPI {
   constructor(config: TestConfiguration) {

yarn.lock

@@ -5666,6 +5666,13 @@
   dependencies:
     undici-types "~5.26.4"
 
+"@types/node@^20.4.5":
+  version "20.11.25"
+  resolved "https://registry.yarnpkg.com/@types/node/-/node-20.11.25.tgz#0f50d62f274e54dd7a49f7704cc16bfbcccaf49f"
+  integrity sha512-TBHyJxk2b7HceLVGFcpAUjsa5zIdsPWlR6XHfyGzd0SFu+/NFgQgMAl96MSDZgQDvJAvV6BKsFOrt6zIL09JDw==
+  dependencies:
+    undici-types "~5.26.4"
+
 "@types/nodemailer@^6.4.4":
   version "6.4.14"
   resolved "https://registry.yarnpkg.com/@types/nodemailer/-/nodemailer-6.4.14.tgz#5c81a5e856db7f8ede80013e6dbad7c5fb2283e2"
@@ -19491,11 +19498,12 @@ schema-utils@^3.1.1, schema-utils@^3.1.2:
     ajv "^6.12.5"
     ajv-keywords "^3.5.2"
 
-scim-patch@^0.7.0:
+scim-patch@^0.8.1:
-  version "0.7.0"
+  version "0.8.1"
-  resolved "https://registry.yarnpkg.com/scim-patch/-/scim-patch-0.7.0.tgz#3f6d94256c07be415a74a49c0ff48dc91e4e0219"
+  resolved "https://registry.yarnpkg.com/scim-patch/-/scim-patch-0.8.1.tgz#611bfdb5538f6d8b97aba0ab0f8bd01055b70c1c"
-  integrity sha512-wXKcsZl+aLfE0yId7MjiOd91v8as6dEYLFvm1gGu3yJxSPhl1Fl3vWiNN4V3D68UKpqO/umK5rwWc8wGpBaOHw==
+  integrity sha512-JRYTA+mJZ8Z5DJGO7kkFc0lGCDs100rNs7iN77mld7gQajTp1R1xjUzMfZTOMAkBDA75GSdbMmfdfMqMJCn/Yg==
   dependencies:
+    "@types/node" "^20.4.5"
     fast-deep-equal "3.1.3"
     scim2-parse-filter "0.2.8"