Merge branch 'master' into cleanup-isolates

2024-03-13 10:30:37 +01:00 · 2024-03-13 10:30:37 +01:00 · cc275983dc
parent 1972008ac2 efe07cef5d
commit cc275983dc
38 changed files with 514 additions and 171 deletions
--- a/lerna.json
+++ b/lerna.json
@ -1,5 +1,5 @@
 {
-  "version": "2.21.4",
+  "version": "2.21.8",
  "npmClient": "yarn",
  "packages": [
    "packages/*",
--- a/packages/backend-core/src/cache/docWritethrough.ts
+++ b/packages/backend-core/src/cache/docWritethrough.ts
@ -10,6 +10,7 @@ interface ProcessDocMessage {
 }

 const PERSIST_MAX_ATTEMPTS = 100
+let processor: DocWritethroughProcessor | undefined

 export const docWritethroughProcessorQueue = createQueue<ProcessDocMessage>(
  JobQueue.DOC_WRITETHROUGH_QUEUE,
@ -61,8 +62,6 @@ class DocWritethroughProcessor {
  }
 }

-export const processor = new DocWritethroughProcessor().init()
-
 export class DocWritethrough {
  private db: Database
  private _docId: string
@ -84,3 +83,15 @@ export class DocWritethrough {
    })
  }
 }
+
+export function init(): DocWritethroughProcessor {
+  processor = new DocWritethroughProcessor().init()
+  return processor
+}
+
+export function getProcessor(): DocWritethroughProcessor {
+  if (!processor) {
+    return init()
+  }
+  return processor
+}
--- a/packages/backend-core/src/cache/tests/docWritethrough.spec.ts
+++ b/packages/backend-core/src/cache/tests/docWritethrough.spec.ts
@ -7,6 +7,7 @@ import { getDB } from "../../db"
 import {
  DocWritethrough,
  docWritethroughProcessorQueue,
+  init,
 } from "../docWritethrough"

 import InMemoryQueue from "../../queue/inMemoryQueue"
@ -19,6 +20,10 @@ async function waitForQueueCompletion() {
 }

 describe("docWritethrough", () => {
+  beforeAll(() => {
+    init()
+  })
+
  const config = new DBTestConfiguration()

  const db = getDB(structures.db.id())
--- a/packages/bbui/src/Icon/IconAvatar.svelte
+++ b/packages/bbui/src/Icon/IconAvatar.svelte
@ -1,22 +1,41 @@
 <script>
  import Icon from "./Icon.svelte"

+  import Tooltip from "../Tooltip/Tooltip.svelte"
+  import { fade } from "svelte/transition"
+
  export let icon
  export let background
  export let color
  export let size = "M"
+  export let tooltip
+
+  let showTooltip = false
 </script>

+<!-- svelte-ignore a11y-no-static-element-interactions -->
+<!-- svelte-ignore a11y-click-events-have-key-events -->
 <div
  class="icon size--{size}"
  style="background: {background || `transparent`};"
  class:filled={!!background}
+  on:mouseover={() => (showTooltip = true)}
+  on:mouseleave={() => (showTooltip = false)}
+  on:focus={() => (showTooltip = true)}
+  on:blur={() => (showTooltip = false)}
+  on:click={() => (showTooltip = false)}
 >
  <Icon name={icon} color={background ? "white" : color} />
+  {#if tooltip && showTooltip}
+    <div class="tooltip" in:fade={{ duration: 130, delay: 250 }}>
+      <Tooltip textWrapping direction="right" text={tooltip} />
+    </div>
+  {/if}
 </div>

 <style>
  .icon {
+    position: relative;
    width: 28px;
    height: 28px;
    flex: 0 0 28px;
@ -32,6 +51,15 @@
    width: 16px;
    height: 16px;
  }
+  .icon.size--XS {
+    width: 18px;
+    height: 18px;
+    flex: 0 0 18px;
+  }
+  .icon.size--XS :global(.spectrum-Icon) {
+    width: 10px;
+    height: 10px;
+  }
  .icon.size--S {
    width: 22px;
    height: 22px;
@ -58,4 +86,14 @@
    width: 22px;
    height: 22px;
  }
+
+  .tooltip {
+    position: absolute;
+    pointer-events: none;
+    left: calc(50% + 8px);
+    bottom: calc(-50% + 6px);
+    /* transform: translateY(-50%); */
+    text-align: center;
+    z-index: 1;
+  }
 </style>
--- a/packages/bbui/src/Table/Table.svelte
+++ b/packages/bbui/src/Table/Table.svelte
@ -470,6 +470,7 @@
    --table-border: 1px solid var(--spectrum-alias-border-color-mid);
    --cell-padding: var(--spectrum-global-dimension-size-250);
    overflow: auto;
+    display: contents;
  }
  .wrapper--quiet {
    --table-bg: var(--spectrum-alias-background-color-transparent);
--- a/packages/builder/src/components/integration/QueryViewer.svelte
+++ b/packages/builder/src/components/integration/QueryViewer.svelte
@ -39,7 +39,7 @@
  let integration
  let schemaType

-  let autoSchema = {}
+  let schema = {}
  let nestedSchemaFields = {}
  let rows = []
  let keys = {}
@ -52,6 +52,8 @@
    schemaType = integration.query[query.queryVerb].type

    newQuery = cloneDeep(query)
+    // init schema from the query if one already exists
+    schema = newQuery.schema
    // Set the location where the query code will be written to an empty string so that it doesn't
    // get changed from undefined -> "" by the input, breaking our unsaved changes checks
    newQuery.fields[schemaType] ??= ""
@ -86,12 +88,7 @@

      nestedSchemaFields = response.nestedSchemaFields

-      if (Object.keys(newQuery.schema).length === 0) {
-        // Assign this to a variable instead of directly to the newQuery.schema so that a user
-        // can change the table they're querying and have the schema update until they first
-        // edit it
-        autoSchema = response.schema
-      }
+      schema = response.schema
      rows = response.rows

      notifications.success("Query executed successfully")
@ -118,10 +115,7 @@
      loading = true
      const response = await queries.save(newQuery.datasourceId, {
        ...newQuery,
-        schema:
-          Object.keys(newQuery.schema).length === 0
-            ? autoSchema
-            : newQuery.schema,
+        schema,
        nestedSchemaFields,
      })

@ -320,12 +314,10 @@
    <QueryViewerSidePanel
      onClose={() => (showSidePanel = false)}
      onSchemaChange={newSchema => {
-        newQuery.schema = newSchema
+        schema = newSchema
      }}
      {rows}
-      schema={Object.keys(newQuery.schema).length === 0
-        ? autoSchema
-        : newQuery.schema}
+      {schema}
    />
  </div>
 </div>
--- a/packages/builder/src/pages/builder/portal/users/_components/ActiveDirectoryInfo.svelte
+++ b/packages/builder/src/pages/builder/portal/users/_components/ActiveDirectoryInfo.svelte
@ -0,0 +1,23 @@
+<script>
+  import { IconAvatar } from "@budibase/bbui"
+
+  export let text = ""
+  export let iconSize = "S"
+</script>
+
+<div class="scim-banner">
+  <IconAvatar
+    icon="Sync"
+    size={iconSize}
+    background={"var(--spectrum-global-color-gray-500)"}
+    tooltip="Synced from your AD"
+  />
+  {text}
+</div>
+
+<style>
+  .scim-banner {
+    display: flex;
+    gap: var(--spacing-s);
+  }
+</style>
--- a/packages/builder/src/pages/builder/portal/users/_components/SCIMBanner.svelte
+++ b/packages/builder/src/pages/builder/portal/users/_components/SCIMBanner.svelte
@ -1,15 +0,0 @@
-<script>
-  import { Icon } from "@budibase/bbui"
-</script>
-
-<div class="scim-banner">
-  <Icon name="Info" size="S" />
-  Users are synced from your AD
-</div>
-
-<style>
-  .scim-banner {
-    display: flex;
-    gap: var(--spacing-s);
-  }
-</style>
--- a/packages/builder/src/pages/builder/portal/users/groups/[groupId].svelte
+++ b/packages/builder/src/pages/builder/portal/users/groups/[groupId].svelte
@ -13,7 +13,7 @@
  import ConfirmDialog from "components/common/ConfirmDialog.svelte"
  import { Breadcrumb, Breadcrumbs } from "components/portal/page"
  import { roles } from "stores/builder"
-  import { apps, auth, features, groups } from "stores/portal"
+  import { apps, auth, groups } from "stores/portal"
  import { onMount, setContext } from "svelte"
  import AppNameTableRenderer from "../users/_components/AppNameTableRenderer.svelte"
  import AppRoleTableRenderer from "../users/_components/AppRoleTableRenderer.svelte"
@ -47,9 +47,10 @@
  let loaded = false
  let editModal, deleteModal

-  $: scimEnabled = $features.isScimEnabled
-  $: readonly = !sdk.users.isAdmin($auth.user) || scimEnabled
  $: group = $groups.find(x => x._id === groupId)
+  $: isScimGroup = group?.scimInfo?.isSync
+  $: isAdmin = sdk.users.isAdmin($auth.user)
+  $: readonly = !isAdmin || isScimGroup
  $: groupApps = $apps
    .filter(app =>
      groups.actions
@ -119,23 +120,31 @@
    <div class="header">
      <GroupIcon {group} size="L" />
      <Heading>{group?.name}</Heading>
-      {#if !readonly}
-        <ActionMenu align="right">
-          <span slot="control">
-            <Icon hoverable name="More" />
-          </span>
-          <MenuItem icon="Refresh" on:click={() => editModal.show()}>
-            Edit
-          </MenuItem>
-          <MenuItem icon="Delete" on:click={() => deleteModal.show()}>
+      <ActionMenu align="right">
+        <span slot="control">
+          <Icon hoverable name="More" />
+        </span>
+        <MenuItem
+          icon="Refresh"
+          on:click={() => editModal.show()}
+          disabled={!isAdmin}
+        >
+          Edit
+        </MenuItem>
+        <div title={isScimGroup && "Group synced from your AD"}>
+          <MenuItem
+            icon="Delete"
+            on:click={() => deleteModal.show()}
+            disabled={readonly}
+          >
            Delete
          </MenuItem>
-        </ActionMenu>
-      {/if}
+        </div>
+      </ActionMenu>
    </div>

    <Layout noPadding gap="S">
-      <GroupUsers {groupId} />
+      <GroupUsers {groupId} {readonly} {isScimGroup} />
    </Layout>

    <Layout noPadding gap="S">
--- a/packages/builder/src/pages/builder/portal/users/groups/_components/CreateEditGroupModal.svelte
+++ b/packages/builder/src/pages/builder/portal/users/groups/_components/CreateEditGroupModal.svelte
@ -11,6 +11,8 @@
  export let group
  export let saveGroup

+  let readonlyTitle = group?.scimInfo?.isSync
+
  let nameError
 </script>

@ -26,7 +28,12 @@
  title={group?._rev ? "Edit group" : "Create group"}
  confirmText="Save"
 >
-  <Input bind:value={group.name} label="Name" error={nameError} />
+  <Input
+    bind:value={group.name}
+    label="Name"
+    error={nameError}
+    disabled={readonlyTitle}
+  />
  <div class="modal-format">
    <div class="modal-inner">
      <Body size="XS">Icon</Body>
--- a/packages/builder/src/pages/builder/portal/users/groups/_components/EditUserPicker.svelte
+++ b/packages/builder/src/pages/builder/portal/users/groups/_components/EditUserPicker.svelte
@ -2,8 +2,7 @@
  import { Button, Popover, notifications } from "@budibase/bbui"
  import UserGroupPicker from "components/settings/UserGroupPicker.svelte"
  import { createPaginationStore } from "helpers/pagination"
-  import { auth, groups, users } from "stores/portal"
-  import { sdk } from "@budibase/shared-core"
+  import { groups, users } from "stores/portal"

  export let groupId
  export let onUsersUpdated
@ -14,7 +13,6 @@
  let prevSearch = undefined
  let pageInfo = createPaginationStore()

-  $: readonly = !sdk.users.isAdmin($auth.user)
  $: page = $pageInfo.page
  $: searchUsers(page, searchTerm)
  $: group = $groups.find(x => x._id === groupId)
@ -43,7 +41,7 @@
 </script>

 <div bind:this={popoverAnchor}>
-  <Button disabled={readonly} on:click={popover.show()} cta>Add user</Button>
+  <Button on:click={popover.show()} cta>Add user</Button>
 </div>
 <Popover align="right" bind:this={popover} anchor={popoverAnchor}>
  <UserGroupPicker
--- a/packages/builder/src/pages/builder/portal/users/groups/_components/GroupIcon.svelte
+++ b/packages/builder/src/pages/builder/portal/users/groups/_components/GroupIcon.svelte
@ -5,4 +5,12 @@
  export let size = "M"
 </script>

-<IconAvatar icon={group?.icon} background={group?.color} {size} />
+<div class="icon-group">
+  <IconAvatar icon={group?.icon} background={group?.color} {size} />
+</div>
+
+<style>
+  .icon-group {
+    position: relative;
+  }
+</style>
--- a/packages/builder/src/pages/builder/portal/users/groups/_components/GroupNameTableRenderer.svelte
+++ b/packages/builder/src/pages/builder/portal/users/groups/_components/GroupNameTableRenderer.svelte
@ -1,5 +1,6 @@
 <script>
  import GroupIcon from "./GroupIcon.svelte"
+  import ActiveDirectoryInfo from "../../_components/ActiveDirectoryInfo.svelte"

  export let value
  export let row
@ -14,13 +15,16 @@
  {:else}
    <div class="text">-</div>
  {/if}
+  {#if row.scimInfo?.isSync}
+    <ActiveDirectoryInfo iconSize="XS" />
+  {/if}
 </div>

 <style>
  .align {
    display: flex;
    align-items: center;
-    overflow: hidden;
+    overflow: visible;
    gap: var(--spacing-m);
    max-width: var(--max-cell-width);
    flex: 1 1 auto;
--- a/packages/builder/src/pages/builder/portal/users/groups/_components/GroupUsers.svelte
+++ b/packages/builder/src/pages/builder/portal/users/groups/_components/GroupUsers.svelte
@ -5,13 +5,15 @@
  import { fetchData } from "@budibase/frontend-core"
  import { goto } from "@roxi/routify"
  import { API } from "api"
-  import { auth, features, groups } from "stores/portal"
+  import { groups } from "stores/portal"
  import { setContext } from "svelte"
-  import ScimBanner from "../../_components/SCIMBanner.svelte"
+
  import RemoveUserTableRenderer from "../_components/RemoveUserTableRenderer.svelte"
-  import { sdk } from "@budibase/shared-core"
+  import ActiveDirectoryInfo from "../../_components/ActiveDirectoryInfo.svelte"

  export let groupId
+  export let readonly
+  export let isScimGroup

  let emailSearch
  let fetchGroupUsers
@ -49,9 +51,6 @@
    },
  ]

-  $: scimEnabled = $features.isScimEnabled
-  $: readonly = !sdk.users.isAdmin($auth.user) || scimEnabled
-
  const removeUser = async id => {
    await groups.actions.removeUser(groupId, id)
    fetchGroupUsers.refresh()
@ -63,10 +62,10 @@
 </script>

 <div class="header">
-  {#if !scimEnabled}
+  {#if isScimGroup}
+    <ActiveDirectoryInfo text="Users synced from your AD" />
+  {:else if !readonly}
    <EditUserPicker {groupId} onUsersUpdated={fetchGroupUsers.getInitialData} />
-  {:else}
-    <ScimBanner />
  {/if}

  <div class="controls-right">
--- a/packages/builder/src/pages/builder/portal/users/groups/index.svelte
+++ b/packages/builder/src/pages/builder/portal/users/groups/index.svelte
@ -13,7 +13,7 @@
    Search,
    notifications,
  } from "@budibase/bbui"
-  import { groups, auth, licensing, admin, features } from "stores/portal"
+  import { groups, auth, licensing, admin } from "stores/portal"
  import { onMount } from "svelte"
  import CreateEditGroupModal from "./_components/CreateEditGroupModal.svelte"
  import { cloneDeep } from "lodash/fp"
@ -21,7 +21,6 @@
  import UsersTableRenderer from "./_components/UsersTableRenderer.svelte"
  import GroupNameTableRenderer from "./_components/GroupNameTableRenderer.svelte"
  import { goto } from "@roxi/routify"
-  import ScimBanner from "../_components/SCIMBanner.svelte"
  import { sdk } from "@budibase/shared-core"

  const DefaultGroup = {
@ -110,14 +109,10 @@
  <div class="controls">
    <ButtonGroup>
      {#if $licensing.groupsEnabled}
-        {#if !$features.isScimEnabled}
-          <!--Show the group create button-->
-          <Button disabled={readonly} cta on:click={showCreateGroupModal}>
-            Add group
-          </Button>
-        {:else}
-          <ScimBanner />
-        {/if}
+        <!--Show the group create button-->
+        <Button disabled={readonly} cta on:click={showCreateGroupModal}>
+          Add group
+        </Button>
      {:else}
        <Button
          primary
--- a/packages/builder/src/pages/builder/portal/users/users/[userId].svelte
+++ b/packages/builder/src/pages/builder/portal/users/users/[userId].svelte
@ -18,7 +18,7 @@
    Table,
  } from "@budibase/bbui"
  import { onMount, setContext } from "svelte"
-  import { users, auth, groups, apps, licensing, features } from "stores/portal"
+  import { users, auth, groups, apps, licensing } from "stores/portal"
  import { roles } from "stores/builder"
  import ForceResetPasswordModal from "./_components/ForceResetPasswordModal.svelte"
  import UserGroupPicker from "components/settings/UserGroupPicker.svelte"
@ -30,8 +30,8 @@
  import GroupNameTableRenderer from "../groups/_components/GroupNameTableRenderer.svelte"
  import AppNameTableRenderer from "./_components/AppNameTableRenderer.svelte"
  import AppRoleTableRenderer from "./_components/AppRoleTableRenderer.svelte"
-  import ScimBanner from "../_components/SCIMBanner.svelte"
  import { sdk } from "@budibase/shared-core"
+  import ActiveDirectoryInfo from "../_components/ActiveDirectoryInfo.svelte"

  export let userId

@ -39,9 +39,10 @@
    name: {
      width: "1fr",
    },
-    ...(readonly
+    ...(!isAdmin
      ? {}
-      : {
+      : // Add
+        {
          _id: {
            displayName: "",
            width: "auto",
@ -87,12 +88,15 @@
  let user
  let loaded = false

-  $: scimEnabled = $features.isScimEnabled
+  $: internalGroups = $groups?.filter(g => !g?.scimInfo?.isSync)
+
  $: isSSO = !!user?.provider
-  $: readonly = !sdk.users.isAdmin($auth.user) || scimEnabled
+  $: isAdmin = sdk.users.isAdmin($auth.user)
+  $: isScim = user?.scimInfo?.isSync
+  $: readonly = !isAdmin || isScim
  $: privileged = sdk.users.isAdminOrGlobalBuilder(user)
  $: nameLabel = getNameLabel(user)
-  $: filteredGroups = getFilteredGroups($groups, searchTerm)
+  $: filteredGroups = getFilteredGroups(internalGroups, searchTerm)
  $: availableApps = getAvailableApps($apps, privileged, user?.roles)
  $: userGroups = $groups.filter(x => {
    return x.users?.find(y => {
@ -274,8 +278,8 @@
    <Layout noPadding gap="S">
      <div class="details-title">
        <Heading size="S">Details</Heading>
-        {#if scimEnabled}
-          <ScimBanner />
+        {#if user?.scimInfo?.isSync}
+          <ActiveDirectoryInfo text="User synced from your AD" />
        {/if}
      </div>
      <div class="fields">
@ -321,23 +325,23 @@
      <Layout gap="S" noPadding>
        <div class="tableTitle">
          <Heading size="S">Groups</Heading>
-          <div bind:this={popoverAnchor}>
-            <Button disabled={readonly} on:click={popover.show()} secondary>
-              Add to group
-            </Button>
-          </div>
-          <Popover align="right" bind:this={popover} anchor={popoverAnchor}>
-            <UserGroupPicker
-              labelKey="name"
-              bind:searchTerm
-              list={filteredGroups}
-              selected={user.userGroups}
-              on:select={e => addGroup(e.detail)}
-              on:deselect={e => removeGroup(e.detail)}
-              iconComponent={GroupIcon}
-              extractIconProps={item => ({ group: item, size: "S" })}
-            />
-          </Popover>
+          {#if internalGroups?.length && isAdmin}
+            <div bind:this={popoverAnchor}>
+              <Button on:click={popover.show()} secondary>Add to group</Button>
+            </div>
+            <Popover align="right" bind:this={popover} anchor={popoverAnchor}>
+              <UserGroupPicker
+                labelKey="name"
+                bind:searchTerm
+                list={filteredGroups}
+                selected={user.userGroups}
+                on:select={e => addGroup(e.detail)}
+                on:deselect={e => removeGroup(e.detail)}
+                iconComponent={GroupIcon}
+                extractIconProps={item => ({ group: item, size: "S" })}
+              />
+            </Popover>
+          {/if}
        </div>
        <Table
          schema={groupSchema}
--- a/packages/builder/src/pages/builder/portal/users/users/_components/AddUserModal.svelte
+++ b/packages/builder/src/pages/builder/portal/users/users/_components/AddUserModal.svelte
@ -33,6 +33,8 @@
  $: reached = licensing.usersLimitReached(userCount)
  $: exceeded = licensing.usersLimitExceeded(userCount)

+  $: internalGroups = $groups?.filter(g => !g?.scimInfo?.isSync)
+
  function removeInput(idx) {
    userData = userData.filter((e, i) => i !== idx)
  }
@ -133,12 +135,12 @@
    {/if}
  </Layout>

-  {#if $licensing.groupsEnabled}
+  {#if $licensing.groupsEnabled && internalGroups?.length}
    <Multiselect
      bind:value={userGroups}
      placeholder="No groups"
      label="Groups"
-      options={$groups}
+      options={internalGroups}
      getOptionLabel={option => option.name}
      getOptionValue={option => option._id}
    />
--- a/packages/builder/src/pages/builder/portal/users/users/_components/EmailTableRenderer.svelte
+++ b/packages/builder/src/pages/builder/portal/users/users/_components/EmailTableRenderer.svelte
@ -0,0 +1,11 @@
+<script>
+  import ActiveDirectoryInfo from "../../_components/ActiveDirectoryInfo.svelte"
+
+  export let value
+  export let row
+</script>
+
+{value}
+{#if row.scimInfo?.isSync}
+  <ActiveDirectoryInfo iconSize="XS" />
+{/if}
--- a/packages/builder/src/pages/builder/portal/users/users/_components/ImportUsersModal.svelte
+++ b/packages/builder/src/pages/builder/portal/users/users/_components/ImportUsersModal.svelte
@ -34,6 +34,8 @@
    label: `${option.label} - ${option.subtitle}`,
  }))

+  $: internalGroups = $groups?.filter(g => !g?.scimInfo?.isSync)
+
  const validEmails = userEmails => {
    if ($admin.cloud && userEmails.length > MAX_USERS_UPLOAD_LIMIT) {
      notifications.error(
@ -106,12 +108,12 @@
  {/if}
  <RadioGroup bind:value={usersRole} options={roleOptions} />

-  {#if $licensing.groupsEnabled}
+  {#if $licensing.groupsEnabled && internalGroups?.length}
    <Multiselect
      bind:value={userGroups}
      placeholder="No groups"
      label="Groups"
-      options={$groups}
+      options={internalGroups}
      getOptionLabel={option => option.name}
      getOptionValue={option => option._id}
    />
--- a/packages/builder/src/pages/builder/portal/users/users/_components/RemoveGroupTableRenderer.svelte
+++ b/packages/builder/src/pages/builder/portal/users/users/_components/RemoveGroupTableRenderer.svelte
@ -5,6 +5,7 @@
  import { sdk } from "@budibase/shared-core"

  export let value
+  export let row

  const groupContext = getContext("groups")

@ -12,12 +13,11 @@
    e.stopPropagation()
    groupContext.removeGroup(value)
  }
+
+  $: disabled = !sdk.users.isAdmin($auth.user) || row?.scimInfo?.isSync
+  $: tooltip = row?.scimInfo?.isSync && "This group is managed via your AD"
 </script>

-<ActionButton
-  disabled={!sdk.users.isAdmin($auth.user)}
-  size="S"
-  on:click={onClick}
+<ActionButton {disabled} size="S" on:click={onClick} {tooltip}
+  >Remove</ActionButton
 >
-  Remove
-</ActionButton>
--- a/packages/builder/src/pages/builder/portal/users/users/index.svelte
+++ b/packages/builder/src/pages/builder/portal/users/users/index.svelte
@ -19,7 +19,6 @@
    auth,
    licensing,
    organisation,
-    features,
    admin,
  } from "stores/portal"
  import { onMount } from "svelte"
@ -28,6 +27,7 @@
  import GroupsTableRenderer from "./_components/GroupsTableRenderer.svelte"
  import AppsTableRenderer from "./_components/AppsTableRenderer.svelte"
  import RoleTableRenderer from "./_components/RoleTableRenderer.svelte"
+  import EmailTableRenderer from "./_components/EmailTableRenderer.svelte"
  import { goto } from "@roxi/routify"
  import OnboardingTypeModal from "./_components/OnboardingTypeModal.svelte"
  import PasswordModal from "./_components/PasswordModal.svelte"
@ -37,7 +37,6 @@
  import { Constants, Utils, fetchData } from "@budibase/frontend-core"
  import { API } from "api"
  import { OnboardingType } from "constants"
-  import ScimBanner from "../_components/SCIMBanner.svelte"
  import { sdk } from "@budibase/shared-core"

  const fetch = fetchData({
@ -63,6 +62,7 @@
  let selectedRows = []
  let bulkSaveResponse
  let customRenderers = [
+    { column: "email", component: EmailTableRenderer },
    { column: "userGroups", component: GroupsTableRenderer },
    { column: "apps", component: AppsTableRenderer },
    { column: "role", component: RoleTableRenderer },
@ -73,7 +73,7 @@
  let parsedInvites = []

  $: isOwner = $auth.accountPortalAccess && $admin.cloud
-  $: readonly = !sdk.users.isAdmin($auth.user) || $features.isScimEnabled
+  $: readonly = !sdk.users.isAdmin($auth.user)
  $: debouncedUpdateFetch(searchEmail)
  $: schema = {
    email: {
@ -247,19 +247,25 @@
    }
  }

-  const deleteRows = async () => {
+  const deleteUsers = async () => {
    try {
      let ids = selectedRows.map(user => user._id)
      if (ids.includes(get(auth).user._id)) {
        notifications.error("You cannot delete yourself")
        return
      }
+
+      if (selectedRows.some(u => u.scimInfo?.isSync)) {
+        notifications.error("You cannot remove users created via your AD")
+        return
+      }
+
      await users.bulkDelete(ids)
      notifications.success(`Successfully deleted ${selectedRows.length} rows`)
      selectedRows = []
      await fetch.refresh()
    } catch (error) {
-      notifications.error("Error deleting rows")
+      notifications.error("Error deleting users")
    }
  }

@ -320,8 +326,6 @@
          Import
        </Button>
      </div>
-    {:else}
-      <ScimBanner />
    {/if}
    <div class="controls-right">
      <Search bind:value={searchEmail} placeholder="Search" />
@ -330,7 +334,7 @@
          item="user"
          on:updaterows
          {selectedRows}
-          {deleteRows}
+          deleteRows={deleteUsers}
        />
      {/if}
    </div>
@ -346,6 +350,7 @@
    {customRenderers}
    loading={!$fetch.loaded || !groupsLoaded}
  />
+
  <div class="pagination">
    <Pagination
      page={$fetch.pageNumber + 1}
@ -355,6 +360,7 @@
      goToNextPage={fetch.nextPage}
    />
  </div>
+
  <Table
    schema={pendingSchema}
    data={parsedInvites}
@ -402,6 +408,7 @@
    display: flex;
    flex-direction: row;
    justify-content: flex-end;
+    margin-left: auto;
  }
  .controls {
    display: flex;
--- a/packages/builder/src/stores/portal/groups.js
+++ b/packages/builder/src/stores/portal/groups.js
@ -35,7 +35,10 @@ export function createGroupsStore() {
    get: getGroup,

    save: async group => {
-      const response = await API.saveGroup(group)
+      const { ...dataToSave } = group
+      delete dataToSave.scimInfo
+      delete dataToSave.userGroups
+      const response = await API.saveGroup(dataToSave)
      group._id = response._id
      group._rev = response._rev
      updateStore(group)
--- a/packages/client/manifest.json
+++ b/packages/client/manifest.json
@ -1137,6 +1137,12 @@
        "key": "color",
        "showInBar": true
      },
+      {
+        "type": "color",
+        "label": "Text Color",
+        "key": "textColor",
+        "showInBar": true
+      },
      {
        "type": "boolean",
        "label": "Allow delete",
--- a/packages/client/src/components/app/Tag.svelte
+++ b/packages/client/src/components/app/Tag.svelte
@ -6,6 +6,7 @@
  export let onClick
  export let text = ""
  export let color
+  export let textColor
  export let closable = false
  export let size = "M"

@ -14,7 +15,7 @@

  // Add color styles to main styles object, otherwise the styleable helper
  // overrides the color when it's passed as inline style.
-  $: styles = enrichStyles($component.styles, color)
+  $: styles = enrichStyles($component.styles, color, textColor)
  $: componentText = getComponentText(text, $builderStore, $component)

  const getComponentText = (text, builderState, componentState) => {
@ -24,7 +25,7 @@
    return text || componentState.name || "Placeholder text"
  }

-  const enrichStyles = (styles, color) => {
+  const enrichStyles = (styles, color, textColor) => {
    if (!color) {
      return styles
    }
@ -34,7 +35,7 @@
        ...styles?.normal,
        "background-color": color,
        "border-color": color,
-        color: "white",
+        color: textColor || "white",
        "--spectrum-clearbutton-medium-icon-color": "white",
      },
    }
--- a/packages/pro
+++ b/packages/pro
@ -1 +1 @@
-Subproject commit 80a4d8ff998895fc298ee510158a82ce7daebc67
+Subproject commit c4c98ae70f2e936009250893898ecf11f4ddf2c3
--- a/packages/server/src/api/controllers/public/utils.ts
+++ b/packages/server/src/api/controllers/public/utils.ts
@ -1,11 +1,12 @@
 import { context } from "@budibase/backend-core"
 import { isExternalTableID } from "../../../integrations/utils"
 import { APP_PREFIX, DocumentType } from "../../../db/utils"
+import { Row } from "@budibase/types"

 export async function addRev(
  body: { _id?: string; _rev?: string },
  tableId?: string
-) {
+): Promise<Row> {
  if (!body._id || (tableId && isExternalTableID(tableId))) {
    return body
  }
--- a/packages/server/src/api/controllers/row/external.ts
+++ b/packages/server/src/api/controllers/row/external.ts
@ -128,7 +128,10 @@ export async function bulkDestroy(ctx: UserCtx) {
    )
  }
  const responses = await Promise.all(promises)
-  return { response: { ok: true }, rows: responses.map(resp => resp.row) }
+  const finalRows = responses
+    .map(resp => resp.row)
+    .filter(row => row && row._id)
+  return { response: { ok: true }, rows: finalRows }
 }

 export async function fetchEnrichedRow(ctx: UserCtx) {
--- a/packages/server/src/api/controllers/row/index.ts
+++ b/packages/server/src/api/controllers/row/index.ts
@ -131,7 +131,10 @@ async function processDeleteRowsRequest(ctx: UserCtx<DeleteRowRequest>) {
      : fixRow(processedRow, ctx.params)
  })

-  return await Promise.all(processedRows)
+  const responses = await Promise.allSettled(processedRows)
+  return responses
+    .filter(resp => resp.status === "fulfilled")
+    .map(resp => (resp as PromiseFulfilledResult<Row>).value)
 }

 async function deleteRows(ctx: UserCtx<DeleteRowRequest>) {
--- a/packages/server/src/api/routes/tests/row.spec.ts
+++ b/packages/server/src/api/routes/tests/row.spec.ts
@ -636,6 +636,17 @@ describe.each([
      expect(res[0]._id).toEqual(createdRow._id)
      await assertRowUsage(rowUsage - 1)
    })
+
+    it("should be able to bulk delete rows, including a row that doesn't exist", async () => {
+      const createdRow = await config.createRow()
+
+      const res = await config.api.row.bulkDelete(table._id!, {
+        rows: [createdRow, { _id: "2" }],
+      })
+
+      expect(res[0]._id).toEqual(createdRow._id)
+      expect(res.length).toEqual(1)
+    })
  })

  describe("validate", () => {
--- a/packages/server/src/startup.ts
+++ b/packages/server/src/startup.ts
@ -7,6 +7,7 @@ import {
  logging,
  tenancy,
  users,
+  cache,
 } from "@budibase/backend-core"
 import fs from "fs"
 import { watch } from "./watch"
@ -74,6 +75,7 @@ export async function startup(app?: Koa, server?: Server) {
  eventEmitter.emitPort(env.PORT)
  fileSystem.init()
  await redis.init()
+  cache.docWritethrough.init()
  eventInit()
  if (app && server) {
    initialiseWebsockets(app, server)
--- a/packages/server/src/threads/query.ts
+++ b/packages/server/src/threads/query.ts
@ -15,7 +15,7 @@ import { context, cache, auth } from "@budibase/backend-core"
 import { getGlobalIDFromUserMetadataID } from "../db/utils"
 import sdk from "../sdk"
 import { cloneDeep } from "lodash/fp"
-import { Datasource, Query, SourceName } from "@budibase/types"
+import { Datasource, Query, SourceName, Row } from "@budibase/types"

 import { isSQL } from "../integrations/utils"
 import { interpolateSQL } from "../integrations/queries/sql"
@ -115,7 +115,7 @@ class QueryRunner {
    }

    let output = threadUtils.formatResponse(await integration[queryVerb](query))
-    let rows = output,
+    let rows = output as Row[],
      info = undefined,
      extra = undefined,
      pagination = undefined
@ -170,7 +170,12 @@ class QueryRunner {
    }

    // get all the potential fields in the schema
-    let keys = rows.flatMap(Object.keys)
+    const keysSet: Set<string> = new Set()
+    rows.forEach(row => {
+      const keys = Object.keys(row)
+      keys.forEach(key => keysSet.add(key))
+    })
+    const keys: string[] = [...keysSet]

    if (integration.end) {
      integration.end()
--- a/packages/types/package.json
+++ b/packages/types/package.json
@ -23,6 +23,6 @@
    "typescript": "5.2.2"
  },
  "dependencies": {
-    "scim-patch": "^0.7.0"
+    "scim-patch": "^0.8.1"
  }
 }
--- a/packages/worker/src/api/routes/global/tests/groups.spec.ts
+++ b/packages/worker/src/api/routes/global/tests/groups.spec.ts
@ -104,17 +104,79 @@ describe("/api/global/groups", () => {
      expect(events.group.permissionsEdited).not.toBeCalled()
    })

-    describe("destroy", () => {
-      it("should be able to delete a basic group", async () => {
-        const group = structures.groups.UserGroup()
-        let oldGroup = await config.api.groups.saveGroup(group)
-        await config.api.groups.deleteGroup(
-          oldGroup.body._id,
-          oldGroup.body._rev
-        )
+    describe("scim", () => {
+      async function createScimGroup() {
+        mocks.licenses.useScimIntegration()
+        await config.setSCIMConfig(true)

-        expect(events.group.deleted).toBeCalledTimes(1)
+        const scimGroup = await config.api.scimGroupsAPI.post({
+          body: structures.scim.createGroupRequest({
+            displayName: generator.word(),
+          }),
+        })
+
+        const { body: group } = await config.api.groups.find(scimGroup.id)
+
+        expect(group).toBeDefined()
+        return group
+      }
+
+      it("update will not allow sending SCIM fields", async () => {
+        const group = await createScimGroup()
+
+        const updatedGroup: UserGroup = {
+          ...group,
+          name: generator.word(),
+        }
+        await config.api.groups.saveGroup(updatedGroup, {
+          expect: {
+            message: 'Invalid body - "scimInfo" is not allowed',
+            status: 400,
+          },
+        })
+
+        expect(events.group.updated).not.toBeCalled()
      })
+
+      it("update will not amend the SCIM fields", async () => {
+        const group: UserGroup = await createScimGroup()
+
+        const updatedGroup: UserGroup = {
+          ...group,
+          name: generator.word(),
+          scimInfo: undefined,
+        }
+
+        await config.api.groups.saveGroup(updatedGroup, {
+          expect: 200,
+        })
+
+        expect(events.group.updated).toBeCalledTimes(1)
+        expect(
+          (
+            await config.api.groups.find(group._id!, {
+              expect: 200,
+            })
+          ).body
+        ).toEqual(
+          expect.objectContaining({
+            ...group,
+            name: updatedGroup.name,
+            scimInfo: group.scimInfo,
+            _rev: expect.any(String),
+          })
+        )
+      })
+    })
+  })
+
+  describe("destroy", () => {
+    it("should be able to delete a basic group", async () => {
+      const group = structures.groups.UserGroup()
+      let oldGroup = await config.api.groups.saveGroup(group)
+      await config.api.groups.deleteGroup(oldGroup.body._id, oldGroup.body._rev)
+
+      expect(events.group.deleted).toBeCalledTimes(1)
    })
  })

@ -147,7 +209,7 @@ describe("/api/global/groups", () => {

        await Promise.all(
          Array.from({ length: 30 }).map(async (_, i) => {
-            const email = `user${i}@example.com`
+            const email = `user${i}+${generator.guid()}@example.com`
            const user = await config.api.users.saveUser({
              ...structures.users.user(),
              email,
@ -257,12 +319,16 @@ describe("/api/global/groups", () => {
      })
    })

-    it("update should return 200", async () => {
+    it("update should return forbidden", async () => {
      await config.withUser(builder, async () => {
-        await config.api.groups.updateGroupUsers(group._id!, {
-          add: [builder._id!],
-          remove: [],
-        })
+        await config.api.groups.updateGroupUsers(
+          group._id!,
+          {
+            add: [builder._id!],
+            remove: [],
+          },
+          { expect: 403 }
+        )
      })
    })
  })
--- a/packages/worker/src/api/routes/global/tests/scim.spec.ts
+++ b/packages/worker/src/api/routes/global/tests/scim.spec.ts
@ -2,6 +2,7 @@ import tk from "timekeeper"
 import _ from "lodash"
 import { generator, mocks, structures } from "@budibase/backend-core/tests"
 import {
+  CloudAccount,
  ScimCreateUserRequest,
  ScimGroupResponse,
  ScimUpdateRequest,
@ -10,25 +11,20 @@ import {
 import { TestConfiguration } from "../../../../tests"
 import { events } from "@budibase/backend-core"

-// this test can 409 - retries reduce issues with this
-jest.retryTimes(2, { logErrorsBeforeRetry: true })
 jest.setTimeout(30000)

 describe("scim", () => {
-  beforeAll(async () => {
-    tk.freeze(mocks.date.MOCK_DATE)
-    mocks.licenses.useScimIntegration()
-
-    await config.setSCIMConfig(true)
-  })
-
-  beforeEach(async () => {
+  async function setup() {
    jest.resetAllMocks()
    tk.freeze(mocks.date.MOCK_DATE)
    mocks.licenses.useScimIntegration()
+    mocks.licenses.useGroups()

    await config.setSCIMConfig(true)
-  })
+  }
+
+  beforeAll(setup)
+  beforeEach(setup)

  const config = new TestConfiguration()

@ -367,13 +363,77 @@ describe("scim", () => {
        })
      })

-      it("creating an existing user name returns a conflict", async () => {
-        const body = structures.scim.createUserRequest()
+      it("creating an external user that conflicts an internal one syncs the existing user", async () => {
+        const { body: internalUser } = await config.api.users.saveUser(
+          structures.users.user()
+        )

-        await postScimUser({ body })
+        const scimUserData = {
+          externalId: structures.uuid(),
+          email: internalUser.email,
+          firstName: structures.generator.first(),
+          lastName: structures.generator.last(),
+          username: structures.generator.name(),
+        }
+        const scimUserRequest = structures.scim.createUserRequest(scimUserData)

-        const res = await postScimUser({ body }, { expect: 409 })
-        expect((res as any).message).toBe("Email already in use")
+        const res = await postScimUser(
+          { body: scimUserRequest },
+          { expect: 200 }
+        )
+
+        const expectedScimUser: ScimUserResponse = {
+          schemas: ["urn:ietf:params:scim:schemas:core:2.0:User"],
+          id: internalUser._id!,
+          externalId: scimUserRequest.externalId,
+          meta: {
+            resourceType: "User",
+            // @ts-ignore
+            created: mocks.date.MOCK_DATE.toISOString(),
+            // @ts-ignore
+            lastModified: mocks.date.MOCK_DATE.toISOString(),
+          },
+          userName: scimUserData.username,
+          name: {
+            formatted: `${scimUserData.firstName} ${scimUserData.lastName}`,
+            familyName: scimUserData.lastName,
+            givenName: scimUserData.firstName,
+          },
+          active: true,
+          emails: [
+            {
+              value: internalUser.email,
+              type: "work",
+              primary: true,
+            },
+          ],
+        }
+
+        expect(res).toEqual(expectedScimUser)
+      })
+
+      it("a user cannot be SCIM synchronised with another SCIM user", async () => {
+        const { body: internalUser } = await config.api.users.saveUser(
+          structures.users.user()
+        )
+
+        await postScimUser(
+          {
+            body: structures.scim.createUserRequest({
+              email: internalUser.email,
+            }),
+          },
+          { expect: 200 }
+        )
+
+        await postScimUser(
+          {
+            body: structures.scim.createUserRequest({
+              email: internalUser.email,
+            }),
+          },
+          { expect: 409 }
+        )
      })
    })

@ -545,6 +605,25 @@ describe("scim", () => {

        expect(events.user.deleted).toBeCalledTimes(1)
      })
+
+      it("an account holder cannot be removed even when synched", async () => {
+        const account: CloudAccount = {
+          ...structures.accounts.account(),
+          budibaseUserId: user.id,
+          email: user.emails![0].value,
+        }
+        mocks.accounts.getAccount.mockResolvedValue(account)
+
+        await deleteScimUser(user.id, {
+          expect: {
+            message: "Account holder cannot be deleted",
+            status: 400,
+            error: { code: "http" },
+          },
+        })
+
+        await config.api.scimUsersAPI.find(user.id, { expect: 200 })
+      })
    })
  })

@ -656,7 +735,6 @@ describe("scim", () => {
        })

        it("can fetch groups even if internal groups exist", async () => {
-          mocks.licenses.useGroups()
          await config.api.groups.saveGroup(structures.userGroups.userGroup())
          await config.api.groups.saveGroup(structures.userGroups.userGroup())

@ -722,6 +800,43 @@ describe("scim", () => {
          )
        })
      })
+
+      it("creating an external group that conflicts an internal one syncs the existing group", async () => {
+        const groupToSave = structures.userGroups.userGroup()
+        const { body: internalGroup } = await config.api.groups.saveGroup(
+          groupToSave
+        )
+
+        const scimGroupData = {
+          externalId: structures.uuid(),
+          displayName: groupToSave.name,
+        }
+
+        const res = await postScimGroup(
+          { body: structures.scim.createGroupRequest(scimGroupData) },
+          { expect: 200 }
+        )
+
+        expect(res).toEqual(
+          expect.objectContaining({
+            id: internalGroup._id!,
+            externalId: scimGroupData.externalId,
+            displayName: scimGroupData.displayName,
+          })
+        )
+      })
+
+      it("a group cannot be SCIM synchronised with another SCIM group", async () => {
+        const groupToSave = structures.userGroups.userGroup()
+        await config.api.groups.saveGroup(groupToSave)
+
+        const createGroupRequest = structures.scim.createGroupRequest({
+          displayName: groupToSave.name,
+        })
+        await postScimGroup({ body: createGroupRequest }, { expect: 200 })
+
+        await postScimGroup({ body: createGroupRequest }, { expect: 409 })
+      })
    })

    describe("GET /api/global/scim/v2/groups/:id", () => {
--- a/packages/worker/src/index.ts
+++ b/packages/worker/src/index.ts
@ -17,6 +17,7 @@ import {
  env as coreEnv,
  timers,
  redis,
+  cache,
 } from "@budibase/backend-core"

 db.init()
@ -90,6 +91,7 @@ export default server.listen(parseInt(env.PORT || "4002"), async () => {
  console.log(`Worker running on ${JSON.stringify(server.address())}`)
  await initPro()
  await redis.clients.init()
+  cache.docWritethrough.init()
  // configure events to use the pro audit log write
  // can't integrate directly into backend-core due to cyclic issues
  await events.processors.init(proSdk.auditLogs.write)
--- a/packages/worker/src/tests/api/groups.ts
+++ b/packages/worker/src/tests/api/groups.ts
@ -7,7 +7,10 @@ export class GroupsAPI extends TestAPI {
    super(config)
  }

-  saveGroup = (group: UserGroup, { expect } = { expect: 200 }) => {
+  saveGroup = (
+    group: UserGroup,
+    { expect }: { expect: number | object } = { expect: 200 }
+  ) => {
    return this.request
      .post(`/api/global/groups`)
      .send(group)
@ -44,14 +47,15 @@ export class GroupsAPI extends TestAPI {

  updateGroupUsers = (
    id: string,
-    body: { add: string[]; remove: string[] }
+    body: { add: string[]; remove: string[] },
+    { expect } = { expect: 200 }
  ) => {
    return this.request
      .post(`/api/global/groups/${id}/users`)
      .send(body)
      .set(this.config.defaultHeaders())
      .expect("Content-Type", /json/)
-      .expect(200)
+      .expect(expect)
  }

  fetch = ({ expect } = { expect: 200 }) => {
@ -61,4 +65,12 @@ export class GroupsAPI extends TestAPI {
      .expect("Content-Type", /json/)
      .expect(expect)
  }
+
+  find = (id: string, { expect } = { expect: 200 }) => {
+    return this.request
+      .get(`/api/global/groups/${id}`)
+      .set(this.config.defaultHeaders())
+      .expect("Content-Type", /json/)
+      .expect(expect)
+  }
 }
--- a/packages/worker/src/tests/api/scim/shared.ts
+++ b/packages/worker/src/tests/api/scim/shared.ts
@ -1,13 +1,17 @@
 import TestConfiguration from "../../TestConfiguration"
 import { TestAPI } from "../base"

-const defaultConfig = {
+const defaultConfig: RequestSettings = {
  expect: 200,
  setHeaders: true,
  skipContentTypeCheck: false,
 }

-export type RequestSettings = typeof defaultConfig
+export type RequestSettings = {
+  expect: number | object
+  setHeaders: boolean
+  skipContentTypeCheck: boolean
+}

 export abstract class ScimTestAPI extends TestAPI {
  constructor(config: TestConfiguration) {
--- a/yarn.lock
+++ b/yarn.lock
@ -5666,6 +5666,13 @@
  dependencies:
    undici-types "~5.26.4"

+"@types/node@^20.4.5":
+  version "20.11.25"
+  resolved "https://registry.yarnpkg.com/@types/node/-/node-20.11.25.tgz#0f50d62f274e54dd7a49f7704cc16bfbcccaf49f"
+  integrity sha512-TBHyJxk2b7HceLVGFcpAUjsa5zIdsPWlR6XHfyGzd0SFu+/NFgQgMAl96MSDZgQDvJAvV6BKsFOrt6zIL09JDw==
+  dependencies:
+    undici-types "~5.26.4"
+
 "@types/nodemailer@^6.4.4":
  version "6.4.14"
  resolved "https://registry.yarnpkg.com/@types/nodemailer/-/nodemailer-6.4.14.tgz#5c81a5e856db7f8ede80013e6dbad7c5fb2283e2"
@ -19491,11 +19498,12 @@ schema-utils@^3.1.1, schema-utils@^3.1.2:
    ajv "^6.12.5"
    ajv-keywords "^3.5.2"

-scim-patch@^0.7.0:
-  version "0.7.0"
-  resolved "https://registry.yarnpkg.com/scim-patch/-/scim-patch-0.7.0.tgz#3f6d94256c07be415a74a49c0ff48dc91e4e0219"
-  integrity sha512-wXKcsZl+aLfE0yId7MjiOd91v8as6dEYLFvm1gGu3yJxSPhl1Fl3vWiNN4V3D68UKpqO/umK5rwWc8wGpBaOHw==
+scim-patch@^0.8.1:
+  version "0.8.1"
+  resolved "https://registry.yarnpkg.com/scim-patch/-/scim-patch-0.8.1.tgz#611bfdb5538f6d8b97aba0ab0f8bd01055b70c1c"
+  integrity sha512-JRYTA+mJZ8Z5DJGO7kkFc0lGCDs100rNs7iN77mld7gQajTp1R1xjUzMfZTOMAkBDA75GSdbMmfdfMqMJCn/Yg==
  dependencies:
+    "@types/node" "^20.4.5"
    fast-deep-equal "3.1.3"
    scim2-parse-filter "0.2.8"