diff --git a/packages/bbui/src/Actions/click_outside.js b/packages/bbui/src/Actions/click_outside.js
index 12c4c4d002..2c54427b3a 100644
--- a/packages/bbui/src/Actions/click_outside.js
+++ b/packages/bbui/src/Actions/click_outside.js
@@ -39,19 +39,23 @@ const handleClick = event => {
       return
     }
 
+    if (handler.allowedType && event.type !== handler.allowedType) {
+      return
+    }
+
     handler.callback?.(event)
   })
 }
 document.documentElement.addEventListener("click", handleClick, true)
-document.documentElement.addEventListener("contextmenu", handleClick, true)
+document.documentElement.addEventListener("mousedown", handleClick, true)
 
 /**
  * Adds or updates a click handler
  */
-const updateHandler = (id, element, anchor, callback) => {
+const updateHandler = (id, element, anchor, callback, allowedType) => {
   let existingHandler = clickHandlers.find(x => x.id === id)
   if (!existingHandler) {
-    clickHandlers.push({ id, element, anchor, callback })
+    clickHandlers.push({ id, element, anchor, callback, allowedType })
   } else {
     existingHandler.callback = callback
   }
@@ -77,7 +81,8 @@ export default (element, opts) => {
   const update = newOpts => {
     const callback = newOpts?.callback || newOpts
     const anchor = newOpts?.anchor || element
-    updateHandler(id, element, anchor, callback)
+    const allowedType = newOpts?.allowedType || "click"
+    updateHandler(id, element, anchor, callback, allowedType)
   }
   update(opts)
   return {
diff --git a/packages/client/src/components/app/Layout.svelte b/packages/client/src/components/app/Layout.svelte
index 1b92c0fefc..992a166143 100644
--- a/packages/client/src/components/app/Layout.svelte
+++ b/packages/client/src/components/app/Layout.svelte
@@ -291,7 +291,10 @@
   <div
     id="side-panel-container"
     class:open={$sidePanelStore.open}
-    use:clickOutside={autoCloseSidePanel ? sidePanelStore.actions.close : null}
+    use:clickOutside={{
+      callback: autoCloseSidePanel ? sidePanelStore.actions.close : null,
+      allowedType: "mousedown",
+    }}
     class:builder={$builderStore.inBuilder}
   >
     <div class="side-panel-header">