diff --git a/core/modules/layout_builder/js/layout-builder.js b/core/modules/layout_builder/js/layout-builder.js
index 76b751c6ccf..ef85ad8533e 100644
--- a/core/modules/layout_builder/js/layout-builder.js
+++ b/core/modules/layout_builder/js/layout-builder.js
@@ -166,6 +166,7 @@
             draggable: '.js-layout-builder-block',
             ghostClass: 'ui-state-drop',
             group: 'builder-region',
+            filter: '.contextual',
             onEnd: (event) =>
               Drupal.layoutBuilderBlockUpdate(event.item, event.from, event.to),
           });