diff --git a/crates/bevy_ui/src/picking_backend.rs b/crates/bevy_ui/src/picking_backend.rs
index 5647baee12a8c..34efbc962a836 100644
--- a/crates/bevy_ui/src/picking_backend.rs
+++ b/crates/bevy_ui/src/picking_backend.rs
@@ -140,6 +140,10 @@ pub fn ui_picking(
             let mut pointer_pos =
                 pointer_location.position * camera_data.target_scaling_factor().unwrap_or(1.);
             if let Some(viewport) = camera_data.physical_viewport_rect() {
+                if !viewport.as_rect().contains(pointer_pos) {
+                    // The pointer is outside the viewport, skip it
+                    continue;
+                }
                 pointer_pos -= viewport.min.as_vec2();
             }
             pointer_pos_by_camera