Skip to content

Use RayMap and RenderLayers in bevy_sprite/picking_backend #21959

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 4 commits into
base: main
Choose a base branch
from
+150 −115
Open

Use RayMap and RenderLayers in bevy_sprite/picking_backend #21959

Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
5129560
Logic Upgrade: Iterate RayMap to generate pointer hits for each viabl…
bloopyboop Nov 27, 2025
73b859b
Take RenderLayers into account when generating pointer hits.
bloopyboop Nov 27, 2025
1769e43
cargo fmt
bloopyboop Nov 27, 2025
036b335
satisfying clippy
bloopyboop Nov 27, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
265 changes: 150 additions & 115 deletions crates/bevy_sprite/src/picking_backend.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,10 @@
use crate::{Anchor, Sprite};
use bevy_app::prelude::*;
use bevy_asset::prelude::*;
use bevy_camera::{visibility::ViewVisibility, Camera, Projection};
use bevy_camera::{
visibility::{RenderLayers, ViewVisibility},
Camera, Projection,
};
use bevy_color::Alpha;
use bevy_ecs::prelude::*;
use bevy_image::prelude::*;
Expand Down Expand Up @@ -88,6 +91,7 @@ fn sprite_picking(
&GlobalTransform,
&Projection,
Has<SpritePickingCamera>,
Option<&RenderLayers>,
)>,
primary_window: Query<Entity, With<PrimaryWindow>>,
images: Res<Assets<Image>>,
Expand All @@ -100,157 +104,188 @@ fn sprite_picking(
&Anchor,
&Pickable,
&ViewVisibility,
Option<&RenderLayers>,
)>,
mut pointer_hits_writer: MessageWriter<PointerHits>,
ray_map: Res<RayMap>,
) {
let mut sorted_sprites: Vec<_> = sprite_query
.iter()
.filter_map(|(entity, sprite, transform, anchor, pickable, vis)| {
if !transform.affine().is_nan() && vis.get() {
Some((entity, sprite, transform, anchor, pickable))
} else {
None
}
})
.filter_map(
|(entity, sprite, transform, anchor, pickable, vis, render_layers)| {
if !transform.affine().is_nan() && vis.get() {
Some((entity, sprite, transform, anchor, pickable, render_layers))
} else {
None
}
},
)
.collect();

// radsort is a stable radix sort that performed better than `slice::sort_by_key`
radsort::sort_by_key(&mut sorted_sprites, |(_, _, transform, _, _)| {
radsort::sort_by_key(&mut sorted_sprites, |(_, _, transform, _, _, _)| {
-transform.translation().z
});

let primary_window = primary_window.single().ok();

for (pointer, location) in pointers.iter().filter_map(|(pointer, pointer_location)| {
pointer_location.location().map(|loc| (pointer, loc))
}) {
let pick_sets = ray_map.iter().flat_map(|(ray_id, ray)| {
let mut blocked = false;
let Some((cam_entity, camera, cam_transform, Projection::Orthographic(cam_ortho), _)) =
cameras
.iter()
.filter(|(_, camera, _, _, cam_can_pick)| {
let marker_requirement = !settings.require_markers || *cam_can_pick;
camera.is_active && marker_requirement
})
.find(|(_, camera, _, _, _)| {
camera
.target
.normalize(primary_window)
.is_some_and(|x| x == location.target)
})

let Ok((
cam_entity,
camera,
cam_transform,
Projection::Orthographic(cam_ortho),
cam_can_pick,
cam_render_layers,
)) = cameras.get(ray_id.camera)
else {
continue;
return None;
};

let marker_requirement = !settings.require_markers || cam_can_pick;
if !camera.is_active || !marker_requirement {
return None;
}

let location = pointers.iter().find_map(|(id, loc)| {
if *id == ray_id.pointer {
return loc.location.as_ref();
}
None
})?;

if camera
.target
.normalize(primary_window)
.is_none_or(|x| x != location.target)
{
return None;
}

let viewport_pos = location.position;
if let Some(viewport) = camera.logical_viewport_rect()
&& !viewport.contains(viewport_pos)
{
// The pointer is outside the viewport, skip it
continue;
return None;
}

let Ok(cursor_ray_world) = camera.viewport_to_world(cam_transform, viewport_pos) else {
continue;
};
let cursor_ray_len = cam_ortho.far - cam_ortho.near;
let cursor_ray_end = cursor_ray_world.origin + cursor_ray_world.direction * cursor_ray_len;
let cursor_ray_end = ray.origin + ray.direction * cursor_ray_len;

let picks: Vec<(Entity, HitData)> = sorted_sprites
.iter()
.copied()
.filter_map(|(entity, sprite, sprite_transform, anchor, pickable)| {
if blocked {
return None;
}
.filter_map(
|(entity, sprite, sprite_transform, anchor, pickable, sprite_render_layers)| {
if blocked {
return None;
}

// Transform cursor line segment to sprite coordinate system
let world_to_sprite = sprite_transform.affine().inverse();
let cursor_start_sprite = world_to_sprite.transform_point3(cursor_ray_world.origin);
let cursor_end_sprite = world_to_sprite.transform_point3(cursor_ray_end);
// Filter out sprites based on whether they share RenderLayers with the current
// ray's associated camera.
// Any entity without a RenderLayers component will by default be
// on RenderLayers::layer(0) only.
if !cam_render_layers
.unwrap_or_default()
.intersects(sprite_render_layers.unwrap_or_default())
{
return None;
}

// Find where the cursor segment intersects the plane Z=0 (which is the sprite's
// plane in sprite-local space). It may not intersect if, for example, we're
// viewing the sprite side-on
if cursor_start_sprite.z == cursor_end_sprite.z {
// Cursor ray is parallel to the sprite and misses it
return None;
}
let lerp_factor =
f32::inverse_lerp(cursor_start_sprite.z, cursor_end_sprite.z, 0.0);
if !(0.0..=1.0).contains(&lerp_factor) {
// Lerp factor is out of range, meaning that while an infinite line cast by
// the cursor would intersect the sprite, the sprite is not between the
// camera's near and far planes
return None;
}
// Otherwise we can interpolate the xy of the start and end positions by the
// lerp factor to get the cursor position in sprite space!
let cursor_pos_sprite = cursor_start_sprite
.lerp(cursor_end_sprite, lerp_factor)
.xy();
// Transform cursor line segment to sprite coordinate system
let world_to_sprite = sprite_transform.affine().inverse();
let cursor_start_sprite = world_to_sprite.transform_point3(ray.origin);
let cursor_end_sprite = world_to_sprite.transform_point3(cursor_ray_end);

// Find where the cursor segment intersects the plane Z=0 (which is the sprite's
// plane in sprite-local space). It may not intersect if, for example, we're
// viewing the sprite side-on
if cursor_start_sprite.z == cursor_end_sprite.z {
// Cursor ray is parallel to the sprite and misses it
return None;
}
let lerp_factor =
f32::inverse_lerp(cursor_start_sprite.z, cursor_end_sprite.z, 0.0);
if !(0.0..=1.0).contains(&lerp_factor) {
// Lerp factor is out of range, meaning that while an infinite line cast by
// the cursor would intersect the sprite, the sprite is not between the
// camera's near and far planes
return None;
}
// Otherwise we can interpolate the xy of the start and end positions by the
// lerp factor to get the cursor position in sprite space!
let cursor_pos_sprite = cursor_start_sprite
.lerp(cursor_end_sprite, lerp_factor)
.xy();

let Ok(cursor_pixel_space) = sprite.compute_pixel_space_point(
cursor_pos_sprite,
*anchor,
&images,
&texture_atlas_layout,
) else {
return None;
};
let Ok(cursor_pixel_space) = sprite.compute_pixel_space_point(
cursor_pos_sprite,
*anchor,
&images,
&texture_atlas_layout,
) else {
return None;
};

// Since the pixel space coordinate is `Ok`, we know the cursor is in the bounds of
// the sprite.
// Since the pixel space coordinate is `Ok`, we know the cursor is in the bounds of
// the sprite.

let cursor_in_valid_pixels_of_sprite = 'valid_pixel: {
match settings.picking_mode {
SpritePickingMode::AlphaThreshold(cutoff) => {
let Some(image) = images.get(&sprite.image) else {
// [`Sprite::from_color`] returns a defaulted handle.
// This handle doesn't return a valid image, so returning false here would make picking "color sprites" impossible
break 'valid_pixel true;
};
// grab pixel and check alpha
let Ok(color) = image.get_color_at(
cursor_pixel_space.x as u32,
cursor_pixel_space.y as u32,
) else {
// We don't know how to interpret the pixel.
break 'valid_pixel false;
};
// Check the alpha is above the cutoff.
color.alpha() > cutoff
let cursor_in_valid_pixels_of_sprite = 'valid_pixel: {
match settings.picking_mode {
SpritePickingMode::AlphaThreshold(cutoff) => {
let Some(image) = images.get(&sprite.image) else {
// [`Sprite::from_color`] returns a defaulted handle.
// This handle doesn't return a valid image, so returning false here would make picking "color sprites" impossible
break 'valid_pixel true;
};
// grab pixel and check alpha
let Ok(color) = image.get_color_at(
cursor_pixel_space.x as u32,
cursor_pixel_space.y as u32,
) else {
// We don't know how to interpret the pixel.
break 'valid_pixel false;
};
// Check the alpha is above the cutoff.
color.alpha() > cutoff
}
SpritePickingMode::BoundingBox => true,
}
SpritePickingMode::BoundingBox => true,
}
};
};

blocked = cursor_in_valid_pixels_of_sprite && pickable.should_block_lower;
blocked = cursor_in_valid_pixels_of_sprite && pickable.should_block_lower;

cursor_in_valid_pixels_of_sprite.then(|| {
let hit_pos_world =
sprite_transform.transform_point(cursor_pos_sprite.extend(0.0));
// Transform point from world to camera space to get the Z distance
let hit_pos_cam = cam_transform
.affine()
.inverse()
.transform_point3(hit_pos_world);
// HitData requires a depth as calculated from the camera's near clipping plane
let depth = -cam_ortho.near - hit_pos_cam.z;
(
entity,
HitData::new(
cam_entity,
depth,
Some(hit_pos_world),
Some(*sprite_transform.back()),
),
)
})
})
cursor_in_valid_pixels_of_sprite.then(|| {
let hit_pos_world =
sprite_transform.transform_point(cursor_pos_sprite.extend(0.0));
// Transform point from world to camera space to get the Z distance
let hit_pos_cam = cam_transform
.affine()
.inverse()
.transform_point3(hit_pos_world);
// HitData requires a depth as calculated from the camera's near clipping plane
let depth = -cam_ortho.near - hit_pos_cam.z;
(
entity,
HitData::new(
cam_entity,
depth,
Some(hit_pos_world),
Some(*sprite_transform.back()),
),
)
})
},
)
.collect();

let order = camera.order as f32;
pointer_hits_writer.write(PointerHits::new(*pointer, picks, order));
}
Some((ray_id.pointer, picks, camera.order))
});

pick_sets.for_each(|(pointer, picks, order)| {
pointer_hits_writer.write(PointerHits::new(pointer, picks, order as f32));
});
}