3D->2D & 2D->3D selection visualizations stick now around on selection (

#4587)

### What

* Fixes: #3047

Makes the 2D->3D and 3D->2D sticky upon selection!
Going the extra mile to make selection history be aware of this added a
bit more stuff, but the code is better off overall I think :)



https://github.com/rerun-io/rerun/assets/1220815/b59f6f07-b7f4-4e62-b14e-2d8385f49fd5



**Update!** Now also works fine with multielection!
We now associate each item selection with an optional space context.
Right now you can't space context without items, but this is not a
structural limitation.
Toggle (ctrl+cmd select action) behavior has a few new special rules
now: If you change the space view context on an already selected object
its context is updated. If you add a new object, we add it to the list
_with_ the context.
Overall this paves also the way to displaying the context together with
the selected item in a more sane way. Should be easier to reason with in
general I believe.




https://github.com/rerun-io/rerun/assets/1220815/728b569f-e527-4dcd-b346-1007dccea8c7



### Checklist
* [x] I have read and agree to [Contributor
Guide](https://github.com/rerun-io/rerun/blob/main/CONTRIBUTING.md) and
the [Code of
Conduct](https://github.com/rerun-io/rerun/blob/main/CODE_OF_CONDUCT.md)
* [x] I've included a screenshot or gif (if applicable)
* [x] I have tested the web demo (if applicable):
* Using newly built examples:
[app.rerun.io](https://app.rerun.io/pr/4587/index.html)
* Using examples from latest `main` build:
[app.rerun.io](https://app.rerun.io/pr/4587/index.html?manifest_url=https://app.rerun.io/version/main/examples_manifest.json)
* Using full set of examples from `nightly` build:
[app.rerun.io](https://app.rerun.io/pr/4587/index.html?manifest_url=https://app.rerun.io/version/nightly/examples_manifest.json)
* [x] The PR title and labels are set such as to maximize their
usefulness for the next release's CHANGELOG

- [PR Build Summary](https://build.rerun.io/pr/4587)
- [Docs
preview](https://rerun.io/preview/5d4f28365919b9ad57af36e5a7779ec8e9d156a8/docs)
<!--DOCS-PREVIEW-->
- [Examples
preview](https://rerun.io/preview/5d4f28365919b9ad57af36e5a7779ec8e9d156a8/examples)
<!--EXAMPLES-PREVIEW-->
- [Recent benchmark results](https://build.rerun.io/graphs/crates.html)
- [Wasm size tracking](https://build.rerun.io/graphs/sizes.html)

crates/re_data_ui/src/item_ui.rs

@@ -6,7 +6,7 @@ use egui::Ui;
 use re_data_store::{EntityTree, InstancePath};
 use re_log_types::{ComponentPath, EntityPath, TimeInt, Timeline};
 use re_viewer_context::{
-    DataQueryId, HoverHighlight, Item, SpaceViewId, UiVerbosity, ViewerContext,
+    DataQueryId, HoverHighlight, Item, Selection, SpaceViewId, UiVerbosity, ViewerContext,
 };
 
 use super::DataUi;
@@ -97,7 +97,7 @@ pub fn instance_path_button_to(
     let item = Item::InstancePath(space_view_id, instance_path.clone());
 
     let response = ui
-        .selectable_label(ctx.selection().contains(&item), text)
+        .selectable_label(ctx.selection().contains_item(&item), text)
         .on_hover_ui(|ui| {
             instance_hover_card_ui(ui, ctx, instance_path);
         });
@@ -205,7 +205,7 @@ pub fn component_path_button_to(
     component_path: &ComponentPath,
 ) -> egui::Response {
     let item = Item::ComponentPath(component_path.clone());
-    let response = ui.selectable_label(ctx.selection().contains(&item), text);
+    let response = ui.selectable_label(ctx.selection().contains_item(&item), text);
     cursor_interact_with_selectable(ctx, response, item)
 }
 
@@ -224,7 +224,7 @@ pub fn data_blueprint_group_button_to(
             ui,
             &re_ui::icons::CONTAINER,
             text,
-            ctx.selection().contains(&item),
+            ctx.selection().contains_item(&item),
         )
         .on_hover_text("Group");
     cursor_interact_with_selectable(ctx, response, item)
@@ -242,7 +242,7 @@ pub fn data_blueprint_button_to(
         InstancePath::entity_splat(entity_path.clone()),
     );
     let response = ui
-        .selectable_label(ctx.selection().contains(&item), text)
+        .selectable_label(ctx.selection().contains_item(&item), text)
         .on_hover_ui(|ui| {
             entity_hover_card_ui(ui, ctx, entity_path);
         });
@@ -313,7 +313,7 @@ pub fn cursor_interact_with_selectable(
     let is_item_hovered =
         ctx.selection_state().highlight_for_ui_element(&item) == HoverHighlight::Hovered;
 
-    select_hovered_on_click(ctx, &response, &[item]);
+    select_hovered_on_click(ctx, &response, item);
     // TODO(andreas): How to deal with shift click for selecting ranges?
 
     if is_item_hovered {
@@ -324,18 +324,26 @@ pub fn cursor_interact_with_selectable(
 }
 
 // TODO(andreas): Move elsewhere, this is not directly part of the item_ui.
-pub fn select_hovered_on_click(ctx: &ViewerContext<'_>, response: &egui::Response, items: &[Item]) {
+pub fn select_hovered_on_click(
+    ctx: &ViewerContext<'_>,
+    response: &egui::Response,
+    selection: impl Into<Selection>,
+) {
     re_tracing::profile_function!();
 
+    let mut selection = selection.into();
+    selection.resolve_mono_instance_path_items(ctx);
+    let selection_state = ctx.selection_state();
+
     if response.hovered() {
-        ctx.set_hovered(items.iter());
+        selection_state.set_hovered(selection.clone());
     }
 
     if response.clicked() {
         if response.ctx.input(|i| i.modifiers.command) {
-            ctx.selection_state().toggle_selection(items.to_vec());
+            selection_state.toggle_selection(selection);
         } else {
-            ctx.selection_state().set_selection(items.iter().cloned());
+            selection_state.set_selection(selection);
         }
     }
 }

crates/re_space_view_spatial/src/ui.rs

@@ -10,10 +10,9 @@ use re_space_view::ScreenshotMode;
 use re_types::components::{DepthMeter, InstanceKey, TensorData};
 use re_types::tensor_data::TensorDataMeaning;
 use re_viewer_context::{
-    resolve_mono_instance_path, HoverHighlight, HoveredSpace, Item, SelectionHighlight,
-    SpaceViewHighlights, SpaceViewState, SpaceViewSystemExecutionError, TensorDecodeCache,
-    TensorStatsCache, UiVerbosity, ViewContextCollection, ViewPartCollection, ViewQuery,
-    ViewerContext,
+    HoverHighlight, Item, SelectedSpaceContext, SelectionHighlight, SpaceViewHighlights,
+    SpaceViewState, SpaceViewSystemExecutionError, TensorDecodeCache, TensorStatsCache,
+    UiVerbosity, ViewContextCollection, ViewPartCollection, ViewQuery, ViewerContext,
 };
 
 use super::{eye::Eye, ui_2d::View2DState, ui_3d::View3DState};
@@ -564,8 +563,6 @@ pub fn picking(
         if picked_image_with_coords.is_some() {
             // We don't support selecting pixels yet.
             instance_path.instance_key = InstanceKey::SPLAT;
-        } else {
-            instance_path = resolve_mono_instance_path(&ctx.current_query(), store, &instance_path);
         }
 
         hovered_items.push(Item::InstancePath(
@@ -627,36 +624,45 @@ pub fn picking(
         };
     }
 
-    item_ui::select_hovered_on_click(ctx, &response, &hovered_items);
-
-    let hovered_space = match spatial_kind {
-        SpatialSpaceViewKind::TwoD => HoveredSpace::TwoD {
-            space_2d: query.space_origin.clone(),
-            pos: picking_context
-                .pointer_in_space2d
-                .extend(depth_at_pointer.unwrap_or(f32::INFINITY)),
-        },
-        SpatialSpaceViewKind::ThreeD => {
-            let hovered_point = picking_result.space_position();
-            HoveredSpace::ThreeD {
-                space_3d: query.space_origin.clone(),
-                pos: hovered_point,
-                tracked_space_camera: state.state_3d.tracked_camera.clone(),
-                point_in_space_cameras: parts
-                    .get::<CamerasPart>()?
-                    .space_cameras
-                    .iter()
-                    .map(|cam| {
-                        (
-                            cam.ent_path.clone(),
-                            hovered_point.and_then(|pos| cam.project_onto_2d(pos)),
-                        )
-                    })
-                    .collect(),
+    // Associate the hovered space with the first item in the hovered item list.
+    // If we were to add several, space views might render unnecessary additional hints.
+    // TODO(andreas): Should there be context if no item is hovered at all? There's no usecase for that today it seems.
+    let mut hovered_items = hovered_items
+        .into_iter()
+        .map(|item| (item, None))
+        .collect::<Vec<_>>();
+
+    if let Some((_, context)) = hovered_items.first_mut() {
+        *context = Some(match spatial_kind {
+            SpatialSpaceViewKind::TwoD => SelectedSpaceContext::TwoD {
+                space_2d: query.space_origin.clone(),
+                pos: picking_context
+                    .pointer_in_space2d
+                    .extend(depth_at_pointer.unwrap_or(f32::INFINITY)),
+            },
+            SpatialSpaceViewKind::ThreeD => {
+                let hovered_point = picking_result.space_position();
+                SelectedSpaceContext::ThreeD {
+                    space_3d: query.space_origin.clone(),
+                    pos: hovered_point,
+                    tracked_space_camera: state.state_3d.tracked_camera.clone(),
+                    point_in_space_cameras: parts
+                        .get::<CamerasPart>()?
+                        .space_cameras
+                        .iter()
+                        .map(|cam| {
+                            (
+                                cam.ent_path.clone(),
+                                hovered_point.and_then(|pos| cam.project_onto_2d(pos)),
+                            )
+                        })
+                        .collect(),
+                }
             }
-        }
+        });
     };
-    ctx.selection_state().set_hovered_space(hovered_space);
+
+    item_ui::select_hovered_on_click(ctx, &response, re_viewer_context::Selection(hovered_items));
 
     Ok(response)
 }

crates/re_space_view_spatial/src/ui_2d.rs

@@ -9,8 +9,8 @@ use re_renderer::view_builder::{TargetConfiguration, ViewBuilder};
 use re_space_view::controls::{DRAG_PAN2D_BUTTON, RESET_VIEW_BUTTON_TEXT, ZOOM_SCROLL_MODIFIER};
 use re_types::{archetypes::Pinhole, components::ViewCoordinates};
 use re_viewer_context::{
-    gpu_bridge, HoveredSpace, SpaceViewSystemExecutionError, SystemExecutionOutput, ViewQuery,
-    ViewerContext,
+    gpu_bridge, SelectedSpaceContext, SpaceViewSystemExecutionError, SystemExecutionOutput,
+    ViewQuery, ViewerContext,
 };
 
 use super::{
@@ -369,12 +369,25 @@ pub fn view_2d(
             ui.visuals().extreme_bg_color.into(),
         ));
 
-        painter.extend(show_projections_from_3d_space(
-            ctx,
-            ui,
-            query.space_origin,
-            &ui_from_canvas,
-        ));
+        // Make sure to _first_ draw the selected, and *then* the hovered context on top!
+        for selected_context in ctx.selection_state().selected_space_context() {
+            painter.extend(show_projections_from_3d_space(
+                ui,
+                query.space_origin,
+                &ui_from_canvas,
+                selected_context,
+                ui.style().visuals.selection.bg_fill,
+            ));
+        }
+        if let Some(hovered_context) = ctx.selection_state().hovered_space_context() {
+            painter.extend(show_projections_from_3d_space(
+                ui,
+                query.space_origin,
+                &ui_from_canvas,
+                hovered_context,
+                egui::Color32::WHITE,
+            ));
+        }
 
         // Add egui driven labels on top of re_renderer content.
         painter.extend(label_shapes);
@@ -497,16 +510,17 @@ fn re_render_rect_from_egui_rect(rect: egui::Rect) -> re_renderer::RectF32 {
 // ------------------------------------------------------------------------
 
 fn show_projections_from_3d_space(
-    ctx: &ViewerContext<'_>,
     ui: &egui::Ui,
     space: &EntityPath,
     ui_from_canvas: &RectTransform,
+    space_context: &SelectedSpaceContext,
+    color: egui::Color32,
 ) -> Vec<Shape> {
     let mut shapes = Vec::new();
-    if let HoveredSpace::ThreeD {
+    if let SelectedSpaceContext::ThreeD {
         point_in_space_cameras: target_spaces,
         ..
-    } = ctx.selection_state().hovered_space()
+    } = space_context
     {
         for (space_2d, pos_2d) in target_spaces {
             if space_2d == space {
@@ -519,7 +533,7 @@ fn show_projections_from_3d_space(
                         radius + 2.0,
                         Color32::BLACK,
                     ));
-                    shapes.push(Shape::circle_filled(pos_in_ui, radius, Color32::WHITE));
+                    shapes.push(Shape::circle_filled(pos_in_ui, radius, color));
 
                     let text = format!("Depth: {:.3} m", pos_2d.z);
                     let font_id = egui::TextStyle::Body.resolve(ui.style());