Turn on depth cloud backprojection by default (#1710)

* Always backproject depth images by default

* Refactor depth_props_ui

* Calculate the closest pinhole transform on the fly

* Fix: you can turn off depth back-projection

* `backproject_depth` default on for 3D, default off for 2D

* Refactor: return the error description

crates/re_data_store/src/entity_properties.rs

@@ -57,14 +57,10 @@ pub struct EntityProperties {
 
     /// Should the depth texture be backprojected into a point cloud?
     ///
-    /// Only applies to tensors with meaning=depth that are affected by a pinhole transform when
-    /// in a spatial view, using 3D navigation.
-    pub backproject_depth: bool,
-
-    /// Entity path of the pinhole transform used for the backprojection.
+    /// Only applies to tensors with meaning=depth that are affected by a pinhole transform.
     ///
-    /// `None` means backprojection is disabled.
-    pub backproject_pinhole_ent_path: Option<EntityPath>,
+    /// The default for 3D views is `true`, but for 2D views it is `false`.
+    pub backproject_depth: EditableAutoValue<bool>,
 
     /// How many depth units per world-space unit. e.g. 1000 for millimeters.
     ///
@@ -91,11 +87,7 @@ impl EntityProperties {
                 .or(&child.pinhole_image_plane_distance)
                 .clone(),
 
-            backproject_depth: self.backproject_depth || child.backproject_depth,
-            backproject_pinhole_ent_path: self
-                .backproject_pinhole_ent_path
-                .clone()
-                .or(child.backproject_pinhole_ent_path.clone()),
+            backproject_depth: self.backproject_depth.or(&child.backproject_depth).clone(),
             depth_from_world_scale: self
                 .depth_from_world_scale
                 .or(&child.depth_from_world_scale)
@@ -117,8 +109,7 @@ impl Default for EntityProperties {
             interactive: true,
             color_mapper: EditableAutoValue::default(),
             pinhole_image_plane_distance: EditableAutoValue::default(),
-            backproject_depth: false,
-            backproject_pinhole_ent_path: None,
+            backproject_depth: EditableAutoValue::Auto(true),
             depth_from_world_scale: EditableAutoValue::default(),
             backproject_radius_scale: EditableAutoValue::Auto(1.0),
         }

crates/re_viewer/src/misc/mod.rs

@@ -3,6 +3,7 @@ pub mod caches;
 pub mod format_time;
 mod item;
 pub(crate) mod mesh_loader;
+pub mod queries;
 mod selection_state;
 pub(crate) mod space_info;
 pub(crate) mod time_control;

crates/re_viewer/src/misc/queries.rs

@@ -0,0 +1,27 @@
+use re_arrow_store::LatestAtQuery;
+use re_data_store::{query_latest_single, EntityPath};
+use re_log_types::Transform;
+
+use super::ViewerContext;
+
+/// Find closest entity with a pinhole transform.
+pub fn closest_pinhole_transform(
+    ctx: &ViewerContext<'_>,
+    entity_path: &EntityPath,
+    query: &LatestAtQuery,
+) -> Option<EntityPath> {
+    crate::profile_function!();
+
+    let mut pinhole_ent_path = None;
+    let mut cur_path = Some(entity_path.clone());
+    while let Some(path) = cur_path {
+        if let Some(Transform::Pinhole(_)) =
+            query_latest_single::<Transform>(&ctx.log_db.entity_db, &path, query)
+        {
+            pinhole_ent_path = Some(path);
+            break;
+        }
+        cur_path = path.parent();
+    }
+    pinhole_ent_path
+}

crates/re_viewer/src/ui/selection_panel.rs

@@ -487,55 +487,49 @@ fn depth_props_ui(
     ui: &mut egui::Ui,
     entity_path: &EntityPath,
     entity_props: &mut EntityProperties,
-) {
-    let query = ctx.current_query();
+) -> Option<()> {
+    crate::profile_function!();
 
-    // Find closest pinhole transform, if any.
-    let mut pinhole_ent_path = None;
-    let mut cur_path = Some(entity_path.clone());
-    while let Some(path) = cur_path {
-        if let Some(re_log_types::Transform::Pinhole(_)) =
-            query_latest_single::<Transform>(&ctx.log_db.entity_db, &path, &query)
-        {
-            pinhole_ent_path = Some(path);
-            break;
-        }
-        cur_path = path.parent();
+    let query = ctx.current_query();
+    let tensor = query_latest_single::<Tensor>(&ctx.log_db.entity_db, entity_path, &query)?;
+    if tensor.meaning != TensorDataMeaning::Depth {
+        return Some(());
     }
+    let pinhole_ent_path =
+        crate::misc::queries::closest_pinhole_transform(ctx, entity_path, &query)?;
 
-    // Early out if there's no pinhole transform upwards in the tree.
-    let Some(pinhole_ent_path) = pinhole_ent_path else { return; };
+    let mut backproject_depth = *entity_props.backproject_depth.get();
 
-    entity_props.backproject_pinhole_ent_path = Some(pinhole_ent_path.clone());
+    if ui
+        .checkbox(&mut backproject_depth, "Backproject Depth")
+        .on_hover_text(
+            "If enabled, the depth texture will be backprojected into a point cloud rather \
+                than simply displayed as an image.",
+        )
+        .changed()
+    {
+        entity_props.backproject_depth = EditableAutoValue::UserEdited(backproject_depth);
+    }
+    ui.end_row();
 
-    let tensor = query_latest_single::<Tensor>(&ctx.log_db.entity_db, entity_path, &query);
-    if tensor.as_ref().map(|t| t.meaning) == Some(TensorDataMeaning::Depth) {
-        ui.checkbox(&mut entity_props.backproject_depth, "Backproject Depth")
+    if backproject_depth {
+        ui.label("Pinhole");
+        ctx.entity_path_button(ui, None, &pinhole_ent_path)
             .on_hover_text(
-                "If enabled, the depth texture will be backprojected into a point cloud rather \
-                than simply displayed as an image.",
+                "The entity path of the pinhole transform being used to do the backprojection.",
             );
         ui.end_row();
 
-        if entity_props.backproject_depth {
-            ui.label("Pinhole");
-            ctx.entity_path_button(ui, None, &pinhole_ent_path)
-                .on_hover_text(
-                    "The entity path of the pinhole transform being used to do the backprojection.",
-                );
-            ui.end_row();
-
-            depth_from_world_scale_ui(ui, &mut entity_props.depth_from_world_scale);
+        depth_from_world_scale_ui(ui, &mut entity_props.depth_from_world_scale);
 
-            backproject_radius_scale_ui(ui, &mut entity_props.backproject_radius_scale);
+        backproject_radius_scale_ui(ui, &mut entity_props.backproject_radius_scale);
 
-            // TODO(cmc): This should apply to the depth map entity as a whole, but for that we
-            // need to get the current hardcoded colormapping out of the image cache first.
-            colormap_props_ui(ui, entity_props);
-        } else {
-            entity_props.backproject_pinhole_ent_path = None;
-        }
+        // TODO(cmc): This should apply to the depth map entity as a whole, but for that we
+        // need to get the current hardcoded colormapping out of the image cache first.
+        colormap_props_ui(ui, entity_props);
     }
+
+    Some(())
 }
 
 fn depth_from_world_scale_ui(ui: &mut egui::Ui, property: &mut EditableAutoValue<f32>) {

crates/re_viewer/src/ui/view_spatial/scene/scene_part/images.rs

@@ -152,22 +152,30 @@ impl ImagesPart {
 
                 let entity_highlight = highlights.entity_outline_mask(ent_path.hash());
 
-                if tensor.meaning == TensorDataMeaning::Depth {
-                    if let Some(pinhole_ent_path) = properties.backproject_pinhole_ent_path.as_ref()
-                    {
+                if *properties.backproject_depth.get() && tensor.meaning == TensorDataMeaning::Depth
+                {
+                    let query = ctx.current_query();
+                    let pinhole_ent_path =
+                        crate::misc::queries::closest_pinhole_transform(ctx, ent_path, &query);
+
+                    if let Some(pinhole_ent_path) = pinhole_ent_path {
                         // NOTE: we don't pass in `world_from_obj` because this corresponds to the
                         // transform of the projection plane, which is of no use to us here.
                         // What we want are the extrinsics of the depth camera!
-                        Self::process_entity_view_as_depth_cloud(
+                        match Self::process_entity_view_as_depth_cloud(
                             scene,
                             ctx,
                             transforms,
                             properties,
                             &tensor,
-                            pinhole_ent_path,
+                            &pinhole_ent_path,
                             entity_highlight,
-                        );
-                        return Ok(());
+                        ) {
+                            Ok(()) => return Ok(()),
+                            Err(err) => {
+                                re_log::warn_once!("{err}");
+                            }
+                        }
                     };
                 }
 
@@ -264,25 +272,23 @@ impl ImagesPart {
         tensor: &Tensor,
         pinhole_ent_path: &EntityPath,
         entity_highlight: &SpaceViewOutlineMasks,
-    ) {
+    ) -> Result<(), String> {
         crate::profile_function!();
 
         let Some(re_log_types::Transform::Pinhole(intrinsics)) = query_latest_single::<Transform>(
             &ctx.log_db.entity_db,
             pinhole_ent_path,
             &ctx.current_query(),
         ) else {
-            re_log::warn_once!("Couldn't fetch pinhole intrinsics at {pinhole_ent_path:?}");
-            return;
+            return Err(format!("Couldn't fetch pinhole intrinsics at {pinhole_ent_path:?}"));
         };
 
         // TODO(cmc): getting to those extrinsics is no easy task :|
         let world_from_obj = pinhole_ent_path
             .parent()
             .and_then(|ent_path| transforms.reference_from_entity(&ent_path));
         let Some(world_from_obj) = world_from_obj else {
-            re_log::warn_once!("Couldn't fetch pinhole extrinsics at {pinhole_ent_path:?}");
-            return;
+            return Err(format!("Couldn't fetch pinhole extrinsics at {pinhole_ent_path:?}"));
         };
 
         // TODO(cmc): automagically convert as needed for non-natively supported datatypes?
@@ -291,11 +297,10 @@ impl ImagesPart {
             TensorData::U16(data) => DepthCloudDepthData::U16(data.clone()),
             TensorData::F32(data) => DepthCloudDepthData::F32(data.clone()),
             _ => {
-                re_log::warn_once!(
+                return Err(format!(
                     "Tensor datatype {} is not supported for backprojection",
                     tensor.dtype()
-                );
-                return;
+                ));
             }
         };
 
@@ -347,6 +352,8 @@ impl ImagesPart {
             colormap,
             outline_mask_id: entity_highlight.overall,
         });
+
+        Ok(())
     }
 }
 

crates/re_viewer/src/ui/view_spatial/ui.rs

@@ -150,7 +150,7 @@ impl ViewSpatialState {
                 &entity_path,
                 scene_size,
             );
-            Self::update_depth_cloud_property_heuristics(ctx, data_blueprint, &query, &entity_path);
+            self.update_depth_cloud_property_heuristics(ctx, data_blueprint, &query, &entity_path);
         }
     }
 
@@ -186,16 +186,23 @@ impl ViewSpatialState {
     }
 
     fn update_depth_cloud_property_heuristics(
+        &self,
         ctx: &mut ViewerContext<'_>,
         data_blueprint: &mut DataBlueprintTree,
         query: &re_arrow_store::LatestAtQuery,
         entity_path: &EntityPath,
-    ) {
-        let tensor = query_latest_single::<Tensor>(&ctx.log_db.entity_db, entity_path, query);
-        if tensor.as_ref().map(|t| t.meaning) == Some(TensorDataMeaning::Depth) {
-            let tensor = tensor.as_ref().unwrap();
+    ) -> Option<()> {
+        let tensor = query_latest_single::<Tensor>(&ctx.log_db.entity_db, entity_path, query)?;
+
+        let mut properties = data_blueprint.data_blueprints_individual().get(entity_path);
+        if properties.backproject_depth.is_auto() {
+            properties.backproject_depth = EditableAutoValue::Auto(
+                tensor.meaning == TensorDataMeaning::Depth
+                    && *self.nav_mode.get() == SpatialNavigationMode::ThreeD,
+            );
+        }
 
-            let mut properties = data_blueprint.data_blueprints_individual().get(entity_path);
+        if tensor.meaning == TensorDataMeaning::Depth {
             if properties.depth_from_world_scale.is_auto() {
                 let auto = tensor.meter.unwrap_or_else(|| {
                     use re_log_types::component_types::TensorTrait as _;
@@ -216,6 +223,8 @@ impl ViewSpatialState {
                 .data_blueprints_individual()
                 .set(entity_path.clone(), properties);
         }
+
+        Some(())
     }
 
     pub fn selection_ui(