Integrate depth clouds into Rerun (#1421)

* implement depth cloud renderer

* proper rp config

* self-review

* handle breaking changes from main

* upload depth textures through staging belt

* addressing PR comments part 1 of N

* work around wgpu d16 web issue

* addressing PR comments part 2 of N

* big refactoring of the whole quad spanning story

* accomodating breaking changes

* bring back coverage

* todos

* procedural example

* clean

* fix wasm cranky

* set up entity props & selection panel for backprojection

* integrate depth clouds into rerun

* last merge conflicts

* huh, nasty

* everything working as i want

* cleanin up

* self review

* compute auto heuristics properly

* gentle radius scale

crates/re_data_store/src/entity_properties.rs

@@ -36,6 +36,8 @@ impl EntityPropertyMap {
 
 // ----------------------------------------------------------------------------
 
+// TODO(#1423): We need to properly split entity properties that only apply to specific
+// views/primitives.
 #[cfg(feature = "serde")]
 #[derive(Clone, Debug, PartialEq)]
 #[cfg_attr(feature = "serde", derive(serde::Deserialize, serde::Serialize))]
@@ -50,6 +52,20 @@ pub struct EntityProperties {
     /// Only applies to pinhole cameras when in a spatial view, using 3D navigation.
     ///
     pub pinhole_image_plane_distance: EditableAutoValue<f32>,
+
+    /// Should the depth texture be backprojected into a point cloud?
+    ///
+    /// Only applies to tensors with meaning=depth that are affected by a pinhole transform when
+    /// in a spatial view, using 3D navigation.
+    pub backproject_depth: bool,
+    /// Entity path of the pinhole transform used for the backprojection.
+    ///
+    /// `None` means backprojection is disabled.
+    pub backproject_pinhole_ent_path: Option<EntityPath>,
+    /// Used to scale the resulting point cloud.
+    pub backproject_scale: EditableAutoValue<f32>,
+    /// Used to scale the radii of the points in the resulting point cloud.
+    pub backproject_radius_scale: EditableAutoValue<f32>,
 }
 
 #[cfg(feature = "serde")]
@@ -64,6 +80,16 @@ impl EntityProperties {
                 .pinhole_image_plane_distance
                 .or(&child.pinhole_image_plane_distance)
                 .clone(),
+            backproject_depth: self.backproject_depth || child.backproject_depth,
+            backproject_pinhole_ent_path: self
+                .backproject_pinhole_ent_path
+                .clone()
+                .or(child.backproject_pinhole_ent_path.clone()),
+            backproject_scale: self.backproject_scale.or(&child.backproject_scale).clone(),
+            backproject_radius_scale: self
+                .backproject_radius_scale
+                .or(&child.backproject_radius_scale)
+                .clone(),
         }
     }
 }
@@ -76,6 +102,10 @@ impl Default for EntityProperties {
             visible_history: ExtraQueryHistory::default(),
             interactive: true,
             pinhole_image_plane_distance: EditableAutoValue::default(),
+            backproject_depth: false,
+            backproject_pinhole_ent_path: None,
+            backproject_scale: EditableAutoValue::default(),
+            backproject_radius_scale: EditableAutoValue::default(),
         }
     }
 }

crates/re_renderer/examples/depth_cloud.rs

@@ -78,17 +78,14 @@ impl RenderDepthClouds {
         } = self;
 
         let focal_length = glam::Vec2::new(intrinsics.x_axis.x, intrinsics.y_axis.y);
-        let offset = glam::Vec2::new(intrinsics.x_axis.z, intrinsics.y_axis.z);
+        let offset = glam::Vec2::new(intrinsics.z_axis.x, intrinsics.z_axis.y);
 
         let point_cloud_draw_data = {
             let num_points = depth.dimensions.x * depth.dimensions.y;
             let (points, colors, radii): (Vec<_>, Vec<_>, Vec<_>) = (0..depth.dimensions.y)
                 .flat_map(|y| (0..depth.dimensions.x).map(move |x| glam::UVec2::new(x, y)))
                 .map(|texcoords| {
-                    let linear_depth = depth.get_linear(
-                        depth.dimensions.x - texcoords.x - 1,
-                        depth.dimensions.y - texcoords.y - 1,
-                    );
+                    let linear_depth = depth.get_linear(texcoords.x, texcoords.y);
                     let pos_in_world = ((texcoords.as_vec2() - offset) * linear_depth
                         / focal_length)
                         .extend(linear_depth);
@@ -171,17 +168,12 @@ impl RenderDepthClouds {
             ..
         } = self;
 
-        let world_from_obj = glam::Mat4::from_cols(
-            glam::Vec4::NEG_X * *scale,
-            glam::Vec4::NEG_Y * *scale,
-            glam::Vec4::Z * *scale,
-            glam::Vec4::W,
-        );
+        let world_from_obj = glam::Mat4::from_scale(glam::Vec3::splat(*scale));
 
         let depth_cloud_draw_data = DepthCloudDrawData::new(
             re_ctx,
             &[DepthCloud {
-                world_from_obj,
+                depth_camera_extrinsics: world_from_obj,
                 depth_camera_intrinsics: *intrinsics,
                 radius_scale: *radius_scale,
                 depth_dimensions: depth.dimensions,
@@ -261,7 +253,8 @@ impl framework::Example for RenderDepthClouds {
             Vec3::new(focal_length, 0.0, offset.x),
             Vec3::new(0.0, focal_length, offset.y),
             Vec3::new(0.0, 0.0, 1.0),
-        );
+        )
+        .transpose();
 
         RenderDepthClouds {
             depth,

crates/re_renderer/shader/depth_cloud.wgsl

@@ -30,7 +30,7 @@ fn compute_point_data(quad_idx: i32) -> PointData {
     let color = Vec4(linear_from_srgb(Vec3(norm_linear_depth)), 1.0);
 
     // TODO(cmc): This assumes a pinhole camera; need to support other kinds at some point.
-    let intrinsics = transpose(depth_cloud_info.depth_camera_intrinsics);
+    let intrinsics = depth_cloud_info.depth_camera_intrinsics;
     let focal_length = Vec2(intrinsics[0][0], intrinsics[1][1]);
     let offset = Vec2(intrinsics[2][0], intrinsics[2][1]);
 
@@ -39,7 +39,7 @@ fn compute_point_data(quad_idx: i32) -> PointData {
         norm_linear_depth,
     );
 
-    let pos_in_world = depth_cloud_info.world_from_obj * Vec4(pos_in_obj, 1.0);
+    let pos_in_world = depth_cloud_info.extrinsincs * Vec4(pos_in_obj, 1.0);
 
     var data: PointData;
     data.pos_in_world = pos_in_world.xyz;
@@ -52,7 +52,8 @@ fn compute_point_data(quad_idx: i32) -> PointData {
 // ---
 
 struct DepthCloudInfo {
-    world_from_obj: Mat4,
+    /// The extrinsincs of the camera used for the projection.
+    extrinsincs: Mat4,
 
     /// The intrinsics of the camera used for the projection.
     ///

crates/re_renderer/src/renderer/depth_cloud.rs

@@ -39,7 +39,7 @@ mod gpu_data {
     #[repr(C, align(256))]
     #[derive(Clone, Copy, bytemuck::Pod, bytemuck::Zeroable)]
     pub struct DepthCloudInfoUBO {
-        pub world_from_obj: crate::wgpu_buffer_types::Mat4,
+        pub depth_camera_extrinsics: crate::wgpu_buffer_types::Mat4,
         pub depth_camera_intrinsics: crate::wgpu_buffer_types::Mat3,
         pub radius_scale: crate::wgpu_buffer_types::F32RowPadded,
 
@@ -65,7 +65,8 @@ pub enum DepthCloudDepthData {
 }
 
 pub struct DepthCloud {
-    pub world_from_obj: glam::Mat4,
+    /// The extrinsics of the camera used for the projection.
+    pub depth_camera_extrinsics: glam::Mat4,
 
     /// The intrinsics of the camera used for the projection.
     ///
@@ -87,7 +88,7 @@ pub struct DepthCloud {
 impl Default for DepthCloud {
     fn default() -> Self {
         Self {
-            world_from_obj: glam::Mat4::IDENTITY,
+            depth_camera_extrinsics: glam::Mat4::IDENTITY,
             depth_camera_intrinsics: glam::Mat3::IDENTITY,
             radius_scale: 1.0,
             depth_dimensions: glam::UVec2::ZERO,
@@ -113,6 +114,17 @@ impl DepthCloudDrawData {
     ) -> Result<Self, ResourceManagerError> {
         crate::profile_function!();
 
+        let bg_layout = ctx
+            .renderers
+            .write()
+            .get_or_create::<_, DepthCloudRenderer>(
+                &ctx.shared_renderer_data,
+                &mut ctx.gpu_resources,
+                &ctx.device,
+                &mut ctx.resolver,
+            )
+            .bind_group_layout;
+
         if depth_clouds.is_empty() {
             return Ok(DepthCloudDrawData {
                 bind_groups: Vec::new(),
@@ -123,24 +135,13 @@ impl DepthCloudDrawData {
             ctx,
             "depth_cloud_ubos".into(),
             depth_clouds.iter().map(|info| gpu_data::DepthCloudInfoUBO {
-                world_from_obj: info.world_from_obj.into(),
+                depth_camera_extrinsics: info.depth_camera_extrinsics.into(),
                 depth_camera_intrinsics: info.depth_camera_intrinsics.into(),
                 radius_scale: info.radius_scale.into(),
                 end_padding: Default::default(),
             }),
         );
 
-        let bg_layout = ctx
-            .renderers
-            .write()
-            .get_or_create::<_, DepthCloudRenderer>(
-                &ctx.shared_renderer_data,
-                &mut ctx.gpu_resources,
-                &ctx.device,
-                &mut ctx.resolver,
-            )
-            .bind_group_layout;
-
         let mut bind_groups = Vec::with_capacity(depth_clouds.len());
         for (depth_cloud, ubo) in depth_clouds.iter().zip(depth_cloud_ubos.into_iter()) {
             let depth_texture = match &depth_cloud.depth_data {

crates/re_viewer/src/ui/selection_panel.rs

@@ -1,5 +1,9 @@
+use egui::NumExt as _;
 use re_data_store::{query_latest_single, EditableAutoValue, EntityPath, EntityProperties};
-use re_log_types::{TimeType, Transform};
+use re_log_types::{
+    component_types::{Tensor, TensorDataMeaning},
+    TimeType, Transform,
+};
 
 use crate::{
     ui::{view_spatial::SpatialNavigationMode, Blueprint},
@@ -371,8 +375,6 @@ fn entity_props_ui(
     entity_props: &mut EntityProperties,
     view_state: &ViewState,
 ) {
-    use egui::NumExt;
-
     ui.checkbox(&mut entity_props.visible, "Visible");
     ui.checkbox(&mut entity_props.interactive, "Interactive")
         .on_hover_text("If disabled, the entity will not react to any mouse interaction");
@@ -407,31 +409,119 @@ fn entity_props_ui(
             }
             ui.end_row();
 
-            // pinhole_image_plane_distance
             if view_state.state_spatial.nav_mode == SpatialNavigationMode::ThreeD {
                 if let Some(entity_path) = entity_path {
-                    let query = ctx.current_query();
-                    if let Some(re_log_types::Transform::Pinhole(_)) =
-                        query_latest_single::<Transform>(&ctx.log_db.entity_db, entity_path, &query)
-                    {
-                        ui.label("Image plane distance");
-                        let mut distance = *entity_props.pinhole_image_plane_distance.get();
-                        let speed = (distance * 0.05).at_least(0.01);
-                        if ui
-                            .add(
-                                egui::DragValue::new(&mut distance)
-                                    .clamp_range(0.0..=1.0e8)
-                                    .speed(speed),
-                            )
-                            .on_hover_text("Controls how far away the image plane is.")
-                            .changed()
-                        {
-                            entity_props.pinhole_image_plane_distance =
-                                EditableAutoValue::UserEdited(distance);
-                        }
-                        ui.end_row();
-                    }
+                    pinhole_props_ui(ctx, ui, entity_path, entity_props);
+                    depth_props_ui(ctx, ui, entity_path, entity_props);
                 }
             }
         });
 }
+
+fn pinhole_props_ui(
+    ctx: &mut ViewerContext<'_>,
+    ui: &mut egui::Ui,
+    entity_path: &EntityPath,
+    entity_props: &mut EntityProperties,
+) {
+    let query = ctx.current_query();
+    if let Some(re_log_types::Transform::Pinhole(_)) =
+        query_latest_single::<Transform>(&ctx.log_db.entity_db, entity_path, &query)
+    {
+        ui.label("Image plane distance");
+        let mut distance = *entity_props.pinhole_image_plane_distance.get();
+        let speed = (distance * 0.05).at_least(0.01);
+        if ui
+            .add(
+                egui::DragValue::new(&mut distance)
+                    .clamp_range(0.0..=1.0e8)
+                    .speed(speed),
+            )
+            .on_hover_text("Controls how far away the image plane is.")
+            .changed()
+        {
+            entity_props.pinhole_image_plane_distance = EditableAutoValue::UserEdited(distance);
+        }
+        ui.end_row();
+    }
+}
+
+fn depth_props_ui(
+    ctx: &mut ViewerContext<'_>,
+    ui: &mut egui::Ui,
+    entity_path: &EntityPath,
+    entity_props: &mut EntityProperties,
+) {
+    let query = ctx.current_query();
+
+    // Find closest pinhole transform, if any.
+    let mut pinhole_ent_path = None;
+    let mut cur_path = Some(entity_path.clone());
+    while let Some(path) = cur_path {
+        if let Some(re_log_types::Transform::Pinhole(_)) =
+            query_latest_single::<Transform>(&ctx.log_db.entity_db, &path, &query)
+        {
+            pinhole_ent_path = Some(path);
+            break;
+        }
+        cur_path = path.parent();
+    }
+
+    // Early out if there's no pinhole transform upwards in the tree.
+    let Some(pinhole_ent_path) = pinhole_ent_path else { return; };
+
+    entity_props.backproject_pinhole_ent_path = Some(pinhole_ent_path.clone());
+
+    let tensor = query_latest_single::<Tensor>(&ctx.log_db.entity_db, entity_path, &query);
+    if tensor.as_ref().map(|t| t.meaning) == Some(TensorDataMeaning::Depth) {
+        ui.checkbox(&mut entity_props.backproject_depth, "Backproject Depth")
+            .on_hover_text(
+                "If enabled, the depth texture will be backprojected into a point cloud rather \
+                than simply displayed as an image.",
+            );
+        ui.end_row();
+
+        if entity_props.backproject_depth {
+            ui.label("Pinhole");
+            ctx.entity_path_button(ui, None, &pinhole_ent_path)
+                .on_hover_text(
+                    "The entity path of the pinhole transform being used to do the backprojection.",
+                );
+            ui.end_row();
+
+            ui.label("Backproject scale");
+            let mut scale = *entity_props.backproject_scale.get();
+            let speed = (scale * 0.05).at_least(0.01);
+            if ui
+                .add(
+                    egui::DragValue::new(&mut scale)
+                        .clamp_range(0.0..=1.0e8)
+                        .speed(speed),
+                )
+                .on_hover_text("Scales the backprojected point cloud")
+                .changed()
+            {
+                entity_props.backproject_scale = EditableAutoValue::UserEdited(scale);
+            }
+            ui.end_row();
+
+            ui.label("Backproject radius scale");
+            let mut radius_scale = *entity_props.backproject_radius_scale.get();
+            let speed = (radius_scale * 0.001).at_least(0.001);
+            if ui
+                .add(
+                    egui::DragValue::new(&mut radius_scale)
+                        .clamp_range(0.0..=1.0e8)
+                        .speed(speed),
+                )
+                .on_hover_text("Scales the radii of the points in the backprojected point cloud")
+                .changed()
+            {
+                entity_props.backproject_radius_scale = EditableAutoValue::UserEdited(radius_scale);
+            }
+            ui.end_row();
+        } else {
+            entity_props.backproject_pinhole_ent_path = None;
+        }
+    }
+}

crates/re_viewer/src/ui/view_spatial/scene/picking.rs

@@ -166,6 +166,7 @@ pub fn picking(
         line_strips,
         points,
         meshes,
+        depth_clouds: _, // no picking for depth clouds yet
     } = primitives;
 
     picking_points(&context, &mut state, points);