Improve the depth backprojection feature (#1690)

* Replace the backproject scale with the familiar "meter"

* Assume depth images usually start at zero

* Make the depth point radius scale more sane

* Fix an off-by-half error, giving all points an extra 0.5 pixels in radius

* Add a fudge-factor

* Improve tooltip

* Fix typos

* Fix doclink

* Fix the math

* fix typo

Co-authored-by: Andreas Reich <[email protected]>

* #[inline]

* Another typo

* typo

Co-authored-by: Andreas Reich <[email protected]>

* Adjust the feathering margins

* Clean up the depth texture uploading

* Better docs

* fix typos

* Use world depth values in the shader and simplify user code

* Better naming

* Fix bug in the image-hover code

* Use same depth range for the color map in the depth projection shader

* Fix the gamma of the wgsl color map

* Tweak default point radius to avoid Moiré patterns

* Final touches

* spell fix

* more typos

Co-authored-by: Andreas Reich <[email protected]>

* Fix comment about why we use Depth16Unorm

* Add lines to changelog

* max_depth -> max_depth_in_world

---------

Co-authored-by: Andreas Reich <[email protected]>

CHANGELOG.md

@@ -4,6 +4,8 @@
 ## Unreleased
 [Commits since latest release](https://github.com/rerun-io/rerun/compare/latest...HEAD)
 
+* Fixed a bug in the image hover code, causing the wrong RGBA values to be printed 😬 [#1690](https://github.com/rerun-io/rerun/pull/1356)
+* Fixed a bug that caused points to be render too large [#1690](https://github.com/rerun-io/rerun/pull/1690)
 
 ## 0.3.1 - Remove potentially sensitive analytics
 [Commits](https://github.com/rerun-io/rerun/compare/v0.3.1...v0.3.0)

crates/re_data_store/src/entity_properties.rs

@@ -66,8 +66,10 @@ pub struct EntityProperties {
     /// `None` means backprojection is disabled.
     pub backproject_pinhole_ent_path: Option<EntityPath>,
 
-    /// Used to scale the resulting point cloud.
-    pub backproject_scale: EditableAutoValue<f32>,
+    /// How many depth units per world-space unit. e.g. 1000 for millimeters.
+    ///
+    /// This corresponds to [`re_log_types::component_types::Tensor::meter`].
+    pub depth_from_world_scale: EditableAutoValue<f32>,
 
     /// Used to scale the radii of the points in the resulting point cloud.
     pub backproject_radius_scale: EditableAutoValue<f32>,
@@ -94,7 +96,10 @@ impl EntityProperties {
                 .backproject_pinhole_ent_path
                 .clone()
                 .or(child.backproject_pinhole_ent_path.clone()),
-            backproject_scale: self.backproject_scale.or(&child.backproject_scale).clone(),
+            depth_from_world_scale: self
+                .depth_from_world_scale
+                .or(&child.depth_from_world_scale)
+                .clone(),
             backproject_radius_scale: self
                 .backproject_radius_scale
                 .or(&child.backproject_radius_scale)
@@ -114,8 +119,8 @@ impl Default for EntityProperties {
             pinhole_image_plane_distance: EditableAutoValue::default(),
             backproject_depth: false,
             backproject_pinhole_ent_path: None,
-            backproject_scale: EditableAutoValue::default(),
-            backproject_radius_scale: EditableAutoValue::default(),
+            depth_from_world_scale: EditableAutoValue::default(),
+            backproject_radius_scale: EditableAutoValue::Auto(1.0),
         }
     }
 }

crates/re_log_types/src/data.rs

@@ -94,6 +94,12 @@ impl TensorDataType {
         }
     }
 
+    #[inline]
+    pub fn is_integer(&self) -> bool {
+        !self.is_float()
+    }
+
+    #[inline]
     pub fn is_float(&self) -> bool {
         match self {
             Self::U8
@@ -108,6 +114,7 @@ impl TensorDataType {
         }
     }
 
+    #[inline]
     pub fn max_value(&self) -> f64 {
         match self {
             Self::U8 => u8::MAX as _,

crates/re_renderer/examples/depth_cloud.rs

@@ -47,7 +47,7 @@ struct RenderDepthClouds {
     albedo_handle: GpuTexture2DHandle,
 
     scale: f32,
-    radius_scale: f32,
+    point_radius_from_world_depth: f32,
     intrinsics: glam::Mat3,
 
     camera_control: CameraControl,
@@ -72,7 +72,7 @@ impl RenderDepthClouds {
         let Self {
             depth,
             scale,
-            radius_scale,
+            point_radius_from_world_depth,
             intrinsics,
             ..
         } = self;
@@ -93,7 +93,7 @@ impl RenderDepthClouds {
                     (
                         pos_in_world * *scale,
                         Color32::from_gray((linear_depth * 255.0) as u8),
-                        Size(linear_depth * *radius_scale),
+                        Size(linear_depth * *point_radius_from_world_depth),
                     )
                 })
                 .multiunzip();
@@ -163,7 +163,7 @@ impl RenderDepthClouds {
         let Self {
             depth,
             scale,
-            radius_scale,
+            point_radius_from_world_depth,
             intrinsics,
             ..
         } = self;
@@ -173,9 +173,11 @@ impl RenderDepthClouds {
         let depth_cloud_draw_data = DepthCloudDrawData::new(
             re_ctx,
             &[DepthCloud {
-                depth_camera_extrinsics: world_from_obj,
+                world_from_obj,
                 depth_camera_intrinsics: *intrinsics,
-                radius_scale: *radius_scale,
+                world_depth_from_data_depth: 1.0,
+                point_radius_from_world_depth: *point_radius_from_world_depth,
+                max_depth_in_world: 5.0,
                 depth_dimensions: depth.dimensions,
                 depth_data: depth.data.clone(),
                 colormap: re_renderer::ColorMap::ColorMapTurbo,
@@ -246,7 +248,7 @@ impl framework::Example for RenderDepthClouds {
         );
 
         let scale = 50.0;
-        let radius_scale = 0.1;
+        let point_radius_from_world_depth = 0.1;
 
         // hardcoded intrinsics for nyud dataset
         let focal_length = depth.dimensions.x as f32 * 0.7;
@@ -264,7 +266,7 @@ impl framework::Example for RenderDepthClouds {
             albedo_handle,
 
             scale,
-            radius_scale,
+            point_radius_from_world_depth,
             intrinsics,
 
             camera_control: CameraControl::RotateAroundCenter,

crates/re_renderer/shader/colormap.wgsl

@@ -9,6 +9,9 @@ const COLORMAP_PLASMA:  u32 = 3u;
 const COLORMAP_MAGMA:   u32 = 4u;
 const COLORMAP_INFERNO: u32 = 5u;
 
+/// Returns a gamma-space sRGB in 0-1 range.
+///
+/// The input will be saturated to [0, 1] range.
 fn colormap_srgb(which: u32, t: f32) -> Vec3 {
     if which == COLORMAP_TURBO {
         return colormap_turbo_srgb(t);
@@ -25,6 +28,13 @@ fn colormap_srgb(which: u32, t: f32) -> Vec3 {
     }
 }
 
+/// Returns a linear-space sRGB in 0-1 range.
+///
+/// The input will be saturated to [0, 1] range.
+fn colormap_linear(which: u32, t: f32) -> Vec3 {
+    return linear_from_srgb(colormap_srgb(which, t));
+}
+
 // --- Turbo color map ---
 
 // Polynomial approximation in GLSL for the Turbo colormap.
@@ -38,7 +48,8 @@ fn colormap_srgb(which: u32, t: f32) -> Vec3 {
 //   Colormap Design: Anton Mikhailov ([email protected])
 //   GLSL Approximation: Ruofei Du ([email protected])
 
-/// Returns a normalized sRGB polynomial approximation from Turbo color map, assuming `t` is
+/// Returns a gamma-space sRGB in 0-1 range.
+/// This is a polynomial approximation from Turbo color map, assuming `t` is
 /// normalized (it will be saturated no matter what).
 fn colormap_turbo_srgb(t: f32) -> Vec3 {
     let r4 = Vec4(0.13572138, 4.61539260, -42.66032258, 132.13108234);
@@ -73,7 +84,8 @@ fn colormap_turbo_srgb(t: f32) -> Vec3 {
 //
 // Data fitted from https://github.com/BIDS/colormap/blob/master/colormaps.py (CC0).
 
-/// Returns a normalized sRGB polynomial approximation from Viridis color map, assuming `t` is
+/// Returns a gamma-space sRGB in 0-1 range.
+/// This is a polynomial approximation from Viridis color map, assuming `t` is
 /// normalized (it will be saturated no matter what).
 fn colormap_viridis_srgb(t: f32) -> Vec3 {
     let c0 = Vec3(0.2777273272234177, 0.005407344544966578, 0.3340998053353061);
@@ -87,7 +99,8 @@ fn colormap_viridis_srgb(t: f32) -> Vec3 {
     return c0 + t * (c1 + t * (c2 + t * (c3 + t * (c4 + t * (c5 + t * c6)))));
 }
 
-/// Returns a normalized sRGB polynomial approximation from Plasma color map, assuming `t` is
+/// Returns a gamma-space sRGB in 0-1 range.
+/// This is a polynomial approximation from Plasma color map, assuming `t` is
 /// normalized (it will be saturated no matter what).
 fn colormap_plasma_srgb(t: f32) -> Vec3 {
     let c0 = Vec3(0.05873234392399702, 0.02333670892565664, 0.5433401826748754);
@@ -101,7 +114,8 @@ fn colormap_plasma_srgb(t: f32) -> Vec3 {
     return c0 + t * (c1 + t * (c2 + t * (c3 + t * (c4 + t * (c5 + t * c6)))));
 }
 
-/// Returns a normalized sRGB polynomial approximation from Magma color map, assuming `t` is
+/// Returns a gamma-space sRGB in 0-1 range.
+/// This is a polynomial approximation from Magma color map, assuming `t` is
 /// normalized (it will be saturated no matter what).
 fn colormap_magma_srgb(t: f32) -> Vec3 {
     let c0 = Vec3(-0.002136485053939582, -0.000749655052795221, -0.005386127855323933);
@@ -115,7 +129,8 @@ fn colormap_magma_srgb(t: f32) -> Vec3 {
     return c0 + t * (c1 + t * (c2 + t * (c3 + t * (c4 + t * (c5 + t * c6)))));
 }
 
-/// Returns a normalized sRGB polynomial approximation from Inferno color map, assuming `t` is
+/// Returns a gamma-space sRGB in 0-1 range.
+/// This is a polynomial approximation from Inferno color map, assuming `t` is
 /// normalized (it will be saturated no matter what).
 fn colormap_inferno_srgb(t: f32) -> Vec3 {
     let c0 = Vec3(0.0002189403691192265, 0.001651004631001012, -0.01948089843709184);

crates/re_renderer/shader/depth_cloud.wgsl

@@ -25,51 +25,59 @@ fn compute_point_data(quad_idx: i32) -> PointData {
     let texcoords = IVec2(quad_idx % wh.x, quad_idx / wh.x);
 
     // TODO(cmc): expose knobs to linearize/normalize/flip/cam-to-plane depth.
-    let norm_linear_depth = textureLoad(depth_texture, texcoords, 0).x;
+    let world_space_depth = depth_cloud_info.world_depth_from_texture_value * textureLoad(depth_texture, texcoords, 0).x;
 
     // TODO(cmc): albedo textures
-    let color = Vec4(colormap_srgb(depth_cloud_info.colormap, norm_linear_depth), 1.0);
+    let color = Vec4(colormap_linear(depth_cloud_info.colormap, world_space_depth / depth_cloud_info.max_depth_in_world), 1.0);
 
     // TODO(cmc): This assumes a pinhole camera; need to support other kinds at some point.
     let intrinsics = depth_cloud_info.depth_camera_intrinsics;
     let focal_length = Vec2(intrinsics[0][0], intrinsics[1][1]);
     let offset = Vec2(intrinsics[2][0], intrinsics[2][1]);
 
     let pos_in_obj = Vec3(
-        (Vec2(texcoords) - offset) * norm_linear_depth / focal_length,
-        norm_linear_depth,
+        (Vec2(texcoords) - offset) * world_space_depth / focal_length,
+        world_space_depth,
     );
 
-    let pos_in_world = depth_cloud_info.extrinsincs * Vec4(pos_in_obj, 1.0);
+    let pos_in_world = depth_cloud_info.world_from_obj * Vec4(pos_in_obj, 1.0);
 
     var data: PointData;
     data.pos_in_world = pos_in_world.xyz;
-    data.unresolved_radius = norm_linear_depth * depth_cloud_info.radius_scale;
+    data.unresolved_radius = depth_cloud_info.point_radius_from_world_depth * world_space_depth;
     data.color = color;
 
     return data;
 }
 
 // ---
 
+/// Keep in sync with `DepthCloudInfoUBO` in `depth_cloud.rs`.
 struct DepthCloudInfo {
     /// The extrinsincs of the camera used for the projection.
-    extrinsincs: Mat4,
+    world_from_obj: Mat4,
 
     /// The intrinsics of the camera used for the projection.
     ///
     /// Only supports pinhole cameras at the moment.
     depth_camera_intrinsics: Mat3,
 
-    /// The scale to apply to the radii of the backprojected points.
-    radius_scale: f32,
+    /// Outline mask id for the outline mask pass.
+    outline_mask_id: UVec2,
+
+    /// Multiplier to get world-space depth from whatever is in the texture.
+    world_depth_from_texture_value: f32,
+
+    /// Point radius is calculated as world-space depth times this value.
+    point_radius_from_world_depth: f32,
+
+    /// The maximum depth value in world-space, for use with the colormap.
+    max_depth_in_world: f32,
 
     /// Configures color mapping mode, see `colormap.wgsl`.
     colormap: u32,
-
-    /// Outline mask id for the outline mask pass.
-    outline_mask_id: UVec2,
 };
+
 @group(1) @binding(0)
 var<uniform> depth_cloud_info: DepthCloudInfo;
 

crates/re_renderer/shader/utils/sphere_quad.wgsl

@@ -27,9 +27,9 @@ fn sphere_quad_span_perspective(
     let camera_offset = radius_sq * distance_to_camera_inv;
     var modified_radius = point_radius * distance_to_camera_inv * sqrt(distance_to_camera_sq - radius_sq);
 
-    // We're computing a coverage mask in the fragment shader - make sure the quad doesn't cut off our antialiasing.
+    // Add half a pixel of margin for the feathering we do for antialiasing the spheres.
     // It's fairly subtle but if we don't do this our spheres look slightly squarish
-    modified_radius += frame.pixel_world_size_from_camera_distance * camera_distance;
+    modified_radius += 0.5 * frame.pixel_world_size_from_camera_distance * camera_distance;
 
     return point_pos + pos_in_quad * modified_radius + camera_offset * quad_normal;
 
@@ -48,9 +48,9 @@ fn sphere_quad_span_orthographic(point_pos: Vec3, point_radius: f32, top_bottom:
     let quad_up = cross(quad_right, quad_normal);
     let pos_in_quad = top_bottom * quad_up + left_right * quad_right;
 
-    // We're computing a coverage mask in the fragment shader - make sure the quad doesn't cut off our antialiasing.
+    // Add half a pixel of margin for the feathering we do for antialiasing the spheres.
     // It's fairly subtle but if we don't do this our spheres look slightly squarish
-    let radius = point_radius + frame.pixel_world_size_from_camera_distance;
+    let radius = point_radius + 0.5 * frame.pixel_world_size_from_camera_distance;
 
     return point_pos + pos_in_quad * radius;
 }
@@ -100,9 +100,13 @@ fn sphere_quad_coverage(world_position: Vec3, radius: f32, point_center: Vec3) -
     // https://www.shadertoy.com/view/MsSSWV
     // (but rearranged and labeled to it's easier to understand!)
     let d = ray_sphere_distance(ray, point_center, radius);
-    let smallest_distance_to_sphere = d.x;
+    let distance_to_sphere_surface = d.x;
     let closest_ray_dist = d.y;
     let pixel_world_size = approx_pixel_world_size_at(closest_ray_dist);
 
-    return 1.0 - saturate(smallest_distance_to_sphere / pixel_world_size);
+    let distance_to_surface_in_pixels = distance_to_sphere_surface / pixel_world_size;
+
+    // At the surface we have 50% coverage, and it decreases with distance.
+    // Note that we have signed distances to the sphere surface.
+    return saturate(0.5 - distance_to_surface_in_pixels);
 }