Fix projected ray length (#2482)

Closes #2459

<img width="488" alt="Screenshot 2023-06-19 at 20 39 26"
src="https://github.com/rerun-io/rerun/assets/1148717/90e16677-4a86-41b8-b1f5-52642518eac7">

### What
We where taking pinhole matrices and squeezing them into `Affine3A` with
predictably bad results.

The offending methods have been removed and replaced with more direct,
simpler steps.

### Checklist
* [x] I have read and agree to [Contributor
Guide](https://github.com/rerun-io/rerun/blob/main/CONTRIBUTING.md) and
the [Code of
Conduct](https://github.com/rerun-io/rerun/blob/main/CODE_OF_CONDUCT.md)
* [x] I've included a screenshot or gif (if applicable)

<!-- This line will get updated when the PR build summary job finishes.
-->
PR Build Summary: https://build.rerun.io/pr/2482

<!-- pr-link-docs:start -->
Docs preview: https://rerun.io/preview/d330b73/docs
Examples preview: https://rerun.io/preview/d330b73/examples
<!-- pr-link-docs:end -->

---------

Co-authored-by: Andreas Reich <[email protected]>

crates/re_components/src/pinhole.rs

@@ -110,6 +110,28 @@ impl Pinhole {
     pub fn aspect_ratio(&self) -> Option<f32> {
         self.resolution.map(|r| r[0] / r[1])
     }
+
+    /// Project camera-space coordinates into pixel coordinates,
+    /// returning the same z/depth.
+    #[cfg(feature = "glam")]
+    #[inline]
+    pub fn project(&self, pixel: glam::Vec3) -> glam::Vec3 {
+        ((pixel.truncate() * glam::Vec2::from(self.focal_length_in_pixels())) / pixel.z
+            + self.principal_point())
+        .extend(pixel.z)
+    }
+
+    /// Given pixel coordinates and a world-space depth,
+    /// return a position in the camera space.
+    ///
+    /// The returned z is the same as the input z (depth).
+    #[cfg(feature = "glam")]
+    #[inline]
+    pub fn unproject(&self, pixel: glam::Vec3) -> glam::Vec3 {
+        ((pixel.truncate() - self.principal_point()) * pixel.z
+            / glam::Vec2::from(self.focal_length_in_pixels()))
+        .extend(pixel.z)
+    }
 }
 
 #[test]

crates/re_space_view_spatial/src/space_camera_3d.rs

@@ -1,4 +1,4 @@
-use glam::{vec3, Affine3A, Mat3, Quat, Vec2, Vec3};
+use glam::{Quat, Vec2, Vec3};
 use macaw::{IsoTransform, Ray3};
 
 use re_components::{Pinhole, ViewCoordinates};
@@ -52,41 +52,22 @@ impl SpaceCamera3D {
         }
     }
 
-    /// Projects image coordinates into world coordinates
-    pub fn world_from_image(&self) -> Option<Affine3A> {
+    /// Returns x, y, and depth in image/pixel coordinates.
+    pub fn project_onto_2d(&self, point_in_world: Vec3) -> Option<Vec3> {
         let pinhole = self.pinhole?;
-        let world_from_cam = self.world_from_cam();
-        let image_from_cam: Mat3 = pinhole.image_from_cam.into();
-        let cam_from_image = Affine3A::from_mat3(image_from_cam.inverse());
-        Some(world_from_cam * cam_from_image)
-    }
-
-    /// Projects world coordinates onto 2D image coordinates
-    pub fn image_from_world(&self) -> Option<Affine3A> {
-        let pinhole = self.pinhole?;
-        let cam_from_world = self.cam_from_world();
-
-        let image_from_cam = pinhole.image_from_cam.into();
-        let image_from_cam = Affine3A::from_mat3(image_from_cam);
-        Some(image_from_cam * cam_from_world)
-    }
-
-    /// Returns x, y, and depth in image coordinates.
-    pub fn project_onto_2d(&self, pos3d: Vec3) -> Option<Vec3> {
-        self.image_from_world().map(|pixel_from_world| {
-            let point = pixel_from_world.transform_point3(pos3d);
-            vec3(point.x / point.z, point.y / point.z, point.z)
-        })
+        let point_in_cam = self.cam_from_world().transform_point3(point_in_world);
+        let point_in_image = pinhole.project(point_in_cam);
+        Some(point_in_image)
     }
 
     /// Unproject a 2D image coordinate as a ray in 3D space
     pub fn unproject_as_ray(&self, pos2d: Vec2) -> Option<Ray3> {
-        self.world_from_image().map(|world_from_pixel| {
-            let origin = self.position();
-            let stop = world_from_pixel.transform_point3(pos2d.extend(1.0));
-            let dir = (stop - origin).normalize();
-            Ray3::from_origin_dir(origin, dir)
-        })
+        let pinhole = self.pinhole?;
+
+        let depth = 1.0; // whatever will do
+        let stop = pinhole.unproject(pos2d.extend(depth));
+        let ray_in_camera = Ray3::from_origin_dir(Vec3::ZERO, stop);
+        Some(self.world_from_camera * ray_in_camera)
     }
 }
 

crates/re_space_view_spatial/src/ui_3d.rs

@@ -571,15 +571,23 @@ fn show_projections_from_2d_space(
     match ctx.selection_state().hovered_space() {
         HoveredSpace::TwoD { space_2d, pos } => {
             if let Some(cam) = space_cameras.iter().find(|cam| &cam.ent_path == space_2d) {
-                if let Some(ray) = cam.unproject_as_ray(glam::vec2(pos.x, pos.y)) {
+                if let Some(pinhole) = cam.pinhole {
                     // Render a thick line to the actual z value if any and a weaker one as an extension
                     // If we don't have a z value, we only render the thick one.
-                    let thick_ray_length = if pos.z.is_finite() && pos.z > 0.0 {
+                    let depth = if 0.0 < pos.z && pos.z.is_finite() {
                         pos.z
                     } else {
                         cam.picture_plane_distance
                     };
 
+                    let stop_in_cam = pinhole.unproject(glam::vec3(pos.x, pos.y, depth));
+                    let stop_in_world = cam.world_from_cam().transform_point3(stop_in_cam);
+
+                    let origin = cam.position();
+                    let ray =
+                        macaw::Ray3::from_origin_dir(origin, (stop_in_world - origin).normalize());
+
+                    let thick_ray_length = (stop_in_world - origin).length();
                     add_picking_ray(line_builder, ray, scene_bbox_accum, thick_ray_length);
                 }
             }