3d to 2d projections (#2008)

* transform cache now deals with Affine3 matrices only

* use a perspective camera in 2d views that sit at a space camera

* clarify/simplify use of focal length

* refine camera plane distance heuristic for 2D scenes and make it configurable again

* comment wip

* merge fixup and comment improvements

* line & point builder now work with affine transforms

* wip

* improved image plane heuristic for 2D

* hack for image plane distance for in inverse pinhole transforms. remove setting from ui again for 2d views

* add viewport transformation to viewbuilder

* better viewport transform

* limit zoom, correctly handle ui scale under viewport zoom

* 2D points now draw as real 2D circles

* better 2D rendering for lines with perspective camera around

* disable 3D labels in 2D views

* space camera no longer required for correct pinhole camera in ui_2d

* consistent canvas rect handling, take principal point into account when displaying 2d canvas

* comments on the nature of our interim 3D->2D solution

* point out that picking should use same transforms

* easier point/line flag building

* minor cleanup

* doc test fix

* clarify what sphere_quad's coverage methods do

* fix taking only one axis into account for pixel size approximation

* comment explaining how to use FORCE_ORTHO_SPANNING

* remove unnecessary affine3a multiply method

* impl From<glam::Affine3A> for wgpu_buffer_types::Mat4

* rename rect top_left to min

* remove unnecessary quaternion on pinhole transform calc

* make error swallowing on screenshots more explicit

* failure to compute camera now logs error and stops from rendering

* note on non-square pixels

* better handle different x & y focal length + comment

* renaming and tests around RectTransform

* yet another workaround for https://github.com/gfx-rs/naga/issues/1743

* remove h word

.vscode/settings.json

@@ -40,6 +40,7 @@
         "Skybox",
         "smallvec",
         "swapchain",
+        "texcoord",
         "texcoords",
         "Tonemapper",
         "tonemapping",

crates/re_log_types/src/component_types/transform.rs

@@ -163,6 +163,7 @@ impl Pinhole {
     /// Focal length.
     #[inline]
     pub fn focal_length(&self) -> Option<f32> {
+        // Use only the first element of the focal length vector, as we don't support non-square pixels.
         self.resolution.map(|r| self.image_from_cam[0][0] / r[0])
     }
 

crates/re_renderer/examples/2d.rs

@@ -284,6 +284,7 @@ impl framework::Example for Render2D {
                         projection_from_view: Projection::Perspective {
                             vertical_fov: 70.0 * std::f32::consts::TAU / 360.0,
                             near_plane_distance: 0.01,
+                            aspect_ratio: resolution[0] as f32 / resolution[1] as f32,
                         },
                         pixels_from_point,
                         ..Default::default()

crates/re_renderer/examples/depth_cloud.rs

@@ -123,6 +123,7 @@ impl RenderDepthClouds {
                 projection_from_view: Projection::Perspective {
                     vertical_fov: 70.0 * std::f32::consts::TAU / 360.0,
                     near_plane_distance: 0.01,
+                    aspect_ratio: resolution_in_pixel[0] as f32 / resolution_in_pixel[1] as f32,
                 },
                 pixels_from_point,
                 ..Default::default()
@@ -202,6 +203,7 @@ impl RenderDepthClouds {
                 projection_from_view: Projection::Perspective {
                     vertical_fov: 70.0 * std::f32::consts::TAU / 360.0,
                     near_plane_distance: 0.01,
+                    aspect_ratio: resolution_in_pixel[0] as f32 / resolution_in_pixel[1] as f32,
                 },
                 pixels_from_point,
                 ..Default::default()
@@ -287,9 +289,10 @@ impl framework::Example for RenderDepthClouds {
         let splits = framework::split_resolution(resolution, 1, 2).collect::<Vec<_>>();
 
         let frame_size = albedo.dimensions.as_vec2().extend(0.0) / 15.0;
-        let scale = glam::Mat4::from_scale(frame_size);
-        let rotation = glam::Mat4::IDENTITY;
-        let translation_center = glam::Mat4::from_translation(-glam::Vec3::splat(0.5) * frame_size);
+        let scale = glam::Affine3A::from_scale(frame_size);
+        let rotation = glam::Affine3A::IDENTITY;
+        let translation_center =
+            glam::Affine3A::from_translation(-glam::Vec3::splat(0.5) * frame_size);
         let world_from_model = rotation * translation_center * scale;
 
         let frame_draw_data = {

crates/re_renderer/examples/multiview.rs

@@ -113,7 +113,9 @@ fn build_lines(re_ctx: &mut RenderContext, seconds_since_startup: f32) -> LineDr
     // Blue spiral, rotating
     builder
         .batch("blue spiral")
-        .world_from_obj(glam::Mat4::from_rotation_x(seconds_since_startup * 10.0))
+        .world_from_obj(glam::Affine3A::from_rotation_x(
+            seconds_since_startup * 10.0,
+        ))
         .add_strip((0..1000).map(|i| {
             glam::vec3(
                 (i as f32 * 0.01).sin() * 2.0,
@@ -318,7 +320,7 @@ impl Example for Multiview {
         let mut builder = PointCloudBuilder::new(re_ctx);
         builder
             .batch("Random Points")
-            .world_from_obj(glam::Mat4::from_rotation_x(seconds_since_startup))
+            .world_from_obj(glam::Affine3A::from_rotation_x(seconds_since_startup))
             .add_points(
                 self.random_points_positions.len(),
                 self.random_points_positions.iter().cloned(),
@@ -341,6 +343,7 @@ impl Example for Multiview {
             Projection::Perspective {
                 vertical_fov: 70.0 * TAU / 360.0,
                 near_plane_distance: 0.01,
+                aspect_ratio: resolution[0] as f32 / resolution[1] as f32,
             }
         } else {
             Projection::Orthographic {

crates/re_renderer/examples/outlines.rs

@@ -61,6 +61,7 @@ impl framework::Example for Outlines {
                 projection_from_view: Projection::Perspective {
                     vertical_fov: 70.0 * std::f32::consts::TAU / 360.0,
                     near_plane_distance: 0.01,
+                    aspect_ratio: resolution[0] as f32 / resolution[1] as f32,
                 },
                 pixels_from_point,
                 outline_config: Some(OutlineConfig {

crates/re_renderer/examples/picking.rs

@@ -3,8 +3,8 @@ use rand::Rng;
 use re_renderer::{
     renderer::MeshInstance,
     view_builder::{Projection, TargetConfiguration, ViewBuilder},
-    Color32, GpuReadbackIdentifier, IntRect, PickingLayerId, PickingLayerInstanceId,
-    PickingLayerProcessor, PointCloudBuilder, Size,
+    Color32, GpuReadbackIdentifier, PickingLayerId, PickingLayerInstanceId, PickingLayerProcessor,
+    PointCloudBuilder, RectInt, Size,
 };
 
 mod framework;
@@ -135,6 +135,7 @@ impl framework::Example for Picking {
                 projection_from_view: Projection::Perspective {
                     vertical_fov: 70.0 * std::f32::consts::TAU / 360.0,
                     near_plane_distance: 0.01,
+                    aspect_ratio: resolution[0] as f32 / resolution[1] as f32,
                 },
                 pixels_from_point,
                 outline_config: None,
@@ -145,7 +146,7 @@ impl framework::Example for Picking {
         // Use an uneven number of pixels for the picking rect so that there is a clearly defined middle-pixel.
         // (for this sample a size of 1 would be sufficient, but for a real application you'd want to use a larger size to allow snapping)
         let picking_rect_size = 31;
-        let picking_rect = IntRect::from_middle_and_extent(
+        let picking_rect = RectInt::from_middle_and_extent(
             self.picking_position.as_ivec2(),
             glam::uvec2(picking_rect_size, picking_rect_size),
         );

crates/re_renderer/shader/depth_cloud.wgsl

@@ -133,7 +133,8 @@ fn vs_main(@builtin(vertex_index) vertex_idx: u32) -> VertexOut {
 
     if 0.0 < point_data.unresolved_radius {
         // Span quad
-        let quad = sphere_quad_span(vertex_idx, point_data.pos_in_world, point_data.unresolved_radius, depth_cloud_info.radius_boost_in_ui_points);
+        let quad = sphere_or_circle_quad_span(vertex_idx, point_data.pos_in_world, point_data.unresolved_radius,
+                                                depth_cloud_info.radius_boost_in_ui_points, false);
         out.pos_in_clip = frame.projection_from_world * Vec4(quad.pos_in_world, 1.0);
         out.pos_in_world = quad.pos_in_world;
         out.point_radius = quad.point_resolved_radius;

crates/re_renderer/shader/lines.wgsl

@@ -44,6 +44,7 @@ const CAP_START_TRIANGLE: u32 = 8u;
 const CAP_START_ROUND: u32 = 16u;
 const CAP_START_EXTEND_OUTWARDS: u32 = 32u;
 const NO_COLOR_GRADIENT: u32 = 64u;
+const FORCE_ORTHO_SPANNING: u32 = 128u;
 
 // A lot of the attributes don't need to be interpolated across triangles.
 // To document that and safe some time we mark them up with @interpolate(flat)
@@ -199,7 +200,12 @@ fn vs_main(@builtin(vertex_index) vertex_idx: u32) -> VertexOut {
 
     // Resolve radius.
     // (slight inaccuracy: End caps are going to adjust their center_position)
-    let camera_ray = camera_ray_to_world_pos(center_position);
+    var camera_ray: Ray;
+    if has_any_flag(strip_data.flags, FORCE_ORTHO_SPANNING) || is_camera_orthographic() {
+        camera_ray = camera_ray_to_world_pos_orthographic(center_position);
+    } else {
+        camera_ray = camera_ray_to_world_pos_perspective(center_position);
+    }
     let camera_distance = distance(camera_ray.origin, center_position);
     var strip_radius = unresolved_size_to_world(strip_data.unresolved_radius, camera_distance, frame.auto_size_lines);
 

crates/re_renderer/shader/point_cloud.wgsl

@@ -36,6 +36,8 @@ var<uniform> batch: BatchUniformBuffer;
 // Flags
 // See point_cloud.rs#PointCloudBatchFlags
 const ENABLE_SHADING: u32 = 1u;
+const DRAW_AS_CIRCLES: u32 = 2u;
+
 const TEXTURE_SIZE: u32 = 2048u;
 
 struct VertexOut {
@@ -94,7 +96,8 @@ fn vs_main(@builtin(vertex_index) vertex_idx: u32) -> VertexOut {
     let point_data = read_data(quad_idx);
 
     // Span quad
-    let quad = sphere_quad_span(vertex_idx, point_data.pos, point_data.unresolved_radius, draw_data.radius_boost_in_ui_points);
+    let quad = sphere_or_circle_quad_span(vertex_idx, point_data.pos, point_data.unresolved_radius,
+                                          draw_data.radius_boost_in_ui_points, has_any_flag(batch.flags, DRAW_AS_CIRCLES));
 
     // Output, transform to projection space and done.
     var out: VertexOut;
@@ -108,9 +111,32 @@ fn vs_main(@builtin(vertex_index) vertex_idx: u32) -> VertexOut {
     return out;
 }
 
+// TODO(andreas): move this to sphere_quad.wgsl once https://github.com/gfx-rs/naga/issues/1743 is resolved
+// point_cloud.rs has a specific workaround in place so we don't need to split vertex/fragment shader here
+//
+/// Computes coverage of a 2D sphere placed at `circle_center` in the fragment shader using the currently set camera.
+///
+/// 2D primitives are always facing the camera - the difference to sphere_quad_coverage is that
+/// perspective projection is not taken into account.
+fn circle_quad_coverage(world_position: Vec3, radius: f32, circle_center: Vec3) -> f32 {
+    let to_center = circle_center - world_position;
+    let distance = length(to_center);
+    let distance_pixel_difference = fwidth(distance);
+    return smoothstep(radius + distance_pixel_difference, radius - distance_pixel_difference, distance);
+}
+
+fn coverage(world_position: Vec3, radius: f32, point_center: Vec3) -> f32 {
+    if is_camera_orthographic() || has_any_flag(batch.flags, DRAW_AS_CIRCLES) {
+        return circle_quad_coverage(world_position, radius, point_center);
+    } else {
+        return sphere_quad_coverage(world_position, radius, point_center);
+    }
+}
+
+
 @fragment
 fn fs_main(in: VertexOut) -> @location(0) Vec4 {
-    let coverage = sphere_quad_coverage(in.world_position, in.radius, in.point_center);
+    let coverage = coverage(in.world_position, in.radius, in.point_center);
     if coverage < 0.001 {
         discard;
     }
@@ -127,7 +153,7 @@ fn fs_main(in: VertexOut) -> @location(0) Vec4 {
 
 @fragment
 fn fs_main_picking_layer(in: VertexOut) -> @location(0) UVec4 {
-    let coverage = sphere_quad_coverage(in.world_position, in.radius, in.point_center);
+    let coverage = coverage(in.world_position, in.radius, in.point_center);
     if coverage <= 0.5 {
         discard;
     }
@@ -139,7 +165,7 @@ fn fs_main_outline_mask(in: VertexOut) -> @location(0) UVec2 {
     // Output is an integer target, can't use coverage therefore.
     // But we still want to discard fragments where coverage is low.
     // Since the outline extends a bit, a very low cut off tends to look better.
-    let coverage = sphere_quad_coverage(in.world_position, in.radius, in.point_center);
+    let coverage = coverage(in.world_position, in.radius, in.point_center);
     if coverage < 1.0 {
         discard;
     }

crates/re_renderer/shader/utils/camera.wgsl

@@ -15,22 +15,32 @@ struct Ray {
     direction: Vec3,
 }
 
-// Returns the ray from the camera to a given world position.
-fn camera_ray_to_world_pos(world_pos: Vec3) -> Ray {
+// Returns the ray from the camera to a given world position, assuming the camera is perspective
+fn camera_ray_to_world_pos_perspective(world_pos: Vec3) -> Ray {
+    var ray: Ray;
+    ray.origin = frame.camera_position;
+    ray.direction = normalize(world_pos - frame.camera_position);
+    return ray;
+}
+
+// Returns the ray from the camera to a given world position, assuming the camera is orthographic
+fn camera_ray_to_world_pos_orthographic(world_pos: Vec3) -> Ray {
     var ray: Ray;
+    // The ray originates on the camera plane, not from the camera position
+    let to_pos = world_pos - frame.camera_position;
+    let camera_plane_distance = dot(to_pos, frame.camera_forward);
+    ray.origin = world_pos - frame.camera_forward * camera_plane_distance;
+    ray.direction = frame.camera_forward;
+    return ray;
+}
 
+// Returns the ray from the camera to a given world position.
+fn camera_ray_to_world_pos(world_pos: Vec3) -> Ray {
     if is_camera_perspective() {
-        ray.origin = frame.camera_position;
-        ray.direction = normalize(world_pos - frame.camera_position);
+        return camera_ray_to_world_pos_perspective(world_pos);
     } else {
-        // The ray originates on the camera plane, not from the camera position
-        let to_pos = world_pos - frame.camera_position;
-        let camera_plane_distance = dot(to_pos, frame.camera_forward);
-        ray.origin = world_pos - frame.camera_forward * camera_plane_distance;
-        ray.direction = frame.camera_forward;
+        return camera_ray_to_world_pos_orthographic(world_pos);
     }
-
-    return ray;
 }
 
 // Returns the camera ray direction through a given screen uv coordinates (ranging from 0 to 1, i.e. NOT ndc coordinates)

crates/re_renderer/shader/utils/sphere_quad.wgsl

@@ -4,7 +4,7 @@
 
 /// Span a quad in a way that guarantees that we'll be able to draw a perspective correct sphere
 /// on it.
-fn sphere_quad_span_perspective(
+fn sphere_quad(
     point_pos: Vec3,
     point_radius: f32,
     top_bottom: f32,
@@ -42,15 +42,16 @@ fn sphere_quad_span_perspective(
 
 /// Span a quad in a way that guarantees that we'll be able to draw an orthographic correct sphere
 /// on it.
-fn sphere_quad_span_orthographic(point_pos: Vec3, point_radius: f32, top_bottom: f32, left_right: f32) -> Vec3 {
+fn circle_quad(point_pos: Vec3, point_radius: f32, top_bottom: f32, left_right: f32) -> Vec3 {
     let quad_normal = frame.camera_forward;
     let quad_right = normalize(cross(quad_normal, frame.view_from_world[1].xyz)); // It's spheres so any orthogonal vector would do.
     let quad_up = cross(quad_right, quad_normal);
     let pos_in_quad = top_bottom * quad_up + left_right * quad_right;
 
     // Add half a pixel of margin for the feathering we do for antialiasing the spheres.
     // It's fairly subtle but if we don't do this our spheres look slightly squarish
-    let radius = point_radius + 0.5 * frame.pixel_world_size_from_camera_distance;
+    // TODO(andreas): Computing distance to camera here is a bit excessive, should get distance more easily - keep in mind this code runs for ortho & perspective.
+    let radius = point_radius + 0.5 * approx_pixel_world_size_at(distance(point_pos, frame.camera_position));
 
     return point_pos + pos_in_quad * radius;
 }
@@ -65,10 +66,11 @@ struct SphereQuadData {
     point_resolved_radius: f32,
 }
 
-/// Span a quad onto which perspective correct spheres can be drawn.
+/// Span a quad onto which circles or perspective correct spheres can be drawn.
 ///
-/// Spanning is done in perspective or orthographically depending of the state of the global cam.
-fn sphere_quad_span(vertex_idx: u32, point_pos: Vec3, point_unresolved_radius: f32, radius_boost_in_ui_points: f32) -> SphereQuadData {
+/// Note that in orthographic mode, spheres are always circles.
+fn sphere_or_circle_quad_span(vertex_idx: u32, point_pos: Vec3, point_unresolved_radius: f32,
+                                radius_boost_in_ui_points: f32, force_circle: bool) -> SphereQuadData {
     // Resolve radius to a world size. We need the camera distance for this, which is useful later on.
     let to_camera = frame.camera_position - point_pos;
     let camera_distance = length(to_camera);
@@ -82,24 +84,23 @@ fn sphere_quad_span(vertex_idx: u32, point_pos: Vec3, point_unresolved_radius: f
 
     // Span quad
     var pos: Vec3;
-    if is_camera_perspective() {
-        pos = sphere_quad_span_perspective(point_pos, radius, top_bottom, left_right, to_camera, camera_distance);
+    if is_camera_orthographic() || force_circle {
+        pos = circle_quad(point_pos, radius, top_bottom, left_right);
     } else {
-        pos = sphere_quad_span_orthographic(point_pos, radius, top_bottom, left_right);
+        pos = sphere_quad(point_pos, radius, top_bottom, left_right, to_camera, camera_distance);
     }
 
     return SphereQuadData(pos, radius);
 }
 
-fn sphere_quad_coverage(world_position: Vec3, radius: f32, point_center: Vec3) -> f32 {
-    // There's easier ways to compute anti-aliasing for when we are in ortho mode since it's just circles.
-    // But it's very nice to have mostly the same code path and this gives us the sphere world position along the way.
+/// Computes coverage of a 3D sphere placed at `sphere_center` in the fragment shader using the currently set camera.
+fn sphere_quad_coverage(world_position: Vec3, radius: f32, sphere_center: Vec3) -> f32 {
     let ray = camera_ray_to_world_pos(world_position);
 
     // Sphere intersection with anti-aliasing as described by Iq here
     // https://www.shadertoy.com/view/MsSSWV
     // (but rearranged and labeled to it's easier to understand!)
-    let d = ray_sphere_distance(ray, point_center, radius);
+    let d = ray_sphere_distance(ray, sphere_center, radius);
     let distance_to_sphere_surface = d.x;
     let closest_ray_dist = d.y;
     let pixel_world_size = approx_pixel_world_size_at(closest_ray_dist);