re_renderer: Arrow buffers as (optional) first-class citizen

crates/re_data_store/src/entity_properties.rs

@@ -36,6 +36,8 @@ impl EntityPropertyMap {
 
 // ----------------------------------------------------------------------------
 
+// TODO(#1423): We need to properly split entity properties that only apply to specific
+// views/primitives.
 #[cfg(feature = "serde")]
 #[derive(Clone, Debug, PartialEq)]
 #[cfg_attr(feature = "serde", derive(serde::Deserialize, serde::Serialize))]
@@ -50,6 +52,20 @@ pub struct EntityProperties {
     /// Only applies to pinhole cameras when in a spatial view, using 3D navigation.
     ///
     pub pinhole_image_plane_distance: EditableAutoValue<f32>,
+
+    /// Should the depth texture be backprojected into a point cloud?
+    ///
+    /// Only applies to tensors with meaning=depth that are affected by a pinhole transform when
+    /// in a spatial view, using 3D navigation.
+    pub backproject_depth: bool,
+    /// Entity path of the pinhole transform used for the backprojection.
+    ///
+    /// `None` means backprojection is disabled.
+    pub backproject_pinhole_ent_path: Option<EntityPath>,
+    /// Used to scale the resulting point cloud.
+    pub backproject_scale: EditableAutoValue<f32>,
+    /// Used to scale the radii of the points in the resulting point cloud.
+    pub backproject_radius_scale: EditableAutoValue<f32>,
 }
 
 #[cfg(feature = "serde")]
@@ -64,6 +80,16 @@ impl EntityProperties {
                 .pinhole_image_plane_distance
                 .or(&child.pinhole_image_plane_distance)
                 .clone(),
+            backproject_depth: self.backproject_depth || child.backproject_depth,
+            backproject_pinhole_ent_path: child
+                .backproject_pinhole_ent_path
+                .clone()
+                .or(self.backproject_pinhole_ent_path.clone()),
+            backproject_scale: child.backproject_scale.or(&self.backproject_scale).clone(),
+            backproject_radius_scale: child
+                .backproject_radius_scale
+                .or(&self.backproject_radius_scale)
+                .clone(),
         }
     }
 }
@@ -76,6 +102,10 @@ impl Default for EntityProperties {
             visible_history: ExtraQueryHistory::default(),
             interactive: true,
             pinhole_image_plane_distance: EditableAutoValue::default(),
+            backproject_depth: false,
+            backproject_pinhole_ent_path: None,
+            backproject_scale: EditableAutoValue::default(),
+            backproject_radius_scale: EditableAutoValue::default(),
         }
     }
 }

crates/re_renderer/Cargo.toml

@@ -24,7 +24,10 @@ targets = ["x86_64-unknown-linux-gnu", "wasm32-unknown-unknown"]
 
 
 [features]
-default = ["import-obj", "import-gltf"]
+default = ["arrow", "import-obj", "import-gltf"]
+
+## Support for Arrow datatypes for end-to-end zero-copy.
+arrow = ["dep:arrow2"]
 
 ## Support importing .obj files
 import-obj = ["dep:tobj"]
@@ -61,6 +64,7 @@ thiserror.workspace = true
 type-map = "0.5"
 
 # optional
+arrow2 = { workspace = true, optional = true }
 gltf = { workspace = true, optional = true }
 serde = { version = "1", features = ["derive"], optional = true }
 tobj = { version = "3.2", optional = true }

crates/re_renderer/examples/depth_cloud.rs

@@ -78,17 +78,14 @@ impl RenderDepthClouds {
         } = self;
 
         let focal_length = glam::Vec2::new(intrinsics.x_axis.x, intrinsics.y_axis.y);
-        let offset = glam::Vec2::new(intrinsics.x_axis.z, intrinsics.y_axis.z);
+        let offset = glam::Vec2::new(intrinsics.z_axis.x, intrinsics.z_axis.y);
 
         let point_cloud_draw_data = {
             let num_points = depth.dimensions.x * depth.dimensions.y;
             let (points, colors, radii): (Vec<_>, Vec<_>, Vec<_>) = (0..depth.dimensions.y)
                 .flat_map(|y| (0..depth.dimensions.x).map(move |x| glam::UVec2::new(x, y)))
                 .map(|texcoords| {
-                    let linear_depth = depth.get_linear(
-                        depth.dimensions.x - texcoords.x - 1,
-                        depth.dimensions.y - texcoords.y - 1,
-                    );
+                    let linear_depth = depth.get_linear(texcoords.x, texcoords.y);
                     let pos_in_world = ((texcoords.as_vec2() - offset) * linear_depth
                         / focal_length)
                         .extend(linear_depth);
@@ -171,17 +168,12 @@ impl RenderDepthClouds {
             ..
         } = self;
 
-        let world_from_obj = glam::Mat4::from_cols(
-            glam::Vec4::NEG_X * *scale,
-            glam::Vec4::NEG_Y * *scale,
-            glam::Vec4::Z * *scale,
-            glam::Vec4::W,
-        );
+        let world_from_obj = glam::Mat4::from_scale(glam::Vec3::splat(*scale));
 
         let depth_cloud_draw_data = DepthCloudDrawData::new(
             re_ctx,
             &[DepthCloud {
-                world_from_obj,
+                depth_camera_extrinsics: world_from_obj,
                 depth_camera_intrinsics: *intrinsics,
                 radius_scale: *radius_scale,
                 depth_dimensions: depth.dimensions,
@@ -261,7 +253,8 @@ impl framework::Example for RenderDepthClouds {
             Vec3::new(focal_length, 0.0, offset.x),
             Vec3::new(0.0, focal_length, offset.y),
             Vec3::new(0.0, 0.0, 1.0),
-        );
+        )
+        .transpose();
 
         RenderDepthClouds {
             depth,
@@ -447,7 +440,7 @@ impl DepthTexture {
         spiral(dimensions).for_each(|(texcoords, d)| {
             data[(texcoords.x + texcoords.y * dimensions.x) as usize] = d;
         });
-        let data = DepthCloudDepthData::F32(data);
+        let data = DepthCloudDepthData::F32(data.into());
 
         Self { dimensions, data }
     }

crates/re_renderer/shader/depth_cloud.wgsl

@@ -30,7 +30,7 @@ fn compute_point_data(quad_idx: i32) -> PointData {
     let color = Vec4(linear_from_srgb(Vec3(norm_linear_depth)), 1.0);
 
     // TODO(cmc): This assumes a pinhole camera; need to support other kinds at some point.
-    let intrinsics = transpose(depth_cloud_info.depth_camera_intrinsics);
+    let intrinsics = depth_cloud_info.depth_camera_intrinsics;
     let focal_length = Vec2(intrinsics[0][0], intrinsics[1][1]);
     let offset = Vec2(intrinsics[2][0], intrinsics[2][1]);
 
@@ -39,7 +39,7 @@ fn compute_point_data(quad_idx: i32) -> PointData {
         norm_linear_depth,
     );
 
-    let pos_in_world = depth_cloud_info.world_from_obj * Vec4(pos_in_obj, 1.0);
+    let pos_in_world = depth_cloud_info.extrinsincs * Vec4(pos_in_obj, 1.0);
 
     var data: PointData;
     data.pos_in_world = pos_in_world.xyz;
@@ -52,7 +52,8 @@ fn compute_point_data(quad_idx: i32) -> PointData {
 // ---
 
 struct DepthCloudInfo {
-    world_from_obj: Mat4,
+    /// The extrinsincs of the camera used for the projection.
+    extrinsincs: Mat4,
 
     /// The intrinsics of the camera used for the projection.
     ///

crates/re_renderer/src/renderer/depth_cloud.rs

@@ -39,7 +39,7 @@ mod gpu_data {
     #[repr(C, align(256))]
     #[derive(Clone, Copy, bytemuck::Pod, bytemuck::Zeroable)]
     pub struct DepthCloudInfoUBO {
-        pub world_from_obj: crate::wgpu_buffer_types::Mat4,
+        pub depth_camera_extrinsics: crate::wgpu_buffer_types::Mat4,
         pub depth_camera_intrinsics: crate::wgpu_buffer_types::Mat3,
         pub radius_scale: crate::wgpu_buffer_types::F32RowPadded,
 
@@ -60,12 +60,26 @@ mod gpu_data {
 // TODO(cmc): expose knobs to linearize/normalize/flip/cam-to-plane depth.
 #[derive(Debug, Clone)]
 pub enum DepthCloudDepthData {
+    #[cfg(not(feature = "arrow"))]
     U16(Vec<u16>),
+    #[cfg(feature = "arrow")]
+    U16(arrow2::buffer::Buffer<u16>),
+
+    #[cfg(not(feature = "arrow"))]
     F32(Vec<f32>),
+    #[cfg(feature = "arrow")]
+    F32(arrow2::buffer::Buffer<f32>),
+}
+
+impl Default for DepthCloudDepthData {
+    fn default() -> Self {
+        Self::F32(Default::default())
+    }
 }
 
 pub struct DepthCloud {
-    pub world_from_obj: glam::Mat4,
+    /// The extrinsics of the camera used for the projection.
+    pub depth_camera_extrinsics: glam::Mat4,
 
     /// The intrinsics of the camera used for the projection.
     ///
@@ -87,11 +101,11 @@ pub struct DepthCloud {
 impl Default for DepthCloud {
     fn default() -> Self {
         Self {
-            world_from_obj: glam::Mat4::IDENTITY,
+            depth_camera_extrinsics: glam::Mat4::IDENTITY,
             depth_camera_intrinsics: glam::Mat3::IDENTITY,
             radius_scale: 1.0,
             depth_dimensions: glam::UVec2::ZERO,
-            depth_data: DepthCloudDepthData::F32(Vec::new()),
+            depth_data: DepthCloudDepthData::default(),
         }
     }
 }
@@ -113,6 +127,17 @@ impl DepthCloudDrawData {
     ) -> Result<Self, ResourceManagerError> {
         crate::profile_function!();
 
+        let bg_layout = ctx
+            .renderers
+            .write()
+            .get_or_create::<_, DepthCloudRenderer>(
+                &ctx.shared_renderer_data,
+                &mut ctx.gpu_resources,
+                &ctx.device,
+                &mut ctx.resolver,
+            )
+            .bind_group_layout;
+
         if depth_clouds.is_empty() {
             return Ok(DepthCloudDrawData {
                 bind_groups: Vec::new(),
@@ -123,24 +148,13 @@ impl DepthCloudDrawData {
             ctx,
             "depth_cloud_ubos".into(),
             depth_clouds.iter().map(|info| gpu_data::DepthCloudInfoUBO {
-                world_from_obj: info.world_from_obj.into(),
+                depth_camera_extrinsics: info.depth_camera_extrinsics.into(),
                 depth_camera_intrinsics: info.depth_camera_intrinsics.into(),
                 radius_scale: info.radius_scale.into(),
                 end_padding: Default::default(),
             }),
         );
 
-        let bg_layout = ctx
-            .renderers
-            .write()
-            .get_or_create::<_, DepthCloudRenderer>(
-                &ctx.shared_renderer_data,
-                &mut ctx.gpu_resources,
-                &ctx.device,
-                &mut ctx.resolver,
-            )
-            .bind_group_layout;
-
         let mut bind_groups = Vec::with_capacity(depth_clouds.len());
         for (depth_cloud, ubo) in depth_clouds.iter().zip(depth_cloud_ubos.into_iter()) {
             let depth_texture = match &depth_cloud.depth_data {