Roundtrip-able Transform3Ds (#2669)

**Commit by commit!**

This PR makes it so Python and Rust can exchange `Transform3D` data
using the new Archetype-based APIs.

Roughly:
- Fixes serialization issues in Python SDK
- Implements all necessary Rust extensions
- Improves our roundtrip test suite
- Implements cross-language roundtrip tests for `Transform3D`

Fixes #2644 

Requires:
- #2639 
- #2673
- #2708 

---

### Checklist
* [x] I have read and agree to [Contributor
Guide](https://github.com/rerun-io/rerun/blob/main/CONTRIBUTING.md) and
the [Code of
Conduct](https://github.com/rerun-io/rerun/blob/main/CODE_OF_CONDUCT.md)
* [x] I've included a screenshot or gif (if applicable)
* [x] I have tested [demo.rerun.io](https://demo.rerun.io/pr/2669) (if
applicable)

- [PR Build Summary](https://build.rerun.io/pr/2669)
- [Docs
preview](https://rerun.io/preview/pr%3Acmc%2Fhope_rust_transforms/docs)
- [Examples
preview](https://rerun.io/preview/pr%3Acmc%2Fhope_rust_transforms/examples)

---------

Co-authored-by: Jeremy Leibs <[email protected]>

.gitignore

@@ -9,6 +9,7 @@
 **/Makefile
 **/cmake_install.cmake
 _deps
+**/.cache/
 
 # Rust compile target directory:
 **/target

crates/re_log_types/src/data_table.rs

@@ -1127,28 +1127,134 @@ impl DataTable {
                     timepoint: timepoint1,
                     entity_path: entity_path1,
                     num_instances: num_instances1,
-                    cells: cells1,
+                    cells: ref cells1,
                 } = row1;
                 let DataRow {
                     row_id: _,
                     timepoint: timepoint2,
                     entity_path: entity_path2,
                     num_instances: num_instances2,
-                    cells: cells2,
+                    cells: ref cells2,
                 } = row2;
 
+                for (c1, c2) in cells1.0.iter().zip(&cells2.0) {
+                    if c1 != c2 {
+                        anyhow::ensure!(
+                            c1.datatype() == c2.datatype(),
+                            "Found discrepancy in row #{ri}, cells' datatypes don't match!\n{}",
+                            similar_asserts::SimpleDiff::from_str(
+                                &format!("{:?}", c1.datatype()),
+                                &format!("{:?}", c2.datatype()),
+                                "cell1",
+                                "cell2"
+                            )
+                        );
+
+                        let arr1 = c1.as_arrow_ref();
+                        let arr2 = c2.as_arrow_ref();
+
+                        if let (Some(arr1), Some(arr2)) = (
+                            arr1.as_any().downcast_ref::<arrow2::array::UnionArray>(),
+                            arr2.as_any().downcast_ref::<arrow2::array::UnionArray>(),
+                        ) {
+                            anyhow::ensure!(
+                                arr1.validity() == arr2.validity(),
+                                "Found discrepancy in row #{ri}, union arrays' validity bitmaps don't match!\n{}\n{}",
+                                similar_asserts::SimpleDiff::from_str(&row1.to_string(), &row2.to_string(), "row1", "row2"),
+                                similar_asserts::SimpleDiff::from_str(
+                                    &format!("{:?}", arr1.validity()),
+                                    &format!("{:?}", arr2.validity()),
+                                    "cell1",
+                                    "cell2"
+                                )
+                            );
+                            anyhow::ensure!(
+                                arr1.types() == arr2.types(),
+                                "Found discrepancy in row #{ri}, union arrays' type indices don't match!\n{}\n{}",
+                                similar_asserts::SimpleDiff::from_str(&row1.to_string(), &row2.to_string(), "row1", "row2"),
+                                similar_asserts::SimpleDiff::from_str(
+                                    &format!("{:?}", arr1.types()),
+                                    &format!("{:?}", arr2.types()),
+                                    "cell1",
+                                    "cell2"
+                                )
+                            );
+                            anyhow::ensure!(
+                                arr1.offsets() == arr2.offsets(),
+                                "Found discrepancy in row #{ri}, union arrays' offsets don't match!\n{}\n{}",
+                                similar_asserts::SimpleDiff::from_str(&row1.to_string(), &row2.to_string(), "row1", "row2"),
+                                similar_asserts::SimpleDiff::from_str(
+                                    &format!("{:?}", arr1.offsets()),
+                                    &format!("{:?}", arr2.offsets()),
+                                    "cell1",
+                                    "cell2"
+                                )
+                            );
+                        }
+                    }
+                }
+
+                let mut size_mismatches = vec![];
+                for (c1, c2) in cells1.0.iter().zip(&cells2.0) {
+                    if c1.total_size_bytes() != c2.total_size_bytes() {
+                        size_mismatches.push(format!(
+                            "Found discrepancy in row #{ri}, cells' sizes don't match! {} ({}) vs. {} ({}) bytes",
+                            c1.total_size_bytes(),
+                            c1.component_name(),
+                            c2.total_size_bytes(),
+                            c2.component_name(),
+                        ));
+
+                        fn cell_to_bytes(cell: DataCell) -> Vec<u8> {
+                            let row = DataRow::from_cells1(
+                                RowId::random(),
+                                "cell",
+                                TimePoint::default(),
+                                cell.num_instances(),
+                                cell,
+                            );
+                            let table = DataTable::from_rows(TableId::ZERO, [row]);
+
+                            let msg = table.to_arrow_msg().unwrap();
+
+                            use arrow2::io::ipc::write::StreamWriter;
+                            let mut buf = Vec::<u8>::new();
+                            let mut writer = StreamWriter::new(&mut buf, Default::default());
+                            writer.start(&msg.schema, None).unwrap();
+                            writer.write(&msg.chunk, None).unwrap();
+                            writer.finish().unwrap();
+
+                            buf
+                        }
+
+                        let c1_bytes = cell_to_bytes(c1.clone());
+                        let c2_bytes = cell_to_bytes(c1.clone());
+
+                        size_mismatches.push(
+                            similar_asserts::SimpleDiff::from_str(
+                                &format!("{c1_bytes:?}"),
+                                &format!("{c2_bytes:?}"),
+                                "cell1_ipc",
+                                "cell2_ipc",
+                            )
+                            .to_string(),
+                        );
+                    }
+                }
+
                 anyhow::ensure!(
                     timepoint1 == timepoint2
                         && entity_path1 == entity_path2
                         && num_instances1 == num_instances2
                         && cells1 == cells2,
-                    "Found discrepancy in row #{ri}:\n{}",
+                    "Found discrepancy in row #{ri}:\n{}\n{}",
                     similar_asserts::SimpleDiff::from_str(
                         &row1.to_string(),
                         &row2.to_string(),
                         "row1",
                         "row2"
-                    )
+                    ),
+                    size_mismatches.join("\n"),
                 );
             }
         }

crates/re_types/definitions/rerun/archetypes/transform3d.fbs

@@ -6,8 +6,22 @@ include "rerun/components.fbs";
 namespace rerun.archetypes;
 
 
-/// A 3D transform
+/// A 3D transform.
+///
+/// \py Example
+/// \py -------
+/// \py
+/// \py ```python
+/// \py \include:../../../../../docs/code-examples/transform3d_simple_v2.py
+/// \py ```
+///
+/// \rs ## Example
+/// \rs
+/// \rs ```ignore
+/// \rs \include:../../../../../docs/code-examples/transform3d_simple_v2.rs
+/// \rs ```
 table Transform3D (
+  "attr.rust.derive": "PartialEq",
   order: 100
 ) {
   /// The transform

crates/re_types/definitions/rerun/components/transform3d.fbs

@@ -12,8 +12,10 @@ table Transform3D (
   // the arrow schemas. Otherwise the schema-patching in `array_to_rust` will set the
   // wrong schema for this component.
   //"attr.rerun.legacy_fqname": "rerun.transform3d",
+  "attr.rust.derive": "PartialEq",
+  "attr.rust.tuple_struct",
   order: 100
 ) {
   /// Representation of the transform.
-  repr: rerun.datatypes.Transform3D (required, order: 100);
+  repr: rerun.datatypes.Transform3D (order: 100);
 }

crates/re_types/source_hash.txt

@@ -1,4 +1,4 @@
 # This is a sha256 hash for all direct and indirect dependencies of this crate's build script.
 # It can be safely removed at anytime to force the build script to run again.
 # Check out build.rs to see how it's computed.
-ba58a7d123c5fa81f04a58a6325ef65029d93f330d571176101db87596a9c0cf
+e11a5960ff29c75374e3298ae9f2728d9d6d838c517e1b1ee3bc2a18fada0efb

crates/re_types/src/archetypes/transform3d.rs

@@ -12,8 +12,66 @@
 #![allow(clippy::too_many_lines)]
 #![allow(clippy::unnecessary_cast)]
 
-/// A 3D transform
-#[derive(Clone, Debug)]
+/// A 3D transform.
+///
+/// ## Example
+///
+/// ```ignore
+/// //! Log some transforms.
+///
+/// use rerun::{
+///    experimental::{
+///        archetypes::Transform3D,
+///        datatypes::{
+///            Angle, Mat3x3, RotationAxisAngle, Scale3D, TranslationAndMat3x3,
+///            TranslationRotationScale3D,
+///        },
+///    },
+///    MsgSender, RecordingStreamBuilder,
+/// };
+/// use std::f32::consts::PI;
+///
+/// fn main() -> Result<(), Box<dyn std::error::Error>> {
+///    let (rec_stream, storage) = RecordingStreamBuilder::new("transform").memory()?;
+///
+///    let arrow = rerun::components::Arrow3D {
+///        origin: rerun::components::Vec3D::from([0.0, 0.0, 0.0]),
+///        vector: rerun::components::Vec3D::from([0.0, 1.0, 0.0]),
+///    };
+///
+///    MsgSender::new("base")
+///        .with_component(&[arrow])?
+///        .send(&rec_stream)?;
+///
+///    MsgSender::from_archetype(
+///        "base/translated",
+///        &Transform3D::new(TranslationAndMat3x3::new([1.0, 0.0, 0.0], Mat3x3::IDENTITY)),
+///    )?
+///    .send(&rec_stream)?;
+///
+///    MsgSender::new("base/translated")
+///        .with_component(&[arrow])?
+///        .send(&rec_stream)?;
+///
+///    MsgSender::from_archetype(
+///        "base/rotated_scaled",
+///        &Transform3D::new(TranslationRotationScale3D {
+///            rotation: Some(RotationAxisAngle::new([0.0, 0.0, 1.0], Angle::Radians(PI / 4.)).into()),
+///            scale: Some(Scale3D::from(2.0)),
+///            ..Default::default()
+///        }),
+///    )?
+///    .send(&rec_stream)?;
+///
+///    MsgSender::new("base/rotated_scaled")
+///        .with_component(&[arrow])?
+///        .send(&rec_stream)?;
+///
+///    rerun::native_viewer::show(storage.take())?;
+///    Ok(())
+/// }
+/// ```
+#[derive(Clone, Debug, PartialEq)]
 pub struct Transform3D {
     /// The transform
     pub transform: crate::components::Transform3D,