Codegen'd Rust/Arrow (de)ser 9: Rust backport! (#2571)

**Best reviewed on a commit-by-commit basis; in particular the `rerun
codegen` commit is nothing but generated code.**

You can now use archetypes with the existing SDK:
```rust
    use rerun::experimental::archetypes::Points2D;
    MsgSender::from_archetype(
        "2d_layering/points_between_top_and_middle",
        &Points2D::new(
            (0..256).map(|i| (32.0 + (i / 16) as f32 * 16.0, 64.0 + (i % 16) as f32 * 16.0)),
        )
        .with_draw_order(1.51),
    )?
    .send(rec_stream)?;
```

Closes #2387 

---

- #2484
- #2485 
- #2487 
- #2545
- #2546
- #2549
- #2554
- #2570
- #2571

---

* [x] I have read and agree to [Contributor
Guide](https://github.com/rerun-io/rerun/blob/main/CONTRIBUTING.md) and
the [Code of
Conduct](https://github.com/rerun-io/rerun/blob/main/CODE_OF_CONDUCT.md)
* [x] I've included a screenshot or gif (if applicable)
* [x] I have tested [demo.rerun.io](https://demo.rerun.io/pr/{{
pr.number }}) (if applicable)

- [PR Build Summary](https://build.rerun.io/pr/{{ pr.number }})
- [Docs preview](https://rerun.io/preview/{{
"pr:%s"|format(pr.branch)|encode_uri_component }}/docs)
- [Examples preview](https://rerun.io/preview/{{
"pr:%s"|format(pr.branch)|encode_uri_component }}/examples)

crates/re_sdk/Cargo.toml

@@ -38,6 +38,7 @@ re_log_types.workspace = true
 re_log.workspace = true
 re_memory.workspace = true
 re_sdk_comms = { workspace = true, features = ["client"] }
+re_types = { workspace = true, features = ["ecolor", "glam"] }
 
 ahash.workspace = true
 crossbeam.workspace = true

crates/re_sdk/src/lib.rs

@@ -94,6 +94,11 @@ pub mod coordinates {
     pub use re_components::coordinates::{Axis3, Handedness, Sign, SignedAxis3};
 }
 
+/// Experimental APIs, to try out upcoming Rerun features.
+pub mod experimental {
+    pub use re_types::{archetypes, components, datatypes, Archetype, Component, Datatype};
+}
+
 /// Re-exports of other crates.
 pub mod external {
     pub use re_log;

crates/re_sdk/src/msg_sender.rs

@@ -1,4 +1,7 @@
-use re_log_types::{DataRow, DataTableError, InstanceKey, RowId, StoreId};
+use re_log_types::{
+    external::arrow2::datatypes::DataType, DataRow, DataTableError, InstanceKey, RowId, StoreId,
+};
+use re_types::Archetype;
 
 use crate::{
     log::DataCell,
@@ -102,6 +105,32 @@ impl MsgSender {
         }
     }
 
+    /// Starts a new `MsgSender` for the given entity path, and fill it with the contents of the
+    /// passed-in [`Archetype`].
+    ///
+    /// WARNING: This is an experimental feature!
+    pub fn from_archetype(
+        ent_path: impl Into<EntityPath>,
+        arch: &impl Archetype,
+    ) -> Result<Self, MsgSenderError> {
+        let serialized = arch.to_arrow();
+
+        let mut this = Self::new(ent_path);
+        for (field, array) in serialized {
+            // NOTE: Unreachable, a top-level Field will always be a component, and thus an
+            // extension.
+            let DataType::Extension(_, _, legacy_fqname) = field.data_type else { unreachable!() };
+            this = this.with_cell(DataCell::from_arrow(
+                // NOTE: Unwrapping is safe as we always include the legacy fqname into the Field's
+                // metadata while migrating towards HOPE.
+                legacy_fqname.as_deref().unwrap().into(),
+                array,
+            ))?;
+        }
+
+        Ok(this)
+    }
+
     /// Read the file at the given path and log it.
     ///
     /// Supported file extensions are:

crates/re_types/source_hash.txt

@@ -1,4 +1,4 @@
 # This is a sha256 hash for all direct and indirect dependencies of this crate's build script.
 # It can be safely removed at anytime to force the build script to run again.
 # Check out build.rs to see how it's computed.
-128cd542f3f9e6cf3b2a44aeb022cd5f3ad819b00ce5371eeb311300f3e9a7f1
+afc01539cb778ef699e8cef27436420fe8dc4ee078fdd56d874e2409b3749d88

crates/re_types/src/archetypes/points2d.rs

@@ -111,7 +111,11 @@ impl crate::Archetype for Points2D {
                     let array =
                         <crate::components::Point2D>::try_to_arrow(self.points.iter(), None);
                     array.map(|array| {
-                        let datatype = array.data_type().clone();
+                        let datatype = ::arrow2::datatypes::DataType::Extension(
+                            "rerun.components.Point2D".into(),
+                            Box::new(array.data_type().clone()),
+                            Some("rerun.point2d".into()),
+                        );
                         (
                             ::arrow2::datatypes::Field::new("points", datatype, false),
                             array,
@@ -126,7 +130,11 @@ impl crate::Archetype for Points2D {
                     .map(|many| {
                         let array = <crate::components::Radius>::try_to_arrow(many.iter(), None);
                         array.map(|array| {
-                            let datatype = array.data_type().clone();
+                            let datatype = ::arrow2::datatypes::DataType::Extension(
+                                "rerun.components.Radius".into(),
+                                Box::new(array.data_type().clone()),
+                                Some("rerun.radius".into()),
+                            );
                             (
                                 ::arrow2::datatypes::Field::new("radii", datatype, false),
                                 array,
@@ -141,7 +149,11 @@ impl crate::Archetype for Points2D {
                     .map(|many| {
                         let array = <crate::components::Color>::try_to_arrow(many.iter(), None);
                         array.map(|array| {
-                            let datatype = array.data_type().clone();
+                            let datatype = ::arrow2::datatypes::DataType::Extension(
+                                "rerun.components.Color".into(),
+                                Box::new(array.data_type().clone()),
+                                Some("rerun.colorrgba".into()),
+                            );
                             (
                                 ::arrow2::datatypes::Field::new("colors", datatype, false),
                                 array,
@@ -156,7 +168,11 @@ impl crate::Archetype for Points2D {
                     .map(|many| {
                         let array = <crate::components::Label>::try_to_arrow(many.iter(), None);
                         array.map(|array| {
-                            let datatype = array.data_type().clone();
+                            let datatype = ::arrow2::datatypes::DataType::Extension(
+                                "rerun.components.Label".into(),
+                                Box::new(array.data_type().clone()),
+                                Some("rerun.label".into()),
+                            );
                             (
                                 ::arrow2::datatypes::Field::new("labels", datatype, false),
                                 array,
@@ -171,7 +187,11 @@ impl crate::Archetype for Points2D {
                     .map(|single| {
                         let array = <crate::components::DrawOrder>::try_to_arrow([single], None);
                         array.map(|array| {
-                            let datatype = array.data_type().clone();
+                            let datatype = ::arrow2::datatypes::DataType::Extension(
+                                "rerun.components.DrawOrder".into(),
+                                Box::new(array.data_type().clone()),
+                                Some("rerun.draw_order".into()),
+                            );
                             (
                                 ::arrow2::datatypes::Field::new("draw_order", datatype, false),
                                 array,
@@ -186,7 +206,11 @@ impl crate::Archetype for Points2D {
                     .map(|many| {
                         let array = <crate::components::ClassId>::try_to_arrow(many.iter(), None);
                         array.map(|array| {
-                            let datatype = array.data_type().clone();
+                            let datatype = ::arrow2::datatypes::DataType::Extension(
+                                "rerun.components.ClassId".into(),
+                                Box::new(array.data_type().clone()),
+                                Some("rerun.class_id".into()),
+                            );
                             (
                                 ::arrow2::datatypes::Field::new("class_ids", datatype, false),
                                 array,
@@ -202,7 +226,11 @@ impl crate::Archetype for Points2D {
                         let array =
                             <crate::components::KeypointId>::try_to_arrow(many.iter(), None);
                         array.map(|array| {
-                            let datatype = array.data_type().clone();
+                            let datatype = ::arrow2::datatypes::DataType::Extension(
+                                "rerun.components.KeypointId".into(),
+                                Box::new(array.data_type().clone()),
+                                Some("rerun.keypoint_id".into()),
+                            );
                             (
                                 ::arrow2::datatypes::Field::new("keypoint_ids", datatype, false),
                                 array,
@@ -218,7 +246,11 @@ impl crate::Archetype for Points2D {
                         let array =
                             <crate::components::InstanceKey>::try_to_arrow(many.iter(), None);
                         array.map(|array| {
-                            let datatype = array.data_type().clone();
+                            let datatype = ::arrow2::datatypes::DataType::Extension(
+                                "rerun.components.InstanceKey".into(),
+                                Box::new(array.data_type().clone()),
+                                Some("rerun.instance_key".into()),
+                            );
                             (
                                 ::arrow2::datatypes::Field::new("instance_keys", datatype, false),
                                 array,

crates/re_types/src/components/class_id.rs

@@ -38,11 +38,7 @@ impl crate::Component for ClassId {
     #[inline]
     fn to_arrow_datatype() -> arrow2::datatypes::DataType {
         use ::arrow2::datatypes::*;
-        DataType::Extension(
-            "rerun.components.ClassId".to_owned(),
-            Box::new(DataType::UInt16),
-            None,
-        )
+        DataType::UInt16
     }
 
     #[allow(unused_imports, clippy::wildcard_imports)]
@@ -79,6 +75,8 @@ impl crate::Component for ClassId {
                         Box::new(DataType::UInt16),
                         None,
                     )
+                    .to_logical_type()
+                    .clone()
                 },
                 data0.into_iter().map(|v| v.unwrap_or_default()).collect(),
                 data0_bitmap,

crates/re_types/src/components/color.rs

@@ -48,11 +48,7 @@ impl crate::Component for Color {
     #[inline]
     fn to_arrow_datatype() -> arrow2::datatypes::DataType {
         use ::arrow2::datatypes::*;
-        DataType::Extension(
-            "rerun.components.Color".to_owned(),
-            Box::new(DataType::UInt32),
-            None,
-        )
+        DataType::UInt32
     }
 
     #[allow(unused_imports, clippy::wildcard_imports)]
@@ -89,6 +85,8 @@ impl crate::Component for Color {
                         Box::new(DataType::UInt32),
                         None,
                     )
+                    .to_logical_type()
+                    .clone()
                 },
                 data0.into_iter().map(|v| v.unwrap_or_default()).collect(),
                 data0_bitmap,

crates/re_types/src/components/draw_order.rs

@@ -43,11 +43,7 @@ impl crate::Component for DrawOrder {
     #[inline]
     fn to_arrow_datatype() -> arrow2::datatypes::DataType {
         use ::arrow2::datatypes::*;
-        DataType::Extension(
-            "rerun.components.DrawOrder".to_owned(),
-            Box::new(DataType::Float32),
-            None,
-        )
+        DataType::Float32
     }
 
     #[allow(unused_imports, clippy::wildcard_imports)]
@@ -84,6 +80,8 @@ impl crate::Component for DrawOrder {
                         Box::new(DataType::Float32),
                         None,
                     )
+                    .to_logical_type()
+                    .clone()
                 },
                 data0.into_iter().map(|v| v.unwrap_or_default()).collect(),
                 data0_bitmap,