Skip to content

Commit

Permalink
Codegen'd Rust/Arrow (de)ser 8: carry extension metadata across trans…
Browse files Browse the repository at this point in the history 
…parency layers (#2570)

**Best reviewed on a commit-by-commit basis; in particular the `rerun
codegen` commit is nothing but generated code.**

This PR makes it so that arrow extension metadata doesn't get lost
across transparency layers, and adds regression tests to make sure this
doesn't happen again.

It also introduce simple codegen optimizations that greatly reduce the
size of the output code (yay wasm!).

This is Rust only for now, we need things to stabilize on the Python
side before we can apply the same kind of logic and tests.

---

- #2484
- #2485 
- #2487 
- #2545
- #2546
- #2549
- #2554
- #2570
- #2571

---

### Checklist
* [x] I have read and agree to [Contributor
Guide](https://github.com/rerun-io/rerun/blob/main/CONTRIBUTING.md) and
the [Code of
Conduct](https://github.com/rerun-io/rerun/blob/main/CODE_OF_CONDUCT.md)
* [x] I've included a screenshot or gif (if applicable)
* [x] I have tested [demo.rerun.io](https://demo.rerun.io/pr/{{
pr.number }}) (if applicable)

- [PR Build Summary](https://build.rerun.io/pr/{{ pr.number }})
- [Docs preview](https://rerun.io/preview/{{
"pr:%s"|format(pr.branch)|encode_uri_component }}/docs)
- [Examples preview](https://rerun.io/preview/{{
"pr:%s"|format(pr.branch)|encode_uri_component }}/examples)
  • Loading branch information
@teh-cmc
teh-cmc authored Jun 30, 2023
1 parent de97832 commit 9492e87
Show file tree
Hide file tree
Showing 19 changed files with 587 additions and 1,655 deletions.
2 changes: 1 addition & 1 deletion crates/re_types/source_hash.txt
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
# This is a sha256 hash for all direct and indirect dependencies of this crate's build script.
# It can be safely removed at anytime to force the build script to run again.
# Check out build.rs to see how it's computed.
7d42813c538b5c4716e75e225572409d5799dc142ede36243043ef82fc90399e
128cd542f3f9e6cf3b2a44aeb022cd5f3ad819b00ce5371eeb311300f3e9a7f1
70 changes: 42 additions & 28 deletions crates/re_types/src/archetypes/fuzzy.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -140,7 +140,8 @@ impl crate::Archetype for AffixFuzzer1 {
Ok([
{
Some({
let array = <crate::components::AffixFuzzer1>::try_to_arrow([&self.fuzz1001]);
let array =
<crate::components::AffixFuzzer1>::try_to_arrow([&self.fuzz1001], None);
array.map(|array| {
let datatype = array.data_type().clone();
(
Expand All @@ -153,7 +154,8 @@ impl crate::Archetype for AffixFuzzer1 {
},
{
Some({
let array = <crate::components::AffixFuzzer2>::try_to_arrow([&self.fuzz1002]);
let array =
<crate::components::AffixFuzzer2>::try_to_arrow([&self.fuzz1002], None);
array.map(|array| {
let datatype = array.data_type().clone();
(
Expand All @@ -166,7 +168,8 @@ impl crate::Archetype for AffixFuzzer1 {
},
{
Some({
let array = <crate::components::AffixFuzzer3>::try_to_arrow([&self.fuzz1003]);
let array =
<crate::components::AffixFuzzer3>::try_to_arrow([&self.fuzz1003], None);
array.map(|array| {
let datatype = array.data_type().clone();
(
Expand All @@ -179,7 +182,8 @@ impl crate::Archetype for AffixFuzzer1 {
},
{
Some({
let array = <crate::components::AffixFuzzer4>::try_to_arrow([&self.fuzz1004]);
let array =
<crate::components::AffixFuzzer4>::try_to_arrow([&self.fuzz1004], None);
array.map(|array| {
let datatype = array.data_type().clone();
(
Expand All @@ -192,7 +196,8 @@ impl crate::Archetype for AffixFuzzer1 {
},
{
Some({
let array = <crate::components::AffixFuzzer5>::try_to_arrow([&self.fuzz1005]);
let array =
<crate::components::AffixFuzzer5>::try_to_arrow([&self.fuzz1005], None);
array.map(|array| {
let datatype = array.data_type().clone();
(
Expand All @@ -205,7 +210,8 @@ impl crate::Archetype for AffixFuzzer1 {
},
{
Some({
let array = <crate::components::AffixFuzzer6>::try_to_arrow([&self.fuzz1006]);
let array =
<crate::components::AffixFuzzer6>::try_to_arrow([&self.fuzz1006], None);
array.map(|array| {
let datatype = array.data_type().clone();
(
Expand All @@ -218,7 +224,8 @@ impl crate::Archetype for AffixFuzzer1 {
},
{
Some({
let array = <crate::components::AffixFuzzer7>::try_to_arrow([&self.fuzz1007]);
let array =
<crate::components::AffixFuzzer7>::try_to_arrow([&self.fuzz1007], None);
array.map(|array| {
let datatype = array.data_type().clone();
(
Expand All @@ -232,7 +239,7 @@ impl crate::Archetype for AffixFuzzer1 {
{
Some({
let array =
<crate::components::AffixFuzzer1>::try_to_arrow(self.fuzz1101.iter());
<crate::components::AffixFuzzer1>::try_to_arrow(self.fuzz1101.iter(), None);
array.map(|array| {
let datatype = array.data_type().clone();
(
Expand All @@ -246,7 +253,7 @@ impl crate::Archetype for AffixFuzzer1 {
{
Some({
let array =
<crate::components::AffixFuzzer2>::try_to_arrow(self.fuzz1102.iter());
<crate::components::AffixFuzzer2>::try_to_arrow(self.fuzz1102.iter(), None);
array.map(|array| {
let datatype = array.data_type().clone();
(
Expand All @@ -260,7 +267,7 @@ impl crate::Archetype for AffixFuzzer1 {
{
Some({
let array =
<crate::components::AffixFuzzer3>::try_to_arrow(self.fuzz1103.iter());
<crate::components::AffixFuzzer3>::try_to_arrow(self.fuzz1103.iter(), None);
array.map(|array| {
let datatype = array.data_type().clone();
(
Expand All @@ -274,7 +281,7 @@ impl crate::Archetype for AffixFuzzer1 {
{
Some({
let array =
<crate::components::AffixFuzzer4>::try_to_arrow(self.fuzz1104.iter());
<crate::components::AffixFuzzer4>::try_to_arrow(self.fuzz1104.iter(), None);
array.map(|array| {
let datatype = array.data_type().clone();
(
Expand All @@ -288,7 +295,7 @@ impl crate::Archetype for AffixFuzzer1 {
{
Some({
let array =
<crate::components::AffixFuzzer5>::try_to_arrow(self.fuzz1105.iter());
<crate::components::AffixFuzzer5>::try_to_arrow(self.fuzz1105.iter(), None);
array.map(|array| {
let datatype = array.data_type().clone();
(
Expand All @@ -302,7 +309,7 @@ impl crate::Archetype for AffixFuzzer1 {
{
Some({
let array =
<crate::components::AffixFuzzer6>::try_to_arrow(self.fuzz1106.iter());
<crate::components::AffixFuzzer6>::try_to_arrow(self.fuzz1106.iter(), None);
array.map(|array| {
let datatype = array.data_type().clone();
(
Expand All @@ -316,7 +323,7 @@ impl crate::Archetype for AffixFuzzer1 {
{
Some({
let array =
<crate::components::AffixFuzzer7>::try_to_arrow(self.fuzz1107.iter());
<crate::components::AffixFuzzer7>::try_to_arrow(self.fuzz1107.iter(), None);
array.map(|array| {
let datatype = array.data_type().clone();
(
Expand All @@ -331,7 +338,7 @@ impl crate::Archetype for AffixFuzzer1 {
self.fuzz2001
.as_ref()
.map(|single| {
let array = <crate::components::AffixFuzzer1>::try_to_arrow([single]);
let array = <crate::components::AffixFuzzer1>::try_to_arrow([single], None);
array.map(|array| {
let datatype = array.data_type().clone();
(
Expand All @@ -346,7 +353,7 @@ impl crate::Archetype for AffixFuzzer1 {
self.fuzz2002
.as_ref()
.map(|single| {
let array = <crate::components::AffixFuzzer2>::try_to_arrow([single]);
let array = <crate::components::AffixFuzzer2>::try_to_arrow([single], None);
array.map(|array| {
let datatype = array.data_type().clone();
(
Expand All @@ -361,7 +368,7 @@ impl crate::Archetype for AffixFuzzer1 {
self.fuzz2003
.as_ref()
.map(|single| {
let array = <crate::components::AffixFuzzer3>::try_to_arrow([single]);
let array = <crate::components::AffixFuzzer3>::try_to_arrow([single], None);
array.map(|array| {
let datatype = array.data_type().clone();
(
Expand All @@ -376,7 +383,7 @@ impl crate::Archetype for AffixFuzzer1 {
self.fuzz2004
.as_ref()
.map(|single| {
let array = <crate::components::AffixFuzzer4>::try_to_arrow([single]);
let array = <crate::components::AffixFuzzer4>::try_to_arrow([single], None);
array.map(|array| {
let datatype = array.data_type().clone();
(
Expand All @@ -391,7 +398,7 @@ impl crate::Archetype for AffixFuzzer1 {
self.fuzz2005
.as_ref()
.map(|single| {
let array = <crate::components::AffixFuzzer5>::try_to_arrow([single]);
let array = <crate::components::AffixFuzzer5>::try_to_arrow([single], None);
array.map(|array| {
let datatype = array.data_type().clone();
(
Expand All @@ -406,7 +413,7 @@ impl crate::Archetype for AffixFuzzer1 {
self.fuzz2006
.as_ref()
.map(|single| {
let array = <crate::components::AffixFuzzer6>::try_to_arrow([single]);
let array = <crate::components::AffixFuzzer6>::try_to_arrow([single], None);
array.map(|array| {
let datatype = array.data_type().clone();
(
Expand All @@ -421,7 +428,7 @@ impl crate::Archetype for AffixFuzzer1 {
self.fuzz2007
.as_ref()
.map(|single| {
let array = <crate::components::AffixFuzzer7>::try_to_arrow([single]);
let array = <crate::components::AffixFuzzer7>::try_to_arrow([single], None);
array.map(|array| {
let datatype = array.data_type().clone();
(
Expand All @@ -436,7 +443,8 @@ impl crate::Archetype for AffixFuzzer1 {
self.fuzz2101
.as_ref()
.map(|many| {
let array = <crate::components::AffixFuzzer1>::try_to_arrow(many.iter());
let array =
<crate::components::AffixFuzzer1>::try_to_arrow(many.iter(), None);
array.map(|array| {
let datatype = array.data_type().clone();
(
Expand All @@ -451,7 +459,8 @@ impl crate::Archetype for AffixFuzzer1 {
self.fuzz2102
.as_ref()
.map(|many| {
let array = <crate::components::AffixFuzzer2>::try_to_arrow(many.iter());
let array =
<crate::components::AffixFuzzer2>::try_to_arrow(many.iter(), None);
array.map(|array| {
let datatype = array.data_type().clone();
(
Expand All @@ -466,7 +475,8 @@ impl crate::Archetype for AffixFuzzer1 {
self.fuzz2103
.as_ref()
.map(|many| {
let array = <crate::components::AffixFuzzer3>::try_to_arrow(many.iter());
let array =
<crate::components::AffixFuzzer3>::try_to_arrow(many.iter(), None);
array.map(|array| {
let datatype = array.data_type().clone();
(
Expand All @@ -481,7 +491,8 @@ impl crate::Archetype for AffixFuzzer1 {
self.fuzz2104
.as_ref()
.map(|many| {
let array = <crate::components::AffixFuzzer4>::try_to_arrow(many.iter());
let array =
<crate::components::AffixFuzzer4>::try_to_arrow(many.iter(), None);
array.map(|array| {
let datatype = array.data_type().clone();
(
Expand All @@ -496,7 +507,8 @@ impl crate::Archetype for AffixFuzzer1 {
self.fuzz2105
.as_ref()
.map(|many| {
let array = <crate::components::AffixFuzzer5>::try_to_arrow(many.iter());
let array =
<crate::components::AffixFuzzer5>::try_to_arrow(many.iter(), None);
array.map(|array| {
let datatype = array.data_type().clone();
(
Expand All @@ -511,7 +523,8 @@ impl crate::Archetype for AffixFuzzer1 {
self.fuzz2106
.as_ref()
.map(|many| {
let array = <crate::components::AffixFuzzer6>::try_to_arrow(many.iter());
let array =
<crate::components::AffixFuzzer6>::try_to_arrow(many.iter(), None);
array.map(|array| {
let datatype = array.data_type().clone();
(
Expand All @@ -526,7 +539,8 @@ impl crate::Archetype for AffixFuzzer1 {
self.fuzz2107
.as_ref()
.map(|many| {
let array = <crate::components::AffixFuzzer7>::try_to_arrow(many.iter());
let array =
<crate::components::AffixFuzzer7>::try_to_arrow(many.iter(), None);
array.map(|array| {
let datatype = array.data_type().clone();
(
Expand Down
19 changes: 11 additions & 8 deletions crates/re_types/src/archetypes/points2d.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -108,7 +108,8 @@ impl crate::Archetype for Points2D {
Ok([
{
Some({
let array = <crate::components::Point2D>::try_to_arrow(self.points.iter());
let array =
<crate::components::Point2D>::try_to_arrow(self.points.iter(), None);
array.map(|array| {
let datatype = array.data_type().clone();
(
Expand All @@ -123,7 +124,7 @@ impl crate::Archetype for Points2D {
self.radii
.as_ref()
.map(|many| {
let array = <crate::components::Radius>::try_to_arrow(many.iter());
let array = <crate::components::Radius>::try_to_arrow(many.iter(), None);
array.map(|array| {
let datatype = array.data_type().clone();
(
Expand All @@ -138,7 +139,7 @@ impl crate::Archetype for Points2D {
self.colors
.as_ref()
.map(|many| {
let array = <crate::components::Color>::try_to_arrow(many.iter());
let array = <crate::components::Color>::try_to_arrow(many.iter(), None);
array.map(|array| {
let datatype = array.data_type().clone();
(
Expand All @@ -153,7 +154,7 @@ impl crate::Archetype for Points2D {
self.labels
.as_ref()
.map(|many| {
let array = <crate::components::Label>::try_to_arrow(many.iter());
let array = <crate::components::Label>::try_to_arrow(many.iter(), None);
array.map(|array| {
let datatype = array.data_type().clone();
(
Expand All @@ -168,7 +169,7 @@ impl crate::Archetype for Points2D {
self.draw_order
.as_ref()
.map(|single| {
let array = <crate::components::DrawOrder>::try_to_arrow([single]);
let array = <crate::components::DrawOrder>::try_to_arrow([single], None);
array.map(|array| {
let datatype = array.data_type().clone();
(
Expand All @@ -183,7 +184,7 @@ impl crate::Archetype for Points2D {
self.class_ids
.as_ref()
.map(|many| {
let array = <crate::components::ClassId>::try_to_arrow(many.iter());
let array = <crate::components::ClassId>::try_to_arrow(many.iter(), None);
array.map(|array| {
let datatype = array.data_type().clone();
(
Expand All @@ -198,7 +199,8 @@ impl crate::Archetype for Points2D {
self.keypoint_ids
.as_ref()
.map(|many| {
let array = <crate::components::KeypointId>::try_to_arrow(many.iter());
let array =
<crate::components::KeypointId>::try_to_arrow(many.iter(), None);
array.map(|array| {
let datatype = array.data_type().clone();
(
Expand All @@ -213,7 +215,8 @@ impl crate::Archetype for Points2D {
self.instance_keys
.as_ref()
.map(|many| {
let array = <crate::components::InstanceKey>::try_to_arrow(many.iter());
let array =
<crate::components::InstanceKey>::try_to_arrow(many.iter(), None);
array.map(|array| {
let datatype = array.data_type().clone();
(
Expand Down
16 changes: 10 additions & 6 deletions crates/re_types/src/components/class_id.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,7 @@ impl crate::Component for ClassId {
#[allow(unused_imports, clippy::wildcard_imports)]
fn try_to_arrow_opt<'a>(
data: impl IntoIterator<Item = Option<impl Into<::std::borrow::Cow<'a, Self>>>>,
extension_wrapper: Option<&str>,
) -> crate::SerializationResult<Box<dyn ::arrow2::array::Array>>
where
Self: Clone + 'a,
Expand All @@ -71,7 +72,14 @@ impl crate::Component for ClassId {
any_nones.then(|| somes.into())
};
PrimitiveArray::new(
DataType::UInt16,
{
_ = extension_wrapper;
DataType::Extension(
"rerun.components.ClassId".to_owned(),
Box::new(DataType::UInt16),
None,
)
},
data0.into_iter().map(|v| v.unwrap_or_default()).collect(),
data0_bitmap,
)
Expand All @@ -96,11 +104,7 @@ impl crate::Component for ClassId {
.map(|v| v.copied())
.map(|v| {
v.ok_or_else(|| crate::DeserializationError::MissingData {
datatype: DataType::Extension(
"rerun.components.ClassId".to_owned(),
Box::new(DataType::UInt16),
None,
),
datatype: data.data_type().clone(),
})
})
.map(|res| res.map(|v| Some(Self(v))))
Expand Down
Loading

0 comments on commit 9492e87

Please sign in to comment.