Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Codegen'd Rust/Arrow (de)ser 8: carry extension metadata across transparency layers #2570

Merged
merged 3 commits into from
Jun 30, 2023
Merged

Codegen'd Rust/Arrow (de)ser 8: carry extension metadata across transparency layers #2570

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
b0344b5
support carrying extension metadata across transparency layers
teh-cmc Jun 29, 2023
68492bd
update tests to make sure all required extension metadata gets across
teh-cmc Jun 29, 2023
a7c17a1
rerun codegen
teh-cmc Jun 29, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion crates/re_types/source_hash.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
# This is a sha256 hash for all direct and indirect dependencies of this crate's build script.
# It can be safely removed at anytime to force the build script to run again.
# Check out build.rs to see how it's computed.
7d42813c538b5c4716e75e225572409d5799dc142ede36243043ef82fc90399e
128cd542f3f9e6cf3b2a44aeb022cd5f3ad819b00ce5371eeb311300f3e9a7f1
70 changes: 42 additions & 28 deletions crates/re_types/src/archetypes/fuzzy.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -140,7 +140,8 @@ impl crate::Archetype for AffixFuzzer1 {
Ok([
{
Some({
let array = <crate::components::AffixFuzzer1>::try_to_arrow([&self.fuzz1001]);
let array =
<crate::components::AffixFuzzer1>::try_to_arrow([&self.fuzz1001], None);
array.map(|array| {
let datatype = array.data_type().clone();
(
Expand All @@ -153,7 +154,8 @@ impl crate::Archetype for AffixFuzzer1 {
},
{
Some({
let array = <crate::components::AffixFuzzer2>::try_to_arrow([&self.fuzz1002]);
let array =
<crate::components::AffixFuzzer2>::try_to_arrow([&self.fuzz1002], None);
array.map(|array| {
let datatype = array.data_type().clone();
(
Expand All @@ -166,7 +168,8 @@ impl crate::Archetype for AffixFuzzer1 {
},
{
Some({
let array = <crate::components::AffixFuzzer3>::try_to_arrow([&self.fuzz1003]);
let array =
<crate::components::AffixFuzzer3>::try_to_arrow([&self.fuzz1003], None);
array.map(|array| {
let datatype = array.data_type().clone();
(
Expand All @@ -179,7 +182,8 @@ impl crate::Archetype for AffixFuzzer1 {
},
{
Some({
let array = <crate::components::AffixFuzzer4>::try_to_arrow([&self.fuzz1004]);
let array =
<crate::components::AffixFuzzer4>::try_to_arrow([&self.fuzz1004], None);
array.map(|array| {
let datatype = array.data_type().clone();
(
Expand All @@ -192,7 +196,8 @@ impl crate::Archetype for AffixFuzzer1 {
},
{
Some({
let array = <crate::components::AffixFuzzer5>::try_to_arrow([&self.fuzz1005]);
let array =
<crate::components::AffixFuzzer5>::try_to_arrow([&self.fuzz1005], None);
array.map(|array| {
let datatype = array.data_type().clone();
(
Expand All @@ -205,7 +210,8 @@ impl crate::Archetype for AffixFuzzer1 {
},
{
Some({
let array = <crate::components::AffixFuzzer6>::try_to_arrow([&self.fuzz1006]);
let array =
<crate::components::AffixFuzzer6>::try_to_arrow([&self.fuzz1006], None);
array.map(|array| {
let datatype = array.data_type().clone();
(
Expand All @@ -218,7 +224,8 @@ impl crate::Archetype for AffixFuzzer1 {
},
{
Some({
let array = <crate::components::AffixFuzzer7>::try_to_arrow([&self.fuzz1007]);
let array =
<crate::components::AffixFuzzer7>::try_to_arrow([&self.fuzz1007], None);
array.map(|array| {
let datatype = array.data_type().clone();
(
Expand All @@ -232,7 +239,7 @@ impl crate::Archetype for AffixFuzzer1 {
{
Some({
let array =
<crate::components::AffixFuzzer1>::try_to_arrow(self.fuzz1101.iter());
<crate::components::AffixFuzzer1>::try_to_arrow(self.fuzz1101.iter(), None);
array.map(|array| {
let datatype = array.data_type().clone();
(
Expand All @@ -246,7 +253,7 @@ impl crate::Archetype for AffixFuzzer1 {
{
Some({
let array =
<crate::components::AffixFuzzer2>::try_to_arrow(self.fuzz1102.iter());
<crate::components::AffixFuzzer2>::try_to_arrow(self.fuzz1102.iter(), None);
array.map(|array| {
let datatype = array.data_type().clone();
(
Expand All @@ -260,7 +267,7 @@ impl crate::Archetype for AffixFuzzer1 {
{
Some({
let array =
<crate::components::AffixFuzzer3>::try_to_arrow(self.fuzz1103.iter());
<crate::components::AffixFuzzer3>::try_to_arrow(self.fuzz1103.iter(), None);
array.map(|array| {
let datatype = array.data_type().clone();
(
Expand All @@ -274,7 +281,7 @@ impl crate::Archetype for AffixFuzzer1 {
{
Some({
let array =
<crate::components::AffixFuzzer4>::try_to_arrow(self.fuzz1104.iter());
<crate::components::AffixFuzzer4>::try_to_arrow(self.fuzz1104.iter(), None);
array.map(|array| {
let datatype = array.data_type().clone();
(
Expand All @@ -288,7 +295,7 @@ impl crate::Archetype for AffixFuzzer1 {
{
Some({
let array =
<crate::components::AffixFuzzer5>::try_to_arrow(self.fuzz1105.iter());
<crate::components::AffixFuzzer5>::try_to_arrow(self.fuzz1105.iter(), None);
array.map(|array| {
let datatype = array.data_type().clone();
(
Expand All @@ -302,7 +309,7 @@ impl crate::Archetype for AffixFuzzer1 {
{
Some({
let array =
<crate::components::AffixFuzzer6>::try_to_arrow(self.fuzz1106.iter());
<crate::components::AffixFuzzer6>::try_to_arrow(self.fuzz1106.iter(), None);
array.map(|array| {
let datatype = array.data_type().clone();
(
Expand All @@ -316,7 +323,7 @@ impl crate::Archetype for AffixFuzzer1 {
{
Some({
let array =
<crate::components::AffixFuzzer7>::try_to_arrow(self.fuzz1107.iter());
<crate::components::AffixFuzzer7>::try_to_arrow(self.fuzz1107.iter(), None);
array.map(|array| {
let datatype = array.data_type().clone();
(
Expand All @@ -331,7 +338,7 @@ impl crate::Archetype for AffixFuzzer1 {
self.fuzz2001
.as_ref()
.map(|single| {
let array = <crate::components::AffixFuzzer1>::try_to_arrow([single]);
let array = <crate::components::AffixFuzzer1>::try_to_arrow([single], None);
array.map(|array| {
let datatype = array.data_type().clone();
(
Expand All @@ -346,7 +353,7 @@ impl crate::Archetype for AffixFuzzer1 {
self.fuzz2002
.as_ref()
.map(|single| {
let array = <crate::components::AffixFuzzer2>::try_to_arrow([single]);
let array = <crate::components::AffixFuzzer2>::try_to_arrow([single], None);
array.map(|array| {
let datatype = array.data_type().clone();
(
Expand All @@ -361,7 +368,7 @@ impl crate::Archetype for AffixFuzzer1 {
self.fuzz2003
.as_ref()
.map(|single| {
let array = <crate::components::AffixFuzzer3>::try_to_arrow([single]);
let array = <crate::components::AffixFuzzer3>::try_to_arrow([single], None);
array.map(|array| {
let datatype = array.data_type().clone();
(
Expand All @@ -376,7 +383,7 @@ impl crate::Archetype for AffixFuzzer1 {
self.fuzz2004
.as_ref()
.map(|single| {
let array = <crate::components::AffixFuzzer4>::try_to_arrow([single]);
let array = <crate::components::AffixFuzzer4>::try_to_arrow([single], None);
array.map(|array| {
let datatype = array.data_type().clone();
(
Expand All @@ -391,7 +398,7 @@ impl crate::Archetype for AffixFuzzer1 {
self.fuzz2005
.as_ref()
.map(|single| {
let array = <crate::components::AffixFuzzer5>::try_to_arrow([single]);
let array = <crate::components::AffixFuzzer5>::try_to_arrow([single], None);
array.map(|array| {
let datatype = array.data_type().clone();
(
Expand All @@ -406,7 +413,7 @@ impl crate::Archetype for AffixFuzzer1 {
self.fuzz2006
.as_ref()
.map(|single| {
let array = <crate::components::AffixFuzzer6>::try_to_arrow([single]);
let array = <crate::components::AffixFuzzer6>::try_to_arrow([single], None);
array.map(|array| {
let datatype = array.data_type().clone();
(
Expand All @@ -421,7 +428,7 @@ impl crate::Archetype for AffixFuzzer1 {
self.fuzz2007
.as_ref()
.map(|single| {
let array = <crate::components::AffixFuzzer7>::try_to_arrow([single]);
let array = <crate::components::AffixFuzzer7>::try_to_arrow([single], None);
array.map(|array| {
let datatype = array.data_type().clone();
(
Expand All @@ -436,7 +443,8 @@ impl crate::Archetype for AffixFuzzer1 {
self.fuzz2101
.as_ref()
.map(|many| {
let array = <crate::components::AffixFuzzer1>::try_to_arrow(many.iter());
let array =
<crate::components::AffixFuzzer1>::try_to_arrow(many.iter(), None);
array.map(|array| {
let datatype = array.data_type().clone();
(
Expand All @@ -451,7 +459,8 @@ impl crate::Archetype for AffixFuzzer1 {
self.fuzz2102
.as_ref()
.map(|many| {
let array = <crate::components::AffixFuzzer2>::try_to_arrow(many.iter());
let array =
<crate::components::AffixFuzzer2>::try_to_arrow(many.iter(), None);
array.map(|array| {
let datatype = array.data_type().clone();
(
Expand All @@ -466,7 +475,8 @@ impl crate::Archetype for AffixFuzzer1 {
self.fuzz2103
.as_ref()
.map(|many| {
let array = <crate::components::AffixFuzzer3>::try_to_arrow(many.iter());
let array =
<crate::components::AffixFuzzer3>::try_to_arrow(many.iter(), None);
array.map(|array| {
let datatype = array.data_type().clone();
(
Expand All @@ -481,7 +491,8 @@ impl crate::Archetype for AffixFuzzer1 {
self.fuzz2104
.as_ref()
.map(|many| {
let array = <crate::components::AffixFuzzer4>::try_to_arrow(many.iter());
let array =
<crate::components::AffixFuzzer4>::try_to_arrow(many.iter(), None);
array.map(|array| {
let datatype = array.data_type().clone();
(
Expand All @@ -496,7 +507,8 @@ impl crate::Archetype for AffixFuzzer1 {
self.fuzz2105
.as_ref()
.map(|many| {
let array = <crate::components::AffixFuzzer5>::try_to_arrow(many.iter());
let array =
<crate::components::AffixFuzzer5>::try_to_arrow(many.iter(), None);
array.map(|array| {
let datatype = array.data_type().clone();
(
Expand All @@ -511,7 +523,8 @@ impl crate::Archetype for AffixFuzzer1 {
self.fuzz2106
.as_ref()
.map(|many| {
let array = <crate::components::AffixFuzzer6>::try_to_arrow(many.iter());
let array =
<crate::components::AffixFuzzer6>::try_to_arrow(many.iter(), None);
array.map(|array| {
let datatype = array.data_type().clone();
(
Expand All @@ -526,7 +539,8 @@ impl crate::Archetype for AffixFuzzer1 {
self.fuzz2107
.as_ref()
.map(|many| {
let array = <crate::components::AffixFuzzer7>::try_to_arrow(many.iter());
let array =
<crate::components::AffixFuzzer7>::try_to_arrow(many.iter(), None);
array.map(|array| {
let datatype = array.data_type().clone();
(
Expand Down
19 changes: 11 additions & 8 deletions crates/re_types/src/archetypes/points2d.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -108,7 +108,8 @@ impl crate::Archetype for Points2D {
Ok([
{
Some({
let array = <crate::components::Point2D>::try_to_arrow(self.points.iter());
let array =
<crate::components::Point2D>::try_to_arrow(self.points.iter(), None);
array.map(|array| {
let datatype = array.data_type().clone();
(
Expand All @@ -123,7 +124,7 @@ impl crate::Archetype for Points2D {
self.radii
.as_ref()
.map(|many| {
let array = <crate::components::Radius>::try_to_arrow(many.iter());
let array = <crate::components::Radius>::try_to_arrow(many.iter(), None);
array.map(|array| {
let datatype = array.data_type().clone();
(
Expand All @@ -138,7 +139,7 @@ impl crate::Archetype for Points2D {
self.colors
.as_ref()
.map(|many| {
let array = <crate::components::Color>::try_to_arrow(many.iter());
let array = <crate::components::Color>::try_to_arrow(many.iter(), None);
array.map(|array| {
let datatype = array.data_type().clone();
(
Expand All @@ -153,7 +154,7 @@ impl crate::Archetype for Points2D {
self.labels
.as_ref()
.map(|many| {
let array = <crate::components::Label>::try_to_arrow(many.iter());
let array = <crate::components::Label>::try_to_arrow(many.iter(), None);
array.map(|array| {
let datatype = array.data_type().clone();
(
Expand All @@ -168,7 +169,7 @@ impl crate::Archetype for Points2D {
self.draw_order
.as_ref()
.map(|single| {
let array = <crate::components::DrawOrder>::try_to_arrow([single]);
let array = <crate::components::DrawOrder>::try_to_arrow([single], None);
array.map(|array| {
let datatype = array.data_type().clone();
(
Expand All @@ -183,7 +184,7 @@ impl crate::Archetype for Points2D {
self.class_ids
.as_ref()
.map(|many| {
let array = <crate::components::ClassId>::try_to_arrow(many.iter());
let array = <crate::components::ClassId>::try_to_arrow(many.iter(), None);
array.map(|array| {
let datatype = array.data_type().clone();
(
Expand All @@ -198,7 +199,8 @@ impl crate::Archetype for Points2D {
self.keypoint_ids
.as_ref()
.map(|many| {
let array = <crate::components::KeypointId>::try_to_arrow(many.iter());
let array =
<crate::components::KeypointId>::try_to_arrow(many.iter(), None);
array.map(|array| {
let datatype = array.data_type().clone();
(
Expand All @@ -213,7 +215,8 @@ impl crate::Archetype for Points2D {
self.instance_keys
.as_ref()
.map(|many| {
let array = <crate::components::InstanceKey>::try_to_arrow(many.iter());
let array =
<crate::components::InstanceKey>::try_to_arrow(many.iter(), None);
array.map(|array| {
let datatype = array.data_type().clone();
(
Expand Down
16 changes: 10 additions & 6 deletions crates/re_types/src/components/class_id.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,7 @@ impl crate::Component for ClassId {
#[allow(unused_imports, clippy::wildcard_imports)]
fn try_to_arrow_opt<'a>(
data: impl IntoIterator<Item = Option<impl Into<::std::borrow::Cow<'a, Self>>>>,
extension_wrapper: Option<&str>,
) -> crate::SerializationResult<Box<dyn ::arrow2::array::Array>>
where
Self: Clone + 'a,
Expand All @@ -71,7 +72,14 @@ impl crate::Component for ClassId {
any_nones.then(|| somes.into())
};
PrimitiveArray::new(
DataType::UInt16,
{
_ = extension_wrapper;
DataType::Extension(
"rerun.components.ClassId".to_owned(),
Box::new(DataType::UInt16),
None,
)
},
data0.into_iter().map(|v| v.unwrap_or_default()).collect(),
data0_bitmap,
)
Expand All @@ -96,11 +104,7 @@ impl crate::Component for ClassId {
.map(|v| v.copied())
.map(|v| {
v.ok_or_else(|| crate::DeserializationError::MissingData {
datatype: DataType::Extension(
"rerun.components.ClassId".to_owned(),
Box::new(DataType::UInt16),
None,
),
datatype: data.data_type().clone(),
})
})
.map(|res| res.map(|v| Some(Self(v))))
Expand Down
Loading