Skip to content

Commit

Permalink
Derive more field types for Arrow (#739)
Browse files Browse the repository at this point in the history 
* Derive more field types for Arrow
* Update arrow2_convert to point to latest commit on rerun_main
  • Loading branch information
John Hughes authored Jan 13, 2023
1 parent 579b0c6 commit 7030d0f
Show file tree
Hide file tree
Showing 10 changed files with 141 additions and 344 deletions.
4 changes: 2 additions & 2 deletions Cargo.lock
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 1 addition & 1 deletion Cargo.toml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -70,6 +70,6 @@ wgpu-core = { git = "https://github.com/gfx-rs/wgpu.git", ref = "a377ae2b7fe6c1c
# Upstream PR https://github.com/jorgecarleitao/arrow2/pull/1351
arrow2 = { git = "https://github.com/rerun-io/arrow2", rev = "f134e58bb554b069392f6bd495aa43aa06b58944" }
# Upstream PRs https://github.com/DataEngineeringLabs/arrow2-convert/pull/90
arrow2_convert = { git = "https://github.com/rerun-io/arrow2-convert", rev = "dedfa8c43e17438036cfcda127a1afc02f725bff" }
arrow2_convert = { git = "https://github.com/rerun-io/arrow2-convert", rev = "7e0a3a3881eb4577f95d1b7af76e7f943c4fa53d" }
#arrow2 = { path = "../arrow2" }
#arrow2_convert = { path = "../arrow2-convert/arrow2_convert" }
52 changes: 16 additions & 36 deletions crates/re_log_types/src/field_types/class_id.rs
View file
Original file line number Diff line number Diff line change
@@ -1,48 +1,28 @@
use arrow2::array::TryPush;
use arrow2_convert::{deserialize::ArrowDeserialize, field::ArrowField, serialize::ArrowSerialize};
use arrow2_convert::{ArrowDeserialize, ArrowField, ArrowSerialize};

use crate::msg_bundle::Component;

/// A 16-bit ID representing a type of semantic class.
///
/// Used to look up a [`crate::context::ClassDescription`] within the [`crate::context::AnnotationContext`].
#[cfg_attr(feature = "serde", derive(serde::Deserialize, serde::Serialize))]
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, PartialOrd, Ord, Hash)]
#[derive(
Clone,
Copy,
Debug,
Default,
PartialEq,
Eq,
PartialOrd,
Ord,
Hash,
ArrowField,
ArrowSerialize,
ArrowDeserialize,
)]
#[arrow_field(transparent)]
pub struct ClassId(pub u16);

impl ArrowField for ClassId {
type Type = Self;

fn data_type() -> arrow2::datatypes::DataType {
<u16 as ArrowField>::data_type()
}
}

impl ArrowSerialize for ClassId {
type MutableArrayType = <u16 as ArrowSerialize>::MutableArrayType;

#[inline]
fn new_array() -> Self::MutableArrayType {
Self::MutableArrayType::default()
}

#[inline]
fn arrow_serialize(v: &Self, array: &mut Self::MutableArrayType) -> arrow2::error::Result<()> {
array.try_push(Some(v.0))
}
}

impl ArrowDeserialize for ClassId {
type ArrayType = <u16 as ArrowDeserialize>::ArrayType;

#[inline]
fn arrow_deserialize(
v: <&Self::ArrayType as IntoIterator>::Item,
) -> Option<<Self as ArrowField>::Type> {
<u16 as ArrowDeserialize>::arrow_deserialize(v).map(ClassId)
}
}

impl Component for ClassId {
fn name() -> crate::ComponentName {
"rerun.class_id".into()
Expand Down
54 changes: 14 additions & 40 deletions crates/re_log_types/src/field_types/color.rs
View file
Original file line number Diff line number Diff line change
@@ -1,8 +1,4 @@
use arrow2::{array::TryPush, datatypes::DataType};
use arrow2_convert::{
arrow_enable_vec_for_type, deserialize::ArrowDeserialize, field::ArrowField,
serialize::ArrowSerialize,
};
use arrow2_convert::{ArrowDeserialize, ArrowField, ArrowSerialize};

use crate::msg_bundle::Component;

Expand All @@ -15,8 +11,20 @@ use crate::msg_bundle::Component;
///
/// assert_eq!(ColorRGBA::data_type(), DataType::UInt32);
/// ```
#[derive(Clone, Copy, Debug, PartialEq, Eq, derive_more::From, derive_more::Into)]
#[derive(
Clone,
Copy,
Debug,
PartialEq,
Eq,
derive_more::From,
derive_more::Into,
ArrowField,
ArrowSerialize,
ArrowDeserialize,
)]
#[cfg_attr(feature = "serde", derive(serde::Deserialize, serde::Serialize))]
#[arrow_field(transparent)]
pub struct ColorRGBA(pub u32);

impl ColorRGBA {
Expand All @@ -41,40 +49,6 @@ impl From<[u8; 4]> for ColorRGBA {
}
}

arrow_enable_vec_for_type!(ColorRGBA);

impl ArrowField for ColorRGBA {
type Type = Self;
fn data_type() -> DataType {
<u32 as ArrowField>::data_type()
}
}

impl ArrowSerialize for ColorRGBA {
type MutableArrayType = <u32 as ArrowSerialize>::MutableArrayType;

#[inline]
fn new_array() -> Self::MutableArrayType {
Self::MutableArrayType::default()
}

#[inline]
fn arrow_serialize(v: &Self, array: &mut Self::MutableArrayType) -> arrow2::error::Result<()> {
array.try_push(Some(v.0))
}
}

impl ArrowDeserialize for ColorRGBA {
type ArrayType = <u32 as ArrowDeserialize>::ArrayType;

#[inline]
fn arrow_deserialize(
v: <&Self::ArrayType as IntoIterator>::Item,
) -> Option<<Self as ArrowField>::Type> {
<u32 as ArrowDeserialize>::arrow_deserialize(v).map(ColorRGBA)
}
}

impl Component for ColorRGBA {
fn name() -> crate::ComponentName {
"rerun.colorrgba".into()
Expand Down
55 changes: 15 additions & 40 deletions crates/re_log_types/src/field_types/instance.rs
View file
Original file line number Diff line number Diff line change
@@ -1,8 +1,4 @@
use arrow2::{array::TryPush, datatypes::DataType};
use arrow2_convert::{
arrow_enable_vec_for_type, deserialize::ArrowDeserialize, field::ArrowField,
serialize::ArrowSerialize,
};
use arrow2_convert::{ArrowDeserialize, ArrowField, ArrowSerialize};

use crate::msg_bundle::Component;

Expand All @@ -15,8 +11,21 @@ use crate::msg_bundle::Component;
///
/// assert_eq!(Instance::data_type(), DataType::UInt64);
/// ```
#[derive(Copy, Clone, Debug, Hash, PartialEq, Eq, PartialOrd, Ord)]
#[derive(
Copy,
Clone,
Debug,
Hash,
PartialEq,
Eq,
PartialOrd,
Ord,
ArrowField,
ArrowSerialize,
ArrowDeserialize,
)]
#[cfg_attr(feature = "serde", derive(serde::Deserialize, serde::Serialize))]
#[arrow_field(transparent)]
pub struct Instance(pub u64);

impl Instance {
Expand All @@ -31,8 +40,6 @@ impl Instance {
}
}

arrow_enable_vec_for_type!(Instance);

impl std::fmt::Display for Instance {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
if self.is_splat() {
Expand All @@ -44,38 +51,6 @@ impl std::fmt::Display for Instance {
}
}

impl ArrowField for Instance {
type Type = Self;
fn data_type() -> DataType {
<u64 as ArrowField>::data_type()
}
}

impl ArrowSerialize for Instance {
type MutableArrayType = <u64 as ArrowSerialize>::MutableArrayType;

#[inline]
fn new_array() -> Self::MutableArrayType {
Self::MutableArrayType::default()
}

#[inline]
fn arrow_serialize(v: &Self, array: &mut Self::MutableArrayType) -> arrow2::error::Result<()> {
array.try_push(Some(v.0))
}
}

impl ArrowDeserialize for Instance {
type ArrayType = <u64 as ArrowDeserialize>::ArrayType;

#[inline]
fn arrow_deserialize(
v: <&Self::ArrayType as IntoIterator>::Item,
) -> Option<<Self as ArrowField>::Type> {
<u64 as ArrowDeserialize>::arrow_deserialize(v).map(Instance)
}
}

impl Component for Instance {
fn name() -> crate::ComponentName {
"rerun.instance".into()
Expand Down
57 changes: 16 additions & 41 deletions crates/re_log_types/src/field_types/keypoint_id.rs
View file
Original file line number Diff line number Diff line change
@@ -1,8 +1,4 @@
use arrow2::array::TryPush;
use arrow2_convert::{
arrow_enable_vec_for_type, deserialize::ArrowDeserialize, field::ArrowField,
serialize::ArrowSerialize,
};
use arrow2_convert::{ArrowDeserialize, ArrowField, ArrowSerialize};

use crate::msg_bundle::Component;

Expand All @@ -12,46 +8,25 @@ use crate::msg_bundle::Component;
///
/// Used to look up an [`crate::context::AnnotationInfo`] for a Keypoint within the [`crate::context::AnnotationContext`].
#[cfg_attr(feature = "serde", derive(serde::Deserialize, serde::Serialize))]
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, PartialOrd, Ord, Hash)]
#[derive(
Clone,
Copy,
Debug,
Default,
PartialEq,
Eq,
PartialOrd,
Ord,
Hash,
ArrowField,
ArrowSerialize,
ArrowDeserialize,
)]
#[arrow_field(transparent)]
pub struct KeypointId(pub u16);

impl ArrowField for KeypointId {
type Type = Self;

fn data_type() -> arrow2::datatypes::DataType {
<u16 as ArrowField>::data_type()
}
}

impl ArrowSerialize for KeypointId {
type MutableArrayType = <u16 as ArrowSerialize>::MutableArrayType;

#[inline]
fn new_array() -> Self::MutableArrayType {
<u16 as ArrowSerialize>::new_array()
}

#[inline]
fn arrow_serialize(v: &Self, array: &mut Self::MutableArrayType) -> arrow2::error::Result<()> {
array.try_push(Some(v.0))
}
}

impl ArrowDeserialize for KeypointId {
type ArrayType = <u16 as ArrowDeserialize>::ArrayType;

#[inline]
fn arrow_deserialize(
v: <&Self::ArrayType as IntoIterator>::Item,
) -> Option<<Self as ArrowField>::Type> {
<u16 as ArrowDeserialize>::arrow_deserialize(v).map(KeypointId)
}
}

impl Component for KeypointId {
fn name() -> crate::ComponentName {
"rerun.keypoint_id".into()
}
}

arrow_enable_vec_for_type!(KeypointId);
53 changes: 13 additions & 40 deletions crates/re_log_types/src/field_types/label.rs
View file
Original file line number Diff line number Diff line change
@@ -1,8 +1,4 @@
use arrow2::datatypes::DataType;
use arrow2_convert::{
arrow_enable_vec_for_type, deserialize::ArrowDeserialize, field::ArrowField,
serialize::ArrowSerialize,
};
use arrow2_convert::{ArrowDeserialize, ArrowField, ArrowSerialize};

use crate::msg_bundle::Component;

Expand All @@ -15,44 +11,21 @@ use crate::msg_bundle::Component;
///
/// assert_eq!(Label::data_type(), DataType::Utf8);
/// ```
#[derive(Debug, Clone, PartialEq, Eq, derive_more::From, derive_more::Into)]
#[derive(
Debug,
Clone,
PartialEq,
Eq,
derive_more::From,
derive_more::Into,
ArrowField,
ArrowSerialize,
ArrowDeserialize,
)]
#[cfg_attr(feature = "serde", derive(serde::Deserialize, serde::Serialize))]
#[arrow_field(transparent)]
pub struct Label(pub String);

arrow_enable_vec_for_type!(Label);

impl ArrowField for Label {
type Type = Self;
fn data_type() -> DataType {
<String as ArrowField>::data_type()
}
}

impl ArrowSerialize for Label {
type MutableArrayType = <String as ArrowSerialize>::MutableArrayType;

#[inline]
fn new_array() -> Self::MutableArrayType {
Self::MutableArrayType::default()
}

#[inline]
fn arrow_serialize(v: &Self, array: &mut Self::MutableArrayType) -> arrow2::error::Result<()> {
<String as ArrowSerialize>::arrow_serialize(&v.0, array)
}
}

impl ArrowDeserialize for Label {
type ArrayType = <String as ArrowDeserialize>::ArrayType;

#[inline]
fn arrow_deserialize(
v: <&Self::ArrayType as IntoIterator>::Item,
) -> Option<<Self as ArrowField>::Type> {
<String as ArrowDeserialize>::arrow_deserialize(v).map(Label)
}
}

impl Component for Label {
fn name() -> crate::ComponentName {
"rerun.label".into()
Expand Down
Loading

1 comment on commit 7030d0f

@github-actions
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Rust Benchmark

Benchmark suite Current: 7030d0f Previous: 579b0c6 Ratio
datastore/insert/batch/rects/insert 272622 ns/iter (± 815) 275411 ns/iter (± 707) 0.99
datastore/latest_at/batch/rects/query 732 ns/iter (± 1) 737 ns/iter (± 1) 0.99
datastore/latest_at/missing_components/primary 302 ns/iter (± 1) 301 ns/iter (± 0) 1.00
datastore/latest_at/missing_components/secondaries 373 ns/iter (± 0) 376 ns/iter (± 1) 0.99
datastore/range/batch/rects/query 45980 ns/iter (± 125) 45465 ns/iter (± 48) 1.01
obj_mono_points/insert 850248825 ns/iter (± 41865119) 843252679 ns/iter (± 4907319) 1.01
obj_mono_points/query 355077 ns/iter (± 129748) 376591 ns/iter (± 2671) 0.94
obj_batch_points/insert 88870340 ns/iter (± 357037) 88364063 ns/iter (± 342626) 1.01
obj_batch_points/query 10966 ns/iter (± 40) 10950 ns/iter (± 61) 1.00
obj_batch_points_sequential/insert 22849231 ns/iter (± 230899) 23066381 ns/iter (± 96587) 0.99
obj_batch_points_sequential/query 7214 ns/iter (± 16) 7177 ns/iter (± 10) 1.01
mono_points_classic/generate_messages 4386209 ns/iter (± 1283207) 4482990 ns/iter (± 163587) 0.98
mono_points_classic/encode_log_msg 11825865 ns/iter (± 760975) 13022711 ns/iter (± 515598) 0.91
mono_points_classic/encode_total 16144324 ns/iter (± 899597) 17670262 ns/iter (± 790810) 0.91
mono_points_classic/decode_total 37481782 ns/iter (± 722406) 37677499 ns/iter (± 349936) 0.99
mono_points_arrow/generate_message_bundles 49252913 ns/iter (± 522319) 48540414 ns/iter (± 442626) 1.01
mono_points_arrow/generate_messages 126666288 ns/iter (± 1046879) 125345805 ns/iter (± 1090676) 1.01
mono_points_arrow/encode_log_msg 154547632 ns/iter (± 16448447) 152359193 ns/iter (± 969621) 1.01
mono_points_arrow/encode_total 330224979 ns/iter (± 1745219) 325907132 ns/iter (± 1389702) 1.01
mono_points_arrow/decode_log_msg 179505077 ns/iter (± 24309416) 177747814 ns/iter (± 755205) 1.01
mono_points_arrow/decode_message_bundles 68750759 ns/iter (± 773223) 68924711 ns/iter (± 807919) 1.00
mono_points_arrow/decode_total 246372569 ns/iter (± 1437486) 243166581 ns/iter (± 1367063) 1.01
batch_points_classic/generate_messages 3435 ns/iter (± 30) 3493 ns/iter (± 33) 0.98
batch_points_classic/encode_log_msg 399625 ns/iter (± 29947) 393056 ns/iter (± 526) 1.02
batch_points_classic/encode_total 405572 ns/iter (± 1269) 399956 ns/iter (± 859) 1.01
batch_points_classic/decode_total 747626 ns/iter (± 4040) 740896 ns/iter (± 1447) 1.01
batch_points_arrow/generate_message_bundles 317389 ns/iter (± 1570) 311566 ns/iter (± 325) 1.02
batch_points_arrow/generate_messages 6224 ns/iter (± 23) 6183 ns/iter (± 13) 1.01
batch_points_arrow/encode_log_msg 357845 ns/iter (± 1029) 363300 ns/iter (± 1439) 0.98
batch_points_arrow/encode_total 700714 ns/iter (± 192890) 714105 ns/iter (± 2331) 0.98
batch_points_arrow/decode_log_msg 347896 ns/iter (± 1099) 351968 ns/iter (± 1641) 0.99
batch_points_arrow/decode_message_bundles 2134 ns/iter (± 9) 2106 ns/iter (± 6) 1.01
batch_points_arrow/decode_total 351914 ns/iter (± 1396) 354660 ns/iter (± 1335) 0.99
arrow_mono_points/insert 6114208949 ns/iter (± 341500088) 6052298815 ns/iter (± 14855747) 1.01
arrow_mono_points/query 1653034 ns/iter (± 9821) 1667313 ns/iter (± 17152) 0.99
arrow_batch_points/insert 2660473 ns/iter (± 18709) 2666941 ns/iter (± 17729) 1.00
arrow_batch_points/query 15955 ns/iter (± 57) 15624 ns/iter (± 21) 1.02
obj_batch_points_sequential/Tuid::random 37 ns/iter (± 0) 37 ns/iter (± 0) 1

This comment was automatically generated by workflow using github-action-benchmark.

Please sign in to comment.