diff --git a/crates/re_types/definitions/fbs/attributes.fbs b/crates/re_types/definitions/fbs/attributes.fbs index d431691e8d96..d8b50532e3d5 100644 --- a/crates/re_types/definitions/fbs/attributes.fbs +++ b/crates/re_types/definitions/fbs/attributes.fbs @@ -14,3 +14,19 @@ namespace fbs.attributes; /// NOTE: We do not use flatbuffers' builtin `id` attribute as it only works on `table`s, whereas we /// need a stable order for all kinds of things. attribute "order"; + +/// This attribute enables _semantic_ transparency. +/// +/// * When applied to a table's field that is itself a table, this removes the extra table +/// layer during the semantic pass, leaving just the inner field's type in its stead. +/// +/// * When applied to a table object, this indicates that the object is only ever used as a +/// field type, and thus should not generate any standalone code for itself. +/// +/// Since all of this is done during the semantic pass, none of the other passes will ever know +/// that the extra layer ever existed in the first place. +/// +/// This allows to bypass all of Flatbuffers artificial limitations (have to wrap scalars in a +/// table when defining a union, cannot put arrays (as opposed to vectors) into tables, etc) that +/// do not apply in our case. +attribute "transparent"; diff --git a/crates/re_types/definitions/fbs/scalars.fbs b/crates/re_types/definitions/fbs/scalars.fbs deleted file mode 100644 index 085df42e26db..000000000000 --- a/crates/re_types/definitions/fbs/scalars.fbs +++ /dev/null @@ -1,13 +0,0 @@ -/// Unions cannot directly refer to scalar types, they need to be wrapped in a struct or table -/// first. -/// This package provides pre-wrapped scalars that will be automatically flattened down to their -/// inner type by our parsers. -/// -/// Look e.g. for `fbs.scalars.Float32` in `objects.rs` to see this flatenning in action. - -namespace fbs.scalars; - -/// Flattens down to a 32-bit float. -struct Float32 { - v: float; -} diff --git a/crates/re_types/definitions/rerun/testing/datatypes/fuzzy.fbs b/crates/re_types/definitions/rerun/testing/datatypes/fuzzy.fbs index 6bc62ddd44f1..3671982ecbfd 100644 --- a/crates/re_types/definitions/rerun/testing/datatypes/fuzzy.fbs +++ b/crates/re_types/definitions/rerun/testing/datatypes/fuzzy.fbs @@ -6,6 +6,22 @@ namespace rerun.testing.datatypes; // --- +table FlattenedScalar ("attr.rust.derive": "PartialEq", order: 001) { + value: float (order: 100); +} + +table DeeplyFlattenedScalar (transparent, order: 001) { + value: FlattenedScalar (order: 100, transparent); +} + +table VeryDeeplyFlattenedScalar (transparent, order: 001) { + value: DeeplyFlattenedScalar (order: 100, transparent); +} + +table SurprisinglyShallowScalar (transparent, order: 001) { + value: FlattenedScalar (required, order: 100); +} + table AffixFuzzer1 ( "attr.rust.derive": "PartialEq", order: 100 @@ -16,6 +32,8 @@ table AffixFuzzer1 ( many_floats_optional: [float] (order: 104); many_strings_required: [string] (order: 105, required); many_strings_optional: [string] (order: 106); + flattened_scalar: VeryDeeplyFlattenedScalar (order: 107, required, transparent); + almost_flattened_scalar: SurprisinglyShallowScalar (order: 108, required, transparent); } table AffixFuzzer2 ( diff --git a/crates/re_types/source_hash.txt b/crates/re_types/source_hash.txt index 4b15921d0d74..a2f233144148 100644 --- a/crates/re_types/source_hash.txt +++ b/crates/re_types/source_hash.txt @@ -1,4 +1,4 @@ # This is a sha256 hash for all direct and indirect dependencies of this crate's build script. # It can be safely removed at anytime to force the build script to run again. # Check out build.rs to see how it's computed. -65d6bf828303169287dd4a8df605e50acbde791fa0f2e8167ea751fd23d3a56f \ No newline at end of file +d05383876b33ae3b108ceffac46e0292a0dbc2253e3e777028181b4ed338315f \ No newline at end of file diff --git a/crates/re_types/src/components/fuzzy.rs b/crates/re_types/src/components/fuzzy.rs index d5e9d32ac390..a1d1ef6d2105 100644 --- a/crates/re_types/src/components/fuzzy.rs +++ b/crates/re_types/src/components/fuzzy.rs @@ -90,6 +90,23 @@ impl crate::Component for AffixFuzzer1 { is_nullable: true, metadata: [].into(), }, + Field { + name: "flattened_scalar".to_owned(), + data_type: DataType::Float32, + is_nullable: false, + metadata: [].into(), + }, + Field { + name: "almost_flattened_scalar".to_owned(), + data_type: DataType::Struct(vec![Field { + name: "value".to_owned(), + data_type: DataType::Float32, + is_nullable: true, + metadata: [].into(), + }]), + is_nullable: false, + metadata: [].into(), + }, ]) } @@ -230,6 +247,23 @@ impl crate::Component for AffixFuzzer2 { is_nullable: true, metadata: [].into(), }, + Field { + name: "flattened_scalar".to_owned(), + data_type: DataType::Float32, + is_nullable: false, + metadata: [].into(), + }, + Field { + name: "almost_flattened_scalar".to_owned(), + data_type: DataType::Struct(vec![Field { + name: "value".to_owned(), + data_type: DataType::Float32, + is_nullable: true, + metadata: [].into(), + }]), + is_nullable: false, + metadata: [].into(), + }, ]) } @@ -372,6 +406,23 @@ impl crate::Component for AffixFuzzer3 { is_nullable: true, metadata: [].into(), }, + Field { + name: "flattened_scalar".to_owned(), + data_type: DataType::Float32, + is_nullable: false, + metadata: [].into(), + }, + Field { + name: "almost_flattened_scalar".to_owned(), + data_type: DataType::Struct(vec![Field { + name: "value".to_owned(), + data_type: DataType::Float32, + is_nullable: true, + metadata: [].into(), + }]), + is_nullable: false, + metadata: [].into(), + }, ]) } @@ -514,6 +565,23 @@ impl crate::Component for AffixFuzzer4 { is_nullable: true, metadata: [].into(), }, + Field { + name: "flattened_scalar".to_owned(), + data_type: DataType::Float32, + is_nullable: false, + metadata: [].into(), + }, + Field { + name: "almost_flattened_scalar".to_owned(), + data_type: DataType::Struct(vec![Field { + name: "value".to_owned(), + data_type: DataType::Float32, + is_nullable: true, + metadata: [].into(), + }]), + is_nullable: false, + metadata: [].into(), + }, ]) } @@ -652,6 +720,23 @@ impl crate::Component for AffixFuzzer5 { is_nullable: true, metadata: [].into(), }, + Field { + name: "flattened_scalar".to_owned(), + data_type: DataType::Float32, + is_nullable: false, + metadata: [].into(), + }, + Field { + name: "almost_flattened_scalar".to_owned(), + data_type: DataType::Struct(vec![Field { + name: "value".to_owned(), + data_type: DataType::Float32, + is_nullable: true, + metadata: [].into(), + }]), + is_nullable: false, + metadata: [].into(), + }, ]) } @@ -792,6 +877,23 @@ impl crate::Component for AffixFuzzer6 { is_nullable: true, metadata: [].into(), }, + Field { + name: "flattened_scalar".to_owned(), + data_type: DataType::Float32, + is_nullable: false, + metadata: [].into(), + }, + Field { + name: "almost_flattened_scalar".to_owned(), + data_type: DataType::Struct(vec![Field { + name: "value".to_owned(), + data_type: DataType::Float32, + is_nullable: true, + metadata: [].into(), + }]), + is_nullable: false, + metadata: [].into(), + }, ]) } @@ -934,6 +1036,23 @@ impl crate::Component for AffixFuzzer7 { is_nullable: true, metadata: [].into(), }, + Field { + name: "flattened_scalar".to_owned(), + data_type: DataType::Float32, + is_nullable: false, + metadata: [].into(), + }, + Field { + name: "almost_flattened_scalar".to_owned(), + data_type: DataType::Struct(vec![Field { + name: "value".to_owned(), + data_type: DataType::Float32, + is_nullable: true, + metadata: [].into(), + }]), + is_nullable: false, + metadata: [].into(), + }, ]), is_nullable: true, metadata: [].into(), @@ -1052,6 +1171,23 @@ impl crate::Component for AffixFuzzer7 { is_nullable: true, metadata: [].into(), }, + Field { + name: "flattened_scalar".to_owned(), + data_type: DataType::Float32, + is_nullable: false, + metadata: [].into(), + }, + Field { + name: "almost_flattened_scalar".to_owned(), + data_type: DataType::Struct(vec![Field { + name: "value".to_owned(), + data_type: DataType::Float32, + is_nullable: true, + metadata: [].into(), + }]), + is_nullable: false, + metadata: [].into(), + }, ]), is_nullable: true, metadata: [].into(), @@ -1154,6 +1290,23 @@ impl crate::Component for AffixFuzzer7 { is_nullable: true, metadata: [].into(), }, + Field { + name: "flattened_scalar".to_owned(), + data_type: DataType::Float32, + is_nullable: false, + metadata: [].into(), + }, + Field { + name: "almost_flattened_scalar".to_owned(), + data_type: DataType::Struct(vec![Field { + name: "value".to_owned(), + data_type: DataType::Float32, + is_nullable: true, + metadata: [].into(), + }]), + is_nullable: false, + metadata: [].into(), + }, ]), }) }) diff --git a/crates/re_types/src/datatypes/fuzzy.rs b/crates/re_types/src/datatypes/fuzzy.rs index 1bcced7de879..567010f20a43 100644 --- a/crates/re_types/src/datatypes/fuzzy.rs +++ b/crates/re_types/src/datatypes/fuzzy.rs @@ -9,6 +9,153 @@ #![allow(clippy::too_many_lines)] #![allow(clippy::unnecessary_cast)] +#[derive(Debug, Clone, PartialEq)] +pub struct FlattenedScalar { + pub value: Option, +} + +impl<'a> From for ::std::borrow::Cow<'a, FlattenedScalar> { + #[inline] + fn from(value: FlattenedScalar) -> Self { + std::borrow::Cow::Owned(value) + } +} + +impl<'a> From<&'a FlattenedScalar> for ::std::borrow::Cow<'a, FlattenedScalar> { + #[inline] + fn from(value: &'a FlattenedScalar) -> Self { + std::borrow::Cow::Borrowed(value) + } +} + +impl crate::Datatype for FlattenedScalar { + #[inline] + fn name() -> crate::DatatypeName { + crate::DatatypeName::Borrowed("rerun.testing.datatypes.FlattenedScalar") + } + + #[allow(unused_imports, clippy::wildcard_imports)] + #[inline] + fn to_arrow_datatype() -> arrow2::datatypes::DataType { + use ::arrow2::datatypes::*; + DataType::Struct(vec![Field { + name: "value".to_owned(), + data_type: DataType::Float32, + is_nullable: true, + metadata: [].into(), + }]) + } + + #[allow(unused_imports, clippy::wildcard_imports)] + fn try_to_arrow_opt<'a>( + data: impl IntoIterator>>>, + extension_wrapper: Option<&str>, + ) -> crate::SerializationResult> + where + Self: Clone + 'a, + { + use crate::{Component as _, Datatype as _}; + use ::arrow2::{array::*, datatypes::*}; + Ok({ + let (somes, data): (Vec<_>, Vec<_>) = data + .into_iter() + .map(|datum| { + let datum: Option<::std::borrow::Cow<'a, Self>> = datum.map(Into::into); + (datum.is_some(), datum) + }) + .unzip(); + let bitmap: Option<::arrow2::bitmap::Bitmap> = { + let any_nones = somes.iter().any(|some| !*some); + any_nones.then(|| somes.into()) + }; + StructArray::new( + (if let Some(ext) = extension_wrapper { + DataType::Extension( + ext.to_owned(), + Box::new(::to_arrow_datatype()), + None, + ) + } else { + ::to_arrow_datatype() + }) + .to_logical_type() + .clone(), + vec![{ + let (somes, value): (Vec<_>, Vec<_>) = data + .iter() + .map(|datum| { + let datum = datum + .as_ref() + .map(|datum| { + let Self { value, .. } = &**datum; + value.clone() + }) + .flatten(); + (datum.is_some(), datum) + }) + .unzip(); + let value_bitmap: Option<::arrow2::bitmap::Bitmap> = { + let any_nones = somes.iter().any(|some| !*some); + any_nones.then(|| somes.into()) + }; + PrimitiveArray::new( + { + _ = extension_wrapper; + DataType::Float32.to_logical_type().clone() + }, + value.into_iter().map(|v| v.unwrap_or_default()).collect(), + value_bitmap, + ) + .boxed() + }], + bitmap, + ) + .boxed() + }) + } + + #[allow(unused_imports, clippy::wildcard_imports)] + fn try_from_arrow_opt( + data: &dyn ::arrow2::array::Array, + ) -> crate::DeserializationResult>> + where + Self: Sized, + { + use crate::{Component as _, Datatype as _}; + use ::arrow2::{array::*, datatypes::*}; + Ok({ + let data = data + .as_any() + .downcast_ref::<::arrow2::array::StructArray>() + .ok_or_else(|| crate::DeserializationError::SchemaMismatch { + expected: data.data_type().clone(), + got: data.data_type().clone(), + })?; + let (data_fields, data_arrays, data_bitmap) = + (data.fields(), data.values(), data.validity()); + let is_valid = |i| data_bitmap.map_or(true, |bitmap| bitmap.get_bit(i)); + let arrays_by_name: ::std::collections::HashMap<_, _> = data_fields + .iter() + .map(|field| field.name.as_str()) + .zip(data_arrays) + .collect(); + let value = { + let data = &**arrays_by_name["value"]; + + data.as_any() + .downcast_ref::() + .unwrap() + .into_iter() + .map(|v| v.copied()) + }; + ::itertools::izip!(value) + .enumerate() + .map(|(i, (value))| is_valid(i).then(|| Ok(Self { value })).transpose()) + .collect::>>()? + }) + } +} + #[derive(Debug, Clone, PartialEq)] pub struct AffixFuzzer1 { pub single_float_optional: Option, @@ -17,6 +164,8 @@ pub struct AffixFuzzer1 { pub many_floats_optional: Option>, pub many_strings_required: Vec, pub many_strings_optional: Option>, + pub flattened_scalar: f32, + pub almost_flattened_scalar: crate::datatypes::FlattenedScalar, } impl<'a> From for ::std::borrow::Cow<'a, AffixFuzzer1> { @@ -95,6 +244,23 @@ impl crate::Datatype for AffixFuzzer1 { is_nullable: true, metadata: [].into(), }, + Field { + name: "flattened_scalar".to_owned(), + data_type: DataType::Float32, + is_nullable: false, + metadata: [].into(), + }, + Field { + name: "almost_flattened_scalar".to_owned(), + data_type: DataType::Struct(vec![Field { + name: "value".to_owned(), + data_type: DataType::Float32, + is_nullable: true, + metadata: [].into(), + }]), + is_nullable: false, + metadata: [].into(), + }, ]) } @@ -534,6 +700,63 @@ impl crate::Datatype for AffixFuzzer1 { .boxed() } }, + { + let (somes, flattened_scalar): (Vec<_>, Vec<_>) = data + .iter() + .map(|datum| { + let datum = datum.as_ref().map(|datum| { + let Self { + flattened_scalar, .. + } = &**datum; + flattened_scalar.clone() + }); + (datum.is_some(), datum) + }) + .unzip(); + let flattened_scalar_bitmap: Option<::arrow2::bitmap::Bitmap> = { + let any_nones = somes.iter().any(|some| !*some); + any_nones.then(|| somes.into()) + }; + PrimitiveArray::new( + { + _ = extension_wrapper; + DataType::Float32.to_logical_type().clone() + }, + flattened_scalar + .into_iter() + .map(|v| v.unwrap_or_default()) + .collect(), + flattened_scalar_bitmap, + ) + .boxed() + }, + { + let (somes, almost_flattened_scalar): (Vec<_>, Vec<_>) = data + .iter() + .map(|datum| { + let datum = datum.as_ref().map(|datum| { + let Self { + almost_flattened_scalar, + .. + } = &**datum; + almost_flattened_scalar.clone() + }); + (datum.is_some(), datum) + }) + .unzip(); + let almost_flattened_scalar_bitmap: Option<::arrow2::bitmap::Bitmap> = { + let any_nones = somes.iter().any(|some| !*some); + any_nones.then(|| somes.into()) + }; + { + _ = almost_flattened_scalar_bitmap; + _ = extension_wrapper; + crate::datatypes::FlattenedScalar::try_to_arrow_opt( + almost_flattened_scalar, + None::<&str>, + )? + } + }, ], bitmap, ) @@ -746,13 +969,29 @@ impl crate::Datatype for AffixFuzzer1 { .into_iter() } }; + let flattened_scalar = { + let data = &**arrays_by_name["flattened_scalar"]; + + data.as_any() + .downcast_ref::() + .unwrap() + .into_iter() + .map(|v| v.copied()) + }; + let almost_flattened_scalar = { + let data = &**arrays_by_name["almost_flattened_scalar"]; + + crate::datatypes::FlattenedScalar::try_from_arrow_opt(data)?.into_iter() + }; ::itertools::izip!( single_float_optional, single_string_required, single_string_optional, many_floats_optional, many_strings_required, - many_strings_optional + many_strings_optional, + flattened_scalar, + almost_flattened_scalar ) .enumerate() .map( @@ -765,6 +1004,8 @@ impl crate::Datatype for AffixFuzzer1 { many_floats_optional, many_strings_required, many_strings_optional, + flattened_scalar, + almost_flattened_scalar, ), )| { is_valid(i) @@ -784,6 +1025,16 @@ impl crate::Datatype for AffixFuzzer1 { } })?, many_strings_optional, + flattened_scalar: flattened_scalar.ok_or_else(|| { + crate::DeserializationError::MissingData { + datatype: data.data_type().clone(), + } + })?, + almost_flattened_scalar: almost_flattened_scalar.ok_or_else( + || crate::DeserializationError::MissingData { + datatype: data.data_type().clone(), + }, + )?, }) }) .transpose() diff --git a/crates/re_types/src/datatypes/mod.rs b/crates/re_types/src/datatypes/mod.rs index 6567c6a2fc17..57ee71907bf6 100644 --- a/crates/re_types/src/datatypes/mod.rs +++ b/crates/re_types/src/datatypes/mod.rs @@ -6,6 +6,6 @@ mod point2d_ext; mod vec2d; mod vec2d_ext; -pub use self::fuzzy::{AffixFuzzer1, AffixFuzzer2}; +pub use self::fuzzy::{AffixFuzzer1, AffixFuzzer2, FlattenedScalar}; pub use self::point2d::Point2D; pub use self::vec2d::Vec2D; diff --git a/crates/re_types/tests/fuzzy.rs b/crates/re_types/tests/fuzzy.rs index 910a1100f696..37d78baf8950 100644 --- a/crates/re_types/tests/fuzzy.rs +++ b/crates/re_types/tests/fuzzy.rs @@ -14,6 +14,8 @@ fn roundtrip() { many_floats_optional: Some(vec![1.0, 10.0, 100.0]), many_strings_required: vec!["1".into(), "2".into()], many_strings_optional: Some(vec!["10".into(), "20".into()]), + flattened_scalar: 42.0, + almost_flattened_scalar: re_types::datatypes::FlattenedScalar { value: Some(42.0) }, }, }; @@ -24,6 +26,8 @@ fn roundtrip() { many_floats_optional: Some(vec![2.0, 20.0, 200.0]), many_strings_required: vec!["3".into(), "4".into()], many_strings_optional: None, + flattened_scalar: 43.0, + almost_flattened_scalar: re_types::datatypes::FlattenedScalar { value: Some(43.0) }, }); let fuzzy3 = re_types::components::AffixFuzzer3 { @@ -34,6 +38,8 @@ fn roundtrip() { many_floats_optional: Some(vec![3.0, 30.0, 300.0]), many_strings_required: vec!["5".into(), "6".into()], many_strings_optional: Some(vec!["50".into(), "60".into()]), + flattened_scalar: 44.0, + almost_flattened_scalar: re_types::datatypes::FlattenedScalar { value: Some(44.0) }, }, }; @@ -45,6 +51,8 @@ fn roundtrip() { many_floats_optional: Some(vec![4.0, 40.0, 400.0]), many_strings_required: vec!["7".into(), "8".into()], many_strings_optional: None, + flattened_scalar: 45.0, + almost_flattened_scalar: re_types::datatypes::FlattenedScalar { value: Some(45.0) }, }), }; @@ -83,6 +91,8 @@ fn roundtrip() { many_floats_optional: Some(vec![4.0, 40.0, 400.0]), many_strings_required: vec!["7".into(), "8".into()], many_strings_optional: None, + flattened_scalar: 46.0, + almost_flattened_scalar: re_types::datatypes::FlattenedScalar { value: Some(46.0) }, }]), }; diff --git a/crates/re_types_builder/src/lib.rs b/crates/re_types_builder/src/lib.rs index 50d21171afae..d1bfe57fea5c 100644 --- a/crates/re_types_builder/src/lib.rs +++ b/crates/re_types_builder/src/lib.rs @@ -138,6 +138,9 @@ pub use self::objects::{ // --- Attributes --- +pub const ATTR_ORDER: &str = "order"; +pub const ATTR_TRANSPARENT: &str = "transparent"; + pub const ATTR_ARROW_TRANSPARENT: &str = "attr.arrow.transparent"; pub const ATTR_ARROW_SPARSE_UNION: &str = "attr.arrow.sparse_union"; diff --git a/crates/re_types_builder/src/objects.rs b/crates/re_types_builder/src/objects.rs index 22f74bc89164..c0eeb37e2763 100644 --- a/crates/re_types_builder/src/objects.rs +++ b/crates/re_types_builder/src/objects.rs @@ -51,19 +51,80 @@ impl Objects { } // resolve objects - for obj in schema - .objects() - .iter() - // NOTE: Wrapped scalar types used by unions, not actual objects: ignore. - .filter(|obj| !obj.name().starts_with("fbs.scalars.")) - { + for obj in schema.objects().iter() { let resolved_obj = Object::from_raw_object(include_dir_path, &enums, &objs, &obj); resolved_objs.insert(resolved_obj.fqname.clone(), resolved_obj); } - Self { + let mut this = Self { objects: resolved_enums.into_iter().chain(resolved_objs).collect(), + }; + + // Resolve field-level semantic transparency recursively. + let mut done = false; + while !done { + done = true; + let mut objects_copy = this.objects.clone(); // borrowck, the lazy way + for obj in this.objects.values_mut() { + for field in &mut obj.fields { + if field.is_transparent() { + if let Some(target_fqname) = field.typ.fqname() { + let mut target_obj = objects_copy.remove(target_fqname).unwrap(); + assert!( + target_obj.fields.len() == 1, + "field '{}' is marked transparent but points to object '{}' which \ + doesn't have exactly one field (found {} fields instead)", + field.fqname, + target_obj.fqname, + target_obj.fields.len(), + ); + + let ObjectField { + virtpath: _, + filepath: _, + fqname, + pkg_name: _, + name: _, + docs: _, + typ, + attrs, + is_nullable: _, + is_deprecated: _, + datatype, + } = target_obj.fields.pop().unwrap(); + + field.typ = typ; + field.datatype = datatype; + + // TODO(cmc): might want to do something smarter at some point regarding attrs. + + // NOTE: Transparency (or lack thereof) of the target field takes precedence. + if let transparency @ Some(_) = + attrs.try_get::(&fqname, crate::ATTR_TRANSPARENT) + { + field.attrs.0.insert( + crate::ATTR_TRANSPARENT.to_owned(), + transparency.clone(), + ); + } else { + field.attrs.0.remove(crate::ATTR_TRANSPARENT); + } + + done = false; + } + } + } + } } + + // Remove whole objects marked as transparent. + this.objects = this + .objects + .drain() + .filter(|(_, obj)| !obj.is_transparent()) + .collect(); + + this } } @@ -473,7 +534,7 @@ impl Object { /// /// Panics if no order has been set. pub fn order(&self) -> u32 { - self.attrs.get::(&self.fqname, "order") + self.attrs.get::(&self.fqname, crate::ATTR_ORDER) } pub fn is_struct(&self) -> bool { @@ -503,6 +564,12 @@ impl Object { .try_get_attr::(crate::ATTR_ARROW_TRANSPARENT) .is_some() } + + fn is_transparent(&self) -> bool { + self.attrs + .try_get::(&self.fqname, crate::ATTR_TRANSPARENT) + .is_some() + } } /// Properties specific to either structs or unions, but not both. @@ -667,6 +734,12 @@ impl ObjectField { self.attrs.get::(&self.fqname, "order") } + fn is_transparent(&self) -> bool { + self.attrs + .try_get::(&self.fqname, crate::ATTR_TRANSPARENT) + .is_some() + } + pub fn get_attr(&self, name: impl AsRef) -> T where T: std::str::FromStr, @@ -754,7 +827,7 @@ impl Type { FbsBaseType::String => Self::String, FbsBaseType::Obj => { let obj = &objs[field_type.index() as usize]; - flatten_scalar_wrappers(obj).into() + Self::Object(obj.name().to_owned()) } FbsBaseType::Union => { let union = &enums[field_type.index() as usize]; @@ -869,7 +942,7 @@ impl ElementType { FbsBaseType::String => Self::String, FbsBaseType::Obj => { let obj = &objs[outer_type.index() as usize]; - flatten_scalar_wrappers(obj) + Self::Object(obj.name().to_owned()) } FbsBaseType::Union => unimplemented!("{inner_type:#?}"), // NOLINT FbsBaseType::None @@ -972,19 +1045,6 @@ impl Attributes { } } -/// Helper to turn wrapped scalars into actual scalars. -fn flatten_scalar_wrappers(obj: &FbsObject<'_>) -> ElementType { - let name = obj.name(); - if name.starts_with("fbs.scalars.") { - match name { - "fbs.scalars.Float32" => ElementType::Float32, - _ => unimplemented!("{name:#?}"), // NOLINT - } - } else { - ElementType::Object(name.to_owned()) - } -} - fn filepath_from_declaration_file( include_dir_path: impl AsRef, declaration_file: impl AsRef, diff --git a/rerun_py/rerun_sdk/rerun/_rerun2/components/fuzzy.py b/rerun_py/rerun_sdk/rerun/_rerun2/components/fuzzy.py index 3b49fdbb2505..2ba67c186a25 100644 --- a/rerun_py/rerun_sdk/rerun/_rerun2/components/fuzzy.py +++ b/rerun_py/rerun_sdk/rerun/_rerun2/components/fuzzy.py @@ -209,6 +209,13 @@ def __init__(self) -> None: pa.field( "many_strings_optional", pa.list_(pa.field("item", pa.utf8(), True, {})), True, {} ), + pa.field("flattened_scalar", pa.float32(), False, {}), + pa.field( + "almost_flattened_scalar", + pa.struct([pa.field("value", pa.float32(), True, {})]), + False, + {}, + ), ] ), True, diff --git a/rerun_py/rerun_sdk/rerun/_rerun2/datatypes/__init__.py b/rerun_py/rerun_sdk/rerun/_rerun2/datatypes/__init__.py index ca164f0ad4db..6881e833958a 100644 --- a/rerun_py/rerun_sdk/rerun/_rerun2/datatypes/__init__.py +++ b/rerun_py/rerun_sdk/rerun/_rerun2/datatypes/__init__.py @@ -13,6 +13,11 @@ AffixFuzzer2ArrayLike, AffixFuzzer2Like, AffixFuzzer2Type, + FlattenedScalar, + FlattenedScalarArray, + FlattenedScalarArrayLike, + FlattenedScalarLike, + FlattenedScalarType, ) from .point2d import Point2D, Point2DArray, Point2DArrayLike, Point2DLike, Point2DType from .vec2d import Vec2D, Vec2DArray, Vec2DArrayLike, Vec2DLike, Vec2DType @@ -28,6 +33,11 @@ "AffixFuzzer2ArrayLike", "AffixFuzzer2Like", "AffixFuzzer2Type", + "FlattenedScalar", + "FlattenedScalarArray", + "FlattenedScalarArrayLike", + "FlattenedScalarLike", + "FlattenedScalarType", "Point2D", "Point2DArray", "Point2DArrayLike", diff --git a/rerun_py/rerun_sdk/rerun/_rerun2/datatypes/fuzzy.py b/rerun_py/rerun_sdk/rerun/_rerun2/datatypes/fuzzy.py index c73509a3dcdf..88c3b03e587e 100644 --- a/rerun_py/rerun_sdk/rerun/_rerun2/datatypes/fuzzy.py +++ b/rerun_py/rerun_sdk/rerun/_rerun2/datatypes/fuzzy.py @@ -9,6 +9,7 @@ import pyarrow as pa from attrs import define, field +from .. import datatypes from .._baseclasses import ( BaseExtensionArray, BaseExtensionType, @@ -28,13 +29,60 @@ "AffixFuzzer2ArrayLike", "AffixFuzzer2Like", "AffixFuzzer2Type", + "FlattenedScalar", + "FlattenedScalarArray", + "FlattenedScalarArrayLike", + "FlattenedScalarLike", + "FlattenedScalarType", ] +@define +class FlattenedScalar: + value: float | None = field(default=None) + + def __array__(self, dtype: npt.DTypeLike = None) -> npt.ArrayLike: + return np.asarray(self.value, dtype=dtype) + + +FlattenedScalarLike = FlattenedScalar +FlattenedScalarArrayLike = Union[ + FlattenedScalar, + Sequence[FlattenedScalarLike], +] + + +# --- Arrow support --- + + +class FlattenedScalarType(BaseExtensionType): + def __init__(self) -> None: + pa.ExtensionType.__init__( + self, pa.struct([pa.field("value", pa.float32(), True, {})]), "rerun.testing.datatypes.FlattenedScalar" + ) + + +class FlattenedScalarArray(BaseExtensionArray[FlattenedScalarArrayLike]): + _EXTENSION_NAME = "rerun.testing.datatypes.FlattenedScalar" + _EXTENSION_TYPE = FlattenedScalarType + + @staticmethod + def _native_to_pa_array(data: FlattenedScalarArrayLike, data_type: pa.DataType) -> pa.Array: + raise NotImplementedError + + +FlattenedScalarType._ARRAY_TYPE = FlattenedScalarArray + +# TODO(cmc): bring back registration to pyarrow once legacy types are gone +# pa.register_extension_type(FlattenedScalarType()) + + @define class AffixFuzzer1: single_string_required: str = field() many_strings_required: list[str] = field() + flattened_scalar: float = field() + almost_flattened_scalar: datatypes.FlattenedScalar = field() single_float_optional: float | None = field(default=None) single_string_optional: str | None = field(default=None) many_floats_optional: npt.NDArray[np.float32] | None = field(default=None, converter=to_np_float32) @@ -63,6 +111,10 @@ def __init__(self) -> None: pa.field("many_floats_optional", pa.list_(pa.field("item", pa.float32(), True, {})), True, {}), pa.field("many_strings_required", pa.list_(pa.field("item", pa.utf8(), False, {})), False, {}), pa.field("many_strings_optional", pa.list_(pa.field("item", pa.utf8(), True, {})), True, {}), + pa.field("flattened_scalar", pa.float32(), False, {}), + pa.field( + "almost_flattened_scalar", pa.struct([pa.field("value", pa.float32(), True, {})]), False, {} + ), ] ), "rerun.testing.datatypes.AffixFuzzer1",