From 456e0baa28894887e012e01a1f87793a4c211670 Mon Sep 17 00:00:00 2001 From: Emil Ernerfeldt Date: Wed, 12 Jul 2023 15:33:19 +0200 Subject: [PATCH] Rust codegen: generate proper docstrings (#2668) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ### What Replace `#[doc = "…"]` with `/// …`. This is so that the generated code is easier to read, which is nice for us, and nice for our users. ### Checklist * [x] I have read and agree to [Contributor Guide](https://github.com/rerun-io/rerun/blob/main/CONTRIBUTING.md) and the [Code of Conduct](https://github.com/rerun-io/rerun/blob/main/CODE_OF_CONDUCT.md) * [x] I've included a screenshot or gif (if applicable) * [x] I have tested [demo.rerun.io](https://demo.rerun.io/pr/2668) (if applicable) - [PR Build Summary](https://build.rerun.io/pr/2668) - [Docs preview](https://rerun.io/preview/pr%3Aemilk%2Fcodegen-docstrings/docs) - [Examples preview](https://rerun.io/preview/pr%3Aemilk%2Fcodegen-docstrings/examples) --------- Co-authored-by: Clement Rey --- crates/re_types/build.rs | 64 ++++-------- crates/re_types/source_hash.txt | 2 +- crates/re_types/src/archetypes/points2d.rs | 98 +++++++++---------- crates/re_types/src/archetypes/transform3d.rs | 4 +- crates/re_types/src/components/class_id.rs | 6 +- crates/re_types/src/components/color.rs | 2 +- crates/re_types/src/components/draw_order.rs | 14 +-- .../re_types/src/components/instance_key.rs | 2 +- crates/re_types/src/components/keypoint_id.rs | 10 +- crates/re_types/src/components/label.rs | 2 +- crates/re_types/src/components/point2d.rs | 2 +- crates/re_types/src/components/radius.rs | 2 +- crates/re_types/src/components/transform3d.rs | 4 +- crates/re_types/src/datatypes/angle.rs | 2 +- crates/re_types/src/datatypes/mat3x3.rs | 2 +- crates/re_types/src/datatypes/mat4x4.rs | 2 +- crates/re_types/src/datatypes/point2d.rs | 2 +- crates/re_types/src/datatypes/quaternion.rs | 2 +- crates/re_types/src/datatypes/rotation3d.rs | 6 +- .../src/datatypes/rotation_axis_angle.rs | 14 +-- crates/re_types/src/datatypes/scale3d.rs | 6 +- crates/re_types/src/datatypes/transform3d.rs | 8 +- .../src/datatypes/translation_and_mat3x3.rs | 14 +-- .../datatypes/translation_rotation_scale3d.rs | 12 +-- crates/re_types/src/datatypes/vec2d.rs | 2 +- crates/re_types/src/datatypes/vec3d.rs | 2 +- crates/re_types/src/datatypes/vec4d.rs | 2 +- crates/re_types_builder/src/codegen/mod.rs | 2 +- crates/re_types_builder/src/codegen/python.rs | 20 ++-- crates/re_types_builder/src/codegen/rust.rs | 95 ++++++++++++++++-- crates/re_types_builder/src/lib.rs | 50 +++++----- crates/re_types_builder/src/objects.rs | 10 +- 32 files changed, 262 insertions(+), 203 deletions(-) diff --git a/crates/re_types/build.rs b/crates/re_types/build.rs index 1fd4bbd20ed3..431aeba6c0af 100644 --- a/crates/re_types/build.rs +++ b/crates/re_types/build.rs @@ -16,6 +16,7 @@ use re_build_tools::{ const SOURCE_HASH_PATH: &str = "./source_hash.txt"; const DEFINITIONS_DIR_PATH: &str = "./definitions"; +const ENTRYPOINT_PATH: &str = "./definitions/rerun/archetypes.fbs"; const DOC_EXAMPLES_DIR_PATH: &str = "../../docs/code-examples"; const RUST_OUTPUT_DIR_PATH: &str = "."; const PYTHON_OUTPUT_DIR_PATH: &str = "../../rerun_py/rerun_sdk/rerun/_rerun2"; @@ -101,52 +102,25 @@ fn main() { let sh = Shell::new().unwrap(); - re_types_builder::generate_rust_code( - DEFINITIONS_DIR_PATH, - RUST_OUTPUT_DIR_PATH, - "./definitions/rerun/archetypes.fbs", - ); - - // NOTE: We're purposefully ignoring the error here. - // - // In the very unlikely chance that the user doesn't have the `fmt` component installed, - // there's still no good reason to fail the build. - // - // The CI will catch the unformatted file at PR time and complain appropriately anyhow. - cmd!(sh, "cargo fmt -p re_types").run().ok(); - // RUN IT TWICE!!! `rustfmt` is __not__ idempotent until its second run! - // - // You can try it out yourself with e.g. this snippet: - // ``` - // # [derive (Clone , Debug)] - // - // # [derive (Default , Copy , PartialEq , PartialOrd)] - // pub struct Vec2D (pub [f32 ; 2usize] ,) ; - // ``` - // - // First run will take care of most things, but since `rustfmt` isn't recursive, it will also - // miss the opportunity to merge the two #derive clauses after it took care of removing the - // superfluous linefeeds: - // ``` - // #[derive(Clone, Debug)] - // #[derive(Default, Copy, PartialEq, PartialOrd)] - // pub struct Vec2D(pub [f32; 2usize]); - // ``` - // - // Now if you run it a second time on the other hand...: - // ``` - // #[derive(Clone, Debug, Default, Copy, PartialEq, PartialOrd)] - // pub struct Vec2D(pub [f32; 2usize]); - // ``` - // - // And finally things are idempotent, for real this time. - cmd!(sh, "cargo fmt -p re_types").run().ok(); + // passes 1 through 3: bfbs, semantic, arrow registry + let (objects, arrow_registry) = + re_types_builder::generate_lang_agnostic(DEFINITIONS_DIR_PATH, ENTRYPOINT_PATH); + + re_types_builder::generate_rust_code(RUST_OUTPUT_DIR_PATH, &objects, &arrow_registry); + + // We need to run `cago fmt` several times because it is not idempotent! + // See https://github.com/rust-lang/rustfmt/issues/5824 + for _ in 0..2 { + // NOTE: We're purposefully ignoring the error here. + // + // In the very unlikely chance that the user doesn't have the `fmt` component installed, + // there's still no good reason to fail the build. + // + // The CI will catch the unformatted file at PR time and complain appropriately anyhow. + cmd!(sh, "cargo fmt -p re_types").run().ok(); + } - re_types_builder::generate_python_code( - DEFINITIONS_DIR_PATH, - PYTHON_OUTPUT_DIR_PATH, - "./definitions/rerun/archetypes.fbs", - ); + re_types_builder::generate_python_code(PYTHON_OUTPUT_DIR_PATH, &objects, &arrow_registry); let pyproject_path = PathBuf::from(PYTHON_OUTPUT_DIR_PATH) .parent() diff --git a/crates/re_types/source_hash.txt b/crates/re_types/source_hash.txt index 823411f345c5..e8389b6668e0 100644 --- a/crates/re_types/source_hash.txt +++ b/crates/re_types/source_hash.txt @@ -1,4 +1,4 @@ # This is a sha256 hash for all direct and indirect dependencies of this crate's build script. # It can be safely removed at anytime to force the build script to run again. # Check out build.rs to see how it's computed. -478bc67049484da40beb2a0e8fa97aae5ffe1d0a29bd1e30024815ae844956be \ No newline at end of file +73588bbdd10b7303d8d04bda573c315fd2acb3cd3edd2ee5244a926064c8dc46 \ No newline at end of file diff --git a/crates/re_types/src/archetypes/points2d.rs b/crates/re_types/src/archetypes/points2d.rs index 9ba720585438..1f2f88fb25ba 100644 --- a/crates/re_types/src/archetypes/points2d.rs +++ b/crates/re_types/src/archetypes/points2d.rs @@ -12,71 +12,71 @@ #![allow(clippy::too_many_lines)] #![allow(clippy::unnecessary_cast)] -#[doc = "A 2D point cloud with positions and optional colors, radii, labels, etc."] -#[doc = ""] -#[doc = "## Example"] -#[doc = ""] -#[doc = "```ignore"] -#[doc = "//! Log some very simple points."] -#[doc = ""] -#[doc = "use rerun::{"] -#[doc = " components::{Rect2D, Vec4D},"] -#[doc = " experimental::archetypes::Points2D,"] -#[doc = " MsgSender, RecordingStreamBuilder,"] -#[doc = "};"] -#[doc = ""] -#[doc = "fn main() -> Result<(), Box> {"] -#[doc = " let (rec_stream, storage) = RecordingStreamBuilder::new(\"points\").memory()?;"] -#[doc = ""] -#[doc = " MsgSender::from_archetype(\"points\", &Points2D::new([(0.0, 0.0), (1.0, 1.0)]))?"] -#[doc = " .send(&rec_stream)?;"] -#[doc = ""] -#[doc = " // Log an extra rect to set the view bounds"] -#[doc = " MsgSender::new(\"bounds\")"] -#[doc = " .with_component(&[Rect2D::XCYCWH(Vec4D([0.0, 0.0, 4.0, 3.0]))])?"] -#[doc = " .send(&rec_stream)?;"] -#[doc = ""] -#[doc = " rerun::native_viewer::show(storage.take())?;"] -#[doc = ""] -#[doc = " Ok(())"] -#[doc = "}"] -#[doc = "```"] +/// A 2D point cloud with positions and optional colors, radii, labels, etc. +/// +/// ## Example +/// +/// ```ignore +/// //! Log some very simple points. +/// +/// use rerun::{ +/// components::{Rect2D, Vec4D}, +/// experimental::archetypes::Points2D, +/// MsgSender, RecordingStreamBuilder, +/// }; +/// +/// fn main() -> Result<(), Box> { +/// let (rec_stream, storage) = RecordingStreamBuilder::new("points").memory()?; +/// +/// MsgSender::from_archetype("points", &Points2D::new([(0.0, 0.0), (1.0, 1.0)]))? +/// .send(&rec_stream)?; +/// +/// // Log an extra rect to set the view bounds +/// MsgSender::new("bounds") +/// .with_component(&[Rect2D::XCYCWH(Vec4D([0.0, 0.0, 4.0, 3.0]))])? +/// .send(&rec_stream)?; +/// +/// rerun::native_viewer::show(storage.take())?; +/// +/// Ok(()) +/// } +/// ``` #[derive(Clone, Debug, PartialEq)] pub struct Points2D { - #[doc = "All the actual 2D points that make up the point cloud."] + /// All the actual 2D points that make up the point cloud. pub points: Vec, - #[doc = "Optional radii for the points, effectively turning them into circles."] + /// Optional radii for the points, effectively turning them into circles. pub radii: Option>, - #[doc = "Optional colors for the points."] + /// Optional colors for the points. pub colors: Option>, - #[doc = "Optional text labels for the points."] + /// Optional text labels for the points. pub labels: Option>, - #[doc = "An optional floating point value that specifies the 2D drawing order."] - #[doc = "Objects with higher values are drawn on top of those with lower values."] - #[doc = ""] - #[doc = "The default for 2D points is 30.0."] + /// An optional floating point value that specifies the 2D drawing order. + /// Objects with higher values are drawn on top of those with lower values. + /// + /// The default for 2D points is 30.0. pub draw_order: Option, - #[doc = "Optional class Ids for the points."] - #[doc = ""] - #[doc = "The class ID provides colors and labels if not specified explicitly."] + /// Optional class Ids for the points. + /// + /// The class ID provides colors and labels if not specified explicitly. pub class_ids: Option>, - #[doc = "Optional keypoint IDs for the points, identifying them within a class."] - #[doc = ""] - #[doc = "If keypoint IDs are passed in but no class IDs were specified, the class ID will"] - #[doc = "default to 0."] - #[doc = "This is useful to identify points within a single classification (which is identified"] - #[doc = "with `class_id`)."] - #[doc = "E.g. the classification might be 'Person' and the keypoints refer to joints on a"] - #[doc = "detected skeleton."] + /// Optional keypoint IDs for the points, identifying them within a class. + /// + /// If keypoint IDs are passed in but no class IDs were specified, the class ID will + /// default to 0. + /// This is useful to identify points within a single classification (which is identified + /// with `class_id`). + /// E.g. the classification might be 'Person' and the keypoints refer to joints on a + /// detected skeleton. pub keypoint_ids: Option>, - #[doc = "Unique identifiers for each individual point in the batch."] + /// Unique identifiers for each individual point in the batch. pub instance_keys: Option>, } diff --git a/crates/re_types/src/archetypes/transform3d.rs b/crates/re_types/src/archetypes/transform3d.rs index 82739091e564..211258c83b5f 100644 --- a/crates/re_types/src/archetypes/transform3d.rs +++ b/crates/re_types/src/archetypes/transform3d.rs @@ -12,10 +12,10 @@ #![allow(clippy::too_many_lines)] #![allow(clippy::unnecessary_cast)] -#[doc = "A 3D transform"] +/// A 3D transform #[derive(Clone, Debug)] pub struct Transform3D { - #[doc = "The transform"] + /// The transform pub transform: crate::components::Transform3D, } diff --git a/crates/re_types/src/components/class_id.rs b/crates/re_types/src/components/class_id.rs index 0a519fc4c3df..ae158fd51690 100644 --- a/crates/re_types/src/components/class_id.rs +++ b/crates/re_types/src/components/class_id.rs @@ -12,9 +12,9 @@ #![allow(clippy::too_many_lines)] #![allow(clippy::unnecessary_cast)] -#[doc = "A 16-bit ID representing a type of semantic class."] -#[doc = ""] -#[doc = "Used to look up a `crate::components::ClassDescription` within the `crate::components::AnnotationContext`."] +/// A 16-bit ID representing a type of semantic class. +/// +/// Used to look up a `crate::components::ClassDescription` within the `crate::components::AnnotationContext`. #[derive(Clone, Debug, Copy, Default, PartialEq, Eq, PartialOrd, Ord, Hash)] pub struct ClassId(pub u16); diff --git a/crates/re_types/src/components/color.rs b/crates/re_types/src/components/color.rs index 5d2f193001e2..2fbba30af4ed 100644 --- a/crates/re_types/src/components/color.rs +++ b/crates/re_types/src/components/color.rs @@ -12,7 +12,7 @@ #![allow(clippy::too_many_lines)] #![allow(clippy::unnecessary_cast)] -#[doc = "An RGBA color tuple with unmultiplied/separate alpha, in sRGB gamma space with linear alpha."] +/// An RGBA color tuple with unmultiplied/separate alpha, in sRGB gamma space with linear alpha. #[derive( Clone, Debug, diff --git a/crates/re_types/src/components/draw_order.rs b/crates/re_types/src/components/draw_order.rs index 88fcf52b68f3..7a9d3e7e3433 100644 --- a/crates/re_types/src/components/draw_order.rs +++ b/crates/re_types/src/components/draw_order.rs @@ -12,13 +12,13 @@ #![allow(clippy::too_many_lines)] #![allow(clippy::unnecessary_cast)] -#[doc = "Draw order used for the display order of 2D elements."] -#[doc = ""] -#[doc = "Higher values are drawn on top of lower values."] -#[doc = "An entity can have only a single draw order component."] -#[doc = "Within an entity draw order is governed by the order of the components."] -#[doc = ""] -#[doc = "Draw order for entities with the same draw order is generally undefined."] +/// Draw order used for the display order of 2D elements. +/// +/// Higher values are drawn on top of lower values. +/// An entity can have only a single draw order component. +/// Within an entity draw order is governed by the order of the components. +/// +/// Draw order for entities with the same draw order is generally undefined. #[derive(Clone, Debug, Copy)] #[repr(transparent)] pub struct DrawOrder(pub f32); diff --git a/crates/re_types/src/components/instance_key.rs b/crates/re_types/src/components/instance_key.rs index 13bc30955f02..570a3eb97ca6 100644 --- a/crates/re_types/src/components/instance_key.rs +++ b/crates/re_types/src/components/instance_key.rs @@ -12,7 +12,7 @@ #![allow(clippy::too_many_lines)] #![allow(clippy::unnecessary_cast)] -#[doc = "A unique numeric identifier for each individual instance within a batch."] +/// A unique numeric identifier for each individual instance within a batch. #[derive(Clone, Debug, Copy, PartialEq, Eq, PartialOrd, Ord)] pub struct InstanceKey(pub u64); diff --git a/crates/re_types/src/components/keypoint_id.rs b/crates/re_types/src/components/keypoint_id.rs index f81f4cc3eeb3..a98d9004610d 100644 --- a/crates/re_types/src/components/keypoint_id.rs +++ b/crates/re_types/src/components/keypoint_id.rs @@ -12,11 +12,11 @@ #![allow(clippy::too_many_lines)] #![allow(clippy::unnecessary_cast)] -#[doc = "A 16-bit ID representing a type of semantic keypoint within a class."] -#[doc = ""] -#[doc = "`KeypointId`s are only meaningful within the context of a `crate::components::ClassDescription`."] -#[doc = ""] -#[doc = "Used to look up an `crate::components::AnnotationInfo` for a Keypoint within the `crate::components::AnnotationContext`."] +/// A 16-bit ID representing a type of semantic keypoint within a class. +/// +/// `KeypointId`s are only meaningful within the context of a `crate::components::ClassDescription`. +/// +/// Used to look up an `crate::components::AnnotationInfo` for a Keypoint within the `crate::components::AnnotationContext`. #[derive(Clone, Debug, Copy, Default, PartialEq, Eq, PartialOrd, Ord, Hash)] pub struct KeypointId(pub u16); diff --git a/crates/re_types/src/components/label.rs b/crates/re_types/src/components/label.rs index d7601d350eee..f9373946bf45 100644 --- a/crates/re_types/src/components/label.rs +++ b/crates/re_types/src/components/label.rs @@ -12,7 +12,7 @@ #![allow(clippy::too_many_lines)] #![allow(clippy::unnecessary_cast)] -#[doc = "A String label component."] +/// A String label component. #[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord)] #[repr(transparent)] pub struct Label(pub String); diff --git a/crates/re_types/src/components/point2d.rs b/crates/re_types/src/components/point2d.rs index fa52b5d21eeb..0fb05dc6b55a 100644 --- a/crates/re_types/src/components/point2d.rs +++ b/crates/re_types/src/components/point2d.rs @@ -12,7 +12,7 @@ #![allow(clippy::too_many_lines)] #![allow(clippy::unnecessary_cast)] -#[doc = "A point in 2D space."] +/// A point in 2D space. #[derive(Clone, Debug, Default, Copy, PartialEq, PartialOrd)] pub struct Point2D { pub xy: crate::datatypes::Point2D, diff --git a/crates/re_types/src/components/radius.rs b/crates/re_types/src/components/radius.rs index 71d0cea69ddb..cfbe5d9dc8a1 100644 --- a/crates/re_types/src/components/radius.rs +++ b/crates/re_types/src/components/radius.rs @@ -12,7 +12,7 @@ #![allow(clippy::too_many_lines)] #![allow(clippy::unnecessary_cast)] -#[doc = "A Radius component."] +/// A Radius component. #[derive(Clone, Debug, Copy, PartialEq, PartialOrd)] pub struct Radius(pub f32); diff --git a/crates/re_types/src/components/transform3d.rs b/crates/re_types/src/components/transform3d.rs index c7888d755e53..d5b45ed36fa0 100644 --- a/crates/re_types/src/components/transform3d.rs +++ b/crates/re_types/src/components/transform3d.rs @@ -12,10 +12,10 @@ #![allow(clippy::too_many_lines)] #![allow(clippy::unnecessary_cast)] -#[doc = "An affine transform between two 3D spaces, represented in a given direction."] +/// An affine transform between two 3D spaces, represented in a given direction. #[derive(Clone, Debug)] pub struct Transform3D { - #[doc = "Representation of the transform."] + /// Representation of the transform. pub repr: crate::datatypes::Transform3D, } diff --git a/crates/re_types/src/datatypes/angle.rs b/crates/re_types/src/datatypes/angle.rs index b5f73371b30a..85d72a0f3859 100644 --- a/crates/re_types/src/datatypes/angle.rs +++ b/crates/re_types/src/datatypes/angle.rs @@ -12,7 +12,7 @@ #![allow(clippy::too_many_lines)] #![allow(clippy::unnecessary_cast)] -#[doc = "Angle in either radians or degrees."] +/// Angle in either radians or degrees. #[derive(Clone, Debug, Copy, PartialEq)] pub enum Angle { Radians(f32), diff --git a/crates/re_types/src/datatypes/mat3x3.rs b/crates/re_types/src/datatypes/mat3x3.rs index 69656a2c469f..c6aa0fdd7b31 100644 --- a/crates/re_types/src/datatypes/mat3x3.rs +++ b/crates/re_types/src/datatypes/mat3x3.rs @@ -12,7 +12,7 @@ #![allow(clippy::too_many_lines)] #![allow(clippy::unnecessary_cast)] -#[doc = "A 3x3 column-major Matrix."] +/// A 3x3 column-major Matrix. #[derive(Clone, Debug, Default, Copy, PartialEq, PartialOrd)] pub struct Mat3x3(pub [f32; 9usize]); diff --git a/crates/re_types/src/datatypes/mat4x4.rs b/crates/re_types/src/datatypes/mat4x4.rs index a7190ade62d7..68859ea35acb 100644 --- a/crates/re_types/src/datatypes/mat4x4.rs +++ b/crates/re_types/src/datatypes/mat4x4.rs @@ -12,7 +12,7 @@ #![allow(clippy::too_many_lines)] #![allow(clippy::unnecessary_cast)] -#[doc = "A 4x4 column-major Matrix."] +/// A 4x4 column-major Matrix. #[derive(Clone, Debug, Default, Copy, PartialEq, PartialOrd)] pub struct Mat4x4(pub [f32; 16usize]); diff --git a/crates/re_types/src/datatypes/point2d.rs b/crates/re_types/src/datatypes/point2d.rs index 250a274b8007..a7f9a2dda922 100644 --- a/crates/re_types/src/datatypes/point2d.rs +++ b/crates/re_types/src/datatypes/point2d.rs @@ -12,7 +12,7 @@ #![allow(clippy::too_many_lines)] #![allow(clippy::unnecessary_cast)] -#[doc = "A point in 2D space."] +/// A point in 2D space. #[derive(Clone, Debug, Default, Copy, PartialEq, PartialOrd)] pub struct Point2D { pub x: f32, diff --git a/crates/re_types/src/datatypes/quaternion.rs b/crates/re_types/src/datatypes/quaternion.rs index de837de147a2..9552537598b9 100644 --- a/crates/re_types/src/datatypes/quaternion.rs +++ b/crates/re_types/src/datatypes/quaternion.rs @@ -12,7 +12,7 @@ #![allow(clippy::too_many_lines)] #![allow(clippy::unnecessary_cast)] -#[doc = "A Quaternion represented by 4 real numbers."] +/// A Quaternion represented by 4 real numbers. #[derive(Clone, Debug, Copy, PartialEq, PartialOrd)] pub struct Quaternion(pub [f32; 4usize]); diff --git a/crates/re_types/src/datatypes/rotation3d.rs b/crates/re_types/src/datatypes/rotation3d.rs index 7c3706d781dc..1a3c4061db9d 100644 --- a/crates/re_types/src/datatypes/rotation3d.rs +++ b/crates/re_types/src/datatypes/rotation3d.rs @@ -12,13 +12,13 @@ #![allow(clippy::too_many_lines)] #![allow(clippy::unnecessary_cast)] -#[doc = "A 3D rotation."] +/// A 3D rotation. #[derive(Clone, Debug, Copy, PartialEq)] pub enum Rotation3D { - #[doc = "Rotation defined by a quaternion."] + /// Rotation defined by a quaternion. Quaternion(crate::datatypes::Quaternion), - #[doc = "Rotation defined with an axis and an angle."] + /// Rotation defined with an axis and an angle. AxisAngle(crate::datatypes::RotationAxisAngle), } diff --git a/crates/re_types/src/datatypes/rotation_axis_angle.rs b/crates/re_types/src/datatypes/rotation_axis_angle.rs index 0c380a063a58..bfe79113bd71 100644 --- a/crates/re_types/src/datatypes/rotation_axis_angle.rs +++ b/crates/re_types/src/datatypes/rotation_axis_angle.rs @@ -12,17 +12,17 @@ #![allow(clippy::too_many_lines)] #![allow(clippy::unnecessary_cast)] -#[doc = "3D rotation represented by a rotation around a given axis."] +/// 3D rotation represented by a rotation around a given axis. #[derive(Clone, Debug, Copy, PartialEq)] pub struct RotationAxisAngle { - #[doc = "Axis to rotate around."] - #[doc = ""] - #[doc = "This is not required to be normalized."] - #[doc = "If normalization fails (typically because the vector is length zero), the rotation is silently"] - #[doc = "ignored."] + /// Axis to rotate around. + /// + /// This is not required to be normalized. + /// If normalization fails (typically because the vector is length zero), the rotation is silently + /// ignored. pub axis: crate::datatypes::Vec3D, - #[doc = "How much to rotate around the axis."] + /// How much to rotate around the axis. pub angle: crate::datatypes::Angle, } diff --git a/crates/re_types/src/datatypes/scale3d.rs b/crates/re_types/src/datatypes/scale3d.rs index 38c5f8012720..aa5a479dd8b2 100644 --- a/crates/re_types/src/datatypes/scale3d.rs +++ b/crates/re_types/src/datatypes/scale3d.rs @@ -12,13 +12,13 @@ #![allow(clippy::too_many_lines)] #![allow(clippy::unnecessary_cast)] -#[doc = "3D scaling factor, part of a transform representation."] +/// 3D scaling factor, part of a transform representation. #[derive(Clone, Debug, Copy, PartialEq)] pub enum Scale3D { - #[doc = "Individual scaling factors for each axis, distorting the original object."] + /// Individual scaling factors for each axis, distorting the original object. ThreeD(crate::datatypes::Vec3D), - #[doc = "Uniform scaling factor along all axis."] + /// Uniform scaling factor along all axis. Uniform(f32), } diff --git a/crates/re_types/src/datatypes/transform3d.rs b/crates/re_types/src/datatypes/transform3d.rs index 19f8cf279565..7b1a3b8e4ca4 100644 --- a/crates/re_types/src/datatypes/transform3d.rs +++ b/crates/re_types/src/datatypes/transform3d.rs @@ -12,10 +12,10 @@ #![allow(clippy::too_many_lines)] #![allow(clippy::unnecessary_cast)] -#[doc = "Representation of a 3D affine transform."] -#[doc = ""] -#[doc = "Rarely used directly, prefer using the underlying representation classes and pass them"] -#[doc = "directly to `Transform3D::child_from_parent` or `Transform3D::parent_from_child`."] +/// Representation of a 3D affine transform. +/// +/// Rarely used directly, prefer using the underlying representation classes and pass them +/// directly to `Transform3D::child_from_parent` or `Transform3D::parent_from_child`. #[derive(Clone, Debug, Copy, PartialEq)] pub enum Transform3D { TranslationAndMat3X3(crate::datatypes::TranslationAndMat3x3), diff --git a/crates/re_types/src/datatypes/translation_and_mat3x3.rs b/crates/re_types/src/datatypes/translation_and_mat3x3.rs index 640e66493eeb..626375cc3807 100644 --- a/crates/re_types/src/datatypes/translation_and_mat3x3.rs +++ b/crates/re_types/src/datatypes/translation_and_mat3x3.rs @@ -12,19 +12,19 @@ #![allow(clippy::too_many_lines)] #![allow(clippy::unnecessary_cast)] -#[doc = "Representation of an affine transform via a 3x3 affine matrix paired with a translation."] -#[doc = ""] -#[doc = "First applies the matrix, then the translation."] +/// Representation of an affine transform via a 3x3 affine matrix paired with a translation. +/// +/// First applies the matrix, then the translation. #[derive(Clone, Debug, Copy, PartialEq)] pub struct TranslationAndMat3x3 { - #[doc = "3D translation, applied after the matrix."] + /// 3D translation, applied after the matrix. pub translation: Option, - #[doc = "3x3 matrix for scale, rotation & shear."] + /// 3x3 matrix for scale, rotation & shear. pub matrix: Option, - #[doc = "If true, the transform maps from the parent space to the space where the transform was logged."] - #[doc = "Otherwise, the transform maps from the space to its parent."] + /// If true, the transform maps from the parent space to the space where the transform was logged. + /// Otherwise, the transform maps from the space to its parent. pub from_parent: Option, } diff --git a/crates/re_types/src/datatypes/translation_rotation_scale3d.rs b/crates/re_types/src/datatypes/translation_rotation_scale3d.rs index a3882588c151..27908dc26ecc 100644 --- a/crates/re_types/src/datatypes/translation_rotation_scale3d.rs +++ b/crates/re_types/src/datatypes/translation_rotation_scale3d.rs @@ -12,20 +12,20 @@ #![allow(clippy::too_many_lines)] #![allow(clippy::unnecessary_cast)] -#[doc = "Representation of an affine transform via separate translation, rotation & scale."] +/// Representation of an affine transform via separate translation, rotation & scale. #[derive(Clone, Debug, Copy, PartialEq)] pub struct TranslationRotationScale3D { - #[doc = "3D translation vector, applied last."] + /// 3D translation vector, applied last. pub translation: Option, - #[doc = "3D rotation, applied second."] + /// 3D rotation, applied second. pub rotation: Option, - #[doc = "3D scale, applied first."] + /// 3D scale, applied first. pub scale: Option, - #[doc = "If true, the transform maps from the parent space to the space where the transform was logged."] - #[doc = "Otherwise, the transform maps from the space to its parent."] + /// If true, the transform maps from the parent space to the space where the transform was logged. + /// Otherwise, the transform maps from the space to its parent. pub from_parent: Option, } diff --git a/crates/re_types/src/datatypes/vec2d.rs b/crates/re_types/src/datatypes/vec2d.rs index 9f8ca4f5af23..0a10e987dd35 100644 --- a/crates/re_types/src/datatypes/vec2d.rs +++ b/crates/re_types/src/datatypes/vec2d.rs @@ -12,7 +12,7 @@ #![allow(clippy::too_many_lines)] #![allow(clippy::unnecessary_cast)] -#[doc = "A vector in 2D space."] +/// A vector in 2D space. #[derive(Clone, Debug, Default, Copy, PartialEq, PartialOrd)] pub struct Vec2D(pub [f32; 2usize]); diff --git a/crates/re_types/src/datatypes/vec3d.rs b/crates/re_types/src/datatypes/vec3d.rs index 37164cfa408b..b2984928b9a0 100644 --- a/crates/re_types/src/datatypes/vec3d.rs +++ b/crates/re_types/src/datatypes/vec3d.rs @@ -12,7 +12,7 @@ #![allow(clippy::too_many_lines)] #![allow(clippy::unnecessary_cast)] -#[doc = "A vector in 3D space."] +/// A vector in 3D space. #[derive(Clone, Debug, Default, Copy, PartialEq, PartialOrd)] pub struct Vec3D(pub [f32; 3usize]); diff --git a/crates/re_types/src/datatypes/vec4d.rs b/crates/re_types/src/datatypes/vec4d.rs index a6029e1d8cde..f19fed2d9913 100644 --- a/crates/re_types/src/datatypes/vec4d.rs +++ b/crates/re_types/src/datatypes/vec4d.rs @@ -12,7 +12,7 @@ #![allow(clippy::too_many_lines)] #![allow(clippy::unnecessary_cast)] -#[doc = "A vector in 4D space."] +/// A vector in 4D space. #[derive(Clone, Debug, Default, Copy, PartialEq, PartialOrd)] pub struct Vec4D(pub [f32; 4usize]); diff --git a/crates/re_types_builder/src/codegen/mod.rs b/crates/re_types_builder/src/codegen/mod.rs index cdc3e87d39a4..0608ace21379 100644 --- a/crates/re_types_builder/src/codegen/mod.rs +++ b/crates/re_types_builder/src/codegen/mod.rs @@ -7,7 +7,7 @@ pub trait CodeGenerator { &mut self, objs: &crate::Objects, arrow_registry: &crate::ArrowRegistry, - ) -> Vec; + ) -> std::collections::BTreeSet; } // --- diff --git a/crates/re_types_builder/src/codegen/python.rs b/crates/re_types_builder/src/codegen/python.rs index 34168845f946..0e73c65e3aae 100644 --- a/crates/re_types_builder/src/codegen/python.rs +++ b/crates/re_types_builder/src/codegen/python.rs @@ -1,7 +1,7 @@ //! Implements the Python codegen pass. use std::{ - collections::{BTreeMap, HashMap, HashSet}, + collections::{BTreeMap, BTreeSet, HashMap, HashSet}, io::Write, }; @@ -85,8 +85,12 @@ fn load_overrides(path: &Utf8Path) -> HashSet { } impl CodeGenerator for PythonCodeGenerator { - fn generate(&mut self, objs: &Objects, arrow_registry: &ArrowRegistry) -> Vec { - let mut filepaths = Vec::new(); + fn generate( + &mut self, + objs: &Objects, + arrow_registry: &ArrowRegistry, + ) -> BTreeSet { + let mut filepaths = BTreeSet::new(); let datatypes_path = self.pkg_path.join("datatypes"); let datatype_overrides = load_overrides(&datatypes_path); @@ -137,7 +141,7 @@ impl CodeGenerator for PythonCodeGenerator { ); filepaths.extend(paths); - filepaths.push(quote_lib(&self.pkg_path, &archetype_names)); + filepaths.insert(quote_lib(&self.pkg_path, &archetype_names)); filepaths } @@ -185,10 +189,10 @@ fn quote_objects( all_objects: &Objects, _kind: ObjectKind, objs: &[&Object], -) -> (Vec, Vec) { +) -> (BTreeSet, Vec) { let out_path = out_path.as_ref(); - let mut filepaths = Vec::new(); + let mut filepaths = BTreeSet::new(); let mut all_names = Vec::new(); let mut files = HashMap::>::new(); @@ -238,7 +242,7 @@ fn quote_objects( .or_default() .extend(names.iter().cloned()); - filepaths.push(filepath.clone()); + filepaths.insert(filepath.clone()); let mut file = std::fs::File::create(&filepath) .with_context(|| format!("{filepath:?}")) .unwrap(); @@ -361,7 +365,7 @@ fn quote_objects( code.push_unindented_text(format!("\n__all__ = [{manifest}]"), 0); - filepaths.push(path.clone()); + filepaths.insert(path.clone()); std::fs::write(&path, code) .with_context(|| format!("{path:?}")) .unwrap(); diff --git a/crates/re_types_builder/src/codegen/rust.rs b/crates/re_types_builder/src/codegen/rust.rs index 9865e68682f8..4ffe98439b29 100644 --- a/crates/re_types_builder/src/codegen/rust.rs +++ b/crates/re_types_builder/src/codegen/rust.rs @@ -1,6 +1,9 @@ //! Implements the Rust codegen pass. -use std::{collections::HashMap, io::Write}; +use std::{ + collections::{BTreeSet, HashMap}, + io::Write, +}; use anyhow::Context as _; use arrow2::datatypes::DataType; @@ -36,8 +39,12 @@ impl RustCodeGenerator { } impl CodeGenerator for RustCodeGenerator { - fn generate(&mut self, objects: &Objects, arrow_registry: &ArrowRegistry) -> Vec { - let mut filepaths = Vec::new(); + fn generate( + &mut self, + objects: &Objects, + arrow_registry: &ArrowRegistry, + ) -> BTreeSet { + let mut filepaths = BTreeSet::new(); let datatypes_path = self.crate_path.join("src/datatypes"); std::fs::create_dir_all(&datatypes_path) @@ -83,10 +90,10 @@ fn create_files( arrow_registry: &ArrowRegistry, objects: &Objects, objs: &[&Object], -) -> Vec { +) -> BTreeSet { let out_path = out_path.as_ref(); - let mut filepaths = Vec::new(); + let mut filepaths = BTreeSet::new(); let mut files = HashMap::>::new(); for obj in objs { @@ -112,7 +119,7 @@ fn create_files( .or_default() .extend(names); - filepaths.push(filepath.clone()); + filepaths.insert(filepath.clone()); let mut file = std::fs::File::create(&filepath) .with_context(|| format!("{filepath:?}")) .unwrap(); @@ -150,7 +157,7 @@ fn create_files( .to_string() .replace('}', "}\n\n") .replace("] ;", "];\n\n") - .replace("# [doc", "\n\n#[doc") + .replace("# [doc", "\n\n# [doc") .replace("impl ", "\n\nimpl "); code.push_text(tokens_str, 1, 0); acc = TokenStream::new(); @@ -169,11 +176,14 @@ fn create_files( .to_string() .replace('}', "}\n\n") .replace("] ;", "];\n\n") - .replace("# [doc", "\n\n#[doc") + .replace("# [doc", "\n\n# [doc") .replace("impl ", "\n\nimpl "); code.push_text(tokens_str, 1, 0); } + + code = replace_doc_attrb_with_doc_comment(&code); + file.write_all(code.as_bytes()) .with_context(|| format!("{filepath:?}")) .unwrap(); @@ -206,7 +216,7 @@ fn create_files( code.push_text(format!("pub use self::{module}::{{{names}}};"), 1, 0); } - filepaths.push(path.clone()); + filepaths.insert(path.clone()); std::fs::write(&path, code) .with_context(|| format!("{path:?}")) .unwrap(); @@ -215,6 +225,71 @@ fn create_files( filepaths } +/// Replace `#[doc = "…"]` attributes with `/// …` doc comments, +/// while also removing trailing whitespace. +fn replace_doc_attrb_with_doc_comment(code: &String) -> String { + // This is difficult to do with regex, because the patterns with newlines overlap. + + let start_pattern = "# [doc = \""; + let end_pattern = "\"]"; // assues there is no escaped quote followed by a bracket + + let problematic = r#"\"]"#; + assert!( + !code.contains(problematic), + "The codegen cannot handle the string {problematic} yet" + ); + + let mut new_code = String::new(); + + let mut i = 0; + while i < code.len() { + if let Some(off) = code[i..].find(start_pattern) { + let doc_start = i + off; + let content_start = doc_start + start_pattern.len(); + if let Some(off) = code[content_start..].find(end_pattern) { + let content_end = content_start + off; + new_code.push_str(&code[i..doc_start]); + new_code.push_str("/// "); + unescape_string_into(&code[content_start..content_end], &mut new_code); + new_code.push('\n'); + + i = content_end + end_pattern.len(); + // Skip trailing whitespace (extra newlines) + while matches!(code.as_bytes().get(i), Some(b'\n' | b' ')) { + i += 1; + } + continue; + } + } + + // No more doc attributes found + new_code.push_str(&code[i..]); + break; + } + new_code +} + +fn unescape_string_into(input: &str, output: &mut String) { + let mut chars = input.chars(); + + while let Some(c) = chars.next() { + if c == '\\' { + let c = chars.next().expect("Trailing backslash"); + match c { + 'n' => output.push('\n'), + 'r' => output.push('\r'), + 't' => output.push('\t'), + '\\' => output.push('\\'), + '"' => output.push('"'), + '\'' => output.push('\''), + _ => panic!("Unknown escape sequence: \\{c}"), + } + } else { + output.push(c); + } + } +} + // --- Codegen core loop --- #[derive(Debug, Clone)] @@ -425,7 +500,7 @@ fn quote_doc_from_docs(docs: &Docs) -> TokenStream { impl quote::ToTokens for DocCommentTokenizer<'_> { fn to_tokens(&self, tokens: &mut TokenStream) { - tokens.extend(self.0.iter().map(|line| quote!(#[doc = #line]))); + tokens.extend(self.0.iter().map(|line| quote!(# [doc = #line]))); } } diff --git a/crates/re_types_builder/src/lib.rs b/crates/re_types_builder/src/lib.rs index dc4f07e7da2c..c4b509d9e1d7 100644 --- a/crates/re_types_builder/src/lib.rs +++ b/crates/re_types_builder/src/lib.rs @@ -205,7 +205,12 @@ pub fn compile_binary_schemas( /// 1. Generate binary reflection dumps for our definitions. /// 2. Run the semantic pass /// 3. Compute the Arrow registry -fn generate_lang_agnostic( +/// +/// Panics on error. +/// +/// - `include_dir_path`: path to the root directory of the fbs definition tree. +/// - `entrypoint_path`: path to the root file of the fbs definition tree. +pub fn generate_lang_agnostic( include_dir_path: impl AsRef, entrypoint_path: impl AsRef, ) -> (Objects, ArrowRegistry) { @@ -247,59 +252,56 @@ fn generate_lang_agnostic( (objects, arrow_registry) } -/// Generates Rust code from a set of flatbuffers definitions. +/// Generates Rust code. /// /// Panics on error. /// -/// - `include_dir_path`: path to the root directory of the fbs definition tree. /// - `output_crate_path`: path to the root of the output crate. -/// - `entrypoint_path`: path to the root file of the fbs definition tree. /// /// E.g.: /// ```no_run -/// re_types_builder::generate_rust_code( +/// let (objects, arrow_registry) = re_types_builder::generate_lang_agnostic( /// "./definitions", -/// ".", /// "./definitions/rerun/archetypes.fbs", /// ); +/// re_types_builder::generate_rust_code( +/// ".", +/// &objects, +/// &arrow_registry, +/// ); /// ``` pub fn generate_rust_code( - include_dir_path: impl AsRef, output_crate_path: impl AsRef, - entrypoint_path: impl AsRef, + objects: &Objects, + arrow_registry: &ArrowRegistry, ) { - // passes 1 through 3: bfbs, semantic, arrow registry - let (objects, arrow_registry) = generate_lang_agnostic(include_dir_path, entrypoint_path); - let mut gen = RustCodeGenerator::new(output_crate_path.as_ref()); - let _filepaths = gen.generate(&objects, &arrow_registry); + let _filepaths = gen.generate(objects, arrow_registry); } -/// Generates Python code from a set of flatbuffers definitions. +/// Generates Python code. /// /// Panics on error. /// -/// - `include_dir_path`: path to the root directory of the fbs definition tree. /// - `output_pkg_path`: path to the root of the output package. -/// - `entrypoint_path`: path to the root file of the fbs definition tree. /// /// E.g.: /// ```no_run -/// re_types_builder::generate_python_code( +/// let (objects, arrow_registry) = re_types_builder::generate_lang_agnostic( /// "./definitions", -/// "./rerun_py", /// "./definitions/rerun/archetypes.fbs", /// ); +/// re_types_builder::generate_python_code( +/// "./rerun_py", +/// &objects, +/// &arrow_registry, +/// ); /// ``` pub fn generate_python_code( - include_dir_path: impl AsRef, output_pkg_path: impl AsRef, - entrypoint_path: impl AsRef, + objects: &Objects, + arrow_registry: &ArrowRegistry, ) { - // passes 1 through 3: bfbs, semantic, arrow registry - let (objects, arrow_registry) = generate_lang_agnostic(include_dir_path, entrypoint_path); - - // generate python code let mut gen = PythonCodeGenerator::new(output_pkg_path.as_ref()); - let _filepaths = gen.generate(&objects, &arrow_registry); + let _filepaths = gen.generate(objects, arrow_registry); } diff --git a/crates/re_types_builder/src/objects.rs b/crates/re_types_builder/src/objects.rs index 09e980039568..e9be98b443f6 100644 --- a/crates/re_types_builder/src/objects.rs +++ b/crates/re_types_builder/src/objects.rs @@ -265,7 +265,7 @@ impl Docs { .with_context(|| format!("couldn't parse included path: {raw_path:?}")) .unwrap(); - let path = filepath.join(path); + let path = filepath.parent().unwrap().join(path); included_files .entry(path.clone()) @@ -416,7 +416,13 @@ impl Object { .map(ToOwned::to_owned) .with_context(|| format!("no declaration_file found for {fqname}")) .unwrap(); + assert!(virtpath.ends_with(".fbs"), "Bad virtpath: {virtpath:?}"); + let filepath = filepath_from_declaration_file(include_dir_path, &virtpath); + assert!( + filepath.to_string().ends_with(".fbs"), + "Bad filepath: {filepath:?}" + ); let docs = Docs::from_raw_docs(&filepath, obj.documentation()); let kind = ObjectKind::from_pkg_name(&pkg_name); @@ -1075,8 +1081,6 @@ fn filepath_from_declaration_file( ) -> Utf8PathBuf { include_dir_path.as_ref().join("rerun").join( Utf8PathBuf::from(declaration_file.as_ref()) - .parent() - .unwrap() // NOTE: safe, this _must_ be a file .to_string() .replace("//", ""), )