Use camino crate for UTF8 paths in re_types_builder

Cargo.toml

@@ -73,6 +73,7 @@ anyhow = "1.0"
 arrow2 = "0.17"
 arrow2_convert = "0.5.0"
 bytemuck = { version = "1.11", features = ["extern_crate_alloc"] }
+camino = "1.1"
 cfg-if = "1.0"
 clap = "4.0"
 comfy-table = { version = "6.1", default-features = false }

crates/re_types/source_hash.txt

@@ -1,4 +1,4 @@
 # This is a sha256 hash for all direct and indirect dependencies of this crate's build script.
 # It can be safely removed at anytime to force the build script to run again.
 # Check out build.rs to see how it's computed.
-4ae47b009f4946234e8d7edd37ce7815ef9f77dd35cfdd83bf328be612b4d54e
+bfac745d1eadd1c774419b33ced26cb6cbf7404036e19725db1dfc4e767e791d

crates/re_types_builder/Cargo.toml

@@ -22,6 +22,7 @@ all-features = true
 # External
 anyhow.workspace = true
 arrow2.workspace = true
+camino.workspace = true
 convert_case = "0.6"
 flatbuffers = "23.0"
 indent = "0.1"

crates/re_types_builder/src/codegen/mod.rs

@@ -7,7 +7,7 @@ pub trait CodeGenerator {
         &mut self,
         objs: &crate::Objects,
         arrow_registry: &crate::ArrowRegistry,
-    ) -> Vec<std::path::PathBuf>;
+    ) -> Vec<camino::Utf8PathBuf>;
 }
 
 // ---

crates/re_types_builder/src/codegen/python.rs

@@ -1,13 +1,14 @@
 //! Implements the Python codegen pass.
 
-use anyhow::Context as _;
-use itertools::Itertools;
 use std::{
     collections::{BTreeMap, HashMap, HashSet},
     io::Write,
-    path::{Path, PathBuf},
 };
 
+use anyhow::Context as _;
+use camino::{Utf8Path, Utf8PathBuf};
+use itertools::Itertools;
+
 use crate::{
     codegen::{StringExt as _, AUTOGEN_WARNING},
     ArrowRegistry, CodeGenerator, Docs, ElementType, Object, ObjectField, ObjectKind, Objects,
@@ -54,11 +55,11 @@ impl PythonObjectExt for Object {
 }
 
 pub struct PythonCodeGenerator {
-    pkg_path: PathBuf,
+    pkg_path: Utf8PathBuf,
 }
 
 impl PythonCodeGenerator {
-    pub fn new(pkg_path: impl Into<PathBuf>) -> Self {
+    pub fn new(pkg_path: impl Into<Utf8PathBuf>) -> Self {
         Self {
             pkg_path: pkg_path.into(),
         }
@@ -69,7 +70,7 @@ impl PythonCodeGenerator {
 ///
 /// This is the hacky way. We extract all identifiers from `__init__.py` which contains, but don't
 /// start with, a underscore (`_`).
-fn load_overrides(path: &Path) -> HashSet<String> {
+fn load_overrides(path: &Utf8Path) -> HashSet<String> {
     let path = path.join("_overrides").join("__init__.py");
     let contents = std::fs::read_to_string(&path)
         .with_context(|| format!("couldn't load overrides module at {path:?}"))
@@ -84,7 +85,7 @@ fn load_overrides(path: &Path) -> HashSet<String> {
 }
 
 impl CodeGenerator for PythonCodeGenerator {
-    fn generate(&mut self, objs: &Objects, arrow_registry: &ArrowRegistry) -> Vec<PathBuf> {
+    fn generate(&mut self, objs: &Objects, arrow_registry: &ArrowRegistry) -> Vec<Utf8PathBuf> {
         let mut filepaths = Vec::new();
 
         let datatypes_path = self.pkg_path.join("datatypes");
@@ -144,7 +145,7 @@ impl CodeGenerator for PythonCodeGenerator {
 
 // --- File management ---
 
-fn quote_lib(out_path: impl AsRef<Path>, archetype_names: &[String]) -> PathBuf {
+fn quote_lib(out_path: impl AsRef<Utf8Path>, archetype_names: &[String]) -> Utf8PathBuf {
     let out_path = out_path.as_ref();
 
     std::fs::create_dir_all(out_path)
@@ -178,19 +179,19 @@ fn quote_lib(out_path: impl AsRef<Path>, archetype_names: &[String]) -> PathBuf
 
 /// Returns all filepaths + all object names.
 fn quote_objects(
-    out_path: impl AsRef<Path>,
+    out_path: impl AsRef<Utf8Path>,
     arrow_registry: &ArrowRegistry,
     overrides: &HashSet<String>,
     all_objects: &Objects,
     _kind: ObjectKind,
     objs: &[&Object],
-) -> (Vec<PathBuf>, Vec<String>) {
+) -> (Vec<Utf8PathBuf>, Vec<String>) {
     let out_path = out_path.as_ref();
 
     let mut filepaths = Vec::new();
     let mut all_names = Vec::new();
 
-    let mut files = HashMap::<PathBuf, Vec<QuotedObject>>::new();
+    let mut files = HashMap::<Utf8PathBuf, Vec<QuotedObject>>::new();
     for obj in objs {
         all_names.push(obj.name.clone());
 
@@ -233,7 +234,7 @@ fn quote_objects(
 
         // NOTE: Isolating the file stem only works because we're handling datatypes, components
         // and archetypes separately (and even then it's a bit shady, eh).
-        mods.entry(filepath.file_stem().unwrap().to_string_lossy().to_string())
+        mods.entry(filepath.file_stem().unwrap().to_owned())
             .or_default()
             .extend(names.iter().cloned());
 
@@ -374,7 +375,7 @@ fn quote_objects(
 #[derive(Debug, Clone)]
 struct QuotedObject {
     object: Object,
-    filepath: PathBuf,
+    filepath: Utf8PathBuf,
     code: String,
 }
 
@@ -543,7 +544,7 @@ impl QuotedObject {
             }
         }
 
-        let mut filepath = PathBuf::from(virtpath);
+        let mut filepath = Utf8PathBuf::from(virtpath);
         filepath.set_extension("py");
 
         Self {
@@ -635,7 +636,7 @@ impl QuotedObject {
             }
         }
 
-        let mut filepath = PathBuf::from(virtpath);
+        let mut filepath = Utf8PathBuf::from(virtpath);
         filepath.set_extension("py");
 
         Self {

crates/re_types_builder/src/codegen/rust.rs

@@ -1,14 +1,13 @@
 //! Implements the Rust codegen pass.
 
+use std::{collections::HashMap, io::Write};
+
 use anyhow::Context as _;
 use arrow2::datatypes::DataType;
 use proc_macro2::TokenStream;
 use quote::{format_ident, quote};
-use std::{
-    collections::HashMap,
-    io::Write,
-    path::{Path, PathBuf},
-};
+
+use camino::{Utf8Path, Utf8PathBuf};
 
 use crate::{
     codegen::{StringExt as _, AUTOGEN_WARNING},
@@ -24,19 +23,19 @@ use crate::{
 // ---
 
 pub struct RustCodeGenerator {
-    crate_path: PathBuf,
+    crate_path: Utf8PathBuf,
 }
 
 impl RustCodeGenerator {
-    pub fn new(crate_path: impl Into<PathBuf>) -> Self {
+    pub fn new(crate_path: impl Into<Utf8PathBuf>) -> Self {
         Self {
             crate_path: crate_path.into(),
         }
     }
 }
 
 impl CodeGenerator for RustCodeGenerator {
-    fn generate(&mut self, objects: &Objects, arrow_registry: &ArrowRegistry) -> Vec<PathBuf> {
+    fn generate(&mut self, objects: &Objects, arrow_registry: &ArrowRegistry) -> Vec<Utf8PathBuf> {
         let mut filepaths = Vec::new();
 
         let datatypes_path = self.crate_path.join("src/datatypes");
@@ -79,16 +78,16 @@ impl CodeGenerator for RustCodeGenerator {
 // --- File management ---
 
 fn create_files(
-    out_path: impl AsRef<Path>,
+    out_path: impl AsRef<Utf8Path>,
     arrow_registry: &ArrowRegistry,
     objects: &Objects,
     objs: &[&Object],
-) -> Vec<PathBuf> {
+) -> Vec<Utf8PathBuf> {
     let out_path = out_path.as_ref();
 
     let mut filepaths = Vec::new();
 
-    let mut files = HashMap::<PathBuf, Vec<QuotedObject>>::new();
+    let mut files = HashMap::<Utf8PathBuf, Vec<QuotedObject>>::new();
     for obj in objs {
         let obj = if obj.is_struct() {
             QuotedObject::from_struct(arrow_registry, objects, obj)
@@ -108,7 +107,7 @@ fn create_files(
         // NOTE: Isolating the file stem only works because we're handling datatypes, components
         // and archetypes separately (and even then it's a bit shady, eh).
         let names = objs.iter().map(|obj| obj.name.clone()).collect::<Vec<_>>();
-        mods.entry(filepath.file_stem().unwrap().to_string_lossy().to_string())
+        mods.entry(filepath.file_stem().unwrap().to_owned())
             .or_default()
             .extend(names);
 
@@ -213,7 +212,7 @@ fn create_files(
 
 #[derive(Debug, Clone)]
 struct QuotedObject {
-    filepath: PathBuf,
+    filepath: Utf8PathBuf,
     name: String,
     tokens: TokenStream,
 }
@@ -271,7 +270,7 @@ impl QuotedObject {
 
         Self {
             filepath: {
-                let mut filepath = PathBuf::from(virtpath);
+                let mut filepath = Utf8PathBuf::from(virtpath);
                 filepath.set_extension("rs");
                 filepath
             },
@@ -352,7 +351,7 @@ impl QuotedObject {
 
         Self {
             filepath: {
-                let mut filepath = PathBuf::from(virtpath);
+                let mut filepath = Utf8PathBuf::from(virtpath);
                 filepath.set_extension("rs");
                 filepath
             },

crates/re_types_builder/src/lib.rs

@@ -159,7 +159,7 @@ pub const ATTR_RUST_TUPLE_STRUCT: &str = "attr.rust.tuple_struct";
 
 // --- Entrypoints ---
 
-use std::path::{Path, PathBuf};
+use camino::{Utf8Path, Utf8PathBuf};
 
 /// Compiles binary reflection dumps from flatbuffers definitions.
 ///
@@ -180,13 +180,13 @@ use std::path::{Path, PathBuf};
 /// );
 /// ```
 pub fn compile_binary_schemas(
-    include_dir_path: impl AsRef<Path>,
-    output_dir_path: impl AsRef<Path>,
-    entrypoint_path: impl AsRef<Path>,
+    include_dir_path: impl AsRef<Utf8Path>,
+    output_dir_path: impl AsRef<Utf8Path>,
+    entrypoint_path: impl AsRef<Utf8Path>,
 ) {
-    let include_dir_path = include_dir_path.as_ref().to_str().unwrap();
-    let output_dir_path = output_dir_path.as_ref().to_str().unwrap();
-    let entrypoint_path = entrypoint_path.as_ref().to_str().unwrap();
+    let include_dir_path = include_dir_path.as_ref().as_str();
+    let output_dir_path = output_dir_path.as_ref().as_str();
+    let entrypoint_path = entrypoint_path.as_ref().as_str();
 
     use xshell::{cmd, Shell};
     let sh = Shell::new().unwrap();
@@ -206,33 +206,34 @@ pub fn compile_binary_schemas(
 /// 2. Run the semantic pass
 /// 3. Compute the Arrow registry
 fn generate_lang_agnostic(
-    include_dir_path: impl AsRef<Path>,
-    entrypoint_path: impl AsRef<Path>,
+    include_dir_path: impl AsRef<Utf8Path>,
+    entrypoint_path: impl AsRef<Utf8Path>,
 ) -> (Objects, ArrowRegistry) {
     use xshell::Shell;
 
     let sh = Shell::new().unwrap();
     let tmp = sh.create_temp_dir().unwrap();
+    let tmp_path = Utf8PathBuf::try_from(tmp.path().to_path_buf()).unwrap();
 
     let entrypoint_path = entrypoint_path.as_ref();
     let entrypoint_filename = entrypoint_path.file_name().unwrap();
 
     let include_dir_path = include_dir_path.as_ref();
     let include_dir_path = include_dir_path
-        .canonicalize()
+        .canonicalize_utf8()
         .with_context(|| format!("failed to canonicalize include path: {include_dir_path:?}"))
         .unwrap();
 
     // generate bfbs definitions
-    compile_binary_schemas(&include_dir_path, tmp.path(), entrypoint_path);
+    compile_binary_schemas(&include_dir_path, &tmp_path, entrypoint_path);
 
-    let mut binary_entrypoint_path = PathBuf::from(entrypoint_filename);
+    let mut binary_entrypoint_path = Utf8PathBuf::from(entrypoint_filename);
     binary_entrypoint_path.set_extension("bfbs");
 
     // semantic pass: high level objects from low-level reflection data
     let mut objects = Objects::from_buf(
         include_dir_path,
-        sh.read_binary_file(tmp.path().join(binary_entrypoint_path))
+        sh.read_binary_file(tmp_path.join(binary_entrypoint_path))
             .unwrap()
             .as_slice(),
     );
@@ -263,9 +264,9 @@ fn generate_lang_agnostic(
 /// );
 /// ```
 pub fn generate_rust_code(
-    include_dir_path: impl AsRef<Path>,
-    output_crate_path: impl AsRef<Path>,
-    entrypoint_path: impl AsRef<Path>,
+    include_dir_path: impl AsRef<Utf8Path>,
+    output_crate_path: impl AsRef<Utf8Path>,
+    entrypoint_path: impl AsRef<Utf8Path>,
 ) {
     // passes 1 through 3: bfbs, semantic, arrow registry
     let (objects, arrow_registry) = generate_lang_agnostic(include_dir_path, entrypoint_path);
@@ -291,9 +292,9 @@ pub fn generate_rust_code(
 /// );
 /// ```
 pub fn generate_python_code(
-    include_dir_path: impl AsRef<Path>,
-    output_pkg_path: impl AsRef<Path>,
-    entrypoint_path: impl AsRef<Path>,
+    include_dir_path: impl AsRef<Utf8Path>,
+    output_pkg_path: impl AsRef<Utf8Path>,
+    entrypoint_path: impl AsRef<Utf8Path>,
 ) {
     // passes 1 through 3: bfbs, semantic, arrow registry
     let (objects, arrow_registry) = generate_lang_agnostic(include_dir_path, entrypoint_path);