From 232d7e59f0620763594f31c58ffc75a0d20d78a1 Mon Sep 17 00:00:00 2001 From: Eashwar Ranganathan Date: Sun, 23 Nov 2025 22:59:44 -0500 Subject: [PATCH 1/4] Document binding generation workflows --- src/binding_generator/mod.rs | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/src/binding_generator/mod.rs b/src/binding_generator/mod.rs index dac25f7bc..3e81425f2 100644 --- a/src/binding_generator/mod.rs +++ b/src/binding_generator/mod.rs @@ -17,6 +17,24 @@ pub use pyo3_binding::write_bindings_module; pub use uniffi_binding::write_uniffi_module; pub use wasm_binding::write_wasm_launcher; +// Every binding generator ultimately has to install the following: +// 1. The python files (if any) +// 2. The artifact +// 3. Additional files +// 4. Type stubs (if any/pure rust only) +// +// Additionally, the above are installed to 2 potential locations: +// 1. The archive +// 2. The filesystem +// +// For editable installs: +// If the project is pure rust, the wheel is built as normal and installed +// If the project has python, the artifact is installed into the project and a pth is written to the archive +// +// So the full matrix comes down to: +// 1. editable, has python => install to fs, write pth to archive +// 2. everything else => install to archive/build as normal + /// Adds a data directory with a scripts directory with the binary inside it pub fn write_bin( writer: &mut impl ModuleWriter, From e9729306125454b885d873cfd9a958b529deedcf Mon Sep 17 00:00:00 2001 From: Eashwar Ranganathan Date: Sun, 23 Nov 2025 23:36:59 -0500 Subject: [PATCH 2/4] Create BindingGenerator trait --- src/binding_generator/mod.rs | 169 +++++++++++++++++++++++++++++++---- 1 file changed, 152 insertions(+), 17 deletions(-) diff --git a/src/binding_generator/mod.rs b/src/binding_generator/mod.rs index 3e81425f2..3007d56fd 100644 --- a/src/binding_generator/mod.rs +++ b/src/binding_generator/mod.rs @@ -1,11 +1,22 @@ +use std::collections::HashMap; +use std::io::Write as _; use std::path::Path; use std::path::PathBuf; +use anyhow::Context as _; use anyhow::Result; +use fs_err as fs; +use fs_err::File; +use tempfile::TempDir; +use tempfile::tempdir; +use crate::BuildArtifact; +use crate::BuildContext; use crate::Metadata24; use crate::ModuleWriter; +use crate::PythonInterpreter; use crate::module_writer::ModuleWriterExt; +use crate::module_writer::write_python_part; mod cffi_binding; mod pyo3_binding; @@ -17,23 +28,147 @@ pub use pyo3_binding::write_bindings_module; pub use uniffi_binding::write_uniffi_module; pub use wasm_binding::write_wasm_launcher; -// Every binding generator ultimately has to install the following: -// 1. The python files (if any) -// 2. The artifact -// 3. Additional files -// 4. Type stubs (if any/pure rust only) -// -// Additionally, the above are installed to 2 potential locations: -// 1. The archive -// 2. The filesystem -// -// For editable installs: -// If the project is pure rust, the wheel is built as normal and installed -// If the project has python, the artifact is installed into the project and a pth is written to the archive -// -// So the full matrix comes down to: -// 1. editable, has python => install to fs, write pth to archive -// 2. everything else => install to archive/build as normal +pub trait BindingGenerator { + fn generate_bindings( + &self, + context: &BuildContext, + interpreter: Option<&PythonInterpreter>, + artifact: &BuildArtifact, + module: &Path, + temp_dir: &TempDir, + ) -> Result; +} + +pub struct GeneratorOutput { + /// The path, relative to the archive root, where the built artifact/module + /// should be installed + artifact_target: PathBuf, + + /// In some cases, the source path of the artifact is altered + /// (e.g. when the build output is an archive which needs to be unpacked) + artifact_source_override: Option, + + /// Additional files to be installed (e.g. __init__.py) + /// The provided PathBuf should be relative to the archive root + additional_files: Option>>, +} + +/// Every binding generator ultimately has to install the following: +/// 1. The python files (if any) +/// 2. The artifact +/// 3. Additional files +/// 4. Type stubs (if any/pure rust only) +/// +/// Additionally, the above are installed to 2 potential locations: +/// 1. The archive +/// 2. The filesystem +/// +/// For editable installs: +/// If the project is pure rust, the wheel is built as normal and installed +/// If the project has python, the artifact is installed into the project and a pth is written to the archive +/// +/// So the full matrix comes down to: +/// 1. editable, has python => install to fs, write pth to archive +/// 2. everything else => install to archive/build as normal +/// +/// Note: Writing the pth to the archive is handled by [BuildContext], not here +pub fn generate_binding( + writer: &mut impl ModuleWriter, + generator: &impl BindingGenerator, + context: &BuildContext, + interpreter: Option<&PythonInterpreter>, + artifact: &BuildArtifact, +) -> Result<()> { + // 1. Install the python files + if !context.editable { + write_python_part( + writer, + &context.project_layout, + context.pyproject_toml.as_ref(), + )?; + } + + let base_path = context + .project_layout + .python_module + .as_ref() + .map(|python_module| python_module.parent().unwrap().to_path_buf()); + + let module = match &base_path { + Some(base_path) => context + .project_layout + .rust_module + .strip_prefix(base_path) + .unwrap() + .to_path_buf(), + None => PathBuf::from(&context.project_layout.extension_name), + }; + + let temp_dir = tempdir()?; + let GeneratorOutput { + artifact_target, + artifact_source_override, + additional_files, + } = generator.generate_bindings(context, interpreter, artifact, &module, &temp_dir)?; + + match (context.editable, &base_path) { + (true, Some(base_path)) => { + let target = base_path.join(&artifact_target); + fs::create_dir_all(target.parent().unwrap())?; + // Remove existing so file to avoid triggering SIGSEV in running process + // See https://github.com/PyO3/maturin/issues/758 + let _ = fs::remove_file(&target); + let source = artifact_source_override.unwrap_or_else(|| artifact.path.clone()); + + // 2. Install the artifact + fs::copy(&source, &target).with_context(|| { + format!( + "Failed to copy {} to {}", + source.display(), + target.display() + ) + })?; + + // 3. Install additional files + if let Some(additional_files) = additional_files { + for (target, data) in additional_files { + let target = base_path.join(target); + fs::create_dir_all(target.parent().unwrap())?; + let mut file = File::options().create(true).truncate(true).open(&target)?; + file.write_all(data.as_slice())?; + } + } + } + _ => { + // 2. Install the artifact + let source = artifact_source_override.unwrap_or_else(|| artifact.path.clone()); + writer.add_file(artifact_target, source, true)?; + + // 3. Install additional files + if let Some(additional_files) = additional_files { + for (target, data) in additional_files { + writer.add_bytes(target, None, data.as_slice(), false)?; + } + } + } + } + + // 4. Install type stubs + if context.project_layout.python_module.is_none() { + let ext_name = &context.project_layout.extension_name; + let type_stub = context + .project_layout + .rust_module + .join(format!("{ext_name}.pyi")); + if type_stub.exists() { + eprintln!("📖 Found type stub file at {ext_name}.pyi"); + writer.add_file(module.join("__init__.pyi"), type_stub, false)?; + writer.add_empty_file(module.join("py.typed"))?; + } + } + + Ok(()) +} /// Adds a data directory with a scripts directory with the binary inside it pub fn write_bin( From fdf65e0ae858c3504521609ee7a7c79eed5fcba6 Mon Sep 17 00:00:00 2001 From: Eashwar Ranganathan Date: Sun, 23 Nov 2025 23:36:59 -0500 Subject: [PATCH 3/4] Add PyO3 impl --- src/binding_generator/mod.rs | 4 +- src/binding_generator/pyo3_binding.rs | 207 +++++++++++--------------- src/build_context.rs | 27 ++-- src/module_writer/path_writer.rs | 1 + 4 files changed, 103 insertions(+), 136 deletions(-) diff --git a/src/binding_generator/mod.rs b/src/binding_generator/mod.rs index 3007d56fd..4b870e311 100644 --- a/src/binding_generator/mod.rs +++ b/src/binding_generator/mod.rs @@ -24,7 +24,7 @@ mod uniffi_binding; mod wasm_binding; pub use cffi_binding::write_cffi_module; -pub use pyo3_binding::write_bindings_module; +pub use pyo3_binding::Pyo3BindingGenerator; pub use uniffi_binding::write_uniffi_module; pub use wasm_binding::write_wasm_launcher; @@ -125,7 +125,7 @@ pub fn generate_binding( format!( "Failed to copy {} to {}", source.display(), - target.display() + target.display(), ) })?; diff --git a/src/binding_generator/pyo3_binding.rs b/src/binding_generator/pyo3_binding.rs index c50b0a838..46dbff5bf 100644 --- a/src/binding_generator/pyo3_binding.rs +++ b/src/binding_generator/pyo3_binding.rs @@ -1,22 +1,102 @@ +use std::collections::HashMap; use std::ffi::OsStr; use std::path::Path; use std::path::PathBuf; use std::process::Command; -use anyhow::Context as _; use anyhow::Result; use anyhow::bail; -use fs_err as fs; +use tempfile::TempDir; use tracing::debug; -use tracing::instrument; -use crate::ModuleWriter; -use crate::PyProjectToml; +use crate::BuildArtifact; +use crate::BuildContext; use crate::PythonInterpreter; use crate::Target; -use crate::module_writer::ModuleWriterExt; -use crate::module_writer::write_python_part; -use crate::project_layout::ProjectLayout; + +use super::BindingGenerator; +use super::GeneratorOutput; + +pub struct Pyo3BindingGenerator { + abi3: bool, +} + +impl Pyo3BindingGenerator { + pub fn new(abi3: bool) -> Self { + Self { abi3 } + } +} + +impl BindingGenerator for Pyo3BindingGenerator { + fn generate_bindings( + &self, + context: &BuildContext, + interpreter: Option<&PythonInterpreter>, + artifact: &BuildArtifact, + module: &Path, + temp_dir: &TempDir, + ) -> Result { + let ext_name = &context.project_layout.extension_name; + let target = &context.target; + + let so_filename = if self.abi3 { + if target.is_unix() { + if target.is_cygwin() { + format!("{ext_name}.abi3.dll") + } else { + format!("{ext_name}.abi3.so") + } + } else { + match interpreter { + Some(interpreter) if interpreter.is_windows_debug() => { + format!("{ext_name}_d.pyd") + } + // Apparently there is no tag for abi3 on windows + _ => format!("{ext_name}.pyd"), + } + } + } else { + let interpreter = + interpreter.expect("A python interpreter is required for non-abi3 build"); + interpreter.get_library_name(ext_name) + }; + let artifact_target = module.join(so_filename); + + let artifact_is_big_ar = target.is_aix() + && artifact.path.extension().unwrap_or(OsStr::new(" ")) == OsStr::new("a"); + + let artifact_source_override = if artifact_is_big_ar { + Some(unpack_big_archive(target, &artifact.path, temp_dir.path())?) + } else { + None + }; + + let additional_files = match context.project_layout.python_module { + Some(_) => None, + None => { + let mut files = HashMap::new(); + files.insert( + module.join("__init__.py"), + format!( + r#"from .{ext_name} import * + +__doc__ = {ext_name}.__doc__ +if hasattr({ext_name}, "__all__"): + __all__ = {ext_name}.__all__"# + ) + .into(), + ); + Some(files) + } + }; + + Ok(GeneratorOutput { + artifact_target, + artifact_source_override, + additional_files, + }) + } +} // Extract the shared object from a AIX big library archive fn unpack_big_archive(target: &Target, artifact: &Path, temp_dir_path: &Path) -> Result { @@ -39,114 +119,3 @@ fn unpack_big_archive(target: &Target, artifact: &Path, temp_dir_path: &Path) -> let unpacked_artifact = temp_dir_path.join(artifact.with_extension("so").file_name().unwrap()); Ok(unpacked_artifact) } - -/// Copies the shared library into the module, which is the only extra file needed with bindings -#[allow(clippy::too_many_arguments)] -#[instrument(skip_all)] -pub fn write_bindings_module( - writer: &mut impl ModuleWriter, - project_layout: &ProjectLayout, - artifact: &Path, - python_interpreter: Option<&PythonInterpreter>, - is_abi3: bool, - target: &Target, - editable: bool, - pyproject_toml: Option<&PyProjectToml>, -) -> Result<()> { - let ext_name = &project_layout.extension_name; - let so_filename = if is_abi3 { - if target.is_unix() { - if target.is_cygwin() { - format!("{ext_name}.abi3.dll") - } else { - format!("{ext_name}.abi3.so") - } - } else { - match python_interpreter { - Some(python_interpreter) if python_interpreter.is_windows_debug() => { - format!("{ext_name}_d.pyd") - } - // Apparently there is no tag for abi3 on windows - _ => format!("{ext_name}.pyd"), - } - } - } else { - let python_interpreter = - python_interpreter.expect("A python interpreter is required for non-abi3 build"); - python_interpreter.get_library_name(ext_name) - }; - - let artifact_is_big_ar = - target.is_aix() && artifact.extension().unwrap_or(OsStr::new(" ")) == OsStr::new("a"); - let temp_dir = if artifact_is_big_ar { - Some(tempfile::tempdir()?) - } else { - None - }; - let artifact_buff = if artifact_is_big_ar { - Some(unpack_big_archive( - target, - artifact, - temp_dir.as_ref().unwrap().path(), - )?) - } else { - None - }; - let artifact = if artifact_is_big_ar { - artifact_buff.as_ref().unwrap() - } else { - artifact - }; - - if !editable { - write_python_part(writer, project_layout, pyproject_toml) - .context("Failed to add the python module to the package")?; - } - if let Some(python_module) = &project_layout.python_module { - if editable { - let target = project_layout.rust_module.join(&so_filename); - // Remove existing so file to avoid triggering SIGSEV in running process - // See https://github.com/PyO3/maturin/issues/758 - debug!("Removing {}", target.display()); - let _ = fs::remove_file(&target); - - debug!("Copying {} to {}", artifact.display(), target.display()); - fs::copy(artifact, &target).context(format!( - "Failed to copy {} to {}", - artifact.display(), - target.display() - ))?; - } else { - let relative = project_layout - .rust_module - .strip_prefix(python_module.parent().unwrap()) - .unwrap(); - writer.add_file(relative.join(&so_filename), artifact, true)?; - } - } else { - let module = PathBuf::from(ext_name); - // Reexport the shared library as if it were the top level module - writer.add_bytes( - module.join("__init__.py"), - None, - format!( - r#"from .{ext_name} import * - -__doc__ = {ext_name}.__doc__ -if hasattr({ext_name}, "__all__"): - __all__ = {ext_name}.__all__"# - ) - .as_bytes(), - false, - )?; - let type_stub = project_layout.rust_module.join(format!("{ext_name}.pyi")); - if type_stub.exists() { - eprintln!("📖 Found type stub file at {ext_name}.pyi"); - writer.add_file(module.join("__init__.pyi"), type_stub, false)?; - writer.add_empty_file(module.join("py.typed"))?; - } - writer.add_file(module.join(so_filename), artifact, true)?; - } - - Ok(()) -} diff --git a/src/build_context.rs b/src/build_context.rs index 1a5879b45..9851d3684 100644 --- a/src/build_context.rs +++ b/src/build_context.rs @@ -1,7 +1,8 @@ use crate::auditwheel::{AuditWheelMode, get_policy_and_libs, patchelf, relpath}; use crate::auditwheel::{PlatformTag, Policy}; use crate::binding_generator::{ - write_bin, write_bindings_module, write_cffi_module, write_uniffi_module, write_wasm_launcher, + Pyo3BindingGenerator, generate_binding, write_bin, write_cffi_module, write_uniffi_module, + write_wasm_launcher, }; use crate::bridge::Abi3Version; use crate::build_options::CargoOptions; @@ -768,15 +769,13 @@ impl BuildContext { )?; self.add_external_libs(&mut writer, &[&artifact], &[ext_libs])?; - write_bindings_module( + let generator = Pyo3BindingGenerator::new(true); + generate_binding( &mut writer, - &self.project_layout, - &artifact.path, + &generator, + self, self.interpreter.first(), - true, - &self.target, - self.editable, - self.pyproject_toml.as_ref(), + &artifact, ) .context("Failed to add the files to the wheel")?; @@ -851,15 +850,13 @@ impl BuildContext { )?; self.add_external_libs(&mut writer, &[&artifact], &[ext_libs])?; - write_bindings_module( + let generator = Pyo3BindingGenerator::new(false); + generate_binding( &mut writer, - &self.project_layout, - &artifact.path, + &generator, + self, Some(python_interpreter), - false, - &self.target, - self.editable, - self.pyproject_toml.as_ref(), + &artifact, ) .context("Failed to add the files to the wheel")?; diff --git a/src/module_writer/path_writer.rs b/src/module_writer/path_writer.rs index 2345781d1..b50609121 100644 --- a/src/module_writer/path_writer.rs +++ b/src/module_writer/path_writer.rs @@ -14,6 +14,7 @@ use fs_err::OpenOptions; use fs_err::os::unix::fs::OpenOptionsExt as _; use super::ModuleWriter; +#[cfg(target_family = "unix")] use super::default_permission; use super::util::FileTracker; From 78fe8c3cc286c4c9360bc1aa808575c62fa8902c Mon Sep 17 00:00:00 2001 From: Eashwar Ranganathan Date: Wed, 26 Nov 2025 11:39:08 -0500 Subject: [PATCH 4/4] Address PR feedback --- src/binding_generator/mod.rs | 19 +++++++++++++++++-- src/binding_generator/pyo3_binding.rs | 5 +++++ 2 files changed, 22 insertions(+), 2 deletions(-) diff --git a/src/binding_generator/mod.rs b/src/binding_generator/mod.rs index 4b870e311..6be87c709 100644 --- a/src/binding_generator/mod.rs +++ b/src/binding_generator/mod.rs @@ -9,6 +9,7 @@ use fs_err as fs; use fs_err::File; use tempfile::TempDir; use tempfile::tempdir; +use tracing::debug; use crate::BuildArtifact; use crate::BuildContext; @@ -28,7 +29,11 @@ pub use pyo3_binding::Pyo3BindingGenerator; pub use uniffi_binding::write_uniffi_module; pub use wasm_binding::write_wasm_launcher; -pub trait BindingGenerator { +///A trait to generate the binding files to be included in the built module +/// +/// This trait is used to generate the support files necessary to build a python +/// module for any [crate::BridgeModel] +pub(crate) trait BindingGenerator { fn generate_bindings( &self, context: &BuildContext, @@ -39,7 +44,8 @@ pub trait BindingGenerator { ) -> Result; } -pub struct GeneratorOutput { +#[derive(Debug)] +pub(crate) struct GeneratorOutput { /// The path, relative to the archive root, where the built artifact/module /// should be installed artifact_target: PathBuf, @@ -114,6 +120,7 @@ pub fn generate_binding( match (context.editable, &base_path) { (true, Some(base_path)) => { let target = base_path.join(&artifact_target); + debug!("Removing previously built module {}", target.display()); fs::create_dir_all(target.parent().unwrap())?; // Remove existing so file to avoid triggering SIGSEV in running process // See https://github.com/PyO3/maturin/issues/758 @@ -121,6 +128,7 @@ pub fn generate_binding( let source = artifact_source_override.unwrap_or_else(|| artifact.path.clone()); // 2. Install the artifact + debug!("Installing {} from {}", target.display(), source.display()); fs::copy(&source, &target).with_context(|| { format!( "Failed to copy {} to {}", @@ -134,6 +142,7 @@ pub fn generate_binding( for (target, data) in additional_files { let target = base_path.join(target); fs::create_dir_all(target.parent().unwrap())?; + debug!("Generating file {}", target.display()); let mut file = File::options().create(true).truncate(true).open(&target)?; file.write_all(data.as_slice())?; } @@ -142,11 +151,17 @@ pub fn generate_binding( _ => { // 2. Install the artifact let source = artifact_source_override.unwrap_or_else(|| artifact.path.clone()); + debug!( + "Adding to archive {} from {}", + artifact_target.display(), + source.display() + ); writer.add_file(artifact_target, source, true)?; // 3. Install additional files if let Some(additional_files) = additional_files { for (target, data) in additional_files { + debug!("Generating archive entry {}", target.display()); writer.add_bytes(target, None, data.as_slice(), false)?; } } diff --git a/src/binding_generator/pyo3_binding.rs b/src/binding_generator/pyo3_binding.rs index 46dbff5bf..4d9dc2d5d 100644 --- a/src/binding_generator/pyo3_binding.rs +++ b/src/binding_generator/pyo3_binding.rs @@ -17,6 +17,11 @@ use crate::Target; use super::BindingGenerator; use super::GeneratorOutput; +/// A generator for producing PyO3 bindings. +/// +/// This struct is responsible for generating Python bindings for modules using PyO3. +/// The `abi3` field determines whether the generated bindings use the stable PyO3 "abi3" interface, +/// which allows compatibility with multiple Python versions. pub struct Pyo3BindingGenerator { abi3: bool, }