diff --git a/src/binding_generator/mod.rs b/src/binding_generator/mod.rs index dac25f7bc..6be87c709 100644 --- a/src/binding_generator/mod.rs +++ b/src/binding_generator/mod.rs @@ -1,11 +1,23 @@ +use std::collections::HashMap; +use std::io::Write as _; use std::path::Path; use std::path::PathBuf; +use anyhow::Context as _; use anyhow::Result; +use fs_err as fs; +use fs_err::File; +use tempfile::TempDir; +use tempfile::tempdir; +use tracing::debug; +use crate::BuildArtifact; +use crate::BuildContext; use crate::Metadata24; use crate::ModuleWriter; +use crate::PythonInterpreter; use crate::module_writer::ModuleWriterExt; +use crate::module_writer::write_python_part; mod cffi_binding; mod pyo3_binding; @@ -13,10 +25,166 @@ mod uniffi_binding; mod wasm_binding; pub use cffi_binding::write_cffi_module; -pub use pyo3_binding::write_bindings_module; +pub use pyo3_binding::Pyo3BindingGenerator; pub use uniffi_binding::write_uniffi_module; pub use wasm_binding::write_wasm_launcher; +///A trait to generate the binding files to be included in the built module +/// +/// This trait is used to generate the support files necessary to build a python +/// module for any [crate::BridgeModel] +pub(crate) trait BindingGenerator { + fn generate_bindings( + &self, + context: &BuildContext, + interpreter: Option<&PythonInterpreter>, + artifact: &BuildArtifact, + module: &Path, + temp_dir: &TempDir, + ) -> Result; +} + +#[derive(Debug)] +pub(crate) struct GeneratorOutput { + /// The path, relative to the archive root, where the built artifact/module + /// should be installed + artifact_target: PathBuf, + + /// In some cases, the source path of the artifact is altered + /// (e.g. when the build output is an archive which needs to be unpacked) + artifact_source_override: Option, + + /// Additional files to be installed (e.g. __init__.py) + /// The provided PathBuf should be relative to the archive root + additional_files: Option>>, +} + +/// Every binding generator ultimately has to install the following: +/// 1. The python files (if any) +/// 2. The artifact +/// 3. Additional files +/// 4. Type stubs (if any/pure rust only) +/// +/// Additionally, the above are installed to 2 potential locations: +/// 1. The archive +/// 2. The filesystem +/// +/// For editable installs: +/// If the project is pure rust, the wheel is built as normal and installed +/// If the project has python, the artifact is installed into the project and a pth is written to the archive +/// +/// So the full matrix comes down to: +/// 1. editable, has python => install to fs, write pth to archive +/// 2. everything else => install to archive/build as normal +/// +/// Note: Writing the pth to the archive is handled by [BuildContext], not here +pub fn generate_binding( + writer: &mut impl ModuleWriter, + generator: &impl BindingGenerator, + context: &BuildContext, + interpreter: Option<&PythonInterpreter>, + artifact: &BuildArtifact, +) -> Result<()> { + // 1. Install the python files + if !context.editable { + write_python_part( + writer, + &context.project_layout, + context.pyproject_toml.as_ref(), + )?; + } + + let base_path = context + .project_layout + .python_module + .as_ref() + .map(|python_module| python_module.parent().unwrap().to_path_buf()); + + let module = match &base_path { + Some(base_path) => context + .project_layout + .rust_module + .strip_prefix(base_path) + .unwrap() + .to_path_buf(), + None => PathBuf::from(&context.project_layout.extension_name), + }; + + let temp_dir = tempdir()?; + let GeneratorOutput { + artifact_target, + artifact_source_override, + additional_files, + } = generator.generate_bindings(context, interpreter, artifact, &module, &temp_dir)?; + + match (context.editable, &base_path) { + (true, Some(base_path)) => { + let target = base_path.join(&artifact_target); + debug!("Removing previously built module {}", target.display()); + fs::create_dir_all(target.parent().unwrap())?; + // Remove existing so file to avoid triggering SIGSEV in running process + // See https://github.com/PyO3/maturin/issues/758 + let _ = fs::remove_file(&target); + let source = artifact_source_override.unwrap_or_else(|| artifact.path.clone()); + + // 2. Install the artifact + debug!("Installing {} from {}", target.display(), source.display()); + fs::copy(&source, &target).with_context(|| { + format!( + "Failed to copy {} to {}", + source.display(), + target.display(), + ) + })?; + + // 3. Install additional files + if let Some(additional_files) = additional_files { + for (target, data) in additional_files { + let target = base_path.join(target); + fs::create_dir_all(target.parent().unwrap())?; + debug!("Generating file {}", target.display()); + let mut file = File::options().create(true).truncate(true).open(&target)?; + file.write_all(data.as_slice())?; + } + } + } + _ => { + // 2. Install the artifact + let source = artifact_source_override.unwrap_or_else(|| artifact.path.clone()); + debug!( + "Adding to archive {} from {}", + artifact_target.display(), + source.display() + ); + writer.add_file(artifact_target, source, true)?; + + // 3. Install additional files + if let Some(additional_files) = additional_files { + for (target, data) in additional_files { + debug!("Generating archive entry {}", target.display()); + writer.add_bytes(target, None, data.as_slice(), false)?; + } + } + } + } + + // 4. Install type stubs + if context.project_layout.python_module.is_none() { + let ext_name = &context.project_layout.extension_name; + let type_stub = context + .project_layout + .rust_module + .join(format!("{ext_name}.pyi")); + if type_stub.exists() { + eprintln!("📖 Found type stub file at {ext_name}.pyi"); + writer.add_file(module.join("__init__.pyi"), type_stub, false)?; + writer.add_empty_file(module.join("py.typed"))?; + } + } + + Ok(()) +} + /// Adds a data directory with a scripts directory with the binary inside it pub fn write_bin( writer: &mut impl ModuleWriter, diff --git a/src/binding_generator/pyo3_binding.rs b/src/binding_generator/pyo3_binding.rs index c50b0a838..4d9dc2d5d 100644 --- a/src/binding_generator/pyo3_binding.rs +++ b/src/binding_generator/pyo3_binding.rs @@ -1,22 +1,107 @@ +use std::collections::HashMap; use std::ffi::OsStr; use std::path::Path; use std::path::PathBuf; use std::process::Command; -use anyhow::Context as _; use anyhow::Result; use anyhow::bail; -use fs_err as fs; +use tempfile::TempDir; use tracing::debug; -use tracing::instrument; -use crate::ModuleWriter; -use crate::PyProjectToml; +use crate::BuildArtifact; +use crate::BuildContext; use crate::PythonInterpreter; use crate::Target; -use crate::module_writer::ModuleWriterExt; -use crate::module_writer::write_python_part; -use crate::project_layout::ProjectLayout; + +use super::BindingGenerator; +use super::GeneratorOutput; + +/// A generator for producing PyO3 bindings. +/// +/// This struct is responsible for generating Python bindings for modules using PyO3. +/// The `abi3` field determines whether the generated bindings use the stable PyO3 "abi3" interface, +/// which allows compatibility with multiple Python versions. +pub struct Pyo3BindingGenerator { + abi3: bool, +} + +impl Pyo3BindingGenerator { + pub fn new(abi3: bool) -> Self { + Self { abi3 } + } +} + +impl BindingGenerator for Pyo3BindingGenerator { + fn generate_bindings( + &self, + context: &BuildContext, + interpreter: Option<&PythonInterpreter>, + artifact: &BuildArtifact, + module: &Path, + temp_dir: &TempDir, + ) -> Result { + let ext_name = &context.project_layout.extension_name; + let target = &context.target; + + let so_filename = if self.abi3 { + if target.is_unix() { + if target.is_cygwin() { + format!("{ext_name}.abi3.dll") + } else { + format!("{ext_name}.abi3.so") + } + } else { + match interpreter { + Some(interpreter) if interpreter.is_windows_debug() => { + format!("{ext_name}_d.pyd") + } + // Apparently there is no tag for abi3 on windows + _ => format!("{ext_name}.pyd"), + } + } + } else { + let interpreter = + interpreter.expect("A python interpreter is required for non-abi3 build"); + interpreter.get_library_name(ext_name) + }; + let artifact_target = module.join(so_filename); + + let artifact_is_big_ar = target.is_aix() + && artifact.path.extension().unwrap_or(OsStr::new(" ")) == OsStr::new("a"); + + let artifact_source_override = if artifact_is_big_ar { + Some(unpack_big_archive(target, &artifact.path, temp_dir.path())?) + } else { + None + }; + + let additional_files = match context.project_layout.python_module { + Some(_) => None, + None => { + let mut files = HashMap::new(); + files.insert( + module.join("__init__.py"), + format!( + r#"from .{ext_name} import * + +__doc__ = {ext_name}.__doc__ +if hasattr({ext_name}, "__all__"): + __all__ = {ext_name}.__all__"# + ) + .into(), + ); + Some(files) + } + }; + + Ok(GeneratorOutput { + artifact_target, + artifact_source_override, + additional_files, + }) + } +} // Extract the shared object from a AIX big library archive fn unpack_big_archive(target: &Target, artifact: &Path, temp_dir_path: &Path) -> Result { @@ -39,114 +124,3 @@ fn unpack_big_archive(target: &Target, artifact: &Path, temp_dir_path: &Path) -> let unpacked_artifact = temp_dir_path.join(artifact.with_extension("so").file_name().unwrap()); Ok(unpacked_artifact) } - -/// Copies the shared library into the module, which is the only extra file needed with bindings -#[allow(clippy::too_many_arguments)] -#[instrument(skip_all)] -pub fn write_bindings_module( - writer: &mut impl ModuleWriter, - project_layout: &ProjectLayout, - artifact: &Path, - python_interpreter: Option<&PythonInterpreter>, - is_abi3: bool, - target: &Target, - editable: bool, - pyproject_toml: Option<&PyProjectToml>, -) -> Result<()> { - let ext_name = &project_layout.extension_name; - let so_filename = if is_abi3 { - if target.is_unix() { - if target.is_cygwin() { - format!("{ext_name}.abi3.dll") - } else { - format!("{ext_name}.abi3.so") - } - } else { - match python_interpreter { - Some(python_interpreter) if python_interpreter.is_windows_debug() => { - format!("{ext_name}_d.pyd") - } - // Apparently there is no tag for abi3 on windows - _ => format!("{ext_name}.pyd"), - } - } - } else { - let python_interpreter = - python_interpreter.expect("A python interpreter is required for non-abi3 build"); - python_interpreter.get_library_name(ext_name) - }; - - let artifact_is_big_ar = - target.is_aix() && artifact.extension().unwrap_or(OsStr::new(" ")) == OsStr::new("a"); - let temp_dir = if artifact_is_big_ar { - Some(tempfile::tempdir()?) - } else { - None - }; - let artifact_buff = if artifact_is_big_ar { - Some(unpack_big_archive( - target, - artifact, - temp_dir.as_ref().unwrap().path(), - )?) - } else { - None - }; - let artifact = if artifact_is_big_ar { - artifact_buff.as_ref().unwrap() - } else { - artifact - }; - - if !editable { - write_python_part(writer, project_layout, pyproject_toml) - .context("Failed to add the python module to the package")?; - } - if let Some(python_module) = &project_layout.python_module { - if editable { - let target = project_layout.rust_module.join(&so_filename); - // Remove existing so file to avoid triggering SIGSEV in running process - // See https://github.com/PyO3/maturin/issues/758 - debug!("Removing {}", target.display()); - let _ = fs::remove_file(&target); - - debug!("Copying {} to {}", artifact.display(), target.display()); - fs::copy(artifact, &target).context(format!( - "Failed to copy {} to {}", - artifact.display(), - target.display() - ))?; - } else { - let relative = project_layout - .rust_module - .strip_prefix(python_module.parent().unwrap()) - .unwrap(); - writer.add_file(relative.join(&so_filename), artifact, true)?; - } - } else { - let module = PathBuf::from(ext_name); - // Reexport the shared library as if it were the top level module - writer.add_bytes( - module.join("__init__.py"), - None, - format!( - r#"from .{ext_name} import * - -__doc__ = {ext_name}.__doc__ -if hasattr({ext_name}, "__all__"): - __all__ = {ext_name}.__all__"# - ) - .as_bytes(), - false, - )?; - let type_stub = project_layout.rust_module.join(format!("{ext_name}.pyi")); - if type_stub.exists() { - eprintln!("📖 Found type stub file at {ext_name}.pyi"); - writer.add_file(module.join("__init__.pyi"), type_stub, false)?; - writer.add_empty_file(module.join("py.typed"))?; - } - writer.add_file(module.join(so_filename), artifact, true)?; - } - - Ok(()) -} diff --git a/src/build_context.rs b/src/build_context.rs index 1a5879b45..9851d3684 100644 --- a/src/build_context.rs +++ b/src/build_context.rs @@ -1,7 +1,8 @@ use crate::auditwheel::{AuditWheelMode, get_policy_and_libs, patchelf, relpath}; use crate::auditwheel::{PlatformTag, Policy}; use crate::binding_generator::{ - write_bin, write_bindings_module, write_cffi_module, write_uniffi_module, write_wasm_launcher, + Pyo3BindingGenerator, generate_binding, write_bin, write_cffi_module, write_uniffi_module, + write_wasm_launcher, }; use crate::bridge::Abi3Version; use crate::build_options::CargoOptions; @@ -768,15 +769,13 @@ impl BuildContext { )?; self.add_external_libs(&mut writer, &[&artifact], &[ext_libs])?; - write_bindings_module( + let generator = Pyo3BindingGenerator::new(true); + generate_binding( &mut writer, - &self.project_layout, - &artifact.path, + &generator, + self, self.interpreter.first(), - true, - &self.target, - self.editable, - self.pyproject_toml.as_ref(), + &artifact, ) .context("Failed to add the files to the wheel")?; @@ -851,15 +850,13 @@ impl BuildContext { )?; self.add_external_libs(&mut writer, &[&artifact], &[ext_libs])?; - write_bindings_module( + let generator = Pyo3BindingGenerator::new(false); + generate_binding( &mut writer, - &self.project_layout, - &artifact.path, + &generator, + self, Some(python_interpreter), - false, - &self.target, - self.editable, - self.pyproject_toml.as_ref(), + &artifact, ) .context("Failed to add the files to the wheel")?; diff --git a/src/module_writer/path_writer.rs b/src/module_writer/path_writer.rs index 2345781d1..b50609121 100644 --- a/src/module_writer/path_writer.rs +++ b/src/module_writer/path_writer.rs @@ -14,6 +14,7 @@ use fs_err::OpenOptions; use fs_err::os::unix::fs::OpenOptionsExt as _; use super::ModuleWriter; +#[cfg(target_family = "unix")] use super::default_permission; use super::util::FileTracker;