diff --git a/crates/uv-cli/src/lib.rs b/crates/uv-cli/src/lib.rs index 5fd42f03bd3..5df014304ab 100644 --- a/crates/uv-cli/src/lib.rs +++ b/crates/uv-cli/src/lib.rs @@ -8029,7 +8029,12 @@ pub struct MetadataArgs { /// /// In dry-run mode, uv will resolve the project's dependencies and report on the resulting /// changes, but will not write the lockfile to disk. - #[arg(long, conflicts_with = "frozen", conflicts_with = "locked")] + #[arg( + long, + conflicts_with = "frozen", + conflicts_with = "locked", + conflicts_with = "sync" + )] pub dry_run: bool, #[command(flatten)] @@ -8041,6 +8046,13 @@ pub struct MetadataArgs { #[command(flatten)] pub refresh: RefreshArgs, + /// Sync the environment to include module ownership metadata in the output. + /// + /// This adds a mapping from importable module names to references to the package nodes + /// that provide them. To do this, the venv will be synced in inexact mode. + #[arg(long)] + pub sync: bool, + /// The Python interpreter to use during resolution. /// /// A Python interpreter is required for building source distributions to determine package diff --git a/crates/uv-configuration/src/dependency_groups.rs b/crates/uv-configuration/src/dependency_groups.rs index 62f51dd7ff9..4196c3a9d48 100644 --- a/crates/uv-configuration/src/dependency_groups.rs +++ b/crates/uv-configuration/src/dependency_groups.rs @@ -140,6 +140,14 @@ impl DependencyGroups { }) } + /// Helper to make a spec from just --all-groups. + pub fn from_all_groups() -> Self { + Self::from_history(DependencyGroupsHistory { + all_groups: true, + ..Default::default() + }) + } + /// Apply defaults to a base [`DependencyGroups`]. /// /// This is appropriate in projects, where the `dev` group is synced by default. diff --git a/crates/uv-distribution-types/src/installed_modules.rs b/crates/uv-distribution-types/src/installed_modules.rs new file mode 100644 index 00000000000..d53dd5cd449 --- /dev/null +++ b/crates/uv-distribution-types/src/installed_modules.rs @@ -0,0 +1,276 @@ +//! Discovers importable modules provided by an installed wheel. +//! +//! Installed wheels record installed paths in `-.dist-info/RECORD`. Python source +//! files, legacy sourceless bytecode, and recognized native extension modules located under the +//! import root contribute a [`ModuleName`] and its parent package prefixes. +//! +//! This is intentionally file-based: it does not infer modules exposed through `.pth` files, +//! legacy namespace declarations in `__init__.py`, or `.pyi`-only stub distributions. + +use std::collections::BTreeSet; +use std::path::{Component, Path}; + +use fs_err::File; +use uv_fs::normalize_path; +use uv_install_wheel::read_record; +use uv_pypi_types::ModuleName; + +use crate::installed::{InstalledDist, InstalledDistError}; + +impl InstalledDist { + /// Read the modules provided by this installed distribution. + pub fn read_modules( + &self, + extension_suffixes: &[Box], + ) -> Result, InstalledDistError> { + let dist_info = self.install_path(); + if !has_extension(dist_info, "dist-info") { + return Ok(BTreeSet::new()); + } + + let record_path = dist_info.join("RECORD"); + let record = read_record(File::open(&record_path)?)?; + + let mut modules = BTreeSet::new(); + for entry in record { + add_record_module(&entry.path, extension_suffixes, &mut modules); + } + + Ok(modules) + } +} + +fn add_record_module( + path: &str, + extension_suffixes: &[Box], + modules: &mut BTreeSet, +) { + let Some(components) = record_path_components(path) else { + return; + }; + let Some((file_name, parents)) = components.split_last() else { + return; + }; + let file_name = file_name.as_ref(); + + // Metadata and other entries under `.dist-info` directories are not modules. + if components + .iter() + .any(|component| has_extension(component.as_ref(), "dist-info")) + { + return; + } + // Files in a `.data` directory that were not relocated into the import root are not modules. + // Relocated files are recorded at their installed paths instead. + if components + .first() + .is_some_and(|component| has_extension(component.as_ref(), "data")) + { + return; + } + + let mut module_components = parents + .iter() + .map(std::convert::AsRef::as_ref) + .collect::>(); + // We intentionally skip `.pyi` files here because we're looking for runtime module ownership. + // Type stubs will require separate ownership modeling. + if file_name == "__init__.py" { + // The parent path is the package. + } else if let Some(stem) = file_name.strip_suffix(".py") { + module_components.push(stem); + } else if let Some(stem) = bytecode_module_stem(file_name, parents) { + if stem != "__init__" { + module_components.push(stem); + } + } else if let Some(stem) = { + // Python reports the recognized suffixes in import lookup order through + // `importlib.machinery.EXTENSION_SUFFIXES`; preserve that order so a generic suffix such + // as `.so` does not consume a more-specific suffix such as `.abi3.so`. + extension_suffixes.iter().find_map(|suffix| { + let stem = file_name.strip_suffix(suffix.as_ref())?; + (!stem.is_empty()).then_some(stem) + }) + } { + if stem != "__init__" { + module_components.push(stem); + } + } else { + return; + } + + add_module_components(&module_components, modules); +} + +fn record_path_components(path: &str) -> Option>> { + let normalized = normalize_path(Path::new(path)); + let path = normalized.as_ref(); + + // `RECORD` can include absolute paths and relative paths that leave the directory containing + // `.dist-info`, for example installed scripts. Those entries cannot describe modules here. + if path.is_absolute() { + return None; + } + + let mut components = Vec::new(); + for component in path.components() { + match component { + Component::Normal(component) => { + components.push(Box::from(component.to_str()?)); + } + Component::CurDir => {} + Component::ParentDir | Component::Prefix(_) | Component::RootDir => return None, + } + } + + Some(components) +} + +/// Return the module stem for importable sourceless bytecode in a `RECORD` path. +/// +/// CPython can import `package/module.pyc` directly when only bytecode is installed. In +/// contrast, `package/__pycache__/module.cpython-312.pyc` is not an import source without +/// `package/module.py`. +fn bytecode_module_stem<'a>(file_name: &'a str, parents: &[Box]) -> Option<&'a str> { + let stem = file_name.strip_suffix(".pyc")?; + if parents + .last() + .is_some_and(|parent| parent.as_ref() == "__pycache__") + { + // A `.pyc` file in `__pycache__` does not make the module importable + // without the corresponding source file. Sourceless imports use the + // legacy `module.pyc` location instead. + return None; + } + + Some(stem) +} + +fn has_extension(path: impl AsRef, extension: &str) -> bool { + path.as_ref() + .extension() + .is_some_and(|candidate| candidate == extension) +} + +fn add_module_components(components: &[&str], modules: &mut BTreeSet) { + let Ok(module) = ModuleName::from_components(components.iter().copied()) else { + return; + }; + + modules.extend(module.prefixes()); +} + +#[cfg(test)] +mod tests { + use std::collections::BTreeSet; + + use uv_pypi_types::ModuleName; + + use super::add_record_module; + + fn extension_suffixes() -> Vec> { + [ + ".cpython-312-darwin.so", + ".cpython-314td-darwin.so", + ".abi3.so", + ".cp312-win_amd64.pyd", + ".so", + ] + .into_iter() + .map(Box::from) + .collect() + } + + fn module_names(modules: BTreeSet) -> String { + modules + .into_iter() + .map(|module| module.to_string()) + .collect::>() + .join("\n") + } + + #[test] + fn record_module_normalizes_record_paths() { + let mut modules = BTreeSet::new(); + add_record_module("./package/../café.py", &[], &mut modules); + + assert_eq!(module_names(modules), "café"); + } + + #[test] + fn record_module_from_legacy_bytecode() { + let mut modules = BTreeSet::new(); + add_record_module("package/module.pyc", &[], &mut modules); + add_record_module("legacy.pyc", &[], &mut modules); + + assert_eq!(module_names(modules), "legacy\npackage\npackage.module"); + } + + #[test] + fn record_module_ignores_pycache_bytecode() { + let mut modules = BTreeSet::new(); + add_record_module( + "package/__pycache__/module.cpython-312.opt-1.pyc", + &[], + &mut modules, + ); + add_record_module( + "package/__pycache__/__init__.cpython-312.pyc", + &[], + &mut modules, + ); + + assert_eq!(module_names(modules), ""); + } + + #[test] + fn record_module_from_extension_module() { + let extension_suffixes = extension_suffixes(); + let mut modules = BTreeSet::new(); + add_record_module( + "package/extension.cpython-312-darwin.so", + &extension_suffixes, + &mut modules, + ); + add_record_module( + "package/free_threaded.cpython-314td-darwin.so", + &extension_suffixes, + &mut modules, + ); + add_record_module("package/limited.abi3.so", &extension_suffixes, &mut modules); + add_record_module( + "package/windows.cp312-win_amd64.pyd", + &extension_suffixes, + &mut modules, + ); + add_record_module( + "package/__init__.cpython-312-darwin.so", + &extension_suffixes, + &mut modules, + ); + add_record_module("plain.so", &extension_suffixes, &mut modules); + + assert_eq!( + module_names(modules), + "package\npackage.extension\npackage.free_threaded\npackage.limited\npackage.windows\nplain" + ); + } + + #[test] + fn record_module_ignores_unrecognized_extension_suffixes() { + let extension_suffixes = extension_suffixes(); + let mut modules = BTreeSet::new(); + add_record_module( + "package/extension.not-an-extension-tag.so", + &extension_suffixes, + &mut modules, + ); + add_record_module( + "package/bogus.pypynonsense.so", + &extension_suffixes, + &mut modules, + ); + + assert_eq!(module_names(modules), ""); + } +} diff --git a/crates/uv-distribution-types/src/lib.rs b/crates/uv-distribution-types/src/lib.rs index d37d0be5472..89270c4ef53 100644 --- a/crates/uv-distribution-types/src/lib.rs +++ b/crates/uv-distribution-types/src/lib.rs @@ -106,6 +106,7 @@ mod index; mod index_name; mod index_url; mod installed; +mod installed_modules; mod known_platform; mod origin; mod pip_index; diff --git a/crates/uv-pypi-types/src/lib.rs b/crates/uv-pypi-types/src/lib.rs index 58f77175332..8bac08563b9 100644 --- a/crates/uv-pypi-types/src/lib.rs +++ b/crates/uv-pypi-types/src/lib.rs @@ -6,6 +6,7 @@ pub use identifier::*; pub use lenient_requirement::*; pub use marker_environment::*; pub use metadata::*; +pub use module_name::*; pub use parsed_url::*; pub use project_status::*; pub use scheme::*; @@ -20,6 +21,7 @@ mod identifier; mod lenient_requirement; mod marker_environment; mod metadata; +mod module_name; mod parsed_url; mod project_status; mod scheme; diff --git a/crates/uv-pypi-types/src/module_name.rs b/crates/uv-pypi-types/src/module_name.rs new file mode 100644 index 00000000000..b1fdadd2e37 --- /dev/null +++ b/crates/uv-pypi-types/src/module_name.rs @@ -0,0 +1,172 @@ +use std::borrow::Cow; +use std::fmt::Display; +use std::str::FromStr; + +use serde::{Serialize, Serializer}; +use thiserror::Error; + +use crate::{Identifier, IdentifierParseError}; + +/// The name of an importable Python module. +/// +/// This is a dotted sequence of Python identifiers, like `foo` or `foo.bar`. +#[derive(Debug, Clone, Hash, PartialEq, Eq, PartialOrd, Ord)] +pub struct ModuleName(Box); + +#[derive(Debug, Clone, Error)] +pub enum ModuleNameParseError { + #[error("A module name must not be empty")] + Empty, + #[error("Invalid module name component `{component}` in `{module}`")] + InvalidComponent { + component: Box, + module: Box, + #[source] + err: IdentifierParseError, + }, +} + +impl ModuleName { + pub fn new(module: impl Into>) -> Result { + let module = module.into(); + if module.is_empty() { + return Err(ModuleNameParseError::Empty); + } + + for component in module.split('.') { + Self::validate_component(&module, component)?; + } + + Ok(Self(module)) + } + + pub fn from_components<'a>( + components: impl IntoIterator, + ) -> Result { + let components = components.into_iter().collect::>(); + if components.is_empty() { + return Err(ModuleNameParseError::Empty); + } + + let module = components.join(".").into_boxed_str(); + for component in components { + Self::validate_component(&module, component)?; + } + + Ok(Self(module)) + } + + /// Iterate over this module and its parent modules. + /// + /// For example, `foo.bar.baz` yields `foo`, `foo.bar`, and `foo.bar.baz`. + pub fn prefixes(&self) -> impl Iterator + '_ { + self.0 + .match_indices('.') + .map(|(index, _)| Self(Box::from(&self.0[..index]))) + .chain(std::iter::once(self.clone())) + } + + fn validate_component(module: &str, component: &str) -> Result<(), ModuleNameParseError> { + Identifier::new(component.to_string()).map_err(|err| { + ModuleNameParseError::InvalidComponent { + component: component.to_string().into_boxed_str(), + module: module.into(), + err, + } + })?; + Ok(()) + } +} + +impl FromStr for ModuleName { + type Err = ModuleNameParseError; + + fn from_str(module: &str) -> Result { + Self::new(module) + } +} + +impl Display for ModuleName { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{}", self.0) + } +} + +impl AsRef for ModuleName { + fn as_ref(&self) -> &str { + &self.0 + } +} + +impl<'de> serde::de::Deserialize<'de> for ModuleName { + fn deserialize(deserializer: D) -> Result + where + D: serde::de::Deserializer<'de>, + { + let s = >::deserialize(deserializer)?; + Self::from_str(&s).map_err(serde::de::Error::custom) + } +} + +impl Serialize for ModuleName { + fn serialize(&self, serializer: S) -> Result + where + S: Serializer, + { + Serialize::serialize(&self.0, serializer) + } +} + +#[cfg(feature = "schemars")] +impl schemars::JsonSchema for ModuleName { + fn schema_name() -> Cow<'static, str> { + Cow::Borrowed("ModuleName") + } + + fn json_schema(_generator: &mut schemars::generate::SchemaGenerator) -> schemars::Schema { + schemars::json_schema!({ + "type": "string", + "pattern": r"^[_\p{Alphabetic}][_0-9\p{Alphabetic}]*(\.[_\p{Alphabetic}][_0-9\p{Alphabetic}]*)*$", + "description": "A dotted Python module name" + }) + } +} + +#[cfg(test)] +mod tests { + use std::str::FromStr; + + use insta::assert_snapshot; + + use super::ModuleName; + + #[test] + fn valid() { + for module_name in ["abc", "abc.def", "_abc", "férrîs", "package.안녕하세요"] { + assert!(ModuleName::from_str(module_name).is_ok(), "{module_name}"); + } + } + + #[test] + fn invalid() { + assert_snapshot!( + ModuleName::from_str("foo-bar").unwrap_err(), + @"Invalid module name component `foo-bar` in `foo-bar`" + ); + assert_snapshot!( + ModuleName::from_str("foo.").unwrap_err(), + @"Invalid module name component `` in `foo.`" + ); + } + + #[test] + fn prefixes() { + let prefixes = ModuleName::from_str("foo.bar.baz") + .expect("valid module name") + .prefixes() + .map(|module| module.to_string()) + .collect::>(); + + assert_eq!(prefixes, ["foo", "foo.bar", "foo.bar.baz"]); + } +} diff --git a/crates/uv-python/python/get_interpreter_info.py b/crates/uv-python/python/get_interpreter_info.py index 30735ae3e5d..fdd9f8d9ec3 100644 --- a/crates/uv-python/python/get_interpreter_info.py +++ b/crates/uv-python/python/get_interpreter_info.py @@ -4,13 +4,13 @@ The script will exit with status 0 on known error that are turned into rust errors. """ -import site -import sys - +import importlib.machinery import json import os import platform +import site import struct +import sys import sysconfig @@ -682,6 +682,7 @@ def main() -> None: "sys_path": sys.path[1:], "site_packages": site.getsitepackages(), "stdlib": sysconfig.get_path("stdlib"), + "extension_suffixes": importlib.machinery.EXTENSION_SUFFIXES, # Prior to the introduction of `sysconfig` patching, python-build-standalone installations would always use # "/install" as the prefix. With `sysconfig` patching, we rewrite the prefix to match the actual installation # location. So in newer versions, we also write a dedicated flag to indicate standalone builds. diff --git a/crates/uv-python/src/interpreter.rs b/crates/uv-python/src/interpreter.rs index 41e3392ad33..6f13c1e81a2 100644 --- a/crates/uv-python/src/interpreter.rs +++ b/crates/uv-python/src/interpreter.rs @@ -54,6 +54,7 @@ pub struct Interpreter { sys_executable: PathBuf, site_packages: Vec, stdlib: PathBuf, + extension_suffixes: Vec>, standalone: bool, tags: OnceLock, target: Option, @@ -90,6 +91,7 @@ impl Interpreter { sys_executable: info.sys_executable, site_packages: info.site_packages, stdlib: info.stdlib, + extension_suffixes: info.extension_suffixes, standalone: info.standalone, tags: OnceLock::new(), target: None, @@ -458,6 +460,11 @@ impl Interpreter { &self.sys_executable } + /// Return the recognized native extension module suffixes for this Python interpreter. + pub fn extension_suffixes(&self) -> &[Box] { + &self.extension_suffixes + } + /// Return the "real" queried executable path for this Python interpreter. pub fn real_executable(&self) -> &Path { &self.real_executable @@ -954,6 +961,7 @@ struct InterpreterInfo { sys_path: Vec, site_packages: Vec, stdlib: PathBuf, + extension_suffixes: Vec>, standalone: bool, pointer_size: PointerSize, gil_disabled: bool, @@ -1384,6 +1392,7 @@ mod tests { "/home/ferris/.pyenv/versions/3.12.0/lib/python3.12/site-packages" ], "stdlib": "/home/ferris/.pyenv/versions/3.12.0/lib/python3.12", + "extension_suffixes": [".cpython-312-x86_64-linux-gnu.so", ".abi3.so", ".so"], "scheme": { "data": "/home/ferris/.pyenv/versions/3.12.0", "include": "/home/ferris/.pyenv/versions/3.12.0/include", diff --git a/crates/uv-python/src/lib.rs b/crates/uv-python/src/lib.rs index e97673e435d..811e98d0e1e 100644 --- a/crates/uv-python/src/lib.rs +++ b/crates/uv-python/src/lib.rs @@ -360,6 +360,7 @@ mod tests { "/home/ferris/.pyenv/versions/{FULL_VERSION}/lib/python{VERSION}/site-packages" ], "stdlib": "/home/ferris/.pyenv/versions/{FULL_VERSION}/lib/python{VERSION}", + "extension_suffixes": [".cpython-{VERSION}-x86_64-linux-gnu.so", ".abi3.so", ".so"], "scheme": { "data": "/home/ferris/.pyenv/versions/{FULL_VERSION}", "include": "/home/ferris/.pyenv/versions/{FULL_VERSION}/include", @@ -455,6 +456,7 @@ mod tests { "/lib/python{VERSION}/site-packages" ], "stdlib": "//lib/python{VERSION}", + "extension_suffixes": [".cpython-{VERSION}-wasm32-emscripten.so", ".so"], "scheme": { "platlib": "//lib/python{VERSION}/site-packages", "purelib": "//lib/python{VERSION}/site-packages", diff --git a/crates/uv-resolver/src/lock/export/metadata.rs b/crates/uv-resolver/src/lock/export/metadata.rs index f618d9a6bc7..eaa69f6658b 100644 --- a/crates/uv-resolver/src/lock/export/metadata.rs +++ b/crates/uv-resolver/src/lock/export/metadata.rs @@ -2,23 +2,25 @@ use std::collections::BTreeMap; use std::fmt::Display; use uv_distribution_filename::WheelFilename; -use uv_distribution_types::{RequiresPython, UrlString}; +use uv_distribution_types::{Name, RequiresPython, ResolvedDist, UrlString}; use uv_fs::PortablePathBuf; use uv_normalize::{ExtraName, GroupName, PackageName}; use uv_pep440::Version; -use uv_pypi_types::{ConflictItem, ConflictKind, ConflictSet, Conflicts}; +use uv_pypi_types::{ConflictItem, ConflictKind, ConflictSet, Conflicts, ModuleName}; use uv_workspace::Workspace; -use crate::Lock; use crate::lock::{ Dependency, DirectSource, PackageId, RegistrySource, Source, SourceDist, SourceDistMetadata, Wheel, WheelWireSource, ZstdWheel, }; +use crate::{Lock, LockError}; #[derive(Debug, thiserror::Error)] enum MetadataErrorKind { #[error(transparent)] Serialize(#[from] serde_json::error::Error), + #[error(transparent)] + Lock(#[from] LockError), } #[derive(Debug)] @@ -66,6 +68,9 @@ pub struct Metadata { requires_python: RequiresPython, /// Info about conflicting packages conflicts: MetadataConflicts, + /// A mapping from importable module names to the package nodes that provide them + #[serde(skip_serializing_if = "BTreeMap::is_empty", default)] + module_owners: BTreeMap>, /// An index of which nodes are workspace members /// /// These entries are often what you should use as the entry-points into the `resolve` graph. @@ -103,6 +108,13 @@ struct MetadataWorkspaceMember { id: MetadataNodeIdFlat, } +/// An installed distribution that provides an importable module. +#[derive(Debug, serde::Serialize)] +struct MetadataModuleOwner { + /// Key for the package node in the `resolution` graph. + package_id: MetadataNodeIdFlat, +} + /// A node in the dependency graph /// /// There are 4 kinds of nodes: @@ -818,6 +830,7 @@ impl Metadata { version: SchemaVersion::Preview, }, conflicts, + module_owners: BTreeMap::new(), workspace_root, requires_python: lock.requires_python.clone(), members, @@ -825,6 +838,36 @@ impl Metadata { }) } + pub fn package_node_id( + workspace_root: &PortablePathBuf, + dist: &ResolvedDist, + ) -> Result { + let source = Source::from_resolved_dist(dist, workspace_root.as_ref())?; + Ok(MetadataNodeId { + name: dist.name().clone(), + version: dist.version().cloned(), + source: MetadataSource::from_source(workspace_root, source), + kind: MetadataNodeKind::Package, + } + .to_flat()) + } + + #[must_use] + pub fn with_module_owners(mut self, module_owners: BTreeMap>) -> Self { + self.module_owners = module_owners + .into_iter() + .filter_map(|(module, owners)| { + let owners = owners + .into_iter() + .filter(|package_id| self.resolution.contains_key(package_id)) + .map(|package_id| MetadataModuleOwner { package_id }) + .collect::>(); + (!owners.is_empty()).then_some((module, owners)) + }) + .collect(); + self + } + pub fn to_json(&self) -> Result { Ok(serde_json::to_string_pretty(self)?) } diff --git a/crates/uv/src/commands/project/mod.rs b/crates/uv/src/commands/project/mod.rs index e1869388ebb..50cd23cf051 100644 --- a/crates/uv/src/commands/project/mod.rs +++ b/crates/uv/src/commands/project/mod.rs @@ -70,7 +70,7 @@ pub(crate) mod environment; pub(crate) mod export; pub(crate) mod format; pub(crate) mod init; -mod install_target; +pub(crate) mod install_target; pub(crate) mod lock; pub(crate) mod lock_target; pub(crate) mod remove; @@ -1402,7 +1402,7 @@ impl ScriptPython { /// The Python environment for a project. #[derive(Debug)] -enum ProjectEnvironment { +pub(crate) enum ProjectEnvironment { /// An existing [`PythonEnvironment`] was discovered, which satisfies the project's requirements. Existing(PythonEnvironment), /// An existing [`PythonEnvironment`] was discovered, but did not satisfy the project's diff --git a/crates/uv/src/commands/project/sync.rs b/crates/uv/src/commands/project/sync.rs index dce812ae017..412fd2651ff 100644 --- a/crates/uv/src/commands/project/sync.rs +++ b/crates/uv/src/commands/project/sync.rs @@ -633,7 +633,7 @@ impl Deref for SyncEnvironment { } /// Sync a lockfile with an environment. -pub(super) async fn do_sync( +pub(crate) async fn do_sync( target: InstallTarget<'_>, venv: &PythonEnvironment, extras: &ExtrasSpecificationWithDefaults, diff --git a/crates/uv/src/commands/workspace/metadata.rs b/crates/uv/src/commands/workspace/metadata.rs index a212c70facd..c839ad0b594 100644 --- a/crates/uv/src/commands/workspace/metadata.rs +++ b/crates/uv/src/commands/workspace/metadata.rs @@ -1,7 +1,7 @@ use std::fmt::Write; use std::path::Path; -use anyhow::Result; +use anyhow::{Context, Result}; use owo_colors::OwoColorize; use uv_cache::{Cache, Refresh}; @@ -9,19 +9,23 @@ use uv_client::BaseClientBuilder; use uv_configuration::{Concurrency, DependencyGroupsWithDefaults, DryRun}; use uv_preview::{Preview, PreviewFeature}; use uv_python::{PythonDownloads, PythonPreference, PythonRequest}; -use uv_resolver::{Lock, Metadata}; -use uv_settings::PythonInstallMirrors; +use uv_resolver::Metadata; +use uv_settings::{MalwareCheckSettings, PythonInstallMirrors}; use uv_warnings::warn_user; -use uv_workspace::{DiscoveryOptions, VirtualProject, Workspace, WorkspaceCache}; +use uv_workspace::{DiscoveryOptions, VirtualProject, WorkspaceCache}; use crate::commands::pip::loggers::DefaultResolveLogger; use crate::commands::project::lock::{LockMode, LockOperation}; use crate::commands::project::lock_target::LockTarget; -use crate::commands::project::{ProjectError, ProjectInterpreter, UniversalState, WorkspacePython}; +use crate::commands::project::{ + ProjectEnvironment, ProjectError, ProjectInterpreter, UniversalState, WorkspacePython, +}; use crate::commands::{ExitStatus, diagnostics}; use crate::printer::Printer; use crate::settings::{FrozenSource, LockCheck, ResolverSettings}; +use super::module_owners::collect_module_owners; + /// Display metadata about the workspace. pub(crate) async fn metadata( project_dir: &Path, @@ -29,8 +33,10 @@ pub(crate) async fn metadata( frozen: Option, dry_run: DryRun, refresh: Refresh, + sync: bool, python: Option, install_mirrors: PythonInstallMirrors, + malware_settings: MalwareCheckSettings, settings: ResolverSettings, client_builder: BaseClientBuilder<'_>, python_preference: PythonPreference, @@ -54,13 +60,14 @@ pub(crate) async fn metadata( .await?; let target = LockTarget::Workspace(virtual_project.workspace()); + // Don't enable any groups' requires-python for interpreter discovery. + let groups = DependencyGroupsWithDefaults::none(); + // Determine the lock mode. let interpreter; let mode = if let Some(frozen_source) = frozen { LockMode::Frozen(frozen_source.into()) } else { - // Don't enable any groups' requires-python for interpreter discovery - let groups = DependencyGroupsWithDefaults::none(); let workspace_python = WorkspacePython::from_request( python.as_deref().map(PythonRequest::parse), Some(virtual_project.workspace()), @@ -117,7 +124,47 @@ pub(crate) async fn metadata( ) .await { - Ok(lock) => print_lock_as_metadata(virtual_project.workspace(), &lock.into_lock(), printer), + Ok(lock) => { + let lock = lock.into_lock(); + let mut export = Metadata::from_lock(virtual_project.workspace(), &lock)?; + if sync { + let environment = ProjectEnvironment::get_or_init( + virtual_project.workspace(), + &groups, + python.as_deref().map(PythonRequest::parse), + &install_mirrors, + &client_builder, + python_preference, + python_downloads, + false, + no_config, + Some(false), + cache, + DryRun::Disabled, + printer, + preview, + ) + .await?; + let module_owners = collect_module_owners( + virtual_project.workspace(), + &lock, + &environment, + &settings, + &client_builder, + &state, + &concurrency, + cache, + workspace_cache, + preview, + &malware_settings, + ) + .await + .context("Failed to collect module owners")?; + export = export.with_module_owners(module_owners); + } + + print_metadata(&export, printer) + } Err(err @ ProjectError::LockMismatch(..)) => { writeln!(printer.stderr(), "{}", err.to_string().bold())?; Ok(ExitStatus::Failure) @@ -131,13 +178,7 @@ pub(crate) async fn metadata( } } -fn print_lock_as_metadata( - workspace: &Workspace, - lock: &Lock, - printer: Printer, -) -> Result { - let export = Metadata::from_lock(workspace, lock)?; - +fn print_metadata(export: &Metadata, printer: Printer) -> Result { writeln!(printer.stdout(), "{}", export.to_json()?)?; Ok(ExitStatus::Success) diff --git a/crates/uv/src/commands/workspace/mod.rs b/crates/uv/src/commands/workspace/mod.rs index 6ee08d3e2b0..e4f1ca0d776 100644 --- a/crates/uv/src/commands/workspace/mod.rs +++ b/crates/uv/src/commands/workspace/mod.rs @@ -1,3 +1,4 @@ pub(crate) mod dir; pub(crate) mod list; pub(crate) mod metadata; +mod module_owners; diff --git a/crates/uv/src/commands/workspace/module_owners.rs b/crates/uv/src/commands/workspace/module_owners.rs new file mode 100644 index 00000000000..473afbef0a9 --- /dev/null +++ b/crates/uv/src/commands/workspace/module_owners.rs @@ -0,0 +1,143 @@ +use std::collections::{BTreeMap, BTreeSet}; + +use anyhow::Result; +use uv_cache::Cache; +use uv_client::BaseClientBuilder; +use uv_configuration::{ + Concurrency, DependencyGroups, DryRun, ExtrasSpecification, InstallOptions, Reinstall, +}; +use uv_distribution_types::{Dist, Name, ResolvedDist}; +use uv_fs::PortablePathBuf; +use uv_installer::SitePackages; +use uv_normalize::{DefaultExtras, DefaultGroups, PackageName}; +use uv_preview::Preview; +use uv_pypi_types::ModuleName; +use uv_python::PythonEnvironment; +use uv_resolver::{Installable, Lock, Metadata}; +use uv_settings::MalwareCheckSettings; +use uv_workspace::{Workspace, WorkspaceCache}; + +use crate::commands::pip::loggers::DefaultInstallLogger; +use crate::commands::pip::operations::Modifications; +use crate::commands::pip::{resolution_markers, resolution_tags}; +use crate::commands::project::UniversalState; +use crate::commands::project::install_target::InstallTarget; +use crate::commands::project::sync::do_sync; +use crate::printer::Printer; +use crate::settings::{InstallerSettingsRef, ResolverSettings}; + +/// Sync all locked extras and groups as needed, then map importable modules to package IDs. +/// +/// This uses a sufficient (inexact) sync so required distributions are available to inspect +/// without removing unrelated packages from an existing environment. Only distributions in the +/// selected resolution are assigned package IDs, so those unrelated packages are not reported. +pub(crate) async fn collect_module_owners( + workspace: &Workspace, + lock: &Lock, + venv: &PythonEnvironment, + settings: &ResolverSettings, + client_builder: &BaseClientBuilder<'_>, + state: &UniversalState, + concurrency: &Concurrency, + cache: &Cache, + workspace_cache: &WorkspaceCache, + preview: Preview, + malware_settings: &MalwareCheckSettings, +) -> Result>> { + let target = InstallTarget::Workspace { workspace, lock }; + let marker_env = resolution_markers(None, None, venv.interpreter()); + let tags = resolution_tags(None, None, venv.interpreter())?; + let extras = ExtrasSpecification::from_all_extras().with_defaults(DefaultExtras::default()); + let groups = DependencyGroups::from_all_groups().with_defaults(DefaultGroups::default()); + + let resolution = target.to_resolution( + &marker_env, + &tags, + &extras, + &groups, + &settings.build_options, + &InstallOptions::default(), + )?; + if resolution.is_empty() { + return Ok(BTreeMap::new()); + } + + let workspace_root = PortablePathBuf::from(workspace.install_path().as_path()); + let mut package_ids = BTreeMap::::new(); + for dist in resolution.distributions().filter(|dist| !is_virtual(dist)) { + package_ids.insert( + dist.name().clone(), + Metadata::package_node_id(&workspace_root, dist)?, + ); + } + + let reinstall = Reinstall::None; + let installer_settings = InstallerSettingsRef { + index_locations: &settings.index_locations, + index_strategy: settings.index_strategy, + keyring_provider: settings.keyring_provider, + dependency_metadata: &settings.dependency_metadata, + config_setting: &settings.config_setting, + config_settings_package: &settings.config_settings_package, + build_isolation: &settings.build_isolation, + extra_build_dependencies: &settings.extra_build_dependencies, + extra_build_variables: &settings.extra_build_variables, + exclude_newer: &settings.exclude_newer, + link_mode: settings.link_mode, + compile_bytecode: false, + reinstall: &reinstall, + build_options: &settings.build_options, + sources: settings.sources.clone(), + }; + + do_sync( + target, + venv, + &extras, + &groups, + None, + InstallOptions::default(), + Modifications::Sufficient, + None, + installer_settings, + client_builder, + &state.fork(), + Box::new(DefaultInstallLogger), + false, + concurrency, + cache, + workspace_cache, + DryRun::Disabled, + Printer::Silent, + preview, + malware_settings, + ) + .await?; + + let mut owners = BTreeMap::>::new(); + for dist in SitePackages::from_environment(venv)?.iter() { + let Some(package_id) = package_ids.get(dist.name()) else { + continue; + }; + // TODO: Editable installs often only record a `.pth` file; we'll + // need to handle them specially. + for module in dist.read_modules(venv.interpreter().extension_suffixes())? { + owners.entry(module).or_default().insert(package_id.clone()); + } + } + + Ok(owners + .into_iter() + .map(|(module, owners)| (module, owners.into_iter().collect())) + .collect()) +} + +fn is_virtual(dist: &ResolvedDist) -> bool { + let ResolvedDist::Installable { dist, .. } = dist else { + return false; + }; + let Dist::Source(source) = dist.as_ref() else { + return false; + }; + source.is_virtual() +} diff --git a/crates/uv/src/lib.rs b/crates/uv/src/lib.rs index 8eb95960a22..65917b0ba0f 100644 --- a/crates/uv/src/lib.rs +++ b/crates/uv/src/lib.rs @@ -1954,8 +1954,10 @@ async fn run(cli: Cli) -> Result { args.frozen, args.dry_run, args.refresh, + args.sync, args.python, args.install_mirrors, + args.malware_settings, args.settings, client_builder.subcommand(vec!["workspace".to_owned(), "metadata".to_owned()]), globals.python_preference, diff --git a/crates/uv/src/settings.rs b/crates/uv/src/settings.rs index e00ca9b8077..09ea285c7bf 100644 --- a/crates/uv/src/settings.rs +++ b/crates/uv/src/settings.rs @@ -1923,10 +1923,12 @@ pub(crate) struct MetadataSettings { pub(crate) lock_check: LockCheck, pub(crate) frozen: Option, pub(crate) dry_run: DryRun, + pub(crate) sync: bool, pub(crate) python: Option, pub(crate) install_mirrors: PythonInstallMirrors, pub(crate) refresh: Refresh, pub(crate) settings: ResolverSettings, + pub(crate) malware_settings: MalwareCheckSettings, } impl MetadataSettings { @@ -1943,6 +1945,7 @@ impl MetadataSettings { resolver, build, refresh, + sync, python, } = *args; @@ -1958,10 +1961,13 @@ impl MetadataSettings { // Check for conflicts between locked and frozen. check_conflicts(locked, frozen); + let malware_settings = MalwareCheckSettings::from(&environment); + Self { lock_check: resolve_lock_check(locked), frozen: resolve_frozen(frozen), dry_run: DryRun::from_args(dry_run), + sync, python: python.and_then(Maybe::into_option), refresh: Refresh::from(refresh), settings: ResolverSettings::combine( @@ -1972,6 +1978,7 @@ impl MetadataSettings { install_mirrors: environment .install_mirrors .combine(filesystem_install_mirrors), + malware_settings, } } } diff --git a/crates/uv/tests/it/workspace_metadata.rs b/crates/uv/tests/it/workspace_metadata.rs index f70af696ad3..a1c970b104f 100644 --- a/crates/uv/tests/it/workspace_metadata.rs +++ b/crates/uv/tests/it/workspace_metadata.rs @@ -1,9 +1,56 @@ +use std::path::Path; + use anyhow::Result; use assert_cmd::assert::OutputAssertExt; -use assert_fs::fixture::PathChild; +use assert_fs::fixture::{FileWriteStr, PathChild}; +use async_zip::base::write::ZipFileWriter; +use async_zip::{Compression, ZipEntryBuilder}; +use futures::executor::block_on; +use url::Url; use uv_test::{copy_dir_ignore, uv_snapshot}; +fn write_wheel( + path: &Path, + name: &str, + dist_info_prefix: &str, + files: &[(&str, &str)], +) -> Result<()> { + let mut writer = ZipFileWriter::new(Vec::new()); + let mut record = Vec::new(); + + for (file_path, contents) in files { + let entry = ZipEntryBuilder::new((*file_path).into(), Compression::Stored); + block_on(writer.write_entry_whole(entry, contents.as_bytes()))?; + record.push(format!("{file_path},,")); + } + + let metadata_path = format!("{dist_info_prefix}.dist-info/METADATA"); + let entry = ZipEntryBuilder::new(metadata_path.clone().into(), Compression::Stored); + block_on(writer.write_entry_whole( + entry, + format!("Metadata-Version: 2.1\nName: {name}\nVersion: 0.1.0\n").as_bytes(), + ))?; + record.push(format!("{metadata_path},,")); + + let wheel_path = format!("{dist_info_prefix}.dist-info/WHEEL"); + let entry = ZipEntryBuilder::new(wheel_path.clone().into(), Compression::Stored); + block_on(writer.write_entry_whole( + entry, + b"Wheel-Version: 1.0\nGenerator: uv-test\nRoot-Is-Purelib: true\nTag: py3-none-any\n", + ))?; + record.push(format!("{wheel_path},,")); + + let record_path = format!("{dist_info_prefix}.dist-info/RECORD"); + record.push(format!("{record_path},,")); + let entry = ZipEntryBuilder::new(record_path.into(), Compression::Stored); + let record = format!("{}\n", record.join("\n")); + block_on(writer.write_entry_whole(entry, record.as_bytes()))?; + + fs_err::write(path, block_on(writer.close())?)?; + Ok(()) +} + /// Test basic metadata output for a simple workspace with one member. #[test] fn workspace_metadata_simple() { @@ -55,6 +102,346 @@ fn workspace_metadata_simple() { ); } +#[test] +fn workspace_metadata_module_owners_from_locked_wheels() -> Result<()> { + let context = uv_test::test_context!("3.12"); + + let gpu_a = context.temp_dir.child("gpu_a-0.1.0-py3-none-any.whl"); + write_wheel(gpu_a.path(), "gpu-a", "gpu_a-0.1.0", &[("gpu/a.py", "")])?; + + let gpu_b = context.temp_dir.child("gpu_b-0.1.0-py3-none-any.whl"); + write_wheel(gpu_b.path(), "gpu-b", "gpu_b-0.1.0", &[("gpu/b.py", "")])?; + + let typing_extensions = context + .temp_dir + .child("typing_extensions-0.1.0-py3-none-any.whl"); + write_wheel( + typing_extensions.path(), + "typing-extensions", + "typing_extensions-0.1.0", + &[ + ("typing_extensions.py", ""), + ("café.py", ""), + ("bogus.pypynonsense.so", ""), + ("bytecode/__pycache__/compiled.cpython-312.pyc", ""), + ], + )?; + + let gpu_a_url = Url::from_file_path(gpu_a.path()) + .map_err(|()| anyhow::anyhow!("failed to convert wheel path to file URL"))?; + let gpu_b_url = Url::from_file_path(gpu_b.path()) + .map_err(|()| anyhow::anyhow!("failed to convert wheel path to file URL"))?; + let typing_extensions_url = Url::from_file_path(typing_extensions.path()) + .map_err(|()| anyhow::anyhow!("failed to convert wheel path to file URL"))?; + + context + .temp_dir + .child("pyproject.toml") + .write_str(&format!( + r#"[project] +name = "module-owner-root" +version = "0.1.0" +requires-python = ">=3.12" +dependencies = [ + "gpu-a @ {gpu_a_url}", + "gpu-b @ {gpu_b_url}", + "typing-extensions @ {typing_extensions_url}", +] +"# + ))?; + + let mut filters = context.filters(); + filters.push((r#""sha256": "[0-9a-f]{64}""#, r#""sha256": "[SHA256]""#)); + + uv_snapshot!(filters, context.workspace_metadata().arg("--sync"), @r#" + success: true + exit_code: 0 + ----- stdout ----- + { + "schema": { + "version": "preview" + }, + "workspace_root": "[TEMP_DIR]/", + "requires_python": ">=3.12", + "conflicts": { + "sets": [] + }, + "module_owners": { + "café": [ + { + "package_id": "typing-extensions==0.1.0@path+[TEMP_DIR]/typing_extensions-0.1.0-py3-none-any.whl" + } + ], + "gpu": [ + { + "package_id": "gpu-a==0.1.0@path+[TEMP_DIR]/gpu_a-0.1.0-py3-none-any.whl" + }, + { + "package_id": "gpu-b==0.1.0@path+[TEMP_DIR]/gpu_b-0.1.0-py3-none-any.whl" + } + ], + "gpu.a": [ + { + "package_id": "gpu-a==0.1.0@path+[TEMP_DIR]/gpu_a-0.1.0-py3-none-any.whl" + } + ], + "gpu.b": [ + { + "package_id": "gpu-b==0.1.0@path+[TEMP_DIR]/gpu_b-0.1.0-py3-none-any.whl" + } + ], + "typing_extensions": [ + { + "package_id": "typing-extensions==0.1.0@path+[TEMP_DIR]/typing_extensions-0.1.0-py3-none-any.whl" + } + ] + }, + "members": [ + { + "name": "module-owner-root", + "path": "[TEMP_DIR]/", + "id": "module-owner-root==0.1.0@virtual+[TEMP_DIR]/" + } + ], + "resolution": { + "gpu-a==0.1.0@path+[TEMP_DIR]/gpu_a-0.1.0-py3-none-any.whl": { + "name": "gpu-a", + "version": "0.1.0", + "source": { + "path": "[TEMP_DIR]/gpu_a-0.1.0-py3-none-any.whl" + }, + "kind": "package", + "dependencies": [], + "wheels": [ + { + "hashes": { + "sha256": "[SHA256]" + }, + "filename": "gpu_a-0.1.0-py3-none-any.whl" + } + ] + }, + "gpu-b==0.1.0@path+[TEMP_DIR]/gpu_b-0.1.0-py3-none-any.whl": { + "name": "gpu-b", + "version": "0.1.0", + "source": { + "path": "[TEMP_DIR]/gpu_b-0.1.0-py3-none-any.whl" + }, + "kind": "package", + "dependencies": [], + "wheels": [ + { + "hashes": { + "sha256": "[SHA256]" + }, + "filename": "gpu_b-0.1.0-py3-none-any.whl" + } + ] + }, + "module-owner-root==0.1.0@virtual+[TEMP_DIR]/": { + "name": "module-owner-root", + "version": "0.1.0", + "source": { + "virtual": "[TEMP_DIR]/" + }, + "kind": "package", + "dependencies": [ + { + "id": "gpu-a==0.1.0@path+[TEMP_DIR]/gpu_a-0.1.0-py3-none-any.whl" + }, + { + "id": "gpu-b==0.1.0@path+[TEMP_DIR]/gpu_b-0.1.0-py3-none-any.whl" + }, + { + "id": "typing-extensions==0.1.0@path+[TEMP_DIR]/typing_extensions-0.1.0-py3-none-any.whl" + } + ] + }, + "typing-extensions==0.1.0@path+[TEMP_DIR]/typing_extensions-0.1.0-py3-none-any.whl": { + "name": "typing-extensions", + "version": "0.1.0", + "source": { + "path": "[TEMP_DIR]/typing_extensions-0.1.0-py3-none-any.whl" + }, + "kind": "package", + "dependencies": [], + "wheels": [ + { + "hashes": { + "sha256": "[SHA256]" + }, + "filename": "typing_extensions-0.1.0-py3-none-any.whl" + } + ] + } + } + } + + ----- stderr ----- + warning: The `uv workspace metadata` command is experimental and may change without warning. Pass `--preview-features workspace-metadata` to disable this warning. + Resolved 4 packages in [TIME] + "#); + + Ok(()) +} + +#[test] +fn workspace_metadata_module_owners_use_installed_package_id() -> Result<()> { + let context = uv_test::test_context!("3.12"); + + let py311_dir = context.temp_dir.child("py311"); + fs_err::create_dir_all(py311_dir.path())?; + let module_owner_311 = py311_dir.child("module_owner-0.1.0-py3-none-any.whl"); + write_wheel( + module_owner_311.path(), + "module-owner", + "module_owner-0.1.0", + &[("shared.py", "")], + )?; + + let py312_dir = context.temp_dir.child("py312"); + fs_err::create_dir_all(py312_dir.path())?; + let module_owner_312 = py312_dir.child("module_owner-0.1.0-py3-none-any.whl"); + write_wheel( + module_owner_312.path(), + "module-owner", + "module_owner-0.1.0", + &[("shared.py", "")], + )?; + + let module_owner_311_url = Url::from_file_path(module_owner_311.path()) + .map_err(|()| anyhow::anyhow!("failed to convert wheel path to file URL"))?; + let module_owner_312_url = Url::from_file_path(module_owner_312.path()) + .map_err(|()| anyhow::anyhow!("failed to convert wheel path to file URL"))?; + + context + .temp_dir + .child("pyproject.toml") + .write_str(&format!( + r#"[project] +name = "module-owner-root" +version = "0.1.0" +requires-python = ">=3.11" +dependencies = [ + "module-owner @ {module_owner_311_url} ; python_version < '3.12'", + "module-owner @ {module_owner_312_url} ; python_version >= '3.12'", +] +"# + ))?; + + let assert = context + .workspace_metadata() + .arg("--sync") + .assert() + .success(); + let metadata: serde_json::Value = serde_json::from_slice(&assert.get_output().stdout)?; + let module_owners = serde_json::to_string_pretty(&metadata["module_owners"])?; + + insta::with_settings!({ filters => context.filters() }, { + insta::assert_snapshot!(module_owners, @r#" + { + "shared": [ + { + "package_id": "module-owner==0.1.0@path+[TEMP_DIR]/py312/module_owner-0.1.0-py3-none-any.whl" + } + ] + } + "#); + }); + + Ok(()) +} + +#[test] +fn workspace_metadata_module_owners_ignore_stale_virtual_package() -> Result<()> { + let context = uv_test::test_context!("3.12"); + + let stale_owner = context + .temp_dir + .child("module_owner_root-0.1.0-py3-none-any.whl"); + write_wheel( + stale_owner.path(), + "module-owner-root", + "module_owner_root-0.1.0", + &[("stale.py", "")], + )?; + + context.temp_dir.child("pyproject.toml").write_str( + r#"[project] +name = "module-owner-root" +version = "0.1.0" +requires-python = ">=3.12" + +[tool.uv] +package = false +"#, + )?; + + context + .pip_install() + .arg(stale_owner.path()) + .assert() + .success(); + + let assert = context + .workspace_metadata() + .arg("--sync") + .assert() + .success(); + let metadata: serde_json::Value = serde_json::from_slice(&assert.get_output().stdout)?; + let module_owners = if let Some(module_owners) = metadata.get("module_owners") { + serde_json::to_string_pretty(module_owners)? + } else { + "".to_string() + }; + + insta::assert_snapshot!(module_owners, @""); + + Ok(()) +} + +#[test] +fn workspace_metadata_module_owners_failure_is_error() -> Result<()> { + let context = uv_test::test_context!("3.12"); + + let gpu_a = context.temp_dir.child("gpu_a-0.1.0-py3-none-any.whl"); + write_wheel(gpu_a.path(), "gpu-a", "gpu_a-0.1.0", &[("gpu/a.py", "")])?; + + let gpu_a_url = Url::from_file_path(gpu_a.path()) + .map_err(|()| anyhow::anyhow!("failed to convert wheel path to file URL"))?; + + context + .temp_dir + .child("pyproject.toml") + .write_str(&format!( + r#"[project] +name = "module-owner-root" +version = "0.1.0" +requires-python = ">=3.12" +dependencies = [ + "gpu-a @ {gpu_a_url}", +] +"# + ))?; + + context.lock().assert().success(); + fs_err::remove_file(gpu_a.path())?; + + uv_snapshot!(context.filters(), context.workspace_metadata().arg("--frozen").arg("--sync"), @r#" + success: false + exit_code: 2 + ----- stdout ----- + + ----- stderr ----- + warning: The `uv workspace metadata` command is experimental and may change without warning. Pass `--preview-features workspace-metadata` to disable this warning. + error: Failed to collect module owners + Caused by: Failed to determine installation plan + Caused by: Distribution not found at: file://[TEMP_DIR]/gpu_a-0.1.0-py3-none-any.whl + "#); + + Ok(()) +} + /// Test metadata for a root workspace (workspace with a root package). #[test] #[cfg(feature = "test-pypi")]