resolvelib: Stop resolving requirements one-by-one

pip_audit/_dependency_source/interface.py

@@ -61,32 +61,20 @@ class DependencyFixError(Exception):
 
 class DependencyResolver(ABC):
     """
-    Represents an abstract resolver of Python dependencies that takes a single
-    dependency and returns all of its transitive dependencies.
+    Represents an abstract resolver of Python dependencies that takes a list of
+    dependencies and returns all of their transitive dependencies.
 
     Concrete dependency sources may use a resolver as part of their
     implementation.
     """
 
     @abstractmethod
-    def resolve(self, req: Requirement) -> list[Dependency]:  # pragma: no cover
+    def resolve(self, reqs: list[Requirement]) -> list[Dependency]:
         """
-        Resolve a single `Requirement` into a list of `Dependency` instances.
+        Resolve a list of `Requirement`s into a list of resolved `Dependency`s.
         """
         raise NotImplementedError
 
-    def resolve_all(
-        self, reqs: Iterator[Requirement]
-    ) -> Iterator[tuple[Requirement, list[Dependency]]]:
-        """
-        Resolve a collection of `Requirement`s into their respective `Dependency` sets.
-
-        `DependencyResolver` implementations can override this implementation with
-        a more optimized one.
-        """
-        for req in reqs:
-            yield (req, self.resolve(req))
-
 
 class DependencyResolverError(Exception):
     """

pip_audit/_dependency_source/pyproject.py

@@ -75,7 +75,7 @@ def collect(self) -> Iterator[Dependency]:
 
             reqs: list[Requirement] = [Requirement(dep) for dep in deps]
             try:
-                for _, deps in self.resolver.resolve_all(iter(reqs)):
+                for deps in self.resolver.resolve(reqs):
                     for dep in deps:
                         # Don't allow duplicate dependencies to be returned
                         if dep in collected:

pip_audit/_dependency_source/requirement.py

@@ -9,6 +9,7 @@
 import re
 import shutil
 from contextlib import ExitStack
+from dataclasses import dataclass, field
 from pathlib import Path
 from tempfile import NamedTemporaryFile
 from typing import IO, Iterator, cast
@@ -70,7 +71,7 @@ def __init__(
         self._require_hashes = require_hashes
         self._no_deps = no_deps
         self.state = state
-        self._dep_cache: dict[Path, dict[Requirement, set[Dependency]]] = {}
+        self._dep_cache: dict[Path, set[Dependency]] = {}
 
     def collect(self) -> Iterator[Dependency]:
         """
@@ -113,7 +114,7 @@ def collect(self) -> Iterator[Dependency]:
                         req_names.add(req.name)
                         reqs.append(req)
 
-                for _, dep in self._collect_cached_deps(filename, reqs):
+                for dep in self._collect_cached_deps(filename, reqs):
                     if dep in collected:
                         continue
                     collected.add(dep)
@@ -201,20 +202,17 @@ def _fix_file(self, filename: Path, fix_version: ResolvedFixVersion) -> None:
             # another.
             try:
                 if not fixed:
-                    installed_reqs: list[InstallRequirement] = [
-                        r for r in reqs if isinstance(r, InstallRequirement)
-                    ]
-                    origin_reqs: set[Requirement] = set()
-                    for req, dep in self._collect_cached_deps(filename, list(installed_reqs)):
-                        if fix_version.dep == dep:
-                            origin_reqs.add(req)
-                    if origin_reqs:
+                    req_dep = cast(RequirementDependency, fix_version.dep)
+                    if req_dep.origin_reqs:
                         logger.warning(
                             "added fixed subdependency explicitly to requirements file "
                             f"{filename}: {fix_version.dep.canonical_name}"
                         )
                         origin_reqs_formatted = ",".join(
-                            [str(req) for req in sorted(list(origin_reqs), key=lambda x: x.name)]
+                            [
+                                str(req)
+                                for req in sorted(list(req_dep.origin_reqs), key=lambda x: x.name)
+                            ]
                         )
                         print(
                             f"    # pip-audit: subdependency fixed via {origin_reqs_formatted}",
@@ -292,19 +290,17 @@ def _build_hash_options_mapping(self, hash_options: list[str]) -> dict[str, list
 
     def _collect_cached_deps(
         self, filename: Path, reqs: list[InstallRequirement]
-    ) -> Iterator[tuple[Requirement, Dependency]]:
+    ) -> Iterator[Dependency]:
         """
         Collect resolved dependencies for a given requirements file, retrieving them from the
         dependency cache if possible.
         """
         # See if we've already have cached dependencies for this file
         cached_deps_for_file = self._dep_cache.get(filename, None)
         if cached_deps_for_file is not None:
-            for req, deps in cached_deps_for_file.items():
-                for dep in deps:
-                    yield req, dep
+            yield from cached_deps_for_file
 
-        new_cached_deps_for_file: dict[Requirement, set[Dependency]] = dict()
+        new_cached_deps_for_file: set[Dependency] = set()
 
         # There are three cases where we skip dependency resolution:
         #
@@ -318,27 +314,22 @@ def _collect_cached_deps(
             for req, dep in self._collect_preresolved_deps(
                 iter(reqs), require_hashes=require_hashes
             ):
-                if req not in new_cached_deps_for_file:
-                    new_cached_deps_for_file[req] = set()
-                new_cached_deps_for_file[req].add(dep)
-                yield req, dep
+                new_cached_deps_for_file.add(dep)
+                yield dep
         else:
             # Invoke the dependency resolver to turn requirements into dependencies
             req_values: list[Requirement] = [r.req for r in reqs]
-            for req, resolved_deps in self._resolver.resolve_all(iter(req_values)):
-                for dep in resolved_deps:
-                    if req not in new_cached_deps_for_file:
-                        new_cached_deps_for_file[req] = set()
-                    new_cached_deps_for_file[req].add(dep)
+            for dep in self._resolver.resolve(req_values):
+                new_cached_deps_for_file.add(dep)
 
-                    if dep.is_skipped():  # pragma: no cover
-                        dep = cast(SkippedDependency, dep)
-                        self.state.update_state(f"Skipping {dep.name}: {dep.skip_reason}")
-                    else:
-                        dep = cast(ResolvedDependency, dep)
-                        self.state.update_state(f"Collecting {dep.name} ({dep.version})")
+                if dep.is_skipped():  # pragma: no cover
+                    dep = cast(SkippedDependency, dep)
+                    self.state.update_state(f"Skipping {dep.name}: {dep.skip_reason}")
+                else:
+                    dep = cast(ResolvedDependency, dep)
+                    self.state.update_state(f"Collecting {dep.name} ({dep.version})")
 
-                    yield req, dep
+                yield dep
 
         # Cache the collected dependencies
         self._dep_cache[filename] = new_cached_deps_for_file
@@ -354,3 +345,12 @@ class RequirementFixError(DependencyFixError):
     """A requirements-fixing specific `DependencyFixError`."""
 
     pass
+
+
+@dataclass(frozen=True)
+class RequirementDependency(ResolvedDependency):
+    """
+    Represents a fully resolved Python package from a requirements file.
+    """
+
+    origin_reqs: set[Requirement] = field(default_factory=set, hash=False)

pip_audit/_dependency_source/resolvelib/pypi_provider.py

@@ -31,6 +31,7 @@
 from resolvelib.resolvers import RequirementInformation
 
 from pip_audit._cache import caching_session
+from pip_audit._service import SkippedDependency
 from pip_audit._state import AuditState
 from pip_audit._util import python_version
 from pip_audit._virtual_env import VirtualEnv, VirtualEnvError
@@ -57,6 +58,7 @@ def __init__(
         url: str,
         extras: set[str],
         is_wheel: bool,
+        reqs: list[Requirement],
         session: CacheControl,
         timeout: int | None = None,
         state: AuditState = AuditState(),
@@ -71,6 +73,7 @@ def __init__(
         self.url = url
         self.extras = extras
         self.is_wheel = is_wheel
+        self.reqs = reqs
         self._session = session
         self._timeout = timeout
         self._state = state
@@ -193,6 +196,7 @@ def get_project_from_indexes(
     index_urls: list[str],
     session: CacheControl,
     project: str,
+    reqs: list[Requirement],
     extras: set[str],
     timeout: int | None,
     state: AuditState,
@@ -203,7 +207,9 @@ def get_project_from_indexes(
         # Not all indexes are guaranteed to have the project so this isn't an error
         # We should only return an error if it can't be found on ANY of the supplied index URLs
         try:
-            yield from get_project_from_index(index_url, session, project, extras, timeout, state)
+            yield from get_project_from_index(
+                index_url, session, project, reqs, extras, timeout, state
+            )
             project_found = True
         except PyPINotFoundError:
             pass
@@ -217,6 +223,7 @@ def get_project_from_index(
     index_url: str,
     session: CacheControl,
     project: str,
+    reqs: list[Requirement],
     extras: set[str],
     timeout: int | None,
     state: AuditState,
@@ -286,6 +293,7 @@ def get_project_from_index(
                 url=dist_url,
                 extras=extras,
                 is_wheel=is_wheel,
+                reqs=reqs,
                 timeout=timeout,
                 state=state,
                 session=session,
@@ -327,6 +335,7 @@ def __init__(
         self.timeout = timeout
         self.session = caching_session(cache_dir, use_pip=True)
         self._state = state
+        self.skip_deps: list[SkippedDependency] = []
 
     def identify(self, requirement_or_candidate: Requirement | Candidate) -> str:
         """
@@ -371,25 +380,37 @@ def find_matches(
         # Need to pass the extras to the search, so they
         # are added to the candidate at creation - we
         # treat candidates as immutable once created.
-        candidates = sorted(
-            [
-                candidate
-                for candidate in get_project_from_indexes(
-                    self.index_urls, self.session, identifier, extras, self.timeout, self._state
-                )
-                if candidate.version not in bad_versions
-                and all(candidate.version in r.specifier for r in requirements)
-                # HACK(ww): Additionally check that each candidate's name matches the
-                # expected project name (identifier).
-                # This technically shouldn't be required, but parsing distribution names
-                # from package indices is imprecise/unreliable when distribution filenames
-                # are PEP 440 compliant but not normalized.
-                # See: https://github.com/pypa/packaging/issues/527
-                and candidate.name == identifier
-            ],
-            key=attrgetter("version", "is_wheel"),
-            reverse=True,
-        )
+        try:
+            candidates = sorted(
+                [
+                    candidate
+                    for candidate in get_project_from_indexes(
+                        self.index_urls,
+                        self.session,
+                        identifier,
+                        requirements,
+                        extras,
+                        self.timeout,
+                        self._state,
+                    )
+                    if candidate.version not in bad_versions
+                    and all(candidate.version in r.specifier for r in requirements)
+                    # HACK(ww): Additionally check that each candidate's name matches the
+                    # expected project name (identifier).
+                    # This technically shouldn't be required, but parsing distribution names
+                    # from package indices is imprecise/unreliable when distribution filenames
+                    # are PEP 440 compliant but not normalized.
+                    # See: https://github.com/pypa/packaging/issues/527
+                    and candidate.name == identifier
+                ],
+                key=attrgetter("version", "is_wheel"),
+                reverse=True,
+            )
+        except PyPINotFoundError as e:
+            skip_reason = str(e)
+            logger.debug(skip_reason)
+            self.skip_deps.append(SkippedDependency(name=identifier, skip_reason=skip_reason))
+            return
 
         # If we have multiple candidates for a single version and some are wheels,
         # yield only the wheels. This keeps us from wasting a large amount of