resolvelib: Stop resolving requirements one-by-one

pip_audit/_dependency_source/interface.py

@@ -61,32 +61,20 @@ class DependencyFixError(Exception):
 
 class DependencyResolver(ABC):
     """
-    Represents an abstract resolver of Python dependencies that takes a single
-    dependency and returns all of its transitive dependencies.
+    Represents an abstract resolver of Python dependencies that takes a list of
+    dependencies and returns all of their transitive dependencies.
 
     Concrete dependency sources may use a resolver as part of their
     implementation.
     """
 
     @abstractmethod
-    def resolve(self, req: Requirement) -> list[Dependency]:  # pragma: no cover
+    def resolve(self, reqs: list[Requirement]) -> list[Dependency]:
         """
-        Resolve a single `Requirement` into a list of `Dependency` instances.
+        Resolve a list of `Requirement`s into a list of resolved `Dependency`s.
         """
         raise NotImplementedError
 
-    def resolve_all(
-        self, reqs: Iterator[Requirement]
-    ) -> Iterator[tuple[Requirement, list[Dependency]]]:
-        """
-        Resolve a collection of `Requirement`s into their respective `Dependency` sets.
-
-        `DependencyResolver` implementations can override this implementation with
-        a more optimized one.
-        """
-        for req in reqs:
-            yield (req, self.resolve(req))
-
 
 class DependencyResolverError(Exception):
     """

pip_audit/_dependency_source/requirement.py

@@ -70,7 +70,7 @@ def __init__(
         self._require_hashes = require_hashes
         self._no_deps = no_deps
         self.state = state
-        self._dep_cache: dict[Path, dict[Requirement, set[Dependency]]] = {}
+        self._dep_cache: dict[Path, set[Dependency]] = {}
 
     def collect(self) -> Iterator[Dependency]:
         """
@@ -113,7 +113,7 @@ def collect(self) -> Iterator[Dependency]:
                         req_names.add(req.name)
                         reqs.append(req)
 
-                for _, dep in self._collect_cached_deps(filename, reqs):
+                for dep in self._collect_cached_deps(filename, reqs):
                     if dep in collected:
                         continue
                     collected.add(dep)
@@ -205,9 +205,12 @@ def _fix_file(self, filename: Path, fix_version: ResolvedFixVersion) -> None:
                         r for r in reqs if isinstance(r, InstallRequirement)
                     ]
                     origin_reqs: set[Requirement] = set()
-                    for req, dep in self._collect_cached_deps(filename, list(installed_reqs)):
+                    for dep in self._collect_cached_deps(filename, list(installed_reqs)):
+                        # NOTE(alex): Now we can't map requirements to dependencies, this is broken.
+                        # We'll have to figure out to reconstruct this information.
                         if fix_version.dep == dep:
-                            origin_reqs.add(req)
+                            # origin_reqs.add(req)
+                            pass
                     if origin_reqs:
                         logger.warning(
                             "added fixed subdependency explicitly to requirements file "
@@ -292,19 +295,17 @@ def _build_hash_options_mapping(self, hash_options: list[str]) -> dict[str, list
 
     def _collect_cached_deps(
         self, filename: Path, reqs: list[InstallRequirement]
-    ) -> Iterator[tuple[Requirement, Dependency]]:
+    ) -> Iterator[Dependency]:
         """
         Collect resolved dependencies for a given requirements file, retrieving them from the
         dependency cache if possible.
         """
         # See if we've already have cached dependencies for this file
         cached_deps_for_file = self._dep_cache.get(filename, None)
         if cached_deps_for_file is not None:
-            for req, deps in cached_deps_for_file.items():
-                for dep in deps:
-                    yield req, dep
+            yield from cached_deps_for_file
 
-        new_cached_deps_for_file: dict[Requirement, set[Dependency]] = dict()
+        new_cached_deps_for_file: set[Dependency] = set()
 
         # There are three cases where we skip dependency resolution:
         #
@@ -318,27 +319,22 @@ def _collect_cached_deps(
             for req, dep in self._collect_preresolved_deps(
                 iter(reqs), require_hashes=require_hashes
             ):
-                if req not in new_cached_deps_for_file:
-                    new_cached_deps_for_file[req] = set()
-                new_cached_deps_for_file[req].add(dep)
-                yield req, dep
+                new_cached_deps_for_file.add(dep)
+                yield dep
         else:
             # Invoke the dependency resolver to turn requirements into dependencies
             req_values: list[Requirement] = [r.req for r in reqs]
-            for req, resolved_deps in self._resolver.resolve_all(iter(req_values)):
-                for dep in resolved_deps:
-                    if req not in new_cached_deps_for_file:
-                        new_cached_deps_for_file[req] = set()
-                    new_cached_deps_for_file[req].add(dep)
-
-                    if dep.is_skipped():  # pragma: no cover
-                        dep = cast(SkippedDependency, dep)
-                        self.state.update_state(f"Skipping {dep.name}: {dep.skip_reason}")
-                    else:
-                        dep = cast(ResolvedDependency, dep)
-                        self.state.update_state(f"Collecting {dep.name} ({dep.version})")
-
-                    yield req, dep
+            for dep in self._resolver.resolve(req_values):
+                new_cached_deps_for_file.add(dep)
+
+                if dep.is_skipped():  # pragma: no cover
+                    dep = cast(SkippedDependency, dep)
+                    self.state.update_state(f"Skipping {dep.name}: {dep.skip_reason}")
+                else:
+                    dep = cast(ResolvedDependency, dep)
+                    self.state.update_state(f"Collecting {dep.name} ({dep.version})")
+
+                yield dep
 
         # Cache the collected dependencies
         self._dep_cache[filename] = new_cached_deps_for_file

pip_audit/_dependency_source/resolvelib/pypi_provider.py

@@ -31,6 +31,7 @@
 from resolvelib.resolvers import RequirementInformation
 
 from pip_audit._cache import caching_session
+from pip_audit._service import SkippedDependency
 from pip_audit._state import AuditState
 from pip_audit._util import python_version
 from pip_audit._virtual_env import VirtualEnv, VirtualEnvError
@@ -327,6 +328,7 @@ def __init__(
         self.timeout = timeout
         self.session = caching_session(cache_dir, use_pip=True)
         self._state = state
+        self.skip_deps: list[SkippedDependency] = []
 
     def identify(self, requirement_or_candidate: Requirement | Candidate) -> str:
         """
@@ -371,25 +373,31 @@ def find_matches(
         # Need to pass the extras to the search, so they
         # are added to the candidate at creation - we
         # treat candidates as immutable once created.
-        candidates = sorted(
-            [
-                candidate
-                for candidate in get_project_from_indexes(
-                    self.index_urls, self.session, identifier, extras, self.timeout, self._state
-                )
-                if candidate.version not in bad_versions
-                and all(candidate.version in r.specifier for r in requirements)
-                # HACK(ww): Additionally check that each candidate's name matches the
-                # expected project name (identifier).
-                # This technically shouldn't be required, but parsing distribution names
-                # from package indices is imprecise/unreliable when distribution filenames
-                # are PEP 440 compliant but not normalized.
-                # See: https://github.com/pypa/packaging/issues/527
-                and candidate.name == identifier
-            ],
-            key=attrgetter("version", "is_wheel"),
-            reverse=True,
-        )
+        try:
+            candidates = sorted(
+                [
+                    candidate
+                    for candidate in get_project_from_indexes(
+                        self.index_urls, self.session, identifier, extras, self.timeout, self._state
+                    )
+                    if candidate.version not in bad_versions
+                    and all(candidate.version in r.specifier for r in requirements)
+                    # HACK(ww): Additionally check that each candidate's name matches the
+                    # expected project name (identifier).
+                    # This technically shouldn't be required, but parsing distribution names
+                    # from package indices is imprecise/unreliable when distribution filenames
+                    # are PEP 440 compliant but not normalized.
+                    # See: https://github.com/pypa/packaging/issues/527
+                    and candidate.name == identifier
+                ],
+                key=attrgetter("version", "is_wheel"),
+                reverse=True,
+            )
+        except PyPINotFoundError as e:
+            skip_reason = str(e)
+            logger.debug(skip_reason)
+            self.skip_deps.append(SkippedDependency(name=identifier, skip_reason=skip_reason))
+            return
 
         # If we have multiple candidates for a single version and some are wheels,
         # yield only the wheels. This keeps us from wasting a large amount of

pip_audit/_dependency_source/resolvelib/resolvelib.py

@@ -17,7 +17,7 @@
 from pip_audit._service.interface import Dependency, ResolvedDependency, SkippedDependency
 from pip_audit._state import AuditState
 
-from .pypi_provider import PyPINotFoundError, PyPIProvider
+from .pypi_provider import PyPIProvider
 
 logger = logging.getLogger(__name__)
 
@@ -58,29 +58,36 @@ def __init__(
         self.resolver: Resolver = Resolver(self.provider, self.reporter)
         self._skip_editable = skip_editable
 
-    def resolve(self, req: Requirement) -> list[Dependency]:
+    def resolve(self, reqs: list[Requirement]) -> list[Dependency]:
         """
         Resolve the given `Requirement` into a `Dependency` list.
         """
 
-        # HACK: `resolve` takes both `packaging.Requirement` and `pip_api.Requirement`,
-        # since the latter is a subclass. But only the latter knows whether the
-        # requirement is editable, so we need to check for it here.
-        if isinstance(req, ParsedRequirement):
-            if req.editable and self._skip_editable:
-                return [
-                    SkippedDependency(name=req.name, skip_reason="requirement marked as editable")
-                ]
-
         deps: list[Dependency] = []
+
+        reqs_to_resolve: list[Requirement] = []
+        for req in reqs:
+            # HACK: `resolve` takes both `packaging.Requirement` and `pip_api.Requirement`,
+            # since the latter is a subclass. But only the latter knows whether the
+            # requirement is editable, so we need to check for it here.
+            if isinstance(req, ParsedRequirement):
+                if req.editable and self._skip_editable:
+                    deps.append(
+                        SkippedDependency(
+                            name=req.name, skip_reason="requirement marked as editable"
+                        )
+                    )
+                continue
+            reqs_to_resolve.append(req)
+
         try:
-            result = self.resolver.resolve([req])
-        except PyPINotFoundError as e:
-            skip_reason = str(e)
-            logger.debug(skip_reason)
-            return [SkippedDependency(name=req.name, skip_reason=skip_reason)]
+            result = self.resolver.resolve(reqs_to_resolve)
         except HTTPError as e:
             raise ResolveLibResolverError("failed to resolve dependencies") from e
+
+        # If the provider encountered any dependencies it couldn't find on PyPI, they'll be here.
+        deps.extend(self.provider.skip_deps)
+
         for name, candidate in result.mapping.items():
             deps.append(ResolvedDependency(name, candidate.version))
         return deps