Merge pull request #8588 from McSinyx/fast-deps

Use lazy wheel to obtain dep info for new resolver

Author: pradyunsg

news/8588.feature

@@ -0,0 +1,3 @@
+Allow the new resolver to obtain dependency information through wheels
+lazily downloaded using HTTP range requests.  To enable this feature,
+invoke ``pip`` with ``--use-feature=fast-deps``.

src/pip/_internal/cli/cmdoptions.py

@@ -913,7 +913,7 @@ def check_list_path_option(options):
     metavar='feature',
     action='append',
     default=[],
-    choices=['2020-resolver'],
+    choices=['2020-resolver', 'fast-deps'],
     help='Enable new functionality, that may be backward incompatible.',
 )  # type: Callable[..., Option]
 

src/pip/_internal/cli/req_command.py

@@ -271,6 +271,7 @@ def make_resolver(
                 force_reinstall=force_reinstall,
                 upgrade_strategy=upgrade_strategy,
                 py_version_info=py_version_info,
+                lazy_wheel='fast-deps' in options.features_enabled,
             )
         import pip._internal.resolution.legacy.resolver
         return pip._internal.resolution.legacy.resolver.Resolver(

src/pip/_internal/resolution/resolvelib/candidates.py

@@ -1,16 +1,22 @@
 import logging
 import sys
 
+from pip._vendor.contextlib2 import suppress
 from pip._vendor.packaging.specifiers import InvalidSpecifier, SpecifierSet
 from pip._vendor.packaging.utils import canonicalize_name
 from pip._vendor.packaging.version import Version
 
 from pip._internal.exceptions import HashError, MetadataInconsistent
+from pip._internal.network.lazy_wheel import (
+    HTTPRangeRequestUnsupported,
+    dist_from_wheel_url,
+)
 from pip._internal.req.constructors import (
     install_req_from_editable,
     install_req_from_line,
 )
 from pip._internal.req.req_install import InstallRequirement
+from pip._internal.utils.logging import indent_log
 from pip._internal.utils.misc import dist_is_editable, normalize_version_info
 from pip._internal.utils.packaging import get_requires_python
 from pip._internal.utils.typing import MYPY_CHECK_RUNNING
@@ -142,6 +148,7 @@ def __init__(
         self._name = name
         self._version = version
         self._dist = None  # type: Optional[Distribution]
+        self._prepared = False
 
     def __repr__(self):
         # type: () -> str
@@ -197,11 +204,23 @@ def _prepare_abstract_distribution(self):
         # type: () -> AbstractDistribution
         raise NotImplementedError("Override in subclass")
 
+    def _check_metadata_consistency(self):
+        # type: () -> None
+        """Check for consistency of project name and version of dist."""
+        # TODO: (Longer term) Rather than abort, reject this candidate
+        #       and backtrack. This would need resolvelib support.
+        dist = self._dist  # type: Distribution
+        name = canonicalize_name(dist.project_name)
+        if self._name is not None and self._name != name:
+            raise MetadataInconsistent(self._ireq, "name", dist.project_name)
+        version = dist.parsed_version
+        if self._version is not None and self._version != version:
+            raise MetadataInconsistent(self._ireq, "version", dist.version)
+
     def _prepare(self):
         # type: () -> None
-        if self._dist is not None:
+        if self._prepared:
             return
-
         try:
             abstract_dist = self._prepare_abstract_distribution()
         except HashError as e:
@@ -210,24 +229,36 @@ def _prepare(self):
 
         self._dist = abstract_dist.get_pkg_resources_distribution()
         assert self._dist is not None, "Distribution already installed"
+        self._check_metadata_consistency()
+        self._prepared = True
 
-        # TODO: (Longer term) Rather than abort, reject this candidate
-        #       and backtrack. This would need resolvelib support.
-        name = canonicalize_name(self._dist.project_name)
-        if self._name is not None and self._name != name:
-            raise MetadataInconsistent(
-                self._ireq, "name", self._dist.project_name,
-            )
-        version = self._dist.parsed_version
-        if self._version is not None and self._version != version:
-            raise MetadataInconsistent(
-                self._ireq, "version", self._dist.version,
-            )
+    def _fetch_metadata(self):
+        # type: () -> None
+        """Fetch metadata, using lazy wheel if possible."""
+        preparer = self._factory.preparer
+        use_lazy_wheel = self._factory.use_lazy_wheel
+        remote_wheel = self._link.is_wheel and not self._link.is_file
+        if use_lazy_wheel and remote_wheel and not preparer.require_hashes:
+            assert self._name is not None
+            logger.info('Collecting %s', self._ireq.req or self._ireq)
+            # If HTTPRangeRequestUnsupported is raised, fallback silently.
+            with indent_log(), suppress(HTTPRangeRequestUnsupported):
+                logger.info(
+                    'Obtaining dependency information from %s %s',
+                    self._name, self._version,
+                )
+                url = self._link.url.split('#', 1)[0]
+                session = preparer.downloader._session
+                self._dist = dist_from_wheel_url(self._name, url, session)
+                self._check_metadata_consistency()
+        if self._dist is None:
+            self._prepare()
 
     @property
     def dist(self):
         # type: () -> Distribution
-        self._prepare()
+        if self._dist is None:
+            self._fetch_metadata()
         return self._dist
 
     def _get_requires_python_specifier(self):

src/pip/_internal/resolution/resolvelib/factory.py

@@ -82,6 +82,7 @@ def __init__(
         ignore_installed,  # type: bool
         ignore_requires_python,  # type: bool
         py_version_info=None,  # type: Optional[Tuple[int, ...]]
+        lazy_wheel=False,  # type: bool
     ):
         # type: (...) -> None
         self._finder = finder
@@ -92,6 +93,7 @@ def __init__(
         self._use_user_site = use_user_site
         self._force_reinstall = force_reinstall
         self._ignore_requires_python = ignore_requires_python
+        self.use_lazy_wheel = lazy_wheel
 
         self._link_candidate_cache = {}  # type: Cache[LinkCandidate]
         self._editable_candidate_cache = {}  # type: Cache[EditableCandidate]

src/pip/_internal/resolution/resolvelib/resolver.py

@@ -49,8 +49,16 @@ def __init__(
         force_reinstall,  # type: bool
         upgrade_strategy,  # type: str
         py_version_info=None,  # type: Optional[Tuple[int, ...]]
+        lazy_wheel=False,  # type: bool
     ):
         super(Resolver, self).__init__()
+        if lazy_wheel:
+            logger.warning(
+                'pip is using lazily downloaded wheels using HTTP '
+                'range requests to obtain dependency information. '
+                'This experimental feature is enabled through '
+                '--use-feature=fast-deps and it is not ready for production.'
+            )
 
         assert upgrade_strategy in self._allowed_strategies
 
@@ -64,6 +72,7 @@ def __init__(
             ignore_installed=ignore_installed,
             ignore_requires_python=ignore_requires_python,
             py_version_info=py_version_info,
+            lazy_wheel=lazy_wheel,
         )
         self.ignore_dependencies = ignore_dependencies
         self.upgrade_strategy = upgrade_strategy

tests/data/packages/requiresPaste/pyproject.toml

@@ -0,0 +1,9 @@
+[build-system]
+requires = ['flit_core >=2,<4']
+build-backend = 'flit_core.buildapi'
+
+[tool.flit.metadata]
+module = 'requiresPaste'
+author = 'A. Random Developer'
+author-email = '[email protected]'
+requires = ['Paste==3.4.2']

tests/data/packages/requiresPaste/requiresPaste.py

@@ -0,0 +1,3 @@
+"""Module requiring Paste to test dependencies download of pip wheel."""
+
+__version__ = '3.1.4'

tests/functional/test_fast_deps.py

@@ -0,0 +1,50 @@
+import fnmatch
+import json
+from os.path import basename
+
+from pip._vendor.packaging.utils import canonicalize_name
+from pytest import mark
+
+
+def pip(script, command, requirement):
+    return script.pip(
+        command, '--prefer-binary', '--no-cache-dir',
+        '--use-feature=fast-deps', requirement,
+        allow_stderr_warning=True,
+    )
+
+
+def assert_installed(script, names):
+    list_output = json.loads(script.pip('list', '--format=json').stdout)
+    installed = {canonicalize_name(item['name']) for item in list_output}
+    assert installed.issuperset(map(canonicalize_name, names))
+
+
+@mark.network
+@mark.parametrize(('requirement', 'expected'), (
+    ('Paste==3.4.2', ('Paste', 'six')),
+    ('Paste[flup]==3.4.2', ('Paste', 'six', 'flup')),
+))
+def test_install_from_pypi(requirement, expected, script):
+    pip(script, 'install', requirement)
+    assert_installed(script, expected)
+
+
+@mark.network
+@mark.parametrize(('requirement', 'expected'), (
+    ('Paste==3.4.2', ('Paste-3.4.2-*.whl', 'six-*.whl')),
+    ('Paste[flup]==3.4.2', ('Paste-3.4.2-*.whl', 'six-*.whl', 'flup-*')),
+))
+def test_download_from_pypi(requirement, expected, script):
+    result = pip(script, 'download', requirement)
+    created = list(map(basename, result.files_created))
+    assert all(fnmatch.filter(created, f) for f in expected)
+
+
+@mark.network
+def test_build_wheel_with_deps(data, script):
+    result = pip(script, 'wheel', data.packages/'requiresPaste')
+    created = list(map(basename, result.files_created))
+    assert fnmatch.filter(created, 'requiresPaste-3.1.4-*.whl')
+    assert fnmatch.filter(created, 'Paste-3.4.2-*.whl')
+    assert fnmatch.filter(created, 'six-*.whl')