Patch piptools to use current environment python

- Fixes #2088, #2234, #1901
- Fully leverage piptools' compile functionality by using constraints
  in the same `RequirementSet` during resolution
- Use `PIP_PYTHON_PATH` for compatibility check to filter out
  `requires_python` markers
- Fix vcs resolution
- Update JSON API endpoints
- Enhance resolution for editable dependencies
- Minor fix for adding packages to pipfiles

Signed-off-by: Dan Ryan <[email protected]>

Author: techalchemy

pipenv/patched/piptools/repositories/pypi.py

@@ -4,6 +4,7 @@
 
 import hashlib
 import os
+import sys
 from contextlib import contextmanager
 from shutil import rmtree
 
@@ -20,14 +21,17 @@
     SafeFileCache,
 )
 
-from notpip._vendor.packaging.requirements import InvalidRequirement
+from notpip._vendor.packaging.requirements import InvalidRequirement, Requirement
+from notpip._vendor.packaging.version import Version, InvalidVersion, parse as parse_version
+from notpip._vendor.packaging.specifiers import SpecifierSet
 from notpip._vendor.pyparsing import ParseException
 
 from ..cache import CACHE_DIR
 from pipenv.environments import PIPENV_CACHE_DIR
 from ..exceptions import NoCandidateFound
-from ..utils import (fs_str, is_pinned_requirement, lookup_table,
-                     make_install_requirement)
+from ..utils import (fs_str, is_pinned_requirement, lookup_table, as_tuple, key_from_req,
+                     make_install_requirement, format_requirement, dedup)
+
 from .base import BaseRepository
 
 
@@ -159,7 +163,15 @@ def find_best_match(self, ireq, prereleases=None):
         if ireq.editable:
             return ireq  # return itself as the best match
 
-        all_candidates = self.find_all_candidates(ireq.name)
+        _all_candidates = self.find_all_candidates(ireq.name)
+        all_candidates = []
+        py_version = parse_version(os.environ.get('PIP_PYTHON_VERSION', str(sys.version_info[:3])))
+        for c in _all_candidates:
+            if c.requires_python:
+                python_specifier = SpecifierSet(c.requires_python)
+                if not python_specifier.contains(py_version):
+                    continue
+            all_candidates.append(c)
         candidates_by_version = lookup_table(all_candidates, key=lambda c: c.version, unique=True)
         try:
             matching_versions = ireq.specifier.filter((candidate.version for candidate in all_candidates),
@@ -194,11 +206,12 @@ def gen(ireq):
                 r = self.session.get(url)
 
                 # TODO: Latest isn't always latest.
-                latest = list(r.json()['releases'].keys())[-1]
-                if str(ireq.req.specifier) == '=={0}'.format(latest):
-                    latest_url = 'https://pypi.org/pypi/{0}/{1}/json'.format(ireq.req.name, latest)
-                    latest_requires = self.session.get(latest_url)
-                    for requires in latest_requires.json().get('info', {}).get('requires_dist', {}):
+                releases = list(r.json()['releases'].keys())
+                match = [r for r in releases if '=={0}'.format(r) == str(ireq.req.specifier)]
+                if match:
+                    release_url = 'https://pypi.org/pypi/{0}/{1}/json'.format(ireq.req.name, match[0])
+                    release_requires = self.session.get(release_url)
+                    for requires in release_requires.json().get('info', {}).get('requires_dist', {}):
                         i = InstallRequirement.from_line(requires)
 
                         if 'extra' not in repr(i.markers):
@@ -245,7 +258,10 @@ def get_legacy_dependencies(self, ireq):
                     setup_requires = self.finder.get_extras_links(
                         dist.get_metadata_lines('requires.txt')
                     )
-            except TypeError:
+                ireq.version = dist.version
+                ireq.project_name = dist.project_name
+                ireq.req = dist.as_requirement()
+            except (TypeError, ValueError):
                 pass
 
         if ireq not in self._dependencies_cache:

pipenv/project.py

@@ -20,7 +20,7 @@
     from pathlib2 import Path
 
 from .cmdparse import Script
-from .vendor.requirementslib import Requirement
+from .vendor.requirementslib.requirements import Requirement
 from .utils import (
     atomic_open_for_write,
     mkdir_p,
@@ -728,8 +728,8 @@ def add_package_to_pipfile(self, package_name, dev=False):
         # Read and append Pipfile.
         p = self.parsed_pipfile
         # Don't re-capitalize file URLs or VCSs.
-        package = Requirement.from_line(package_name)
-        converted = first(package.as_pipfile().values())
+        package = Requirement.from_line(package_name.strip())
+        _, converted = package.pipfile_entry
         key = 'dev-packages' if dev else 'packages'
         # Set empty group if it doesn't exist yet.
         if key not in p:

pipenv/utils.py

@@ -221,6 +221,7 @@ def actually_resolve_reps(
 ):
     from .patched.notpip._internal import basecommand
     from .patched.notpip._internal.req import parse_requirements
+    from .patched.notpip._internal.req.req_install import InstallRequirement
     from .patched.notpip._vendor import requests as pip_requests
     from .patched.notpip._internal.exceptions import DistributionNotFound
     from .patched.notpip._vendor.requests.exceptions import HTTPError
@@ -236,48 +237,37 @@ class PipCommand(basecommand.Command):
         name = 'PipCommand'
 
     constraints = []
-    tmpfile_constraints = []
     cleanup_req_dir = False
     if not req_dir:
         req_dir = TemporaryDirectory(suffix='-requirements', prefix='pipenv-')
         cleanup_req_dir = True
     for dep in deps:
         if dep:
-            if dep.startswith('-e '):
-                constraint = req.InstallRequirement.from_editable(
-                    dep[len('-e '):]
-                )
-            else:
-                tmpfile_constraints.append(dep)
-                req = Requirement.from_line(dep)
-            # extra_constraints = []
+            url = None
             if ' -i ' in dep:
-                index_lookup[req.name] = project.get_source(
-                    url=dep.split(' -i ')[1]
-                ).get(
-                    'name'
-                )
-            if dep.markers:
-                markers_lookup[dep.name] = str(
-                    dep.markers_as_pip
-                ).replace(
-                    '"', "'"
-                )
-            constraints.append(req)
+                dep, url = dep.split(' -i ')
+            req = Requirement.from_line(dep)
+            _line = req.as_line()
+            constraints.append(_line)
+            # extra_constraints = []
+            if url:
+                index_lookup[req.name] = project.get_source(url=url).get('name')
+            if req.markers:
+                markers_lookup[req.name] = req.markers_as_pip
     constraints_file = None
-    with NamedTemporaryFile(mode='w', prefix='pipenv-', suffix='-constraints.txt', dir=req_dir.name, delete=False) as f:
-        f.write('\n'.join(tmpfile_constraints))
-        constraints_file = f.name
     pip_command = get_pip_command()
     pip_args = []
     if sources:
         pip_args = prepare_pip_source_args(sources, pip_args)
+    with NamedTemporaryFile(mode='w', prefix='pipenv-', suffix='-constraints.txt', dir=req_dir.name, delete=False) as f:
+        f.write(u'\n'.join([_constraint for _constraint in constraints]))
+        constraints_file = f.name
     if verbose:
         print('Using pip: {0}'.format(' '.join(pip_args)))
     pip_options, _ = pip_command.parse_args(pip_args)
     session = pip_command._build_session(pip_options)
     pypi = PyPIRepository(
-        pip_options=pip_options, use_json=False, session=session
+        pip_options=pip_options, use_json=True, session=session
     )
     if verbose:
         logging.log.verbose = True
@@ -1138,15 +1128,15 @@ def install_or_update_vcs(vcs_obj, src_dir, name, rev=None):
 
 
 def get_vcs_deps(project, pip_freeze=None, which=None, verbose=False, clear=False, pre=False, allow_global=False, dev=False):
-    from ._compat import vcs
+    from .patched.notpip._internal.vcs import VcsSupport
     section = 'vcs_dev_packages' if dev else 'vcs_packages'
     lines = []
     lockfiles = []
     try:
         packages = getattr(project, section)
     except AttributeError:
         return [], []
-    vcs_registry = vcs()
+    vcs_registry = VcsSupport
     vcs_uri_map = {
         extract_uri_from_vcs_dep(v): {'name': k, 'ref': v.get('ref')}
         for k, v in packages.items()
@@ -1162,13 +1152,15 @@ def get_vcs_deps(project, pip_freeze=None, which=None, verbose=False, clear=Fals
         pipfile_rev = vcs_uri_map[_vcs_match]['ref']
         src_dir = os.environ.get('PIP_SRC', os.path.join(project.virtualenv_location, 'src'))
         mkdir_p(src_dir)
+        pipfile_req = Requirement.from_pipfile(pipfile_name, [], packages[pipfile_name])
         names = {pipfile_name}
-        _pip_uri = line.lstrip('-e ')
-        backend_name = str(_pip_uri.split('+', 1)[0])
-        backend = vcs_registry._registry[first(b for b in vcs_registry if b == backend_name)]
-        __vcs = backend(url=_pip_uri)
-
+        backend = vcs_registry()._registry.get(pipfile_req.vcs)
+        # TODO: Why doesn't pip freeze list 'git+git://' formatted urls?
+        if line.startswith('-e ') and not '{0}+'.format(pipfile_req.vcs) in line:
+            line = line.replace('-e ', '-e {0}+'.format(pipfile_req.vcs))
         installed = Requirement.from_line(line)
+        __vcs = backend(url=installed.req.uri)
+
         names.add(installed.normalized_name)
         locked_rev = None
         for _name in names:

tasks/vendoring/patches/patched/piptools.patch

@@ -19,10 +19,18 @@ index 4e6174c..75f9b49 100644
  # NOTE
  # We used to store the cache dir under ~/.pip-tools, which is not the
 diff --git a/pipenv/patched/piptools/repositories/pypi.py b/pipenv/patched/piptools/repositories/pypi.py
-index 1c4b943..8320e14 100644
+index 1c4b943..858d697 100644
 --- a/pipenv/patched/piptools/repositories/pypi.py
 +++ b/pipenv/patched/piptools/repositories/pypi.py
-@@ -15,10 +15,16 @@ from .._compat import (
+@@ -4,6 +4,7 @@ from __future__ import (absolute_import, division, print_function,
+ 
+ import hashlib
+ import os
++import sys
+ from contextlib import contextmanager
+ from shutil import rmtree
+ 
+@@ -15,13 +16,22 @@ from .._compat import (
      Wheel,
      FAVORITE_HASH,
      TemporaryDirectory,
@@ -32,15 +40,23 @@ index 1c4b943..8320e14 100644
 +    SafeFileCache,
  )
 
-+from pip._vendor.packaging.requirements import InvalidRequirement
++from pip._vendor.packaging.requirements import InvalidRequirement, Requirement
++from pip._vendor.packaging.version import Version, InvalidVersion, parse as parse_version
++from pip._vendor.packaging.specifiers import SpecifierSet
 +from pip._vendor.pyparsing import ParseException
 +
  from ..cache import CACHE_DIR
 +from pipenv.environments import PIPENV_CACHE_DIR
  from ..exceptions import NoCandidateFound
- from ..utils import (fs_str, is_pinned_requirement, lookup_table,
-                      make_install_requirement)
-@@ -37,6 +43,40 @@ except ImportError:
+-from ..utils import (fs_str, is_pinned_requirement, lookup_table,
+-                     make_install_requirement)
++from ..utils import (fs_str, is_pinned_requirement, lookup_table, as_tuple, key_from_req,
++                     make_install_requirement, format_requirement, dedup)
++
+ from .base import BaseRepository
+ 
+ 
+@@ -37,6 +47,40 @@ except ImportError:
      from pip.wheel import WheelCache
 
 
@@ -81,7 +97,7 @@ index 1c4b943..8320e14 100644
  class PyPIRepository(BaseRepository):
      DEFAULT_INDEX_URL = PyPI.simple_url
 
-@@ -46,10 +86,11 @@ class PyPIRepository(BaseRepository):
+@@ -46,10 +90,11 @@ class PyPIRepository(BaseRepository):
      config), but any other PyPI mirror can be used if index_urls is
      changed/configured on the Finder.
      """
@@ -95,7 +111,7 @@ index 1c4b943..8320e14 100644
 
          index_urls = [pip_options.index_url] + pip_options.extra_index_urls
          if pip_options.no_index:
-@@ -74,11 +115,15 @@ class PyPIRepository(BaseRepository):
+@@ -74,11 +119,15 @@ class PyPIRepository(BaseRepository):
          # of all secondary dependencies for the given requirement, so we
          # only have to go to disk once for each requirement
          self._dependencies_cache = {}
@@ -113,9 +129,20 @@ index 1c4b943..8320e14 100644
 
      def freshen_build_caches(self):
          """
-@@ -116,8 +161,11 @@ class PyPIRepository(BaseRepository):
- 
-         all_candidates = self.find_all_candidates(ireq.name)
+@@ -114,10 +163,21 @@ class PyPIRepository(BaseRepository):
+         if ireq.editable:
+             return ireq  # return itself as the best match
+ 
+-        all_candidates = self.find_all_candidates(ireq.name)
++        _all_candidates = self.find_all_candidates(ireq.name)
++        all_candidates = []
++        py_version = parse_version(os.environ.get('PIP_PYTHON_VERSION', str(sys.version_info[:3])))
++        for c in _all_candidates:
++            if c.requires_python:
++                python_specifier = SpecifierSet(c.requires_python)
++                if not python_specifier.contains(py_version):
++                    continue
++            all_candidates.append(c)
          candidates_by_version = lookup_table(all_candidates, key=lambda c: c.version, unique=True)
 -        matching_versions = ireq.specifier.filter((candidate.version for candidate in all_candidates),
 +        try:
@@ -126,7 +153,7 @@ index 1c4b943..8320e14 100644
 
          # Reuses pip's internal candidate sort key to sort
          matching_candidates = [candidates_by_version[ver] for ver in matching_versions]
-@@ -126,11 +174,60 @@ class PyPIRepository(BaseRepository):
+@@ -126,11 +186,61 @@ class PyPIRepository(BaseRepository):
          best_candidate = max(matching_candidates, key=self.finder._candidate_sort_key)
 
          # Turn the candidate into a pinned InstallRequirement
@@ -153,11 +180,12 @@ index 1c4b943..8320e14 100644
 +                r = self.session.get(url)
 +
 +                # TODO: Latest isn't always latest.
-+                latest = list(r.json()['releases'].keys())[-1]
-+                if str(ireq.req.specifier) == '=={0}'.format(latest):
-+                    latest_url = 'https://pypi.org/pypi/{0}/{1}/json'.format(ireq.req.name, latest)
-+                    latest_requires = self.session.get(latest_url)
-+                    for requires in latest_requires.json().get('info', {}).get('requires_dist', {}):
++                releases = list(r.json()['releases'].keys())
++                match = [r for r in releases if '=={0}'.format(r) == str(ireq.req.specifier)]
++                if match:
++                    release_url = 'https://pypi.org/pypi/{0}/{1}/json'.format(ireq.req.name, match[0])
++                    release_requires = self.session.get(release_url)
++                    for requires in release_requires.json().get('info', {}).get('requires_dist', {}):
 +                        i = InstallRequirement.from_line(requires)
 +
 +                        if 'extra' not in repr(i.markers):
@@ -190,7 +218,7 @@ index 1c4b943..8320e14 100644
          """
          Given a pinned or an editable InstallRequirement, returns a set of
          dependencies (also InstallRequirements, but not necessarily pinned).
-@@ -139,6 +236,18 @@ class PyPIRepository(BaseRepository):
+@@ -139,6 +249,21 @@ class PyPIRepository(BaseRepository):
          if not (ireq.editable or is_pinned_requirement(ireq)):
              raise TypeError('Expected pinned or editable InstallRequirement, got {}'.format(ireq))
 
@@ -203,13 +231,16 @@ index 1c4b943..8320e14 100644
 +                    setup_requires = self.finder.get_extras_links(
 +                        dist.get_metadata_lines('requires.txt')
 +                    )
-+            except TypeError:
++                ireq.version = dist.version
++                ireq.project_name = dist.project_name
++                ireq.req = dist.as_requirement()
++            except (TypeError, ValueError):
 +                pass
 +
          if ireq not in self._dependencies_cache:
              if ireq.editable and (ireq.source_dir and os.path.exists(ireq.source_dir)):
                  # No download_dir for locally available editable requirements.
-@@ -164,11 +273,14 @@ class PyPIRepository(BaseRepository):
+@@ -164,11 +289,14 @@ class PyPIRepository(BaseRepository):
                      download_dir=download_dir,
                      wheel_download_dir=self._wheel_download_dir,
                      session=self.session,
@@ -226,7 +257,7 @@ index 1c4b943..8320e14 100644
                  )
              except TypeError:
                  # Pip >= 10 (new resolver!)
-@@ -190,14 +302,44 @@ class PyPIRepository(BaseRepository):
+@@ -190,14 +318,44 @@ class PyPIRepository(BaseRepository):
                      upgrade_strategy="to-satisfy-only",
                      force_reinstall=False,
                      ignore_dependencies=False,
@@ -273,7 +304,7 @@ index 1c4b943..8320e14 100644
              reqset.cleanup_files()
          return set(self._dependencies_cache[ireq])
 
-@@ -224,17 +366,10 @@ class PyPIRepository(BaseRepository):
+@@ -224,17 +382,10 @@ class PyPIRepository(BaseRepository):
          matching_candidates = candidates_by_version[matching_versions[0]]
 
          return {