Merge pull request #2209 from pypa/bugfix/vcs-updates

Fix VCS locking and updating

pipenv/_compat.py

@@ -5,6 +5,7 @@
 operating systems, etc.
 """
 import functools
+import importlib
 import io
 import os
 import six
@@ -50,6 +51,24 @@ def detach(self):
     class ResourceWarning(Warning):
         pass
 
+# -*- coding=utf-8 -*-
+
+
+def pip_import(module_path, subimport=None, old_path=None):
+    internal = 'pip._internal.{0}'.format(module_path)
+    old_path = old_path or module_path
+    pip9 = 'pip.{0}'.format(old_path)
+    try:
+        _tmp = importlib.import_module(internal)
+    except ImportError:
+        _tmp = importlib.import_module(pip9)
+    if subimport:
+        return getattr(_tmp, subimport, _tmp)
+    return _tmp
+
+
+vcs = pip_import('vcs', 'VcsSupport')
+
 
 class TemporaryDirectory(object):
     """Create and return a temporary directory.  This has the same

pipenv/core.py

@@ -16,6 +16,7 @@
 import dotenv
 import delegator
 from .vendor import pexpect
+from first import first
 import pipfile
 from blindspin import spinner
 from requests.packages import urllib3
@@ -46,9 +47,11 @@
     is_star,
     rmtree,
     split_argument,
+    extract_uri_from_vcs_dep,
 )
 from ._compat import (
     TemporaryDirectory,
+    vcs
 )
 from .import pep508checker, progress
 from .environments import (
@@ -999,6 +1002,7 @@ def do_lock(
 ):
     """Executes the freeze functionality."""
     from notpip._vendor.distlib.markers import Evaluator
+    from .utils import get_vcs_deps
     allowed_marker_keys = ['markers'] + [k for k in Evaluator.allowed_values.keys()]
     cached_lockfile = {}
     if not pre:
@@ -1035,6 +1039,9 @@ def do_lock(
         if dev_package in project.packages:
             dev_packages[dev_package] = project.packages[dev_package]
     # Resolve dev-package dependencies, with pip-tools.
+    pip_freeze = delegator.run(
+        '{0} freeze'.format(escape_grouped_arguments(which_pip(allow_global=system)))
+    ).out
     deps = convert_deps_to_pip(
         dev_packages, project, r=False, include_index=True
     )
@@ -1066,24 +1073,14 @@ def do_lock(
             lockfile['develop'][dep['name']]['markers'] = dep['markers']
     # Add refs for VCS installs.
     # TODO: be smarter about this.
-    vcs_deps = convert_deps_to_pip(project.vcs_dev_packages, project, r=False)
-    pip_freeze = delegator.run(
-        '{0} freeze'.format(escape_grouped_arguments(which_pip(allow_global=system)))
-    ).out
-    if vcs_deps:
-        for line in pip_freeze.strip().split('\n'):
-            # if the line doesn't match a vcs dependency in the Pipfile,
-            # ignore it
-            if not any(dep in line for dep in vcs_deps):
-                continue
-
-            try:
-                installed = convert_deps_from_pip(line)
-                name = list(installed.keys())[0]
-                if is_vcs(installed[name]):
-                    lockfile['develop'].update(installed)
-            except IndexError:
-                pass
+    vcs_dev_lines, vcs_dev_lockfiles = get_vcs_deps(project, pip_freeze, which=which, verbose=verbose, clear=clear, pre=pre, allow_global=system, dev=True)
+    for lf in vcs_dev_lockfiles:
+        try:
+            name = first(lf.keys())
+        except AttributeError:
+            continue
+        if hasattr(lf[name], 'keys'):
+            lockfile['develop'].update(lf)
     if write:
         # Alert the user of progress.
         click.echo(
@@ -1132,23 +1129,15 @@ def do_lock(
             lockfile['default'][dep['name']]['markers'] = dep['markers']
     # Add refs for VCS installs.
     # TODO: be smarter about this.
-    vcs_deps = convert_deps_to_pip(project.vcs_packages, project, r=False)
-    if vcs_deps:
-        for line in pip_freeze.strip().split('\n'):
-            # if the line doesn't match a vcs dependency in the Pipfile,
-            # ignore it
-            if not any(dep in line for dep in vcs_deps):
-                continue
+    _vcs_deps, vcs_lockfiles = get_vcs_deps(project, pip_freeze, which=which, verbose=verbose, clear=clear, pre=pre, allow_global=system, dev=False)
+    for lf in vcs_lockfiles:
+        try:
+            name = first(lf.keys())
+        except AttributeError:
+            continue
+        if hasattr(lf[name], 'keys'):
+            lockfile['default'].update(lf)
 
-            try:
-                installed = convert_deps_from_pip(line)
-                name = list(installed.keys())[0]
-                if is_vcs(installed[name]):
-                    # Convert name to PEP 423 name.
-                    installed = {pep423_name(name): installed[name]}
-                    lockfile['default'].update(installed)
-            except IndexError:
-                pass
     # Support for --keep-outdated…
     if keep_outdated:
         for section_name, section in (

pipenv/project.py

@@ -497,41 +497,41 @@ def lockfile_exists(self):
     def lockfile_content(self):
         return self.load_lockfile()
 
-    @property
-    def editable_packages(self):
+    def _get_editable_packages(self, dev=False):
+        section = 'dev-packages' if dev else 'packages'
         packages = {
             k: v
-            for k, v in self.parsed_pipfile.get('packages', {}).items()
+            for k, v in self.parsed_pipfile.get(section, {}).items()
             if is_editable(v)
         }
         return packages
 
-    @property
-    def editable_dev_packages(self):
+    def _get_vcs_packages(self, dev=False):
+        section = 'dev-packages' if dev else 'packages'
         packages = {
             k: v
-            for k, v in self.parsed_pipfile.get('dev-packages', {}).items()
-            if is_editable(v)
+            for k, v in self.parsed_pipfile.get(section, {}).items()
+            if is_vcs(v) or is_vcs(k)
         }
-        return packages
+        return packages or {}
+
+    @property
+    def editable_packages(self):
+        return self._get_editable_packages(dev=False)
+
+    @property
+    def editable_dev_packages(self):
+        return self._get_editable_packages(dev=True)
 
     @property
     def vcs_packages(self):
         """Returns a list of VCS packages, for not pip-tools to consume."""
-        ps = {}
-        for k, v in self.parsed_pipfile.get('packages', {}).items():
-            if is_vcs(v) or is_vcs(k):
-                ps.update({k: v})
-        return ps
+        return self._get_vcs_packages(dev=False)
 
     @property
     def vcs_dev_packages(self):
         """Returns a list of VCS packages, for not pip-tools to consume."""
-        ps = {}
-        for k, v in self.parsed_pipfile.get('dev-packages', {}).items():
-            if is_vcs(v) or is_vcs(k):
-                ps.update({k: v})
-        return ps
+        return self._get_vcs_packages(dev=True)
 
     @property
     def all_packages(self):

pipenv/utils.py

@@ -1358,3 +1358,71 @@ def safe_expandvars(value):
     if isinstance(value, six.string_types):
         return os.path.expandvars(value)
     return value
+
+
+def extract_uri_from_vcs_dep(dep):
+    valid_keys = VCS_LIST + ('uri', 'file')
+    if hasattr(dep, 'keys'):
+        return first(dep[k] for k in valid_keys if k in dep) or None
+    return None
+
+
+def install_or_update_vcs(vcs_obj, src_dir, name, rev=None):    
+    target_dir = os.path.join(src_dir, name)
+    target_rev = vcs_obj.make_rev_options(rev)
+    if not os.path.exists(target_dir):
+        vcs_obj.obtain(target_dir)
+    vcs_obj.update(target_dir, target_rev)
+    return vcs_obj.get_revision(target_dir)
+
+
+def get_vcs_deps(project, pip_freeze=None, which=None, verbose=False, clear=False, pre=False, allow_global=False, dev=False):
+    from ._compat import vcs
+    section = 'vcs_dev_packages' if dev else 'vcs_packages'
+    lines = []
+    lockfiles = []
+    try:
+        packages = getattr(project, section)
+    except AttributeError:
+        return [], []
+    vcs_registry = vcs()
+    vcs_uri_map = {
+        extract_uri_from_vcs_dep(v): {'name': k, 'ref': v.get('ref')}
+        for k, v in packages.items()
+    }
+    for line in pip_freeze.strip().split('\n'):
+        # if the line doesn't match a vcs dependency in the Pipfile,
+        # ignore it
+        _vcs_match = first(_uri for _uri in vcs_uri_map.keys() if _uri in line)
+        if not _vcs_match:
+            continue
+
+        pipfile_name = vcs_uri_map[_vcs_match]['name']
+        pipfile_rev = vcs_uri_map[_vcs_match]['ref']
+        src_dir = os.environ.get('PIP_SRC', os.path.join(project.virtualenv_location, 'src'))
+        mkdir_p(src_dir)
+        names = {pipfile_name}
+        _pip_uri = line.lstrip('-e ')
+        backend_name = str(_pip_uri.split('+', 1)[0])
+        backend = vcs_registry._registry[first(b for b in vcs_registry if b == backend_name)]
+        __vcs = backend(url=_pip_uri)
+
+        installed = convert_deps_from_pip(line)
+        if not hasattr(installed, 'keys'):
+            pass
+        lock_name = first(installed.keys())
+        names.add(lock_name)
+        locked_rev = None
+        for _name in names:
+            locked_rev = install_or_update_vcs(__vcs, src_dir, _name, rev=pipfile_rev)
+        if is_vcs(installed[lock_name]):
+            installed[lock_name]['ref'] = locked_rev
+            lockfiles.append({pipfile_name: installed[lock_name]})
+        pipfile_srcdir = os.path.join(src_dir, pipfile_name)
+        lockfile_srcdir = os.path.join(src_dir, lock_name)
+        lines.append(line)
+        if os.path.exists(pipfile_srcdir):
+            lockfiles.extend(venv_resolve_deps(['-e {0}'.format(pipfile_srcdir)], which=which, verbose=verbose, project=project, clear=clear, pre=pre, allow_global=allow_global))
+        else:
+            lockfiles.extend(venv_resolve_deps(['-e {0}'.format(lockfile_srcdir)], which=which, verbose=verbose, project=project, clear=clear, pre=pre, allow_global=allow_global))
+    return lines, lockfiles

tests/integration/test_install_uri.py

@@ -2,6 +2,10 @@
 import os
 from flaky import flaky
 import delegator
+try:
+    from pathlib import Path
+except ImportError:
+    from pathlib2 import Path
 
 
 @pytest.mark.vcs
@@ -144,3 +148,25 @@ def test_install_local_vcs_not_in_lockfile(PipenvInstance, pip_src_dir):
         assert six_key in p.lockfile['default']
         # Make sure we didn't put six in the lockfile by accident as a vcs ref
         assert 'six' not in p.lockfile['default']
+
+
+@pytest.mark.vcs
+@pytest.mark.install
+@pytest.mark.needs_internet
+def test_get_vcs_refs(PipenvInstance, pip_src_dir):
+    with PipenvInstance(chdir=True) as p:
+        c = p.pipenv('install -e git+https://github.com/hynek/[email protected]#egg=structlog')
+        assert c.return_code == 0
+        assert 'structlog' in p.pipfile['packages']
+        assert 'structlog' in p.lockfile['default']
+        assert 'six' in p.lockfile['default']
+        assert p.lockfile['default']['structlog']['ref'] == 'a39f6906a268fb2f4c365042b31d0200468fb492'
+        pipfile = Path(p.pipfile_path)
+        new_content = pipfile.read_bytes().replace(b'16.1.0', b'18.1.0')
+        pipfile.write_bytes(new_content)
+        c = p.pipenv('lock')
+        assert c.return_code == 0
+        assert p.lockfile['default']['structlog']['ref'] == 'a73fbd3a9c3cafb11f43168582083f839b883034'
+        assert 'structlog' in p.pipfile['packages']
+        assert 'structlog' in p.lockfile['default']
+        assert 'six' in p.lockfile['default']