Merge pull request #15 from suutari/relative-paths

Relative paths support

ChangeLog.rst

@@ -1,3 +1,24 @@
+Unreleased
+----------
+
+Features
+~~~~~~~~
+
+- Add support for unpinned non-editable VCS URLs
+- Make relative paths work as a requirement entry (editable or not)
+
+General Changes
+~~~~~~~~~~~~~~~
+
+- Make "via package" comments work for editable requirers too
+- Speedup: Do not regenerate hashes if they are already known
+
+Bug Fixes
+~~~~~~~~~
+
+- Ignore installed packages when calulating dependencies
+- Generate hashes of artifacts correctly
+
 1.2.2
 -----
 

prequ/cache.py

@@ -82,10 +82,20 @@ def as_cache_key(self, ireq):
         ("ipython", "2.1.0[nbconvert,notebook]")
         """
         name, version, extras = as_tuple(ireq)
+        if not version:
+            if not ireq.link:
+                raise ValueError((
+                    "Cannot cache dependencies of unpinned non-link "
+                    "requirement: {}").format(ireq))
+            version = ':UNPINNED:'
         if not extras:
             extras_string = ""
         else:
             extras_string = "[{}]".format(",".join(extras))
+        if ireq.editable:
+            # Make sure that editables don't end up into the cache with
+            # a version of a real non-editable package
+            extras_string += ':EDITABLE:{}'.format(ireq.link)
         return name, "{}{}".format(version, extras_string)
 
     def read_cache(self):
@@ -99,7 +109,7 @@ def write_cache(self):
         """Writes the cache to disk as JSON."""
         doc = {
             '__format__': 1,
-            'dependencies': self._cache,
+            'dependencies': self._strip_unpinned_and_editables(self._cache),
         }
         with open(self._cache_file, 'w') as f:
             json.dump(doc, f, sort_keys=True)
@@ -162,3 +172,17 @@ def _reverse_dependencies(self, cache_keys):
         return lookup_table((key_from_req(Requirement.parse(dep_name)), name)
                             for name, version_and_extras in cache_keys
                             for dep_name in self.cache[name][version_and_extras])
+
+    @classmethod
+    def _strip_unpinned_and_editables(cls, cache):
+        """
+        Strip out unpinned and editable deps from given dep cache map.
+        """
+        stripped = type(cache)()
+        for (name, dep_map) in cache.items():
+            stripped_dep_map = type(dep_map)()
+            for (version, deps) in dep_map.items():
+                if ':UNPINNED:' not in version and ':EDITABLE:' not in version:
+                    stripped_dep_map[version] = deps
+            stripped[name] = stripped_dep_map
+        return stripped

prequ/repositories/base.py

@@ -6,6 +6,8 @@
 
 from six import add_metaclass
 
+from ..utils import is_pinned_requirement
+
 
 @add_metaclass(ABCMeta)
 class BaseRepository(object):
@@ -23,13 +25,31 @@ def find_best_match(self, ireq):
         InstallRequirement according to the repository.
         """
 
-    @abstractmethod
     def get_dependencies(self, ireq):
         """
         Given a pinned or an editable InstallRequirement, returns a set of
         dependencies (also InstallRequirements, but not necessarily pinned).
         They indicate the secondary dependencies for the given requirement.
         """
+        if not (ireq.editable or is_pinned_requirement(ireq)):
+            raise TypeError('Expected pinned or editable InstallRequirement, got {}'.format(ireq))
+        return self._get_dependencies(ireq)
+
+    def prepare_ireq(self, ireq):
+        """
+        Prepare install requirement for requirement analysis.
+
+        Downloads and unpacks the sources to get the egg_info etc.
+        Note: For URLs or local paths even key_from_ireq(ireq) might
+        fail before this is done.
+
+        :type ireq: pip.req.InstallRequirement
+        """
+        self._get_dependencies(ireq)
+
+    @abstractmethod
+    def _get_dependencies(self, ireq):
+        pass
 
     @abstractmethod
     def get_hashes(self, ireq):

prequ/repositories/local.py

@@ -2,8 +2,8 @@
 from __future__ import (
     absolute_import, division, print_function, unicode_literals)
 
-from prequ.utils import as_tuple, key_from_req, make_install_requirement
-
+from ..utils import (
+    as_tuple, check_is_hashable, key_from_ireq, make_install_requirement)
 from .base import BaseRepository
 
 
@@ -49,7 +49,7 @@ def freshen_build_caches(self):
         self.repository.freshen_build_caches()
 
     def find_best_match(self, ireq, prereleases=None):
-        key = key_from_req(ireq.req)
+        key = key_from_ireq(ireq)
         existing_pin = self.existing_pins.get(key)
         if existing_pin and ireq_satisfied_by_existing_pin(ireq, existing_pin):
             project, version, _ = as_tuple(existing_pin)
@@ -59,8 +59,23 @@ def find_best_match(self, ireq, prereleases=None):
         else:
             return self.repository.find_best_match(ireq, prereleases)
 
-    def get_dependencies(self, ireq):
-        return self.repository.get_dependencies(ireq)
+    def _get_dependencies(self, ireq):
+        return self.repository._get_dependencies(ireq)
 
     def get_hashes(self, ireq):
+        check_is_hashable(ireq)
+        pinned_ireq = self.existing_pins.get(key_from_ireq(ireq))
+        if pinned_ireq and ireq_satisfied_by_existing_pin(ireq, pinned_ireq):
+            if pinned_ireq.has_hash_options:
+                return set(_get_hashes_from_ireq(pinned_ireq))
         return self.repository.get_hashes(ireq)
+
+
+def _get_hashes_from_ireq(ireq):
+    """
+    :type ireq: pip.req.InstallRequirement
+    """
+    hashes = ireq.hashes()
+    for (alg, hash_values) in hashes._allowed.items():
+        for hash_value in hash_values:
+            yield '{}:{}'.format(alg, hash_value)

prequ/repositories/pypi.py

@@ -14,7 +14,7 @@
 from ..cache import CACHE_DIR
 from ..exceptions import NoCandidateFound
 from ..utils import (
-    fs_str, is_pinned_requirement, is_vcs_link, lookup_table,
+    check_is_hashable, fs_str, is_vcs_link, lookup_table,
     make_install_requirement, pip_version_info)
 from .base import BaseRepository
 
@@ -125,16 +125,12 @@ def find_best_match(self, ireq, prereleases=None):
             best_candidate.project, best_candidate.version, ireq.extras, constraint=ireq.constraint
         )
 
-    def get_dependencies(self, ireq):
+    def _get_dependencies(self, ireq):
         """
-        Given a pinned or an editable InstallRequirement, returns a set of
-        dependencies (also InstallRequirements, but not necessarily pinned).
-        They indicate the secondary dependencies for the given requirement.
+        :type ireq: pip.req.InstallRequirement
         """
-        if not (ireq.editable or is_pinned_requirement(ireq)):
-            raise TypeError('Expected pinned or editable InstallRequirement, got {}'.format(ireq))
-
-        if ireq not in self._dependencies_cache:
+        deps = self._dependencies_cache.get(getattr(ireq.link, 'url', None))
+        if not deps:
             if ireq.link and not ireq.link.is_artifact:
                 # No download_dir for VCS sources.  This also works around pip
                 # using git-checkout-index, which gets rid of the .git dir.
@@ -150,19 +146,27 @@ def get_dependencies(self, ireq):
                                     self.source_dir,
                                     download_dir=download_dir,
                                     wheel_download_dir=self._wheel_download_dir,
-                                    session=self.session)
-            self._dependencies_cache[ireq] = reqset._prepare_file(self.finder, ireq)
-        return set(self._dependencies_cache[ireq])
+                                    session=self.session,
+                                    ignore_installed=True)
+            deps = reqset._prepare_file(self.finder, ireq)
+            if ireq.req and ireq._temp_build_dir and ireq._ideal_build_dir:
+                # Move the temporary build directory under self.build_dir
+                ireq.source_dir = None
+                ireq._correct_build_location()
+            assert ireq.link.url
+            self._dependencies_cache[ireq.link.url] = deps
+        return set(deps)
 
     def get_hashes(self, ireq):
         """
         Given a pinned InstallRequire, returns a set of hashes that represent
         all of the files for a given requirement. It is not acceptable for an
         editable or unpinned requirement to be passed to this function.
         """
-        if not is_pinned_requirement(ireq):
-            raise TypeError(
-                "Expected pinned requirement, not unpinned or editable, got {}".format(ireq))
+        check_is_hashable(ireq)
+
+        if ireq.link and ireq.link.is_artifact:
+            return {self._get_file_hash(ireq.link)}
 
         # We need to get all of the candidates that match our current version
         # pin, these will represent all of the files that could possibly

prequ/resolver.py

@@ -11,12 +11,10 @@
 from pip.req import InstallRequirement
 
 from .cache import DependencyCache
-from .exceptions import UnsupportedConstraint
 from .logging import log
 from .utils import (
     UNSAFE_PACKAGES, first, format_requirement, format_specifier, full_groupby,
-    get_pinned_version, is_pinned_requirement, is_vcs_link, key_from_ireq,
-    key_from_req)
+    get_pinned_version, is_pinned_requirement, is_vcs_link, key_from_ireq)
 
 green = partial(click.style, fg='green')
 magenta = partial(click.style, fg='magenta')
@@ -28,7 +26,7 @@ class RequirementSummary(object):
     """
     def __init__(self, ireq):
         self.req = ireq.req
-        self.key = key_from_req(ireq.req)
+        self.key = key_from_ireq(ireq)
         self.extras = str(sorted(ireq.extras))
         self.specifier = str(ireq.specifier)
 
@@ -60,6 +58,20 @@ def __init__(self, constraints, repository, cache=None, prereleases=False, clear
         self.clear_caches = clear_caches
         self.allow_unsafe = allow_unsafe
         self.unsafe_constraints = set()
+        self._prepare_ireqs(self.our_constraints)
+        self._prepare_ireqs(self.limiters)
+
+    def _prepare_ireqs(self, constraints):
+        """
+        Prepare install requirements for analysis.
+
+        :type constraints: Iterable[pip.req.InstallRequirement]
+        """
+        for constraint in constraints:
+            if constraint.link and not constraint.prepared:
+                os.environ[str('PIP_EXISTS_ACTION')] = str('i')
+                self.repository.prepare_ireq(constraint)
+                del os.environ[str('PIP_EXISTS_ACTION')]
 
     @property
     def constraints(self):
@@ -125,11 +137,7 @@ def resolve(self, max_rounds=10):
 
     @staticmethod
     def check_constraints(constraints):
-        for constraint in constraints:
-            if ((is_vcs_link(constraint) and not constraint.editable and
-                 not is_pinned_requirement(constraint))):
-                msg = 'Prequ does not support non-editable vcs URLs that are not pinned to one version.'
-                raise UnsupportedConstraint(msg, constraint)
+        pass
 
     def _group_constraints(self, constraints):
         """
@@ -227,13 +235,13 @@ def _resolve_one_round(self):  # noqa: C901 (too complex)
         if has_changed:
             log.debug('')
             log.debug('New dependencies found in this round:')
-            for new_dependency in sorted(diff, key=lambda req: key_from_req(req.req)):
+            for new_dependency in sorted(diff, key=lambda ireq: key_from_ireq(ireq)):
                 log.debug('  adding {}'.format(new_dependency))
             log.debug('Removed dependencies in this round:')
-            for removed_dependency in sorted(removed, key=lambda req: key_from_req(req.req)):
+            for removed_dependency in sorted(removed, key=lambda ireq: key_from_ireq(ireq)):
                 log.debug('  removing {}'.format(removed_dependency))
             log.debug('Unsafe dependencies in this round:')
-            for unsafe_dependency in sorted(unsafe, key=lambda req: key_from_req(req.req)):
+            for unsafe_dependency in sorted(unsafe, key=lambda ireq: key_from_ireq(ireq)):
                 log.debug('  remembering unsafe {}'.format(unsafe_dependency))
 
         # Store the last round's results in the their_constraints
@@ -282,11 +290,7 @@ def _iter_dependencies(self, ireq):
         Editable requirements will never be looked up, as they may have
         changed at any time.
         """
-        if ireq.editable:
-            for dependency in self.repository.get_dependencies(ireq):
-                yield dependency
-            return
-        elif not is_pinned_requirement(ireq):
+        if not is_pinned_requirement(ireq) and not ireq.editable:
             raise TypeError('Expected pinned or editable requirement, got {}'.format(ireq))
 
         # Now, either get the dependencies from the dependency cache (for
@@ -306,5 +310,4 @@ def _iter_dependencies(self, ireq):
             yield InstallRequirement.from_line(dependency_string, constraint=ireq.constraint)
 
     def reverse_dependencies(self, ireqs):
-        non_editable = [ireq for ireq in ireqs if not ireq.editable]
-        return self.dependency_cache.reverse_dependencies(non_editable)
+        return self.dependency_cache.reverse_dependencies(ireqs)

prequ/scripts/compile_in.py

@@ -16,7 +16,7 @@
 from ..resolver import Resolver
 from ..utils import (
     UNSAFE_PACKAGES, assert_compatible_pip_version, dedup,
-    is_pinned_requirement, key_from_req)
+    is_pinned_requirement, key_from_ireq)
 from ..writer import OutputWriter
 from ._repo import get_pip_options_and_pypi_repository
 
@@ -121,10 +121,10 @@ def cli(verbose, silent, dry_run, pre, rebuild, find_links, index_url,
     if not upgrade and os.path.exists(dst_file):
         ireqs = parse_requirements(dst_file, finder=repository.finder, session=repository.session, options=pip_options)
         # Exclude packages from --upgrade-package/-P from the existing pins: We want to upgrade.
-        upgrade_pkgs_key = {key_from_req(InstallRequirement.from_line(pkg).req) for pkg in upgrade_packages}
-        existing_pins = {key_from_req(ireq.req): ireq
+        upgrade_pkgs_key = {key_from_ireq(InstallRequirement.from_line(pkg)) for pkg in upgrade_packages}
+        existing_pins = {key_from_ireq(ireq): ireq
                          for ireq in ireqs
-                         if is_pinned_requirement(ireq) and key_from_req(ireq.req) not in upgrade_pkgs_key}
+                         if is_pinned_requirement(ireq) and key_from_ireq(ireq) not in upgrade_pkgs_key}
         repository = LocalRequirementsRepository(existing_pins, repository)
 
     log.debug('Using indexes:')
@@ -225,8 +225,8 @@ def cli(verbose, silent, dry_run, pre, rebuild, find_links, index_url,
     writer.write(results=results,
                  unsafe_requirements=resolver.unsafe_constraints,
                  reverse_dependencies=reverse_dependencies,
-                 primary_packages={key_from_req(ireq.req) for ireq in constraints if not ireq.constraint},
-                 markers={key_from_req(ireq.req): ireq.markers
+                 primary_packages={key_from_ireq(ireq) for ireq in constraints if not ireq.constraint},
+                 markers={key_from_ireq(ireq): ireq.markers
                           for ireq in constraints if ireq.markers},
                  hashes=hashes,
                  allow_unsafe=allow_unsafe)