From eae395822702a5cdc1258ce7350d0e48e554c653 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Sun, 27 Jan 2019 02:06:40 -0500 Subject: [PATCH] Fix pep508 direct URL depedencies - Fixes #3148 Signed-off-by: Dan Ryan --- pipenv/core.py | 68 +++--- pipenv/environment.py | 7 +- pipenv/environments.py | 11 + pipenv/utils.py | 230 +++++++++++++----- .../requirementslib/models/requirements.py | 12 + .../requirementslib/models/setup_info.py | 8 +- tests/integration/conftest.py | 3 + 7 files changed, 233 insertions(+), 106 deletions(-) diff --git a/pipenv/core.py b/pipenv/core.py index b4536bb70c..47294258a0 100644 --- a/pipenv/core.py +++ b/pipenv/core.py @@ -722,24 +722,25 @@ def batch_install(deps_list, procs, failed_deps_queue, os.environ["PIP_USER"] = vistir.compat.fs_str("0") if "PYTHONHOME" in os.environ: del os.environ["PYTHONHOME"] - if no_deps: + if not needs_deps: link = getattr(dep.req, "link", None) is_wheel = False if link: is_wheel = link.is_wheel is_non_editable_vcs = (dep.is_vcs and not dep.editable) - no_deps = not (dep.is_file_or_url and not (is_wheel or dep.editable)) + needs_deps = dep.is_file_or_url and not (is_wheel or dep.editable) c = pip_install( dep, ignore_hashes=any([ignore_hashes, dep.editable, dep.is_vcs]), allow_global=allow_global, - no_deps=no_deps, + no_deps=not needs_deps, block=any([dep.editable, dep.is_vcs, blocking]), index=index, requirements_dir=requirements_dir, pypi_mirror=pypi_mirror, trusted_hosts=trusted_hosts, - extra_indexes=extra_indexes + extra_indexes=extra_indexes, + use_pep517=not retry, ) if dep.is_vcs or dep.editable: c.block() @@ -822,33 +823,33 @@ def do_install_dependencies( else: install_kwargs["nprocs"] = 1 - with project.environment.activated(): - batch_install( - deps_list, procs, failed_deps_queue, requirements_dir, **install_kwargs - ) + # with project.environment.activated(): + batch_install( + deps_list, procs, failed_deps_queue, requirements_dir, **install_kwargs + ) - if not procs.empty(): - _cleanup_procs(procs, concurrent, failed_deps_queue) + if not procs.empty(): + _cleanup_procs(procs, concurrent, failed_deps_queue) - # Iterate over the hopefully-poorly-packaged dependencies… - if not failed_deps_queue.empty(): - click.echo( - crayons.normal(fix_utf8("Installing initially failed dependencies…"), bold=True) - ) - retry_list = [] - while not failed_deps_queue.empty(): - failed_dep = failed_deps_queue.get() - retry_list.append(failed_dep) - install_kwargs.update({ - "nprocs": 1, - "retry": False, - "blocking": True, - }) - batch_install( - retry_list, procs, failed_deps_queue, requirements_dir, **install_kwargs - ) - if not procs.empty(): - _cleanup_procs(procs, False, failed_deps_queue, retry=False) + # Iterate over the hopefully-poorly-packaged dependencies… + if not failed_deps_queue.empty(): + click.echo( + crayons.normal(fix_utf8("Installing initially failed dependencies…"), bold=True) + ) + retry_list = [] + while not failed_deps_queue.empty(): + failed_dep = failed_deps_queue.get() + retry_list.append(failed_dep) + install_kwargs.update({ + "nprocs": 1, + "retry": False, + "blocking": True, + }) + batch_install( + retry_list, procs, failed_deps_queue, requirements_dir, **install_kwargs + ) + if not procs.empty(): + _cleanup_procs(procs, False, failed_deps_queue, retry=False) def convert_three_to_python(three, python): @@ -1266,7 +1267,8 @@ def pip_install( requirements_dir=None, extra_indexes=None, pypi_mirror=None, - trusted_hosts=None + trusted_hosts=None, + use_pep517=True ): from pipenv.patched.notpip._internal import logger as piplogger from .utils import Mapping @@ -1450,6 +1452,8 @@ def pip_install( if not ignore_hashes: pip_command.append("--require-hashes") pip_command.append("--no-build-isolation") + if not use_pep517: + pip_command.append("--no-use-pep517") if environments.is_verbose(): click.echo("$ {0}".format(pip_command), err=True) cache_dir = vistir.compat.Path(PIPENV_CACHE_DIR) @@ -1469,8 +1473,8 @@ def pip_install( cmd = Script.parse(pip_command) pip_command = cmd.cmdify() c = None - with project.environment.activated(): - c = delegator.run(pip_command, block=block, env=pip_config) + # with project.environment.activated(): + c = delegator.run(pip_command, block=block, env=pip_config) return c diff --git a/pipenv/environment.py b/pipenv/environment.py index 9397fbfde1..3502107afa 100644 --- a/pipenv/environment.py +++ b/pipenv/environment.py @@ -487,9 +487,8 @@ def activated(self, include_extras=True, extra_dists=None): os.environ["PYTHONDONTWRITEBYTECODE"] = vistir.compat.fs_str("1") from .environments import PIPENV_USE_SYSTEM if self.is_venv: - if not PIPENV_USE_SYSTEM: - os.environ["PYTHONPATH"] = self.base_paths["PYTHONPATH"] - os.environ["VIRTUAL_ENV"] = vistir.compat.fs_str(prefix) + os.environ["PYTHONPATH"] = self.base_paths["PYTHONPATH"] + os.environ["VIRTUAL_ENV"] = vistir.compat.fs_str(prefix) else: if not PIPENV_USE_SYSTEM and not os.environ.get("VIRTUAL_ENV"): os.environ["PYTHONPATH"] = self.base_paths["PYTHONPATH"] @@ -502,7 +501,7 @@ def activated(self, include_extras=True, extra_dists=None): pep517_dir = os.path.join(os.path.dirname(pip_vendor.__file__), "pep517") site.addsitedir(pep517_dir) os.environ["PYTHONPATH"] = os.pathsep.join([ - os.environ["PYTHONPATH"], pep517_dir + os.environ.get("PYTHONPATH", self.base_paths["PYTHONPATH"]), pep517_dir ]) if include_extras: site.addsitedir(parent_path) diff --git a/pipenv/environments.py b/pipenv/environments.py index b72ea5bc1e..f451062160 100644 --- a/pipenv/environments.py +++ b/pipenv/environments.py @@ -303,3 +303,14 @@ def is_in_virtualenv(): PIPENV_SPINNER_FAIL_TEXT = fix_utf8(u"✘ {0}") if not PIPENV_HIDE_EMOJIS else ("{0}") PIPENV_SPINNER_OK_TEXT = fix_utf8(u"✔ {0}") if not PIPENV_HIDE_EMOJIS else ("{0}") + + +def is_type_checking(): + try: + from typing import TYPE_CHECKING + except ImportError: + return False + return TYPE_CHECKING + + +MYPY_RUNNING = is_type_checking() diff --git a/pipenv/utils.py b/pipenv/utils.py index 462d114a0f..bd3c5f7073 100644 --- a/pipenv/utils.py +++ b/pipenv/utils.py @@ -35,6 +35,10 @@ from .pep508checker import lookup +if environments.MYPY_RUNNING: + from typing import Tuple, Dict, Any, List, Union + + logging.basicConfig(level=logging.ERROR) specifiers = [k for k in lookup.keys()] @@ -228,69 +232,79 @@ def prepare_pip_source_args(sources, pip_args=None): return pip_args -def resolve_separate(req): - """ - Resolve a requirement that the normal resolver can't - - This includes non-editable urls to zip or tarballs, non-editable paths, etc. - """ - - from .vendor.requirementslib.models.utils import _requirement_to_str_lowercase_name - from .vendor.requirementslib.models.requirements import Requirement - constraints = set() - lockfile_update = {} - if req.is_file_or_url and not req.is_vcs: - setup_info = req.run_requires() - requirements = [v for v in setup_info.get("requires", {}).values()] - for r in requirements: - if getattr(r, "url", None) and not getattr(r, "editable", False): - requirement = Requirement.from_line(_requirement_to_str_lowercase_name(r)) - constraint_update, child_lockfile = resolve_separate(requirement) - constraints |= constraint_update - lockfile_update.update(child_lockfile) - # for local packages with setup.py files and potential direct url deps: - if req.editable and requirement.is_direct_url: - name, entry = requirement.pipfile_entry - lockfile_update[name] = entry - continue - constraints.add(_requirement_to_str_lowercase_name(r)) - return constraints, lockfile_update - - -def get_resolver_metadata(deps, index_lookup, markers_lookup, project, sources): - from .vendor.requirementslib.models.requirements import Requirement - constraints = set() - skipped = {} - for dep in deps: - if not dep: - continue - url = None - indexes, trusted_hosts, remainder = parse_indexes(dep) - if indexes: - url = indexes[0] - dep = " ".join(remainder) - req = Requirement.from_line(dep) - if req.is_file_or_url and not req.is_vcs: - # TODO: This is a significant hack, should probably be reworked - constraint_update, lockfile_update = resolve_separate(req) - constraints |= constraint_update - name, entry = req.pipfile_entry - skipped[name] = entry - skipped.update(lockfile_update) - continue - constraints.add(req.constraint_line) - - if url: - source = first( - s for s in sources if s.get("url") and url.startswith(s["url"])) - if source: - index_lookup[req.name] = source.get("name") - # strip the marker and re-add it later after resolution - # but we will need a fallback in case resolution fails - # eg pypiwin32 - if req.markers: - markers_lookup[req.name] = req.markers.replace('"', "'") - return constraints, skipped +# def resolve_separate(req): +# # type: ('.vendor.requirementslib.requirements.Requirement') -> Tuple[Set[str], Dict[str, Dict[Any]]] +# """ +# Resolve a requirement that the normal resolver can't + +# This includes non-editable urls to zip or tarballs, non-editable paths, etc. +# """ + +# from .vendor.requirementslib.models.utils import _requirement_to_str_lowercase_name +# from .vendor.requirementslib.models.requirements import Requirement +# constraints = set() +# lockfile_update = {} +# if req.is_file_or_url and not req.is_vcs: +# setup_info = req.run_requires() +# requirements = [v for v in setup_info.get("requires", {}).values()] +# for r in requirements: +# if getattr(r, "url", None) and not getattr(r, "editable", False): +# requirement = Requirement.from_line(_requirement_to_str_lowercase_name(r)) +# constraint_update, child_lockfile = resolve_separate(requirement) +# constraints |= constraint_update +# lockfile_update.update(child_lockfile) +# # for local packages with setup.py files and potential direct url deps: +# if req.editable and requirement.is_direct_url: +# name, entry = requirement.pipfile_entry +# lockfile_update[name] = entry +# continue +# constraints.add(_requirement_to_str_lowercase_name(r)) +# return constraints, lockfile_update + + +# def get_resolver_metadata( +# deps, # type: List[str] +# index_lookup, # type: Dict[str, str] +# markers_lookup, # type: Dict[str, str] +# project, # type: '.project.Project' +# sources # type: Dict[str, str] +# ): +# # type: (...) -> Set() +# from .vendor.requirementslib.models.requirements import Requirement, Line +# constraints = set() +# skipped = {} +# for dep in deps: +# if not dep: +# continue +# url = None +# indexes, trusted_hosts, remainder = parse_indexes(dep) +# if indexes: +# url = indexes[0] +# dep = " ".join(remainder) +# line = Line(dep) +# if ((line.is_direct_url and line.is_vcs) or +# (line.is_file or line.is_url and not (line.is_vcs and line.editable))): +# # TODO: This is a significant hack, should probably be reworked +# constraint_update, lockfile_update = resolve_separate(req) +# constraints |= constraint_update +# req = Requirement.from_line(dep) +# name, entry = req.pipfile_entry +# skipped[name] = entry +# skipped.update(lockfile_update) +# continue +# constraints.add(req.constraint_line) + +# if url: +# source = first( +# s for s in sources if s.get("url") and url.startswith(s["url"])) +# if source: +# index_lookup[req.name] = source.get("name") +# # strip the marker and re-add it later after resolution +# # but we will need a fallback in case resolution fails +# # eg pypiwin32 +# if req.markers: +# markers_lookup[req.name] = req.markers.replace('"', "'") +# return constraints, skipped class Resolver(object): @@ -336,6 +350,85 @@ class PipCommand(Command): from pipenv.patched.piptools.scripts.compile import get_pip_command return get_pip_command() + @classmethod + def get_metadata( + cls, + deps, # type: List[str] + index_lookup, # type: Dict[str, str] + markers_lookup, # type: Dict[str, str] + project, # type: 'pipenv.project.Project' + sources # type: Dict[str, str] + ): + # type: (...) -> Set() + constraints = set() + skipped = {} + for dep in deps: + if not dep: + continue + constraint_update, lockfile_update = cls.get_deps_from_line( + dep, index_lookup=index_lookup, markers_lookup=markers_lookup, sources=sources + ) + constraints |= constraint_update + skipped.update(lockfile_update) + return constraints, skipped + + @classmethod + def parse_line(cls, line, index_lookup=None, markers_lookup=None, sources=None): + from .vendor.requirementslib.models.requirements import Requirement + if sources is None: + sources = [] # type: List[Dict[str, Union[str, bool]]] + url = None + indexes, trusted_hosts, remainder = parse_indexes(line) + if indexes: + url = indexes[0] + line = " ".join(remainder) + req = Requirement.from_line(line) + if url: + source = first( + s for s in sources if s.get("url") and url.startswith(s["url"])) + if source and index_lookup is not None: + index_lookup[req.name] = source.get("name") + # strip the marker and re-add it later after resolution + # but we will need a fallback in case resolution fails + # eg pypiwin32 + if req.markers and markers_lookup is not None: + markers_lookup[req.name] = req.markers.replace('"', "'") + return req + + @classmethod + def get_deps_from_line(cls, line, index_lookup=None, markers_lookup=None, sources=None): + from .vendor.requirementslib.models.utils import _requirement_to_str_lowercase_name + if sources is None: + sources = [] # type: List[Dict[str, Union[str, bool]]] + req = cls.parse_line( + line, index_lookup=index_lookup, markers_lookup=markers_lookup, sources=sources + ) + parsed_line = req.line_instance + constraints = set() + locked_deps = {} + if ((parsed_line.is_direct_url and parsed_line.is_vcs) or + (parsed_line.is_file or parsed_line.is_url and not + (parsed_line.is_vcs and parsed_line.editable)) + ): + # for local packages with setup.py files and potential direct url deps: + name, entry = req.pipfile_entry + # TODO: This might belong as a conditional include after we do the other logic in the for loop (line 427) + setup_info = parsed_line.setup_info + requirements = [v for v in setup_info.get_info().get("requires", {}).values()] + for r in requirements: + if getattr(r, "url", None) and not getattr(r, "editable", False): + new_constraints, new_lock = cls.get_deps_from_line( + _requirement_to_str_lowercase_name(r) + ) + constraints |= new_constraints + locked_deps.update(new_lock) + continue + constraints.add(_requirement_to_str_lowercase_name(r)) + locked_deps[name] = entry + else: + constraints.add(req.constraint_line) + return constraints, locked_deps + @property def pip_command(self): if self._pip_command is None: @@ -546,7 +639,7 @@ def actually_resolve_deps( warning_list = [] with warnings.catch_warnings(record=True) as warning_list: - constraints, skipped = get_resolver_metadata( + constraints, skipped = Resolver.get_metadata( deps, index_lookup, markers_lookup, project, sources, ) resolver = Resolver(constraints, req_dir, project, sources, clear=clear, pre=pre) @@ -557,8 +650,11 @@ def actually_resolve_deps( is_url = url and not url.startswith("file:") path = v.get("path") if not is_url and not path: - path = pip_shims.shims.url_to_path(url) - if is_url or (path and os.path.exists(path) and not os.path.isdir(path)): + try: + path = pip_shims.shims.url_to_path(url) + except AttributeError: + path = None + if is_url or (path and os.path.exists(path) and not os.path.isdir(path)) or req.is_vcs: existing = next(iter(req for req in resolved_tree if req.name == k), None) if existing: resolved_tree.remove(existing) @@ -902,7 +998,7 @@ def resolve_deps( entry.update(pf_entry) if version is not None: entry["version"] = version - if req.is_direct_url: + if req.line_instance.is_direct_url: entry["file"] = req.req.uri if collected_hashes: entry["hashes"] = sorted(set(collected_hashes)) diff --git a/pipenv/vendor/requirementslib/models/requirements.py b/pipenv/vendor/requirementslib/models/requirements.py index 4403950a60..84d13f978f 100644 --- a/pipenv/vendor/requirementslib/models/requirements.py +++ b/pipenv/vendor/requirementslib/models/requirements.py @@ -113,6 +113,7 @@ def __init__(self, line): self._pyproject_backend = None # type: Optional[str] self._wheel_kwargs = None # type: Dict[str, str] self._vcsrepo = None # type: Optional[VCSRepository] + self._setup_info = None # type: Optional[SetupInfo] self._ref = None # type: Optional[str] self._ireq = None # type: Optional[InstallRequirement] self._src_root = None # type: Optional[str] @@ -384,6 +385,13 @@ def is_installable(self): return True return False + @property + def setup_info(self): + # type: () -> Optional[SetupInfo] + if self._setup_info is None: + self._setup_info = SetupInfo.from_ireq(self.ireq) + return self._setup_info + def _get_vcsrepo(self): # type: () -> Optional[VCSRepository] from .vcs import VCSRepository @@ -1717,6 +1725,7 @@ class Requirement(object): hashes = attr.ib(default=attr.Factory(list), converter=list) extras = attr.ib(default=attr.Factory(list)) abstract_dep = attr.ib(default=None) + line_instance = attr.ib(default=None) # type: Optional[Line] _ireq = None @name.default @@ -1818,6 +1827,7 @@ def from_line(cls, line): if "--hash=" in line: hashes = line.split(" --hash=") line, hashes = hashes[0], hashes[1:] + line_instance = Line(line) editable = line.startswith("-e ") line = line.split(" ", 1)[1] if editable else line line, markers = split_markers_from_line(line) @@ -1888,6 +1898,7 @@ def from_line(cls, line): "req": r, "markers": markers, "editable": editable, + "line_instance": line_instance } if extras: extras = sorted(dedup([extra.lower() for extra in extras])) @@ -1956,6 +1967,7 @@ def from_pipfile(cls, name, pipfile): if any(key in _pipfile for key in ["hash", "hashes"]): args["hashes"] = _pipfile.get("hashes", [pipfile.get("hash")]) cls_inst = cls(**args) + cls_inst.line_instance = Line(cls_inst.as_line()) return cls_inst def as_line( diff --git a/pipenv/vendor/requirementslib/models/setup_info.py b/pipenv/vendor/requirementslib/models/setup_info.py index 13147c4f77..7c03a7e9aa 100644 --- a/pipenv/vendor/requirementslib/models/setup_info.py +++ b/pipenv/vendor/requirementslib/models/setup_info.py @@ -75,12 +75,14 @@ def _get_src_dir(root): if src: return src virtual_env = os.environ.get("VIRTUAL_ENV") - if virtual_env: + if virtual_env is not None: return os.path.join(virtual_env, "src") if not root: # Intentionally don't match pip's behavior here -- this is a temporary copy - root = create_tracked_tempdir(prefix="requirementslib-", suffix="-src") - return os.path.join(root, "src") + src_dir = create_tracked_tempdir(prefix="requirementslib-", suffix="-src") + else: + src_dir = os.path.join(root, "src") + return src_dir def ensure_reqs(reqs): diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index f577190149..b8b03eb070 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -233,6 +233,9 @@ def __init__( def __enter__(self): if self.chdir: os.chdir(self.path) + os.environ['PIPENV_PIPFILE'] = fs_str(self.pipfile_path) + c = delegator.run("pipenv run pip install /home/hawk/git/pip") + assert c.return_code == 0 return self def __exit__(self, *args):