From afc344258e0d0d9a093e51a9be40d47c4bb91eb1 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Fri, 2 Nov 2018 10:34:18 -0400 Subject: [PATCH 1/7] Include hashes from local artifacts - Fixes #2394 Signed-off-by: Dan Ryan --- news/2394.bugfix.rst | 1 + pipenv/utils.py | 16 ++++++++++++++-- 2 files changed, 15 insertions(+), 2 deletions(-) create mode 100644 news/2394.bugfix.rst diff --git a/news/2394.bugfix.rst b/news/2394.bugfix.rst new file mode 100644 index 0000000000..78868381b3 --- /dev/null +++ b/news/2394.bugfix.rst @@ -0,0 +1 @@ +Local wheels and artifacts will now include their own hashes rather than including hashes from ``PyPI``. diff --git a/pipenv/utils.py b/pipenv/utils.py index 33e930d65f..ed91387dc4 100644 --- a/pipenv/utils.py +++ b/pipenv/utils.py @@ -229,6 +229,7 @@ class PipCommand(Command): name = "PipCommand" constraints = [] + needs_hash = [] if not req_dir: req_dir = create_tracked_tempdir(suffix="-requirements", prefix="pipenv-") for dep in deps: @@ -240,6 +241,9 @@ class PipCommand(Command): url = indexes[0] dep = " ".join(remainder) req = Requirement.from_line(dep) + new_ireq = req.as_ireq() + if getattr(new_ireq, "link", None) and new_ireq.link.is_wheel and new_ireq.link.scheme == 'file': + needs_hash.append(new_ireq) # extra_constraints = [] @@ -288,10 +292,13 @@ class PipCommand(Command): constraints=constraints, repository=pypi, clear_caches=clear, prereleases=pre ) # pre-resolve instead of iterating to avoid asking pypi for hashes of editable packages - hashes = None + hashes = { + ireq: pypi._hash_cache.get_hash(ireq.link) + for ireq in constraints if getattr(ireq, "link", None) + and ireq.link.scheme == "file" and ireq.link.is_artifact + } try: results = resolver.resolve(max_rounds=environments.PIPENV_MAX_ROUNDS) - hashes = resolver.resolve_hashes(results) resolved_tree.update(results) except (NoCandidateFound, DistributionNotFound, HTTPError) as e: click_echo( @@ -318,6 +325,11 @@ class PipCommand(Command): ), err=True ) raise RuntimeError + else: + resolved_hashes = resolver.resolve_hashes(results) + for ireq, ireq_hashes in resolved_hashes.items(): + if ireq not in hashes: + hashes[ireq] = ireq_hashes return (resolved_tree, hashes, markers_lookup, resolver) From 740c589ef4be38604682faa95e074f7576a5334b Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Fri, 2 Nov 2018 10:44:18 -0400 Subject: [PATCH 2/7] Also prioritize remote non-pypi hashes if given - Fixes #3145 Signed-off-by: Dan Ryan --- news/2394.bugfix.rst | 2 +- news/3145.bugfix.rst | 1 + pipenv/utils.py | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) create mode 100644 news/3145.bugfix.rst diff --git a/news/2394.bugfix.rst b/news/2394.bugfix.rst index 78868381b3..61ec9445a0 100644 --- a/news/2394.bugfix.rst +++ b/news/2394.bugfix.rst @@ -1 +1 @@ -Local wheels and artifacts will now include their own hashes rather than including hashes from ``PyPI``. +Remote non-PyPI artifacts and local wheels and artifacts will now include their own hashes rather than including hashes from ``PyPI``. diff --git a/news/3145.bugfix.rst b/news/3145.bugfix.rst new file mode 100644 index 0000000000..61ec9445a0 --- /dev/null +++ b/news/3145.bugfix.rst @@ -0,0 +1 @@ +Remote non-PyPI artifacts and local wheels and artifacts will now include their own hashes rather than including hashes from ``PyPI``. diff --git a/pipenv/utils.py b/pipenv/utils.py index ed91387dc4..84c729791e 100644 --- a/pipenv/utils.py +++ b/pipenv/utils.py @@ -295,7 +295,7 @@ class PipCommand(Command): hashes = { ireq: pypi._hash_cache.get_hash(ireq.link) for ireq in constraints if getattr(ireq, "link", None) - and ireq.link.scheme == "file" and ireq.link.is_artifact + and ireq.link.is_artifact and not is_pypi_url(ireq.link.url) } try: results = resolver.resolve(max_rounds=environments.PIPENV_MAX_ROUNDS) From b1d3aec5d2da9d011892131e7af425b2422d5d2f Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Fri, 2 Nov 2018 11:17:33 -0400 Subject: [PATCH 3/7] Don't write build artifacts to cwd - Fixes #3106 Signed-off-by: Dan Ryan --- news/3106.bugfix.rst | 1 + pipenv/utils.py | 6 +++++- pipenv/vendor/requirementslib/utils.py | 7 +++++++ 3 files changed, 13 insertions(+), 1 deletion(-) create mode 100644 news/3106.bugfix.rst diff --git a/news/3106.bugfix.rst b/news/3106.bugfix.rst new file mode 100644 index 0000000000..70c0917fb4 --- /dev/null +++ b/news/3106.bugfix.rst @@ -0,0 +1 @@ +Pipenv will avoid leaving build artifacts in the current working directory. diff --git a/pipenv/utils.py b/pipenv/utils.py index 84c729791e..c2f2a9d8fe 100644 --- a/pipenv/utils.py +++ b/pipenv/utils.py @@ -295,7 +295,11 @@ class PipCommand(Command): hashes = { ireq: pypi._hash_cache.get_hash(ireq.link) for ireq in constraints if getattr(ireq, "link", None) - and ireq.link.is_artifact and not is_pypi_url(ireq.link.url) + # We can only hash artifacts, but we don't want normal pypi artifcats since the + # normal resolver handles those + and ireq.link.is_artifact and not is_pypi_url(ireq.link.url) and not + # We also don't want to try to hash directories as this will fail (editable deps) + (ireq.link.scheme == "file" and os.path.isdir(ireq.link.path)) } try: results = resolver.resolve(max_rounds=environments.PIPENV_MAX_ROUNDS) diff --git a/pipenv/vendor/requirementslib/utils.py b/pipenv/vendor/requirementslib/utils.py index 75a05ce05b..291c30ae6d 100644 --- a/pipenv/vendor/requirementslib/utils.py +++ b/pipenv/vendor/requirementslib/utils.py @@ -186,7 +186,14 @@ def ensure_setup_py(base_dir): if not base_dir: base_dir = create_tracked_tempdir(prefix="requirementslib-setup") base_dir = Path(base_dir) + if base_dir.exists() and base_dir.name == "setup.py": + base_dir = base_dir.parent + elif not (base_dir.exists() and base_dir.is_dir()): + base_dir = base_dir.parent + if not (base_dir.exists() and base_dir.is_dir()): + base_dir = base_dir.parent setup_py = base_dir.joinpath("setup.py") + is_new = False if setup_py.exists() else True if not setup_py.exists(): setup_py.write_text(u"") From 77b3edd3020feba434a7724ee1a224ff50342a1d Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Fri, 2 Nov 2018 11:35:20 -0400 Subject: [PATCH 4/7] Update requirementslib version Signed-off-by: Dan Ryan --- pipenv/vendor/requirementslib/__init__.py | 2 +- pipenv/vendor/vendor.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pipenv/vendor/requirementslib/__init__.py b/pipenv/vendor/requirementslib/__init__.py index bcbff09e7e..5429d6ee00 100644 --- a/pipenv/vendor/requirementslib/__init__.py +++ b/pipenv/vendor/requirementslib/__init__.py @@ -1,5 +1,5 @@ # -*- coding=utf-8 -*- -__version__ = '1.2.3' +__version__ = '1.2.4' import logging diff --git a/pipenv/vendor/vendor.txt b/pipenv/vendor/vendor.txt index 2e00af0c0f..2fe5e47f34 100644 --- a/pipenv/vendor/vendor.txt +++ b/pipenv/vendor/vendor.txt @@ -27,7 +27,7 @@ requests==2.20.0 idna==2.7 urllib3==1.24 certifi==2018.10.15 -requirementslib==1.2.3 +requirementslib==1.2.4 attrs==18.2.0 distlib==0.2.8 packaging==18.0 From d5680fb61792cc3a7400e985936e3f82214bba89 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Fri, 2 Nov 2018 17:57:50 -0400 Subject: [PATCH 5/7] Fix hash function Signed-off-by: Dan Ryan --- pipenv/patched/piptools/repositories/pypi.py | 2 +- tasks/vendoring/patches/patched/piptools.patch | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pipenv/patched/piptools/repositories/pypi.py b/pipenv/patched/piptools/repositories/pypi.py index 2a0743a338..61250f2b94 100644 --- a/pipenv/patched/piptools/repositories/pypi.py +++ b/pipenv/patched/piptools/repositories/pypi.py @@ -65,7 +65,7 @@ def get_hash(self, location): if can_hash: # hash url WITH fragment hash_value = self.get(new_location.url) - if not hash_value: + if not hash_value and ((new_location.scheme == "file" and not os.path.isdir(new_location.path)) or new_location.scheme != "file"): hash_value = self._get_file_hash(new_location) if not new_location.url.startswith("ssh") else None hash_value = hash_value.encode('utf8') if hash_value else None if can_hash: diff --git a/tasks/vendoring/patches/patched/piptools.patch b/tasks/vendoring/patches/patched/piptools.patch index 3799ccf49a..a96bf56804 100644 --- a/tasks/vendoring/patches/patched/piptools.patch +++ b/tasks/vendoring/patches/patched/piptools.patch @@ -194,7 +194,7 @@ index bf69803..31b85b9 100644 + if can_hash: + # hash url WITH fragment + hash_value = self.get(new_location.url) -+ if not hash_value: ++ if not hash_value and ((new_location.scheme == "file" and not os.path.isdir(new_location.path)) or new_location.scheme != "file"): + hash_value = self._get_file_hash(new_location) if not new_location.url.startswith("ssh") else None + hash_value = hash_value.encode('utf8') if hash_value else None + if can_hash: From 02b0de9f7bde9ffdd7de2dd95d31fd4ab0705aa6 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Fri, 2 Nov 2018 18:23:26 -0400 Subject: [PATCH 6/7] Revert "Fix hash function" This reverts commit d5680fb61792cc3a7400e985936e3f82214bba89. --- pipenv/patched/piptools/repositories/pypi.py | 2 +- tasks/vendoring/patches/patched/piptools.patch | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pipenv/patched/piptools/repositories/pypi.py b/pipenv/patched/piptools/repositories/pypi.py index 61250f2b94..2a0743a338 100644 --- a/pipenv/patched/piptools/repositories/pypi.py +++ b/pipenv/patched/piptools/repositories/pypi.py @@ -65,7 +65,7 @@ def get_hash(self, location): if can_hash: # hash url WITH fragment hash_value = self.get(new_location.url) - if not hash_value and ((new_location.scheme == "file" and not os.path.isdir(new_location.path)) or new_location.scheme != "file"): + if not hash_value: hash_value = self._get_file_hash(new_location) if not new_location.url.startswith("ssh") else None hash_value = hash_value.encode('utf8') if hash_value else None if can_hash: diff --git a/tasks/vendoring/patches/patched/piptools.patch b/tasks/vendoring/patches/patched/piptools.patch index a96bf56804..3799ccf49a 100644 --- a/tasks/vendoring/patches/patched/piptools.patch +++ b/tasks/vendoring/patches/patched/piptools.patch @@ -194,7 +194,7 @@ index bf69803..31b85b9 100644 + if can_hash: + # hash url WITH fragment + hash_value = self.get(new_location.url) -+ if not hash_value and ((new_location.scheme == "file" and not os.path.isdir(new_location.path)) or new_location.scheme != "file"): ++ if not hash_value: + hash_value = self._get_file_hash(new_location) if not new_location.url.startswith("ssh") else None + hash_value = hash_value.encode('utf8') if hash_value else None + if can_hash: From 2632d0bd112882a4b640280c6045fcb303921a82 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Fri, 2 Nov 2018 18:23:55 -0400 Subject: [PATCH 7/7] Fix url to path comparison Signed-off-by: Dan Ryan --- pipenv/utils.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/pipenv/utils.py b/pipenv/utils.py index c2f2a9d8fe..dd107db021 100644 --- a/pipenv/utils.py +++ b/pipenv/utils.py @@ -221,7 +221,10 @@ def actually_resolve_deps( from pipenv.patched.piptools import logging as piptools_logging from pipenv.patched.piptools.exceptions import NoCandidateFound from .vendor.requirementslib.models.requirements import Requirement - from .vendor.vistir.path import create_tracked_tempdir, create_tracked_tempfile + from .vendor.vistir.path import ( + create_tracked_tempdir, create_tracked_tempfile, url_to_path, + ) + from .vendor.vistir.compat import Path, to_native_string class PipCommand(Command): """Needed for pip-tools.""" @@ -294,12 +297,12 @@ class PipCommand(Command): # pre-resolve instead of iterating to avoid asking pypi for hashes of editable packages hashes = { ireq: pypi._hash_cache.get_hash(ireq.link) - for ireq in constraints if getattr(ireq, "link", None) + for ireq in constraints if (getattr(ireq, "link", None) # We can only hash artifacts, but we don't want normal pypi artifcats since the # normal resolver handles those - and ireq.link.is_artifact and not is_pypi_url(ireq.link.url) and not + and ireq.link.is_artifact and not (is_pypi_url(ireq.link.url) or # We also don't want to try to hash directories as this will fail (editable deps) - (ireq.link.scheme == "file" and os.path.isdir(ireq.link.path)) + (ireq.link.scheme == "file" and Path(to_native_string(url_to_path(ireq.link.url))).is_dir()))) } try: results = resolver.resolve(max_rounds=environments.PIPENV_MAX_ROUNDS)