diff --git a/news/5809.bugfix.rst b/news/5809.bugfix.rst new file mode 100644 index 0000000000..935fc73a7c --- /dev/null +++ b/news/5809.bugfix.rst @@ -0,0 +1 @@ +Restore running Resolver in sub-process using the project python by default; maintains ability to run directly by setting ``PIPENV_RESOLVER_PARENT_PYTHON`` environment variable to 1 (useful for internal debugging). diff --git a/pipenv/environments.py b/pipenv/environments.py index 3b8718bd66..d71a499e26 100644 --- a/pipenv/environments.py +++ b/pipenv/environments.py @@ -362,6 +362,9 @@ def __init__(self) -> None: # Internal, overwrite all index functionality. self.PIPENV_TEST_INDEX = get_from_env("TEST_INDEX", check_for_negation=False) + # Internal, for testing the resolver without using subprocess + self.PIPENV_RESOLVER_PARENT_PYTHON = get_from_env("RESOLVER_PARENT_PYTHON") + # Internal, tells Pipenv about the surrounding environment. self.PIPENV_USE_SYSTEM = False self.PIPENV_VIRTUALENV = None diff --git a/pipenv/resolver.py b/pipenv/resolver.py index f7d08d0f2f..a9288e7573 100644 --- a/pipenv/resolver.py +++ b/pipenv/resolver.py @@ -614,7 +614,15 @@ def parse_packages(packages, pre, clear, system, requirements_dir=None): def resolve_packages( - pre, clear, verbose, system, requirements_dir, packages, category, constraints=None + pre, + clear, + verbose, + system, + write, + requirements_dir, + packages, + category, + constraints=None, ): from pipenv.utils.internet import create_mirror_source, replace_pypi_sources from pipenv.utils.resolver import resolve_deps @@ -662,11 +670,42 @@ def resolve( requirements_dir=requirements_dir, ) results = clean_results(results, resolver, project, category) + if write: + with open(write, "w") as fh: + if not results: + json.dump([], fh) + else: + json.dump(results, fh) if results: return results return [] +def _main( + pre, + clear, + verbose, + system, + write, + requirements_dir, + packages, + parse_only=False, + category=None, +): + if parse_only: + parse_packages( + packages, + pre=pre, + clear=clear, + system=system, + requirements_dir=requirements_dir, + ) + else: + resolve_packages( + pre, clear, verbose, system, write, requirements_dir, packages, category + ) + + def main(argv=None): parser = get_parser() parsed, remaining = parser.parse_known_args(argv) @@ -682,13 +721,15 @@ def main(argv=None): os.environ["PYTHONIOENCODING"] = "utf-8" os.environ["PYTHONUNBUFFERED"] = "1" parsed = handle_parsed_args(parsed) - resolve_packages( + _main( parsed.pre, parsed.clear, parsed.verbose, parsed.system, + parsed.write, parsed.requirements_dir, parsed.packages, + parse_only=parsed.parse_only, category=parsed.category, ) diff --git a/pipenv/utils/resolver.py b/pipenv/utils/resolver.py index 4568b30f41..25564e3908 100644 --- a/pipenv/utils/resolver.py +++ b/pipenv/utils/resolver.py @@ -1,15 +1,18 @@ import contextlib import hashlib +import json import os import subprocess import sys +import tempfile import warnings from functools import lru_cache from html.parser import HTMLParser +from pathlib import Path from typing import Dict, List, Optional, Set, Tuple, Union from urllib import parse -from pipenv import environments +from pipenv import environments, resolver from pipenv.exceptions import RequirementError, ResolutionFailure from pipenv.patched.pip._internal.cache import WheelCache from pipenv.patched.pip._internal.commands.install import InstallCommand @@ -27,7 +30,6 @@ from pipenv.patched.pip._internal.utils.temp_dir import global_tempdir_manager from pipenv.patched.pip._vendor import pkg_resources, rich from pipenv.project import Project -from pipenv.resolver import resolve_packages from pipenv.vendor import click from pipenv.vendor.requirementslib.fileutils import create_tracked_tempdir, open_file from pipenv.vendor.requirementslib.models.requirements import Line, Requirement @@ -55,7 +57,7 @@ from .indexes import parse_indexes, prepare_pip_source_args from .internet import _get_requests_session, is_pypi_url from .locking import format_requirement_for_lockfile, prepare_lockfile -from .shell import subprocess_run, temp_environ +from .shell import make_posix, subprocess_run, temp_environ console = rich.console.Console() err = rich.console.Console(stderr=True) @@ -1073,29 +1075,83 @@ def venv_resolve_deps( # dependency resolution on them, so we are including this step inside the # spinner context manager for the UX improvement st.console.print("Building requirements...") - deps = convert_deps_to_pip(deps, project) + deps = convert_deps_to_pip(deps, project, include_index=True) constraints = set(deps) st.console.print("Resolving dependencies...") - try: - results = resolve_packages( - pre, - clear, - project.s.is_verbose(), - allow_global, - req_dir, - packages=deps, - category=category, - constraints=constraints, + # Useful for debugging and hitting breakpoints in the resolver + if project.s.PIPENV_RESOLVER_PARENT_PYTHON: + try: + results = resolver.resolve_packages( + pre, + clear, + project.s.is_verbose(), + system=allow_global, + write=False, + requirements_dir=req_dir, + packages=deps, + category=category, + constraints=constraints, + ) + if results: + st.console.print( + environments.PIPENV_SPINNER_OK_TEXT.format("Success!") + ) + except Exception: + st.console.print( + environments.PIPENV_SPINNER_FAIL_TEXT.format("Locking Failed!") + ) + raise + else: # Default/Production behavior is to use project python's resolver + cmd = [ + which("python", allow_global=allow_global), + Path(resolver.__file__.rstrip("co")).as_posix(), + ] + if pre: + cmd.append("--pre") + if clear: + cmd.append("--clear") + if allow_global: + cmd.append("--system") + if category: + cmd.append("--category") + cmd.append(category) + target_file = tempfile.NamedTemporaryFile( + prefix="resolver", suffix=".json", delete=False ) - if results: + target_file.close() + cmd.extend(["--write", make_posix(target_file.name)]) + + with tempfile.NamedTemporaryFile( + mode="w+", prefix="pipenv", suffix="constraints.txt", delete=False + ) as constraints_file: + constraints_file.write(str("\n".join(constraints))) + cmd.append("--constraints-file") + cmd.append(constraints_file.name) + st.console.print("Resolving dependencies...") + c = resolve(cmd, st, project=project) + if c.returncode == 0: + try: + with open(target_file.name) as fh: + results = json.load(fh) + except (IndexError, json.JSONDecodeError): + click.echo(c.stdout.strip(), err=True) + click.echo(c.stderr.strip(), err=True) + if os.path.exists(target_file.name): + os.unlink(target_file.name) + raise RuntimeError("There was a problem with locking.") + if os.path.exists(target_file.name): + os.unlink(target_file.name) st.console.print( environments.PIPENV_SPINNER_OK_TEXT.format("Success!") ) - except Exception: - st.console.print( - environments.PIPENV_SPINNER_FAIL_TEXT.format("Locking Failed!") - ) - raise RuntimeError("There was a problem with locking.") + if not project.s.is_verbose() and c.stderr.strip(): + click.echo(click.style(f"Warning: {c.stderr.strip()}"), err=True) + else: + st.console.print( + environments.PIPENV_SPINNER_FAIL_TEXT.format("Locking Failed!") + ) + click.echo(f"Output: {c.stdout.strip()}", err=True) + click.echo(f"Error: {c.stderr.strip()}", err=True) if lockfile_section not in lockfile: lockfile[lockfile_section] = {} return prepare_lockfile(results, pipfile, lockfile[lockfile_section]) diff --git a/pipenv/vendor/requirementslib/models/setup_info.py b/pipenv/vendor/requirementslib/models/setup_info.py index 73eb25c1ea..ef77db38bf 100644 --- a/pipenv/vendor/requirementslib/models/setup_info.py +++ b/pipenv/vendor/requirementslib/models/setup_info.py @@ -1133,8 +1133,9 @@ def run_setup(script_path, egg_base=None): if egg_base: args += ["--egg-base", egg_base] + python = os.environ.get("PIP_PYTHON_PATH", sys.executable) sp.run( - [sys.executable, "setup.py"] + args, + [python, "setup.py"] + args, capture_output=True, ) dist = get_metadata(egg_base, metadata_type="egg")