diff --git a/pex/environment.py b/pex/environment.py index c659ca408..82c818f29 100644 --- a/pex/environment.py +++ b/pex/environment.py @@ -27,12 +27,14 @@ if TYPE_CHECKING: from typing import ( DefaultDict, + Dict, FrozenSet, Iterable, Iterator, List, MutableMapping, Optional, + Tuple, Union, ) @@ -206,6 +208,8 @@ def satisfied_keys(self): class PEXEnvironment(object): + _CACHE = {} # type: Dict[Tuple[str, str, Target], PEXEnvironment] + @classmethod def mount( cls, @@ -214,18 +218,24 @@ def mount( target=None, # type: Optional[Target] ): # type: (...) -> PEXEnvironment + pex_file = os.path.realpath(pex) if not pex_info: - pex_info = PexInfo.from_pex(pex) + pex_info = PexInfo.from_pex(pex_file) pex_info.update(PexInfo.from_env()) pex_hash = pex_info.pex_hash if pex_hash is None: raise AssertionError( "There was no pex_hash stored in {} for {}.".format(PexInfo.PATH, pex) ) - pex_root = pex_info.pex_root - pex = maybe_install(pex=pex, pex_root=pex_root, pex_hash=pex_hash) or pex target = target or targets.current() - return cls(pex=pex, pex_info=pex_info, target=target) + key = (pex_file, pex_hash, target) + mounted = cls._CACHE.get(key) + if mounted is None: + pex_root = pex_info.pex_root + pex = maybe_install(pex=pex, pex_root=pex_root, pex_hash=pex_hash) or pex + mounted = cls(pex=pex, pex_info=pex_info, target=target) + cls._CACHE[key] = mounted + return mounted def __init__( self, diff --git a/pex/pex_bootstrapper.py b/pex/pex_bootstrapper.py index 0f1f434aa..af51e638e 100644 --- a/pex/pex_bootstrapper.py +++ b/pex/pex_bootstrapper.py @@ -9,11 +9,13 @@ from pex import pex_warnings from pex.common import atomic_directory, die, pluralize +from pex.environment import ResolveError from pex.inherit_path import InheritPath from pex.interpreter import PythonInterpreter from pex.interpreter_constraints import UnsatisfiableInterpreterConstraintsError from pex.orderedset import OrderedSet from pex.pex_info import PexInfo +from pex.targets import LocalInterpreter from pex.tracer import TRACER from pex.typing import TYPE_CHECKING, cast from pex.variables import ENV @@ -21,9 +23,13 @@ if TYPE_CHECKING: from typing import Iterable, Iterator, List, NoReturn, Optional, Set, Tuple, Union + import attr # vendor:skip + from pex.dist_metadata import Requirement from pex.interpreter import InterpreterIdentificationError, InterpreterOrError, PathFilter from pex.pex import PEX +else: + from pex.third_party import attr def parse_path(path): @@ -36,6 +42,38 @@ def parse_path(path): ) +@attr.s(frozen=True) +class InterpreterTest(object): + entry_point = attr.ib() # type: str + pex_info = attr.ib() # type: PexInfo + + @property + def interpreter_constraints(self): + # type: () -> List[str] + return self.pex_info.interpreter_constraints + + def test_resolve(self, interpreter): + # type: (PythonInterpreter) -> bool + """Checks if `interpreter` can resolve all required distributions for the PEX under test.""" + with TRACER.timed( + "Testing {python} can resolve PEX at {pex}".format( + python=interpreter.binary, pex=self.entry_point + ) + ): + from pex.environment import PEXEnvironment + + pex_environment = PEXEnvironment.mount( + self.entry_point, + pex_info=self.pex_info, + target=LocalInterpreter.create(interpreter), + ) + try: + pex_environment.resolve() + return True + except ResolveError as e: + return False + + # TODO(John Sirois): Move this to interpreter_constraints.py. As things stand, both pex/bin/pex.py # and this file use this function. The Pex CLI should not depend on this file which hosts code # used at PEX runtime. @@ -44,6 +82,7 @@ def iter_compatible_interpreters( valid_basenames=None, # type: Optional[Iterable[str]] interpreter_constraints=None, # type: Optional[Iterable[Union[str, Requirement]]] preferred_interpreter=None, # type: Optional[PythonInterpreter] + interpreter_test=None, # type: Optional[InterpreterTest] ): # type: (...) -> Iterator[PythonInterpreter] """Find all compatible interpreters on the system within the supplied constraints. @@ -56,7 +95,7 @@ def iter_compatible_interpreters( `--interpreter-constraint`. :param preferred_interpreter: For testing - an interpreter to prefer amongst all others. Defaults to the current running interpreter. - + :param interpreter_test: Optional test to verify selected interpreters can boot a given PEX. Interpreters are searched for in `path` if specified and $PATH if not. If no interpreters are found and there are no further constraints (neither `valid_basenames` nor @@ -110,10 +149,9 @@ def _valid_interpreter(interp_or_error): if not isinstance(interp_or_error, PythonInterpreter): return False + interp = interp_or_error if not interpreter_constraints: - return True - - interp = cast(PythonInterpreter, interp_or_error) + return interpreter_test.test_resolve(interp) if interpreter_test else True if any( interp.identity.matches(interpreter_constraint) @@ -125,7 +163,7 @@ def _valid_interpreter(interp_or_error): ), V=3, ) - return True + return interpreter_test.test_resolve(interp) if interpreter_test else True return False @@ -135,9 +173,9 @@ def _valid_interpreter(interp_or_error): for interpreter_or_error in _iter_interpreters(): if isinstance(interpreter_or_error, PythonInterpreter): - interpreter = cast(PythonInterpreter, interpreter_or_error) + interpreter = interpreter_or_error candidates.append(interpreter) - if _valid_interpreter(interpreter_or_error): + if _valid_interpreter(interpreter): found = True yield interpreter else: @@ -160,13 +198,16 @@ def _select_path_interpreter( valid_basenames=None, # type: Optional[Tuple[str, ...]] interpreter_constraints=None, # type: Optional[Iterable[str]] preferred_interpreter=None, # type: Optional[PythonInterpreter] + interpreter_test=None, # type: Optional[InterpreterTest] ): # type: (...) -> Optional[PythonInterpreter] + candidate_interpreters_iter = iter_compatible_interpreters( path=path, valid_basenames=valid_basenames, interpreter_constraints=interpreter_constraints, preferred_interpreter=preferred_interpreter, + interpreter_test=interpreter_test, ) current_interpreter = PythonInterpreter.get() # type: PythonInterpreter preferred_interpreter = preferred_interpreter or current_interpreter @@ -188,8 +229,10 @@ def _select_path_interpreter( return PythonInterpreter.latest_release_of_min_compatible_version(candidate_interpreters) -def find_compatible_interpreter(interpreter_constraints=None): - # type: (Optional[Iterable[str]]) -> PythonInterpreter +def find_compatible_interpreter(interpreter_test=None): + # type: (Optional[InterpreterTest]) -> PythonInterpreter + + interpreter_constraints = interpreter_test.interpreter_constraints if interpreter_test else None def gather_constraints(): # type: () -> Iterable[str] @@ -233,11 +276,13 @@ def gather_constraints(): path=ENV.PEX_PYTHON, interpreter_constraints=interpreter_constraints, preferred_interpreter=preferred_interpreter, + interpreter_test=interpreter_test, ) else: target = _select_path_interpreter( valid_basenames=(os.path.basename(ENV.PEX_PYTHON),), interpreter_constraints=interpreter_constraints, + interpreter_test=interpreter_test, ) except UnsatisfiableInterpreterConstraintsError as e: raise e.with_preamble( @@ -245,7 +290,7 @@ def gather_constraints(): pex_python=ENV.PEX_PYTHON ) ) - elif ENV.PEX_PYTHON_PATH or interpreter_constraints: + else: TRACER.log( "Using {path} constrained by {constraints}".format( path="PEX_PYTHON_PATH={}".format(ENV.PEX_PYTHON_PATH) @@ -260,6 +305,7 @@ def gather_constraints(): path=ENV.PEX_PYTHON_PATH, interpreter_constraints=interpreter_constraints, preferred_interpreter=preferred_interpreter, + interpreter_test=interpreter_test, ) except UnsatisfiableInterpreterConstraintsError as e: raise e.with_preamble( @@ -293,8 +339,8 @@ def gather_constraints(): return target -def maybe_reexec_pex(interpreter_constraints=None): - # type: (Optional[Iterable[str]]) -> Union[None, NoReturn] +def maybe_reexec_pex(interpreter_test): + # type: (InterpreterTest) -> Union[None, NoReturn] """Handle environment overrides for the Python interpreter to use when executing this pex. This function supports interpreter filtering based on interpreter constraints stored in PEX-INFO @@ -308,8 +354,7 @@ def maybe_reexec_pex(interpreter_constraints=None): currently executing interpreter. If compatibility constraints are used, we match those constraints against these interpreters. - :param interpreter_constraints: Optional list of requirements-style strings that constrain the - Python interpreter to re-exec this pex with. + :param interpreter_test: Optional test to verify selected interpreters can boot a given PEX. """ current_interpreter = PythonInterpreter.get() @@ -327,9 +372,7 @@ def maybe_reexec_pex(interpreter_constraints=None): return None try: - target = find_compatible_interpreter( - interpreter_constraints=interpreter_constraints, - ) + target = find_compatible_interpreter(interpreter_test=interpreter_test) except UnsatisfiableInterpreterConstraintsError as e: die(str(e)) @@ -376,7 +419,7 @@ def maybe_reexec_pex(interpreter_constraints=None): python=sys.executable, pex_python=ENV.PEX_PYTHON, pex_python_path=ENV.PEX_PYTHON_PATH, - interpreter_constraints=interpreter_constraints, + interpreter_constraints=interpreter_test.interpreter_constraints, pythonpath=', (stashed) PYTHONPATH="{}"'.format(pythonpath) if pythonpath is not None else "", @@ -496,15 +539,14 @@ def ensure_venv( def bootstrap_pex(entry_point): # type: (str) -> None pex_info = _bootstrap(entry_point) + interpreter_test = InterpreterTest(entry_point=entry_point, pex_info=pex_info) # ENV.PEX_ROOT is consulted by PythonInterpreter and Platform so set that up as early as # possible in the run. with ENV.patch(PEX_ROOT=pex_info.pex_root): if not (ENV.PEX_UNZIP or ENV.PEX_TOOLS) and pex_info.venv: try: - target = find_compatible_interpreter( - interpreter_constraints=pex_info.interpreter_constraints, - ) + target = find_compatible_interpreter(interpreter_test=interpreter_test) except UnsatisfiableInterpreterConstraintsError as e: die(str(e)) from . import pex @@ -515,7 +557,7 @@ def bootstrap_pex(entry_point): die(str(e)) os.execv(venv_pex, [venv_pex] + sys.argv[1:]) else: - maybe_reexec_pex(pex_info.interpreter_constraints) + maybe_reexec_pex(interpreter_test=interpreter_test) from . import pex pex.PEX(entry_point).execute() diff --git a/pex/pex_info.py b/pex/pex_info.py index 3e8a6063e..c753da715 100644 --- a/pex/pex_info.py +++ b/pex/pex_info.py @@ -19,7 +19,7 @@ from pex.version import __version__ as pex_version if TYPE_CHECKING: - from typing import Any, Dict, Mapping, Optional, Text, Union + from typing import Any, Dict, List, Mapping, Optional, Text, Union from pex.interpreter import PythonInterpreter @@ -288,6 +288,7 @@ def inherit_path(self, value): @property def interpreter_constraints(self): + # type: () -> List[str] """A list of constraints that determine the interpreter compatibility for this pex, using the Requirement-style format, e.g. ``'CPython>=3', or just '>=2.7,<3'`` for requirements agnostic to interpreter class. diff --git a/pex/tools/commands/interpreter.py b/pex/tools/commands/interpreter.py index 0061d0e09..03df95c44 100644 --- a/pex/tools/commands/interpreter.py +++ b/pex/tools/commands/interpreter.py @@ -11,6 +11,7 @@ from pex.interpreter import PythonInterpreter from pex.interpreter_constraints import UnsatisfiableInterpreterConstraintsError from pex.pex import PEX +from pex.pex_bootstrapper import InterpreterTest from pex.result import Error, Ok, Result from pex.tools.command import PEXCommand from pex.typing import TYPE_CHECKING @@ -62,9 +63,11 @@ def _find_interpreters(self, pex): "Ignoring PEX_PYTHON={} in order to scan for all compatible " "interpreters.".format(ENV.PEX_PYTHON) ) + pex_info = pex.pex_info() for interpreter in pex_bootstrapper.iter_compatible_interpreters( path=ENV.PEX_PYTHON_PATH, - interpreter_constraints=pex.pex_info().interpreter_constraints, + interpreter_constraints=pex_info.interpreter_constraints, + interpreter_test=InterpreterTest(entry_point=pex.path(), pex_info=pex_info), ): yield interpreter diff --git a/pex/tools/main.py b/pex/tools/main.py index b9a7637cf..5e1b5fa35 100644 --- a/pex/tools/main.py +++ b/pex/tools/main.py @@ -9,6 +9,7 @@ from pex import pex_bootstrapper from pex.commands.command import GlobalConfigurationError, Main from pex.pex import PEX +from pex.pex_bootstrapper import InterpreterTest from pex.pex_info import PexInfo from pex.result import Result, catch from pex.tools import commands @@ -84,7 +85,7 @@ def main(pex=None): pex_info = PexInfo.from_pex(pex_file_path) pex_info.update(PexInfo.from_env()) interpreter = pex_bootstrapper.find_compatible_interpreter( - interpreter_constraints=pex_info.interpreter_constraints + interpreter_test=InterpreterTest(entry_point=pex_file_path, pex_info=pex_info) ) pex = PEX(pex_file_path, interpreter=interpreter) diff --git a/tests/integration/test_pex_bootstrapper.py b/tests/integration/test_pex_bootstrapper.py index 15181aa65..c6f8b5ef3 100644 --- a/tests/integration/test_pex_bootstrapper.py +++ b/tests/integration/test_pex_bootstrapper.py @@ -4,21 +4,23 @@ import os.path import re import subprocess +import sys from textwrap import dedent import pytest from pex.common import safe_open +from pex.interpreter import PythonInterpreter from pex.pex import PEX from pex.pex_bootstrapper import ensure_venv from pex.pex_info import PexInfo -from pex.testing import make_env, run_pex_command +from pex.testing import PY27, PY37, PY_VER, ensure_python_interpreter, make_env, run_pex_command from pex.typing import TYPE_CHECKING from pex.venv.pex import CollisionError from pex.venv.virtualenv import Virtualenv if TYPE_CHECKING: - from typing import Any, Set, Text + from typing import Any, Optional, Set, Text def test_ensure_venv_short_link( @@ -258,3 +260,98 @@ def assert_venv_site_packages_copies(copies): assert_venv_site_packages_copies(copies=True) assert_venv_site_packages_copies(copies=False) + + +def test_boot_compatible_issue_1020_no_ic(tmpdir): + # type: (Any) -> None + + pex = os.path.join(str(tmpdir), "pex") + run_pex_command(args=["psutil==5.9.0", "-o", pex]).assert_success() + + def assert_boot(python=None): + # type: (Optional[str]) -> None + args = [python] if python else [] + args.extend([pex, "-c", "import psutil, sys; print(sys.executable)"]) + output = subprocess.check_output(args=args, stderr=subprocess.PIPE) + + # N.B.: We expect the current interpreter the PEX was built with to be selected since the + # PEX contains a single platform specific distribution that only works with that + # interpreter. If the current interpreter is in a venv though, we expect the PEX bootstrap + # to have broken out of the venv and used its base system interpreter. + # See: + # https://github.com/pantsbuild/pex/pull/1130 + # https://github.com/pantsbuild/pex/issues/1031 + assert ( + PythonInterpreter.get().resolve_base_interpreter() + == PythonInterpreter.from_binary( + str(output.decode("ascii").strip()) + ).resolve_base_interpreter() + ) + + assert_boot() + assert_boot(sys.executable) + + other_interpreter = ( + ensure_python_interpreter(PY27) if PY_VER != (2, 7) else ensure_python_interpreter(PY37) + ) + assert_boot(other_interpreter) + + +def test_boot_compatible_issue_1020_ic_min_compatible_build_time_hole(tmpdir): + # type: (Any) -> None + other_interpreter = PythonInterpreter.from_binary( + ensure_python_interpreter(PY27) if PY_VER != (2, 7) else ensure_python_interpreter(PY37) + ) + current_interpreter = PythonInterpreter.get() + + min_interpreter, max_interpreter = ( + (other_interpreter, current_interpreter) + if other_interpreter.version < current_interpreter.version + else (current_interpreter, other_interpreter) + ) + assert min_interpreter.version < max_interpreter.version + + # Try to build a PEX that works for min and max, but only find max locally. + pex = os.path.join(str(tmpdir), "pex") + run_pex_command( + args=[ + "psutil==5.9.0", + "-o", + pex, + "--python-path", + max_interpreter.binary, + "--interpreter-constraint", + "{python}=={major}.{minor}.*".format( + python=max_interpreter.identity.interpreter, + major=min_interpreter.version[0], + minor=min_interpreter.version[1], + ), + "--interpreter-constraint", + "{python}=={major}.{minor}.*".format( + python=max_interpreter.identity.interpreter, + major=max_interpreter.version[0], + minor=max_interpreter.version[1], + ), + ] + ).assert_success() + + # Now try to run the PEX remotely where both min and max exist. + output = subprocess.check_output( + args=[min_interpreter.binary, pex, "-c", "import psutil, sys; print(sys.executable)"], + env=make_env(PEX_PYTHON_PATH=":".join((min_interpreter.binary, max_interpreter.binary))), + stderr=subprocess.PIPE, + ) + + # N.B.: We expect the max interpreter the PEX was built with to be selected since the + # PEX contains a single platform specific distribution that only works with that + # interpreter. If the max interpreter is in a venv though, we expect the PEX bootstrap + # to have broken out of the venv and used its base system interpreter. + # See: + # https://github.com/pantsbuild/pex/pull/1130 + # https://github.com/pantsbuild/pex/issues/1031 + assert ( + max_interpreter.resolve_base_interpreter() + == PythonInterpreter.from_binary( + str(output.decode("ascii").strip()) + ).resolve_base_interpreter() + ) diff --git a/tests/test_pex_bootstrapper.py b/tests/test_pex_bootstrapper.py index e588edff2..c6b5e0c3f 100644 --- a/tests/test_pex_bootstrapper.py +++ b/tests/test_pex_bootstrapper.py @@ -14,6 +14,7 @@ from pex.interpreter_constraints import UnsatisfiableInterpreterConstraintsError from pex.pex import PEX from pex.pex_bootstrapper import ( + InterpreterTest, ensure_venv, find_compatible_interpreter, iter_compatible_interpreters, @@ -280,9 +281,13 @@ def test_pp_exact_satisfies_constraints(): py310 = ensure_python_interpreter(PY310) + pb = PEXBuilder() + pb.info.add_interpreter_constraint(">=3.7") + pb.freeze() + with ENV.patch(PEX_PYTHON=py310): assert PythonInterpreter.from_binary(py310) == find_compatible_interpreter( - interpreter_constraints=[">=3.7"] + interpreter_test=InterpreterTest(entry_point=pb.path(), pex_info=pb.info), ) @@ -291,12 +296,18 @@ def test_pp_exact_does_not_satisfy_constraints(): py310 = ensure_python_interpreter(PY310) + pb = PEXBuilder() + pb.info.add_interpreter_constraint("<=3.7") + pb.freeze() + with ENV.patch(PEX_PYTHON=py310): with pytest.raises( UnsatisfiableInterpreterConstraintsError, match=r"Failed to find a compatible PEX_PYTHON={pp}.".format(pp=py310), ): - find_compatible_interpreter(interpreter_constraints=["<=3.7"]) + find_compatible_interpreter( + interpreter_test=InterpreterTest(entry_point=pb.path(), pex_info=pb.info), + ) def test_pp_exact_not_on_ppp():