Skip to content

Commit

Permalink
Improve UnsatisfiableInterpreterConstraintsError. (#1028)
Browse files Browse the repository at this point in the history 
Surface all broken interpreters and a link to known breaks and
workarounds tracked in #1027.
  • Loading branch information
@jsirois
jsirois authored Sep 11, 2020
1 parent 8cf8011 commit f44cba1
Show file tree
Hide file tree
Showing 5 changed files with 282 additions and 96 deletions.
59 changes: 53 additions & 6 deletions pex/interpreter.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,13 +12,14 @@
import re
import subprocess
import sys
from collections import OrderedDict
from textwrap import dedent

from pex import third_party
from pex.common import safe_rmtree
from pex.compatibility import string
from pex.executor import Executor
from pex.jobs import Job, SpawnedJob, execute_parallel
from pex.jobs import Job, Retain, SpawnedJob, execute_parallel
from pex.platforms import Platform
from pex.third_party.packaging import markers, tags
from pex.third_party.pkg_resources import Distribution, Requirement
Expand Down Expand Up @@ -283,16 +284,56 @@ def _paths(paths=None):

@classmethod
def iter(cls, paths=None):
"""Iterate all interpreters found in `paths`.
"""Iterate all valid interpreters found in `paths`.
NB: The paths can either be directories to search for python binaries or the paths of python
binaries themselves.
:param paths: The paths to look for python interpreters; by default the `PATH`.
:type paths: list str
:return: An iterator of PythonInterpreter.
"""
return cls._filter(cls._find(cls._paths(paths=paths)))

@classmethod
def iter_candidates(cls, paths=None):
"""Iterate all likely interpreters found in `paths`.
NB: The paths can either be directories to search for python binaries or the paths of python
binaries themselves.
:param paths: The paths to look for python interpreters; by default the `PATH`.
:type paths: list str
:return: A heterogeneous iterator over valid interpreters and (python, error) invalid
python binary tuples.
:rtype: A heterogeneous iterator over :class:`PythonInterpreter` and (str, str).
"""
failed_interpreters = OrderedDict()

def iter_interpreters():
for candidate in cls._find(cls._paths(paths=paths), error_handler=Retain()):
if isinstance(candidate, cls):
yield candidate
else:
python, exception = candidate
if isinstance(exception, Job.Error):
# We spawned a subprocess to identify the interpreter but the interpreter
# could not run our identification code meaning the interpreter is either
# broken or old enough that it either can't parse our identification code
# or else provide stdlib modules we expect. The stderr should indicate the
# broken-ness appropriately.
failed_interpreters[python] = exception.stderr.strip()
else:
# We couldn't even spawn a subprocess to identify the interpreter. The
# likely OSError should help identify the underlying issue.
failed_interpreters[python] = repr(exception)

for interpreter in cls._filter(iter_interpreters()):
yield interpreter

for python, error in failed_interpreters.items():
yield python, error

@classmethod
def all(cls, paths=None):
return list(cls.iter(paths=paths))
Expand Down Expand Up @@ -463,22 +504,28 @@ def _matches_binary_name(cls, path):
return any(matcher.match(basefile) is not None for matcher in cls._REGEXEN)

@classmethod
def _find(cls, paths):
def _find(cls, paths, error_handler=None):
"""Given a list of files or directories, try to detect python interpreters amongst them.
Returns an iterator over PythonInterpreter objects.
"""
return cls._identify_interpreters(filter=cls._matches_binary_name, paths=paths)
return cls._identify_interpreters(
filter=cls._matches_binary_name, paths=paths, error_handler=error_handler
)

@classmethod
def _identify_interpreters(cls, filter, paths=None):
def _identify_interpreters(cls, filter, paths=None, error_handler=None):
def iter_candidates():
for path in cls._paths(paths=paths):
for fn in cls._expand_path(path):
if filter(fn):
yield fn

return execute_parallel(inputs=list(iter_candidates()), spawn_func=cls._spawn_from_binary)
return execute_parallel(
inputs=list(iter_candidates()),
spawn_func=cls._spawn_from_binary,
error_handler=error_handler,
)

@classmethod
def _filter(cls, pythons):
Expand Down
95 changes: 53 additions & 42 deletions pex/interpreter_constraints.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,11 +3,12 @@

# A library of functions for filtering Python interpreters based on compatibility constraints

from __future__ import absolute_import
from __future__ import absolute_import, print_function

import os

from pex.common import die
from pex.interpreter import PythonIdentity
from pex.tracer import TRACER


def validate_constraints(constraints):
Expand All @@ -24,7 +25,7 @@ def validate_constraints(constraints):
class UnsatisfiableInterpreterConstraintsError(Exception):
"""Indicates interpreter constraints could not be satisfied."""

def __init__(self, constraints, candidates):
def __init__(self, constraints, candidates, failures):
"""
:param constraints: The constraints that could not be satisfied.
:type constraints: iterable of str
Expand All @@ -33,6 +34,7 @@ def __init__(self, constraints, candidates):
"""
self.constraints = tuple(constraints)
self.candidates = tuple(candidates)
self.failures = tuple(failures)
super(UnsatisfiableInterpreterConstraintsError, self).__init__(self.create_message())

def create_message(self, preamble=None):
Expand All @@ -45,54 +47,63 @@ def create_message(self, preamble=None):
:rtype: str
"""
preamble = "{}\n\n".format(preamble) if preamble else ""

failures_message = ""
if self.failures:
seen = set()
broken_interpreters = []
for python, error in self.failures:
canonical_python = os.path.realpath(python)
if canonical_python not in seen:
broken_interpreters.append((canonical_python, error))
seen.add(canonical_python)

failures_message = (
"{}\n"
"\n"
"(See https://github.com/pantsbuild/pex/issues/1027 for a list of known breaks and "
"workarounds.)"
).format(
"\n".join(
"{index}.) {binary}:\n{error}".format(index=i, binary=python, error=error)
for i, (python, error) in enumerate(broken_interpreters, start=1)
)
)

if not self.candidates:
if failures_message:
return (
"{preamble}"
"Interpreters were found but they all appear to be broken:\n"
"{failures}"
).format(preamble=preamble, failures=failures_message)
return "{}No interpreters could be found on the system.".format(preamble)

binary_column_width = max(len(candidate.binary) for candidate in self.candidates)
interpreters_format = "{{binary: >{}}} {{requirement}}".format(binary_column_width)
interpreters_format = "{{index}}.) {{binary: >{}}} {{requirement}}".format(
binary_column_width
)
if failures_message:
failures_message = (
"\n" "\n" "Skipped the following broken interpreters:\n" "{}"
).format(failures_message)
return (
"{preamble}"
"Examined the following interpreters:\n {interpreters}\n\n"
"None were compatible with the requested constraints:\n {constraints}"
"Examined the following {qualifier}interpreters:\n"
"{interpreters}"
"{failures_message}\n"
"\n"
"No {qualifier}interpreter compatible with the requested constraints was found:\n"
" {constraints}"
).format(
preamble=preamble,
interpreters="\n ".join(
qualifier="working " if failures_message else "",
interpreters="\n".join(
interpreters_format.format(
binary=candidate.binary, requirement=candidate.identity.requirement
index=i, binary=candidate.binary, requirement=candidate.identity.requirement
)
for candidate in self.candidates
for i, candidate in enumerate(self.candidates, start=1)
),
constraints="\n ".join(self.constraints),
failures_message=failures_message,
)


def matched_interpreters_iter(interpreters_iter, constraints):
"""Given some filters, yield any interpreter that matches at least one of them.
:param interpreters_iter: A `PythonInterpreter` iterable for filtering.
:param constraints: A sequence of strings that constrain the interpreter compatibility for this
pex. Each string uses the Requirement-style format, e.g. 'CPython>=3' or '>=2.7,<3' for
requirements agnostic to interpreter class. Multiple requirement strings may be combined
into a list to OR the constraints, such as ['CPython>=2.7,<3', 'CPython>=3.4'].
:return: returns a generator that yields compatible interpreters
:raises: :class:`UnsatisfiableInterpreterConstraintsError` if constraints were given and could
not be satisfied. The exception will only be raised when the returned generator is fully
consumed.
"""
candidates = []
found = False

for interpreter in interpreters_iter:
if any(interpreter.identity.matches(filt) for filt in constraints):
TRACER.log(
"Constraints on interpreters: %s, Matching Interpreter: %s"
% (constraints, interpreter.binary),
V=3,
)
found = True
yield interpreter

if not found:
candidates.append(interpreter)

if not found:
raise UnsatisfiableInterpreterConstraintsError(constraints, candidates)
Loading

0 comments on commit f44cba1

Please sign in to comment.