Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Improve UnsatisfiableInterpreterConstraintsError. #1028

Merged
merged 3 commits into from
Sep 11, 2020
Merged

Improve UnsatisfiableInterpreterConstraintsError. #1028

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
47c23b5
Improve UnsatisfiableInterpreterConstraintsError.
jsirois Sep 1, 2020
ae95ab3
Fix inscrutable type bug.
jsirois Sep 11, 2020
8daa08c
Button up error message matrix.
jsirois Sep 11, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
59 changes: 53 additions & 6 deletions pex/interpreter.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,13 +12,14 @@
import re
import subprocess
import sys
from collections import OrderedDict
from textwrap import dedent

from pex import third_party
from pex.common import safe_rmtree
from pex.compatibility import string
from pex.executor import Executor
from pex.jobs import Job, SpawnedJob, execute_parallel
from pex.jobs import Job, Retain, SpawnedJob, execute_parallel
from pex.platforms import Platform
from pex.third_party.packaging import markers, tags
from pex.third_party.pkg_resources import Distribution, Requirement
Expand Down Expand Up @@ -283,16 +284,56 @@ def _paths(paths=None):

@classmethod
def iter(cls, paths=None):
"""Iterate all interpreters found in `paths`.
"""Iterate all valid interpreters found in `paths`.
NB: The paths can either be directories to search for python binaries or the paths of python
binaries themselves.
:param paths: The paths to look for python interpreters; by default the `PATH`.
:type paths: list str
:return: An iterator of PythonInterpreter.
"""
return cls._filter(cls._find(cls._paths(paths=paths)))

@classmethod
def iter_candidates(cls, paths=None):
"""Iterate all likely interpreters found in `paths`.
NB: The paths can either be directories to search for python binaries or the paths of python
binaries themselves.
:param paths: The paths to look for python interpreters; by default the `PATH`.
:type paths: list str
:return: A heterogeneous iterator over valid interpreters and (python, error) invalid
python binary tuples.
:rtype: A heterogeneous iterator over :class:`PythonInterpreter` and (str, str).
"""
failed_interpreters = OrderedDict()

def iter_interpreters():
for candidate in cls._find(cls._paths(paths=paths), error_handler=Retain()):
if isinstance(candidate, cls):
yield candidate
else:
python, exception = candidate
if isinstance(exception, Job.Error):
# We spawned a subprocess to identify the interpreter but the interpreter
# could not run our identification code meaning the interpreter is either
# broken or old enough that it either can't parse our identification code
# or else provide stdlib modules we expect. The stderr should indicate the
# broken-ness appropriately.
failed_interpreters[python] = exception.stderr.strip()
else:
# We couldn't even spawn a subprocess to identify the interpreter. The
# likely OSError should help identify the underlying issue.
failed_interpreters[python] = repr(exception)

for interpreter in cls._filter(iter_interpreters()):
yield interpreter

for python, error in failed_interpreters.items():
yield python, error

@classmethod
def all(cls, paths=None):
return list(cls.iter(paths=paths))
Expand Down Expand Up @@ -463,22 +504,28 @@ def _matches_binary_name(cls, path):
return any(matcher.match(basefile) is not None for matcher in cls._REGEXEN)

@classmethod
def _find(cls, paths):
def _find(cls, paths, error_handler=None):
"""Given a list of files or directories, try to detect python interpreters amongst them.
Returns an iterator over PythonInterpreter objects.
"""
return cls._identify_interpreters(filter=cls._matches_binary_name, paths=paths)
return cls._identify_interpreters(
filter=cls._matches_binary_name, paths=paths, error_handler=error_handler
)

@classmethod
def _identify_interpreters(cls, filter, paths=None):
def _identify_interpreters(cls, filter, paths=None, error_handler=None):
def iter_candidates():
for path in cls._paths(paths=paths):
for fn in cls._expand_path(path):
if filter(fn):
yield fn

return execute_parallel(inputs=list(iter_candidates()), spawn_func=cls._spawn_from_binary)
return execute_parallel(
inputs=list(iter_candidates()),
spawn_func=cls._spawn_from_binary,
error_handler=error_handler,
)

@classmethod
def _filter(cls, pythons):
Expand Down
95 changes: 53 additions & 42 deletions pex/interpreter_constraints.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,11 +3,12 @@

# A library of functions for filtering Python interpreters based on compatibility constraints

from __future__ import absolute_import
from __future__ import absolute_import, print_function

import os

from pex.common import die
from pex.interpreter import PythonIdentity
from pex.tracer import TRACER


def validate_constraints(constraints):
Expand All @@ -24,7 +25,7 @@ def validate_constraints(constraints):
class UnsatisfiableInterpreterConstraintsError(Exception):
"""Indicates interpreter constraints could not be satisfied."""

def __init__(self, constraints, candidates):
def __init__(self, constraints, candidates, failures):
"""
:param constraints: The constraints that could not be satisfied.
:type constraints: iterable of str
Expand All @@ -33,6 +34,7 @@ def __init__(self, constraints, candidates):
"""
self.constraints = tuple(constraints)
self.candidates = tuple(candidates)
self.failures = tuple(failures)
super(UnsatisfiableInterpreterConstraintsError, self).__init__(self.create_message())

def create_message(self, preamble=None):
Expand All @@ -45,54 +47,63 @@ def create_message(self, preamble=None):
:rtype: str
"""
preamble = "{}\n\n".format(preamble) if preamble else ""

failures_message = ""
if self.failures:
seen = set()
broken_interpreters = []
for python, error in self.failures:
canonical_python = os.path.realpath(python)
if canonical_python not in seen:
broken_interpreters.append((canonical_python, error))
seen.add(canonical_python)

failures_message = (
"{}\n"
"\n"
"(See https://github.com/pantsbuild/pex/issues/1027 for a list of known breaks and "
"workarounds.)"
).format(
"\n".join(
"{index}.) {binary}:\n{error}".format(index=i, binary=python, error=error)
for i, (python, error) in enumerate(broken_interpreters, start=1)
)
)

if not self.candidates:
if failures_message:
return (
"{preamble}"
"Interpreters were found but they all appear to be broken:\n"
"{failures}"
).format(preamble=preamble, failures=failures_message)
return "{}No interpreters could be found on the system.".format(preamble)

binary_column_width = max(len(candidate.binary) for candidate in self.candidates)
interpreters_format = "{{binary: >{}}} {{requirement}}".format(binary_column_width)
interpreters_format = "{{index}}.) {{binary: >{}}} {{requirement}}".format(
binary_column_width
)
if failures_message:
failures_message = (
"\n" "\n" "Skipped the following broken interpreters:\n" "{}"
).format(failures_message)
return (
"{preamble}"
"Examined the following interpreters:\n {interpreters}\n\n"
"None were compatible with the requested constraints:\n {constraints}"
"Examined the following {qualifier}interpreters:\n"
"{interpreters}"
"{failures_message}\n"
"\n"
"No {qualifier}interpreter compatible with the requested constraints was found:\n"
" {constraints}"
).format(
preamble=preamble,
interpreters="\n ".join(
qualifier="working " if failures_message else "",
interpreters="\n".join(
interpreters_format.format(
binary=candidate.binary, requirement=candidate.identity.requirement
index=i, binary=candidate.binary, requirement=candidate.identity.requirement
)
for candidate in self.candidates
for i, candidate in enumerate(self.candidates, start=1)
),
constraints="\n ".join(self.constraints),
failures_message=failures_message,
)


def matched_interpreters_iter(interpreters_iter, constraints):
"""Given some filters, yield any interpreter that matches at least one of them.
:param interpreters_iter: A `PythonInterpreter` iterable for filtering.
:param constraints: A sequence of strings that constrain the interpreter compatibility for this
pex. Each string uses the Requirement-style format, e.g. 'CPython>=3' or '>=2.7,<3' for
requirements agnostic to interpreter class. Multiple requirement strings may be combined
into a list to OR the constraints, such as ['CPython>=2.7,<3', 'CPython>=3.4'].
:return: returns a generator that yields compatible interpreters
:raises: :class:`UnsatisfiableInterpreterConstraintsError` if constraints were given and could
not be satisfied. The exception will only be raised when the returned generator is fully
consumed.
"""
candidates = []
found = False

for interpreter in interpreters_iter:
if any(interpreter.identity.matches(filt) for filt in constraints):
TRACER.log(
"Constraints on interpreters: %s, Matching Interpreter: %s"
% (constraints, interpreter.binary),
V=3,
)
found = True
yield interpreter

if not found:
candidates.append(interpreter)

if not found:
raise UnsatisfiableInterpreterConstraintsError(constraints, candidates)
Loading