Skip to content

Commit

Permalink
Fixup single target Pip log case.
Browse files Browse the repository at this point in the history
  • Loading branch information
jsirois committed Sep 17, 2024
1 parent acab547 commit a1ec512
Show file tree
Hide file tree
Showing 3 changed files with 77 additions and 57 deletions.
16 changes: 8 additions & 8 deletions pex/platforms.py
Original file line number Diff line number Diff line change
Expand Up @@ -28,8 +28,6 @@ def _normalize_platform(platform):
class PlatformSpec(object):
"""Represents a target Python platform, implementation, version and ABI."""

SEP = "-"

class InvalidSpecError(Exception):
"""Indicates an invalid platform string."""

Expand Down Expand Up @@ -82,6 +80,8 @@ def create(cls, platform, cause=None):
)
return cls("\n\n".join(message_parts))

SEP = "-"

@classmethod
def parse(cls, platform_spec):
# type: (str) -> PlatformSpec
Expand Down Expand Up @@ -205,13 +205,13 @@ class Platform(PlatformSpec):
@classmethod
def from_tags(cls, compatibility_tags):
# type: (CompatibilityTags) -> Platform
abbreviated_platform = PlatformSpec.from_tag(compatibility_tags[0])
platform_spec = PlatformSpec.from_tag(compatibility_tags[0])
return Platform(
platform=abbreviated_platform.platform,
impl=abbreviated_platform.impl,
version=abbreviated_platform.version,
version_info=abbreviated_platform.version_info,
abi=abbreviated_platform.abi,
platform=platform_spec.platform,
impl=platform_spec.impl,
version=platform_spec.version,
version_info=platform_spec.version_info,
abi=platform_spec.abi,
supported_tags=compatibility_tags,
)

Expand Down
114 changes: 67 additions & 47 deletions pex/resolver.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,6 @@
import hashlib
import itertools
import os
import shutil
import zipfile
from abc import abstractmethod
from collections import OrderedDict, defaultdict
Expand Down Expand Up @@ -76,6 +75,68 @@ def _uniqued_targets(targets=None):
return tuple(OrderedSet(targets)) if targets is not None else ()


@attr.s(frozen=True)
class PipLogManager(object):
@classmethod
def create(
cls,
log, # type: Optional[str]
targets, # type: Sequence[Target]
):
# type: (...) -> PipLogManager
log_by_target = {} # type: Dict[Target, str]
if log and len(targets) == 1:
log_by_target[targets[0]] = log
elif log:
log_dir = safe_mkdtemp(prefix="pex-pip-log.")
log_by_target.update(
(target, os.path.join(log_dir, "pip.{target}.log".format(target=target.id)))
for target in targets
)
return cls(log=log, log_by_target=log_by_target)

log = attr.ib() # type: Optional[str]
_log_by_target = attr.ib() # type: Mapping[Target, str]

@staticmethod
def _target_id(target):
# type: (Target) -> str
if isinstance(target, LocalInterpreter):
# e.g.: CPython 2.7.18
return target.interpreter.version_string
if isinstance(target, AbbreviatedPlatform):
return str(target.platform)
if isinstance(target, CompletePlatform):
return str(target.platform.tag)
return target.id

def finalize_log(self):
# type: () -> None
target_count = len(self._log_by_target)
if target_count <= 1:
return

with safe_open(self.log, "a") as out_fp:
for index, (target, log) in enumerate(self._log_by_target.items(), start=1):
prefix = "{index}/{count}]{target}".format(
index=index, count=target_count, target=self._target_id(target)
)
if not os.path.exists(log):
print(
"{prefix}: WARNING: no Pip log was generated!".format(prefix=prefix),
file=out_fp,
)
continue

with open(log) as in_fp:
for line in in_fp:
out_fp.write("{prefix}: {line}".format(prefix=prefix, line=line))

def get_log(self, target):
# type: (Target) -> Optional[str]
return self._log_by_target.get(target)


@attr.s(frozen=True)
class DownloadRequest(object):
targets = attr.ib(converter=_uniqued_targets) # type: Tuple[Target, ...]
Expand Down Expand Up @@ -111,14 +172,8 @@ def download_distributions(self, dest=None, max_parallel_jobs=None):
dest = dest or safe_mkdtemp(
prefix="resolver_download.", dir=safe_mkdir(get_downloads_dir())
)
log_by_target = {} # type: Dict[Target, str]
if self.pip_log:
log_dir = safe_mkdtemp(prefix="pex-pip-log.")
log_by_target.update(
(target, os.path.join(log_dir, "pip.{target}.log".format(target=target.id)))
for target in self.targets
)
spawn_download = functools.partial(self._spawn_download, dest, log_by_target)
log_manager = PipLogManager.create(self.pip_log, self.targets)
spawn_download = functools.partial(self._spawn_download, dest, log_manager)
with TRACER.timed(
"Resolving for:\n {}".format(
"\n ".join(target.render_description() for target in self.targets)
Expand All @@ -134,47 +189,12 @@ def download_distributions(self, dest=None, max_parallel_jobs=None):
)
)
finally:
if self.pip_log:
if len(log_by_target) == 1:
_, log = log_by_target.popitem()
if os.path.exists(log):
with safe_open(self.pip_log, "a") as out_fp, open(log) as in_fp:
shutil.copyfileobj(in_fp, out_fp)
elif log_by_target:
with safe_open(self.pip_log, "a") as out_fp:
for index, (target, log) in enumerate(log_by_target.items(), start=1):
if isinstance(target, LocalInterpreter):
# e.g.: CPython 2.7.18
target_id = target.interpreter.version_string
elif isinstance(target, AbbreviatedPlatform):
target_id = str(target.platform)
elif isinstance(target, CompletePlatform):
target_id = str(target.platform.tag)
else:
target_id = target.id
prefix = "{index}/{count}]{target}".format(
index=index, count=len(self.targets), target=target_id
)

if not os.path.exists(log):
print(
"{prefix}: WARNING: no Pip log was generated!".format(
prefix=prefix
),
file=out_fp,
)
continue

with open(log) as in_fp:
for line in in_fp:
out_fp.write(
"{prefix}: {line}".format(prefix=prefix, line=line)
)
log_manager.finalize_log()

def _spawn_download(
self,
resolved_dists_dir, # type: str
log_by_target, # type: Mapping[Target, str]
log_manager, # type: PipLogManager
target, # type: Target
):
# type: (...) -> SpawnedJob[DownloadResult]
Expand Down Expand Up @@ -207,7 +227,7 @@ def _spawn_download(
build_configuration=self.build_configuration,
observer=observer,
dependency_configuration=self.dependency_configuration,
log=log_by_target.get(target, None),
log=log_manager.get_log(target),
)

return SpawnedJob.wait(job=download_job, result=download_result)
Expand Down
4 changes: 2 additions & 2 deletions tests/integration/cli/commands/test_issue_1801.py
Original file line number Diff line number Diff line change
Expand Up @@ -17,9 +17,9 @@ def test_preserve_pip_download_log():
match = re.search(
r"^pex: Preserving `pip download` log at (?P<log_path>.*)$", result.error, re.MULTILINE
)
assert match is not None
assert match is not None, result.error
log_path = match.group("log_path")
assert os.path.exists(log_path)
assert os.path.exists(log_path), result.error
expected_url_suffix = "ansicolors-1.1.8-py2.py3-none-any.whl"
expected_algorithm = "sha256"
expected_hash = "00d2dde5a675579325902536738dd27e4fac1fd68f773fe36c21044eb559e187"
Expand Down

0 comments on commit a1ec512

Please sign in to comment.