diff --git a/src/python/pants/backend/python/goals/export.py b/src/python/pants/backend/python/goals/export.py new file mode 100644 index 000000000000..8d589e6d1672 --- /dev/null +++ b/src/python/pants/backend/python/goals/export.py @@ -0,0 +1,82 @@ +# Copyright 2021 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +from __future__ import annotations + +import os +from dataclasses import dataclass + +from pants.backend.python.subsystems.setup import PythonSetup +from pants.backend.python.util_rules.interpreter_constraints import InterpreterConstraints +from pants.backend.python.util_rules.pex import VenvPex, VenvPexProcess +from pants.backend.python.util_rules.pex_environment import PexEnvironment +from pants.backend.python.util_rules.pex_from_targets import PexFromTargetsRequest +from pants.core.goals.export import ExportableData, ExportableDataRequest, ExportError, Symlink +from pants.engine.internals.selectors import Get +from pants.engine.process import ProcessResult +from pants.engine.rules import collect_rules, rule +from pants.engine.unions import UnionRule + + +@dataclass(frozen=True) +class ExportedVenvRequest(ExportableDataRequest): + pass + + +@rule +async def export_venv( + request: ExportedVenvRequest, python_setup: PythonSetup, pex_env: PexEnvironment +) -> ExportableData: + # Pick a single interpreter for the venv. + interpreter_constraints = InterpreterConstraints.create_from_targets( + request.targets, python_setup + ) + min_interpreter = interpreter_constraints.snap_to_minimum(python_setup.interpreter_universe) + if not min_interpreter: + raise ExportError( + "There is no single Python interpreter compatible with all the " + "targets for which export was requested. Please restrict the target set " + "to one that shares a compatible interpreter." + ) + + venv_pex = await Get( + VenvPex, + PexFromTargetsRequest, + PexFromTargetsRequest.for_requirements( + (tgt.address for tgt in request.targets), + internal_only=True, + hardcoded_interpreter_constraints=min_interpreter, + ), + ) + + complete_pex_env = pex_env.in_workspace() + venv_abspath = os.path.join(complete_pex_env.pex_root, venv_pex.venv_rel_dir) + + # Run the venv_pex to ensure that the underlying venv is created if necessary. + # We also use this to get the full python version (including patch #), so we + # can use it in the symlink name (not critical, but nice to have). + res = await Get( + ProcessResult, + VenvPexProcess( + venv_pex=venv_pex, + description="Create virtualenv", + argv=["-c", "import sys; print('.'.join(str(x) for x in sys.version_info[0:3]))"], + input_digest=venv_pex.digest, + # TODO: Is there always a python_configured? + extra_env=complete_pex_env.environment_dict(python_configured=True), + ), + ) + py_version = res.stdout.strip().decode() + + return ExportableData( + f"virtualenv for {min_interpreter}", + os.path.join("python", "virtualenv"), + symlinks=[Symlink(venv_abspath, py_version)], + ) + + +def rules(): + return [ + *collect_rules(), + UnionRule(ExportableDataRequest, ExportedVenvRequest), + ] diff --git a/src/python/pants/backend/python/register.py b/src/python/pants/backend/python/register.py index 2a3285cbbc30..5cfb386817de 100644 --- a/src/python/pants/backend/python/register.py +++ b/src/python/pants/backend/python/register.py @@ -10,6 +10,7 @@ from pants.backend.python.dependency_inference import rules as dependency_inference_rules from pants.backend.python.goals import ( coverage_py, + export, lockfile, package_pex_binary, pytest_runner, @@ -62,6 +63,7 @@ def build_file_aliases(): def rules(): return ( *coverage_py.rules(), + *export.rules(), *lockfile.rules(), *tailor.rules(), *ancestor_files.rules(), diff --git a/src/python/pants/backend/python/util_rules/interpreter_constraints.py b/src/python/pants/backend/python/util_rules/interpreter_constraints.py index 1cd21f5b3a1c..d8af499b40b4 100644 --- a/src/python/pants/backend/python/util_rules/interpreter_constraints.py +++ b/src/python/pants/backend/python/util_rules/interpreter_constraints.py @@ -35,12 +35,18 @@ def interpreter_constraints(self) -> InterpreterConstraintsField: _FS = TypeVar("_FS", bound=FieldSetWithInterpreterConstraints) -# The current maxes are 2.7.18 and 3.6.13. -_EXPECTED_LAST_PATCH_VERSION = 18 +# The current maxes are 2.7.18 and 3.6.15. We go much higher, for safety. +_PATCH_VERSION_UPPER_BOUND = 30 # Normally we would subclass `DeduplicatedCollection`, but we want a custom constructor. class InterpreterConstraints(FrozenOrderedSet[Requirement], EngineAwareParameter): + @classmethod + def for_fixed_python_version( + cls, cpython_version_str: str, interpreter_type: str = "CPython" + ) -> InterpreterConstraints: + return cls([f"{interpreter_type}=={cpython_version_str}"]) + def __init__(self, constraints: Iterable[str | Requirement] = ()) -> None: # #12578 `parse_constraint` will sort the requirement's component constraints into a stable form. # We need to sort the component constraints for each requirement _before_ sorting the entire list @@ -180,7 +186,7 @@ def generate_pex_arg_list(self) -> list[str]: return args def _valid_patch_versions(self, major: int, minor: int) -> Iterator[int]: - for p in range(0, _EXPECTED_LAST_PATCH_VERSION + 1): + for p in range(0, _PATCH_VERSION_UPPER_BOUND + 1): for req in self: if req.specifier.contains(f"{major}.{minor}.{p}"): # type: ignore[attr-defined] yield p @@ -202,9 +208,21 @@ def minimum_python_version(self, interpreter_universe: Iterable[str]) -> str | N The constraints may also be compatible with later versions; this is the lowest version that still works. """ + snapped = self.snap_to_minimum(interpreter_universe) + if snapped: + # We know by construction that snapped has a single requirement, and that has a single + # (op, version) spec whose version is "major.minor.*", so we just strip off the ".*" . + return next(iter(snapped)).specs[0][1][:-2] + return None + + def snap_to_minimum(self, interpreter_universe: Iterable[str]) -> InterpreterConstraints | None: + """Snap to the lowest Python major.minor version that works with these constraints.""" for major, minor in sorted(_major_minor_to_int(s) for s in interpreter_universe): - if self._includes_version(major, minor): - return f"{major}.{minor}" + for p in range(0, _PATCH_VERSION_UPPER_BOUND + 1): + for req in self: + if req.specifier.contains(f"{major}.{minor}.{p}"): # type: ignore[attr-defined] + snapped = Requirement.parse(f"{req.project_name}=={major}.{minor}.*") + return InterpreterConstraints([snapped]) return None def _requires_python3_version_or_newer( @@ -212,7 +230,7 @@ def _requires_python3_version_or_newer( ) -> bool: if not self: return False - patch_versions = list(reversed(range(0, _EXPECTED_LAST_PATCH_VERSION))) + patch_versions = list(reversed(range(0, _PATCH_VERSION_UPPER_BOUND))) # We only look at the prior Python release. For example, consider Python 3.8+ # looking at 3.7. If using something like `>=3.5`, Py37 will be included. # `==3.6.*,!=3.7.*,==3.8.*` is unlikely, and even that will work correctly as diff --git a/src/python/pants/backend/python/util_rules/interpreter_constraints_test.py b/src/python/pants/backend/python/util_rules/interpreter_constraints_test.py index 89d464173880..68ac792cfbe3 100644 --- a/src/python/pants/backend/python/util_rules/interpreter_constraints_test.py +++ b/src/python/pants/backend/python/util_rules/interpreter_constraints_test.py @@ -12,7 +12,7 @@ from pants.backend.python.subsystems.setup import PythonSetup from pants.backend.python.target_types import InterpreterConstraintsField from pants.backend.python.util_rules.interpreter_constraints import ( - _EXPECTED_LAST_PATCH_VERSION, + _PATCH_VERSION_UPPER_BOUND, InterpreterConstraints, ) from pants.build_graph.address import Address @@ -192,6 +192,29 @@ def test_interpreter_constraints_minimum_python_version( assert ics.minimum_python_version(sorted(universe)) == expected +@pytest.mark.parametrize( + "constraints,expected", + [ + (["CPython>=2.7"], "CPython==2.7.*"), + (["CPython>=3.7"], "CPython==3.7.*"), + (["CPython>=3.8.6"], "CPython==3.8.*"), + (["CPython==3.5.*", "CPython>=3.6"], "CPython==3.5.*"), + (["CPython==3.7.*", "PyPy==3.6.*"], "PyPy==3.6.*"), + (["CPython"], "CPython==2.7.*"), + (["CPython==3.7.*,<3.6"], None), + ([], None), + ], +) +def test_snap_to_minimum(constraints, expected) -> None: + universe = ["2.7", "3.5", "3.6", "3.7", "3.8", "3.9", "3.10"] + ics = InterpreterConstraints(constraints) + snapped = ics.snap_to_minimum(universe) + if expected is None: + assert snapped is None + else: + assert snapped == InterpreterConstraints([expected]) + + @pytest.mark.parametrize( "constraints", [ @@ -304,7 +327,7 @@ def test_to_poetry_constraint(constraints: list[str], expected: str) -> None: assert InterpreterConstraints(constraints).to_poetry_constraint() == expected -_ALL_PATCHES = list(range(_EXPECTED_LAST_PATCH_VERSION + 1)) +_ALL_PATCHES = list(range(_PATCH_VERSION_UPPER_BOUND + 1)) def patches( @@ -324,7 +347,7 @@ def patches( ["==2.7.1", ">=3.6.15"], ( [(2, 7, 1)] - + patches(3, 6, range(15, _EXPECTED_LAST_PATCH_VERSION + 1)) + + patches(3, 6, range(15, _PATCH_VERSION_UPPER_BOUND + 1)) + patches(3, 7, _ALL_PATCHES) + patches(3, 8, _ALL_PATCHES) + patches(3, 9, _ALL_PATCHES) diff --git a/src/python/pants/backend/python/util_rules/pex.py b/src/python/pants/backend/python/util_rules/pex.py index 67a933874fa5..ee813d5f6849 100644 --- a/src/python/pants/backend/python/util_rules/pex.py +++ b/src/python/pants/backend/python/util_rules/pex.py @@ -832,7 +832,7 @@ class VenvPexRequest: bin_names: tuple[str, ...] = () def __init__(self, pex_request: PexRequest, bin_names: Iterable[str] = ()) -> None: - """A request for a PEX that runs in a venv and optionally exposes select vanv `bin` scripts. + """A request for a PEX that runs in a venv and optionally exposes select venv `bin` scripts. :param pex_request: The details of the desired PEX. :param bin_names: The names of venv `bin` scripts to expose for execution. diff --git a/src/python/pants/core/goals/export.py b/src/python/pants/core/goals/export.py new file mode 100644 index 000000000000..7d368509f305 --- /dev/null +++ b/src/python/pants/core/goals/export.py @@ -0,0 +1,115 @@ +# Copyright 2021 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +from __future__ import annotations + +import os +from dataclasses import dataclass +from typing import Iterable, cast + +from pants.core.util_rules.distdir import DistDir +from pants.engine.console import Console +from pants.engine.fs import EMPTY_DIGEST, AddPrefix, Digest, MergeDigests, Workspace +from pants.engine.goal import Goal, GoalSubsystem +from pants.engine.internals.selectors import Get, MultiGet +from pants.engine.rules import collect_rules, goal_rule +from pants.engine.target import Targets +from pants.engine.unions import UnionMembership, union +from pants.util.dirutil import absolute_symlink +from pants.util.meta import frozen_after_init + + +class ExportError(Exception): + pass + + +@union +@dataclass(frozen=True) +class ExportableDataRequest: + """A union for exportable data provided by a backend. + + Subclass and install a member of this type to export data. + """ + + targets: Targets + + +@dataclass(frozen=True) +class Symlink: + """A symlink from link_rel_path pointing to source_abs_path. + + link_rel_path is relative to the enclosing ExportableData's reldir, and will be absolutized when + a location for that dir is chosen. + """ + + source_abs_path: str + link_rel_path: str + + +@frozen_after_init +@dataclass(unsafe_hash=True) +class ExportableData: + description: str + # Materialize digests and create symlinks under this reldir. + reldir: str + # Materialize this digest. + digest: Digest + # Create these symlinks. Symlinks are created after the digest is materialized, + # so may reference files/dirs in the digest. + symlinks: tuple[Symlink, ...] + + def __init__( + self, + description: str, + reldir: str, + *, + digest: Digest = EMPTY_DIGEST, + symlinks: Iterable[Symlink] = tuple(), + ): + self.description = description + self.reldir = reldir + self.digest = digest + self.symlinks = tuple(symlinks) + + +class ExportSubsystem(GoalSubsystem): + name = "export" + help = "Export Pants data for use in other tools, such as IDEs." + + +class Export(Goal): + subsystem_cls = ExportSubsystem + + +@goal_rule +async def export( + console: Console, + targets: Targets, + export_subsystem: ExportSubsystem, + workspace: Workspace, + union_membership: UnionMembership, + dist_dir: DistDir, +) -> Export: + request_types = cast( + "Iterable[type[ExportableDataRequest]]", union_membership.get(ExportableDataRequest) + ) + requests = tuple(request_type(targets) for request_type in request_types) + exportables = await MultiGet( + Get(ExportableData, ExportableDataRequest, request) for request in requests + ) + output_dir = os.path.join(str(dist_dir.relpath), "export") + merged_digest = await Get(Digest, MergeDigests(exp.digest for exp in exportables)) + dist_digest = await Get(Digest, AddPrefix(merged_digest, output_dir)) + workspace.write_digest(dist_digest) + for exp in exportables: + for symlink in exp.symlinks: + link_abspath = os.path.abspath( + os.path.join(output_dir, exp.reldir, symlink.link_rel_path) + ) + absolute_symlink(symlink.source_abs_path, link_abspath) + console.print_stdout(f"Wrote {exp.description} to {os.path.join(output_dir, exp.reldir)}") + return Export(exit_code=0) + + +def rules(): + return collect_rules() diff --git a/src/python/pants/core/register.py b/src/python/pants/core/register.py index 76bfb3fef129..5c6e9685f7d2 100644 --- a/src/python/pants/core/register.py +++ b/src/python/pants/core/register.py @@ -8,6 +8,7 @@ from pants.core.goals import ( check, + export, fmt, lint, package, @@ -48,6 +49,7 @@ def rules(): return [ # goals *check.rules(), + *export.rules(), *fmt.rules(), *lint.rules(), *update_build_files.rules(),