diff --git a/src/python/pants/backend/python/goals/export.py b/src/python/pants/backend/python/goals/export.py index 1953847c8f6..fbcfe900d34 100644 --- a/src/python/pants/backend/python/goals/export.py +++ b/src/python/pants/backend/python/goals/export.py @@ -15,6 +15,10 @@ from pants.backend.python.subsystems.setup import PythonSetup from pants.backend.python.target_types import PexLayout, PythonResolveField from pants.backend.python.util_rules.interpreter_constraints import InterpreterConstraints +from pants.backend.python.util_rules.local_dists_pep660 import ( + EditableLocalDists, + EditableLocalDistsRequest, +) from pants.backend.python.util_rules.pex import Pex, PexProcess, PexRequest, VenvPex, VenvPexProcess from pants.backend.python.util_rules.pex_cli import PexPEX from pants.backend.python.util_rules.pex_environment import PexEnvironment @@ -33,13 +37,13 @@ from pants.core.util_rules.distdir import DistDir from pants.engine.engine_aware import EngineAwareParameter from pants.engine.environment import EnvironmentName -from pants.engine.internals.native_engine import AddPrefix, Digest, MergeDigests +from pants.engine.internals.native_engine import AddPrefix, Digest, MergeDigests, Snapshot from pants.engine.internals.selectors import Get, MultiGet from pants.engine.process import ProcessCacheScope, ProcessResult from pants.engine.rules import collect_rules, rule from pants.engine.target import Target from pants.engine.unions import UnionMembership, UnionRule, union -from pants.option.option_types import BoolOption, EnumOption +from pants.option.option_types import BoolOption, EnumOption, StrListOption from pants.util.docutil import bin_name from pants.util.strutil import path_safe, softwrap @@ -123,6 +127,30 @@ class ExportPluginOptions: removal_hint="Set the `[export].py_resolve_format` option to 'symlinked_immutable_virtualenv'", ) + py_editable_in_resolve = StrListOption( + # TODO: Is there a way to get [python].resolves in a memoized_property here? + # If so, then we can validate that all resolves here are defined there. + help=softwrap( + """ + When exporting a mutable virtualenv for a resolve, do PEP-660 editable installs + of all 'python_distribution' targets that own code in the exported resolve. + + If a resolve name is not in this list, 'python_distribution' targets will not + be installed in the virtualenv. This defaults to an empty list for backwards + compatibility and to prevent unnecessary work to generate and install the + PEP-660 editable wheels. + + This only applies when '[python].enable_resolves' is true and when exporting a + 'mutable_virtualenv' ('symlinked_immutable_virtualenv' exports are not "full" + virtualenvs because they must not be edited, and do not include 'pip'). + + NOTE: If you are using legacy exports (not using the '--resolve' option), then + this option has no effect. Legacy exports will not include any editable installs. + """ + ), + advanced=True, + ) + async def _get_full_python_version(pex_or_venv_pex: Pex | VenvPex) -> str: # Get the full python version (including patch #). @@ -150,6 +178,7 @@ class VenvExportRequest: dest_prefix: str resolve_name: str qualify_path_with_python_version: bool + editable_local_dists_digest: Digest | None = None @rule @@ -230,30 +259,92 @@ async def do_export( tmpdir_under_digest_root = os.path.join("{digest_root}", tmpdir_prefix) merged_digest_under_tmpdir = await Get(Digest, AddPrefix(merged_digest, tmpdir_prefix)) + post_processing_cmds = [ + PostProcessingCommand( + complete_pex_env.create_argv( + os.path.join(tmpdir_under_digest_root, pex_pex.exe), + *( + os.path.join(tmpdir_under_digest_root, requirements_pex.name), + "venv", + "--pip", + "--collisions-ok", + output_path, + ), + ), + { + **complete_pex_env.environment_dict(python=requirements_pex.python), + "PEX_MODULE": "pex.tools", + }, + ), + # Remove the requirements and pex pexes, to avoid confusion. + PostProcessingCommand(["rm", "-rf", tmpdir_under_digest_root]), + ] + + # Insert editable wheel post processing commands if needed. + if req.editable_local_dists_digest is not None: + # We need the snapshot to get the wheel file names which are something like: + # - pkg_name-1.2.3-0.editable-py3-none-any.whl + wheels_snapshot = await Get(Snapshot, Digest, req.editable_local_dists_digest) + # We need the paths to the installed .dist-info directories to finish installation. + py_major_minor_version = ".".join(py_version.split(".", 2)[:2]) + lib_dir = os.path.join( + output_path, "lib", f"python{py_major_minor_version}", "site-packages" + ) + dist_info_dirs = [ + # This builds: dist/.../resolve/3.8.9/lib/python3.8/site-packages/pkg_name-1.2.3.dist-info + os.path.join(lib_dir, "-".join(f.split("-")[:2]) + ".dist-info") + for f in wheels_snapshot.files + ] + # We use slice assignment to insert multiple elements at index 1. + post_processing_cmds[1:1] = [ + PostProcessingCommand( + [ + # The wheels are "sources" in the pex and get dumped in lib_dir + # so we move them to tmpdir where they will be removed at the end. + "mv", + *(os.path.join(lib_dir, f) for f in wheels_snapshot.files), + tmpdir_under_digest_root, + ] + ), + PostProcessingCommand( + [ + # Now install the editable wheels. + os.path.join(output_path, "bin", "pip"), + "install", + "--no-deps", # The deps were already installed via requirements.pex. + "--no-build-isolation", # Avoid VCS dep downloads (as they are installed). + *(os.path.join(tmpdir_under_digest_root, f) for f in wheels_snapshot.files), + ] + ), + PostProcessingCommand( + [ + # Replace pip's direct_url.json (which points to the temp editable wheel) + # with ours (which points to build_dir sources and is marked "editable"). + # Also update INSTALLER file to indicate that pants installed it. + "sh", + "-c", + " ".join( + [ + f"mv -f {src} {dst}; echo pants > {installer};" + for src, dst, installer in zip( + [ + os.path.join(d, "direct_url__pants__.json") + for d in dist_info_dirs + ], + [os.path.join(d, "direct_url.json") for d in dist_info_dirs], + [os.path.join(d, "INSTALLER") for d in dist_info_dirs], + ) + ] + ), + ] + ), + ] + return ExportResult( description, dest, digest=merged_digest_under_tmpdir, - post_processing_cmds=[ - PostProcessingCommand( - complete_pex_env.create_argv( - os.path.join(tmpdir_under_digest_root, pex_pex.exe), - *[ - os.path.join(tmpdir_under_digest_root, requirements_pex.name), - "venv", - "--pip", - "--collisions-ok", - output_path, - ], - ), - { - **complete_pex_env.environment_dict(python=requirements_pex.python), - "PEX_MODULE": "pex.tools", - }, - ), - # Remove the requirements and pex pexes, to avoid confusion. - PostProcessingCommand(["rm", "-rf", tmpdir_under_digest_root]), - ], + post_processing_cmds=post_processing_cmds, resolve=req.resolve_name or None, ) else: @@ -269,8 +360,10 @@ class MaybeExportResult: async def export_virtualenv_for_resolve( request: _ExportVenvForResolveRequest, python_setup: PythonSetup, + export_subsys: ExportSubsystem, union_membership: UnionMembership, ) -> MaybeExportResult: + editable_local_dists_digest: Digest | None = None resolve = request.resolve lockfile_path = python_setup.resolves.get(resolve) if lockfile_path: @@ -287,11 +380,20 @@ async def export_virtualenv_for_resolve( ) ) + if resolve in export_subsys.options.py_editable_in_resolve: + editable_local_dists = await Get( + EditableLocalDists, EditableLocalDistsRequest(resolve=resolve) + ) + editable_local_dists_digest = editable_local_dists.optional_digest + else: + editable_local_dists_digest = None + pex_request = PexRequest( description=f"Build pex for resolve `{resolve}`", output_filename=f"{path_safe(resolve)}.pex", internal_only=True, requirements=EntireLockfile(lockfile), + sources=editable_local_dists_digest, interpreter_constraints=interpreter_constraints, # Packed layout should lead to the best performance in this use case. layout=PexLayout.PACKED, @@ -337,6 +439,7 @@ async def export_virtualenv_for_resolve( dest_prefix, resolve, qualify_path_with_python_version=True, + editable_local_dists_digest=editable_local_dists_digest, ), ) return MaybeExportResult(export_result) diff --git a/src/python/pants/backend/python/goals/export_integration_test.py b/src/python/pants/backend/python/goals/export_integration_test.py index c91c35094b7..5a73b5aada1 100644 --- a/src/python/pants/backend/python/goals/export_integration_test.py +++ b/src/python/pants/backend/python/goals/export_integration_test.py @@ -24,7 +24,16 @@ """ ), "src/python/foo.py": "from colors import *", - "src/python/BUILD": "python_source(name='foo', source='foo.py', resolve=parametrize('a', 'b'))", + "src/python/BUILD": dedent( + """\ + python_source(name='foo', source='foo.py', resolve=parametrize('a', 'b')) + python_distribution( + name='dist', + provides=python_artifact(name='foo-dist', version='1.2.3'), + dependencies=[':foo@resolve=a'], + ) + """ + ), } @@ -97,7 +106,12 @@ def test_export(py_resolve_format: PythonResolveExportFormat) -> None: with setup_tmpdir(SOURCES) as tmpdir: resolve_names = ["a", "b", *(tool.name for tool in EXPORTED_TOOLS)] run_pants( - ["generate-lockfiles", "export", *(f"--resolve={name}" for name in resolve_names)], + [ + "generate-lockfiles", + "export", + *(f"--resolve={name}" for name in resolve_names), + "--export-py-editable-in-resolve=['a']", + ], config=build_config(tmpdir, py_resolve_format), ).assert_success() @@ -128,6 +142,29 @@ def test_export(py_resolve_format: PythonResolveExportFormat) -> None: expected_ansicolors_dir ), f"expected dist-info for ansicolors '{expected_ansicolors_dir}' does not exist" + if py_resolve_format == PythonResolveExportFormat.mutable_virtualenv: + expected_foo_dir = os.path.join(lib_dir, "foo_dist-1.2.3.dist-info") + if resolve == "b": + assert not os.path.isdir( + expected_foo_dir + ), f"unexpected dist-info for foo-dist '{expected_foo_dir}' exists" + elif resolve == "a": + # make sure the editable wheel for the python_distribution is installed + assert os.path.isdir( + expected_foo_dir + ), f"expected dist-info for foo-dist '{expected_foo_dir}' does not exist" + # direct_url__pants__.json should be moved to direct_url.json + expected_foo_direct_url_pants = os.path.join( + expected_foo_dir, "direct_url__pants__.json" + ) + assert not os.path.isfile( + expected_foo_direct_url_pants + ), f"expected direct_url__pants__.json for foo-dist '{expected_foo_direct_url_pants}' was not removed" + expected_foo_direct_url = os.path.join(expected_foo_dir, "direct_url.json") + assert os.path.isfile( + expected_foo_direct_url + ), f"expected direct_url.json for foo-dist '{expected_foo_direct_url}' does not exist" + for tool_config in EXPORTED_TOOLS: export_dir = os.path.join(export_prefix, tool_config.name, platform.python_version()) assert os.path.isdir(export_dir), f"expected export dir '{export_dir}' does not exist" diff --git a/src/python/pants/backend/python/goals/export_test.py b/src/python/pants/backend/python/goals/export_test.py index 331ae86c2bd..3d4b6c76696 100644 --- a/src/python/pants/backend/python/goals/export_test.py +++ b/src/python/pants/backend/python/goals/export_test.py @@ -11,11 +11,17 @@ from pants.backend.python.goals import export from pants.backend.python.goals.export import ExportVenvsRequest, PythonResolveExportFormat from pants.backend.python.lint.flake8 import subsystem as flake8_subsystem -from pants.backend.python.target_types import PythonRequirementTarget -from pants.backend.python.util_rules import pex_from_targets +from pants.backend.python.macros.python_artifact import PythonArtifact +from pants.backend.python.target_types import ( + PythonDistribution, + PythonRequirementTarget, + PythonSourcesGeneratorTarget, +) +from pants.backend.python.util_rules import local_dists_pep660, pex_from_targets from pants.base.specs import RawSpecs, RecursiveGlobSpec from pants.core.goals.export import ExportResults from pants.core.util_rules import distdir +from pants.engine.internals.parametrize import Parametrize from pants.engine.rules import QueryRule from pants.engine.target import Targets from pants.testutil.rule_runner import RuleRunner @@ -30,11 +36,13 @@ def rule_runner() -> RuleRunner: *pex_from_targets.rules(), *target_types_rules.rules(), *distdir.rules(), + *local_dists_pep660.rules(), *flake8_subsystem.rules(), QueryRule(Targets, [RawSpecs]), QueryRule(ExportResults, [ExportVenvsRequest]), ], - target_types=[PythonRequirementTarget], + target_types=[PythonRequirementTarget, PythonSourcesGeneratorTarget, PythonDistribution], + objects={"python_artifact": PythonArtifact, "parametrize": Parametrize}, ) @@ -170,8 +178,15 @@ def test_export_venv_new_codepath( current_interpreter = f"{vinfo.major}.{vinfo.minor}.{vinfo.micro}" rule_runner.write_files( { + "src/foo/__init__.py": "from colors import *", "src/foo/BUILD": dedent( """\ + python_sources(name='foo', resolve=parametrize('a', 'b')) + python_distribution( + name='dist', + provides=python_artifact(name='foo', version='1.2.3'), + dependencies=[':foo@resolve=a'], + ) python_requirement(name='req1', requirements=['ansicolors==1.1.8'], resolve='a') python_requirement(name='req2', requirements=['ansicolors==1.1.8'], resolve='b') """ @@ -184,12 +199,16 @@ def test_export_venv_new_codepath( rule_runner.set_options( [ f"--python-interpreter-constraints=['=={current_interpreter}']", + "--python-enable-resolves=True", "--python-resolves={'a': 'lock.txt', 'b': 'lock.txt'}", "--export-resolve=a", "--export-resolve=b", "--export-resolve=flake8", # Turn off lockfile validation to make the test simpler. "--python-invalid-lockfile-behavior=ignore", + # Turn off python synthetic lockfile targets to make the test simpler. + "--no-python-enable-lockfile-targets", + "--export-py-editable-in-resolve=['a', 'b']", format_flag, ], env_inherit={"PATH", "PYENV_ROOT"}, @@ -208,7 +227,13 @@ def test_export_venv_new_codepath( assert ppc1.argv[3] == "{digest_root}" assert ppc1.extra_env == FrozenDict() else: - assert len(result.post_processing_cmds) == 2 + if resolve == "a": + # editable wheels are installed for a user resolve that has dists + assert len(result.post_processing_cmds) == 5 + else: + # tool resolves (flake8) and user resolves w/o dists (b) + # do not run the commands to do editable installs + assert len(result.post_processing_cmds) == 2 ppc0 = result.post_processing_cmds[0] # The first arg is the full path to the python interpreter, which we @@ -233,7 +258,7 @@ def test_export_venv_new_codepath( assert ppc0.extra_env["PEX_MODULE"] == "pex.tools" assert ppc0.extra_env.get("PEX_ROOT") is not None - ppc1 = result.post_processing_cmds[1] + ppc1 = result.post_processing_cmds[-1] assert ppc1.argv == ("rm", "-rf", tmpdir) assert ppc1.extra_env == FrozenDict() diff --git a/src/python/pants/backend/python/register.py b/src/python/pants/backend/python/register.py index 51dc645df53..38ee925abec 100644 --- a/src/python/pants/backend/python/register.py +++ b/src/python/pants/backend/python/register.py @@ -46,6 +46,7 @@ from pants.backend.python.util_rules import ( ancestor_files, local_dists, + local_dists_pep660, pex, pex_from_targets, python_sources, @@ -70,6 +71,7 @@ def rules(): # Util rules *ancestor_files.rules(), *dependency_inference_rules.rules(), + *local_dists_pep660.rules(), *pex.rules(), *pex_from_targets.rules(), *python_sources.rules(), diff --git a/src/python/pants/backend/python/util_rules/BUILD b/src/python/pants/backend/python/util_rules/BUILD index c4b26a9160b..d8b31bdafde 100644 --- a/src/python/pants/backend/python/util_rules/BUILD +++ b/src/python/pants/backend/python/util_rules/BUILD @@ -1,7 +1,11 @@ # Copyright 2020 Pants project contributors (see CONTRIBUTORS.md). # Licensed under the Apache License, Version 2.0 (see LICENSE). -python_sources() +python_sources( + overrides={ + "local_dists_pep660.py": dict(dependencies=["./scripts/pep660_backend_wrapper.py"]), + } +) python_tests( name="tests", diff --git a/src/python/pants/backend/python/util_rules/dists.py b/src/python/pants/backend/python/util_rules/dists.py index 4088713cd28..dfdb768a190 100644 --- a/src/python/pants/backend/python/util_rules/dists.py +++ b/src/python/pants/backend/python/util_rules/dists.py @@ -146,6 +146,7 @@ class DistBuildResult: _BACKEND_SHIM_BOILERPLATE = """ # DO NOT EDIT THIS FILE -- AUTOGENERATED BY PANTS +import errno import os import {build_backend_module} diff --git a/src/python/pants/backend/python/util_rules/local_dists_pep660.py b/src/python/pants/backend/python/util_rules/local_dists_pep660.py new file mode 100644 index 00000000000..72ddbe479fa --- /dev/null +++ b/src/python/pants/backend/python/util_rules/local_dists_pep660.py @@ -0,0 +1,430 @@ +# Copyright 2023 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +from __future__ import annotations + +import json +import logging +import os +import shlex +from collections import defaultdict +from dataclasses import dataclass + +from pants.backend.python.subsystems.setup import PythonSetup +from pants.backend.python.subsystems.setuptools import PythonDistributionFieldSet +from pants.backend.python.target_types import PythonProvidesField, PythonResolveField +from pants.backend.python.util_rules import package_dists +from pants.backend.python.util_rules.dists import BuildBackendError, DistBuildRequest +from pants.backend.python.util_rules.dists import rules as dists_rules +from pants.backend.python.util_rules.package_dists import ( + DependencyOwner, + ExportedTarget, + OwnedDependencies, + create_dist_build_request, +) +from pants.backend.python.util_rules.pex import PexRequest, VenvPex, VenvPexProcess +from pants.base.build_root import BuildRoot +from pants.core.util_rules import system_binaries +from pants.core.util_rules.system_binaries import BashBinary, UnzipBinary +from pants.engine.fs import ( + CreateDigest, + Digest, + DigestSubset, + FileContent, + MergeDigests, + PathGlobs, + RemovePrefix, + Snapshot, +) +from pants.engine.process import Process, ProcessResult +from pants.engine.rules import Get, MultiGet, collect_rules, rule +from pants.engine.target import AllTargets, Target, Targets, WrappedTarget, WrappedTargetRequest +from pants.engine.unions import UnionMembership +from pants.util.docutil import doc_url +from pants.util.frozendict import FrozenDict +from pants.util.logging import LogLevel +from pants.util.osutil import is_macos_big_sur +from pants.util.resources import read_resource +from pants.util.strutil import softwrap + +logger = logging.getLogger(__name__) + + +_scripts_package = "pants.backend.python.util_rules.scripts" + + +@dataclass(frozen=True) +class PEP660BuildResult: + output: Digest + # Relpaths in the output digest. + editable_wheel_path: str | None + + +def dump_backend_wrapper_json( + dist_dir: str, + pth_file_path: str, + direct_url: str, + request: DistBuildRequest, +) -> bytes: + """Build the settings json for our PEP 517 / PEP 660 wrapper script.""" + + def clean_config_settings( + cs: FrozenDict[str, tuple[str, ...]] | None + ) -> dict[str, list[str]] | None: + # setuptools.build_meta expects list values and chokes on tuples. + # We assume/hope that other backends accept lists as well. + return None if cs is None else {k: list(v) for k, v in cs.items()} + + # tag the editable wheel as widely compatible + lang_tag, abi_tag, platform_tag = "py3", "none", "any" + if request.interpreter_constraints.includes_python2(): + # Assume everything has py3 support. If not, we'll need a new includes_python3 method. + lang_tag = "py2.py3" + + settings = { + "build_backend": request.build_system.build_backend, + "dist_dir": dist_dir, + "pth_file_path": pth_file_path, + "wheel_config_settings": clean_config_settings(request.wheel_config_settings), + "tags": "-".join([lang_tag, abi_tag, platform_tag]), + "direct_url": direct_url, + } + return json.dumps(settings).encode() + + +@rule +async def run_pep660_build( + request: DistBuildRequest, python_setup: PythonSetup, build_root: BuildRoot +) -> PEP660BuildResult: + """Run our PEP 517 / PEP 660 wrapper script to generate an editable wheel. + + The PEP 517 / PEP 660 wraper script is responsible for building the editable wheel. + The backend wrapper script, along with the commands that install the editable wheel, + need to conform to the following specs so that Pants is a PEP 660 compliant frontend, + a PEP 660 compliant backend, and that it builds a compliant wheel and install. + + NOTE: PEP 660 does not address the `.data` directory, so the wrapper ignores it. + + Relevant Specs: + https://peps.python.org/pep-0517/ + https://peps.python.org/pep-0660/ + https://packaging.python.org/en/latest/specifications/platform-compatibility-tags/ + https://packaging.python.org/en/latest/specifications/recording-installed-packages/ + https://packaging.python.org/en/latest/specifications/direct-url-data-structure/ + https://packaging.python.org/en/latest/specifications/binary-distribution-format/ + """ + + # Create the .pth files to add the relevant source root to sys.path. + # We cannot use the build backend to do this because we do not want to tell + # it where the workspace is and risk it adding anything there. + # NOTE: We use .pth files to support ICs less than python3.7. + # A future enhancement might be to provide more precise editable + # wheels based on https://pypi.org/project/editables/, but that only + # supports python3.7+ (what pip supports as of April 2023). + # Or maybe do something like setuptools strict editable wheel. + pth_file_contents = "" + direct_url = "" + for source_root in request.build_time_source_roots: + # can we use just the first one to only have the dist's source root? + abs_path = ( + build_root.path if source_root == "." else str(build_root.pathlib_path / source_root) + ) + pth_file_contents += f"{abs_path}\n" + if not direct_url: # use just the first source_root + direct_url = "file://" + abs_path.replace(os.path.sep, "/") + pth_file_name = "__pants__.pth" + pth_file_path = os.path.join(request.working_directory, pth_file_name) + + # This is the setuptools dist directory, not Pants's, so we hardcode to dist/. + dist_dir = "dist" + dist_output_dir = os.path.join(dist_dir, request.output_path) + + backend_wrapper_json = "backend_wrapper.json" + backend_wrapper_json_path = os.path.join(request.working_directory, backend_wrapper_json) + backend_wrapper_name = "backend_wrapper.py" + backend_wrapper_path = os.path.join(request.working_directory, backend_wrapper_name) + backend_wrapper_content = read_resource(_scripts_package, "pep660_backend_wrapper.py") + assert backend_wrapper_content is not None + + conf_digest, backend_wrapper_digest, build_backend_pex = await MultiGet( + Get( + Digest, + CreateDigest( + [ + FileContent(pth_file_path, pth_file_contents.encode()), + FileContent( + backend_wrapper_json_path, + dump_backend_wrapper_json( + dist_output_dir, pth_file_name, direct_url, request + ), + ), + ] + ), + ), + # The backend_wrapper has its own digest for cache reuse. + Get( + Digest, + CreateDigest([FileContent(backend_wrapper_path, backend_wrapper_content)]), + ), + # Note that this pex has no entrypoint. We use it to run our wrapper, which + # in turn imports from and invokes the build backend. + Get( + VenvPex, + PexRequest( + output_filename="build_backend.pex", + internal_only=True, + requirements=request.build_system.requires, + pex_path=request.extra_build_time_requirements, + interpreter_constraints=request.interpreter_constraints, + ), + ), + ) + + merged_digest = await Get( + Digest, MergeDigests((request.input, conf_digest, backend_wrapper_digest)) + ) + + extra_env = { + **(request.extra_build_time_env or {}), + "PEX_EXTRA_SYS_PATH": os.pathsep.join(request.build_time_source_roots), + } + if python_setup.macos_big_sur_compatibility and is_macos_big_sur(): + extra_env["MACOSX_DEPLOYMENT_TARGET"] = "10.16" + + result = await Get( + ProcessResult, + VenvPexProcess( + build_backend_pex, + argv=(backend_wrapper_name, backend_wrapper_json), + input_digest=merged_digest, + extra_env=extra_env, + working_directory=request.working_directory, + output_directories=(dist_dir,), # Relative to the working_directory. + description=( + f"Run {request.build_system.build_backend} to gather .dist-info for {request.target_address_spec}" + if request.target_address_spec + else f"Run {request.build_system.build_backend} to gather .dist-info" + ), + level=LogLevel.DEBUG, + ), + ) + output_lines = result.stdout.decode().splitlines() + line_prefix = "editable_path: " + editable_path = "" + for line in output_lines: + if line.startswith(line_prefix): + editable_path = os.path.join(request.output_path, line[len(line_prefix) :].strip()) + break + + # Note that output_digest paths are relative to the working_directory. + output_digest = await Get(Digest, RemovePrefix(result.output_digest, dist_dir)) + output_snapshot = await Get(Snapshot, Digest, output_digest) + if editable_path not in output_snapshot.files: + raise BuildBackendError( + softwrap( + f""" + Failed to build PEP 660 editable wheel {editable_path} + (extracted dist-info from PEP 517 build backend + {request.build_system.build_backend}). + """ + ) + ) + return PEP660BuildResult(output_digest, editable_wheel_path=editable_path) + + +@dataclass(frozen=True) +class LocalDistPEP660Wheels: + """Contains the PEP 660 "editable" wheels isolated from a single local Python distribution.""" + + pep660_wheel_paths: tuple[str, ...] + pep660_wheels_digest: Digest + provided_files: frozenset[str] + + +@rule +async def isolate_local_dist_pep660_wheels( + dist_field_set: PythonDistributionFieldSet, + bash: BashBinary, + unzip_binary: UnzipBinary, + python_setup: PythonSetup, + union_membership: UnionMembership, +) -> LocalDistPEP660Wheels: + dist_build_request = await create_dist_build_request( + field_set=dist_field_set, + python_setup=python_setup, + union_membership=union_membership, + # editable wheel ignores build_wheel+build_sdist args + validate_wheel_sdist=False, + ) + pep660_result = await Get(PEP660BuildResult, DistBuildRequest, dist_build_request) + + # the output digest should only contain wheels, but filter to be safe. + wheels_snapshot = await Get( + Snapshot, DigestSubset(pep660_result.output, PathGlobs(["**/*.whl"])) + ) + + wheels = tuple(wheels_snapshot.files) + + if not wheels: + tgt = await Get( + WrappedTarget, + WrappedTargetRequest(dist_field_set.address, description_of_origin=""), + ) + logger.warning( + softwrap( + f""" + Encountered a dependency on the {tgt.target.alias} target at {dist_field_set.address}, + but this target does not produce a Python wheel artifact. Therefore this target's + code will be used directly from sources, without a distribution being built, + and any native extensions in it will not be built. + + See {doc_url('python-distributions')} for details on how to set up a + {tgt.target.alias} target to produce a wheel. + """ + ) + ) + + wheels_listing_result = await Get( + ProcessResult, + Process( + argv=[ + bash.path, + "-c", + f""" + set -ex + for f in {' '.join(shlex.quote(f) for f in wheels)}; do + {unzip_binary.path} -Z1 "$f" + done + """, + ], + input_digest=wheels_snapshot.digest, + description=f"List contents of editable artifacts produced by {dist_field_set.address}", + ), + ) + provided_files = set(wheels_listing_result.stdout.decode().splitlines()) + + return LocalDistPEP660Wheels(wheels, wheels_snapshot.digest, frozenset(provided_files)) + + +@dataclass(frozen=True) +class AllPythonDistributionTargets: + targets: Targets + + +@rule(desc="Find all Python Distribution targets in project", level=LogLevel.DEBUG) +def find_all_python_distributions( + all_targets: AllTargets, +) -> AllPythonDistributionTargets: + return AllPythonDistributionTargets( + # 'provides' is the field used in PythonDistributionFieldSet + Targets(tgt for tgt in all_targets if tgt.has_field(PythonProvidesField)) + ) + + +@dataclass(frozen=True) +class ResolveSortedPythonDistributionTargets: + targets: FrozenDict[str | None, tuple[Target, ...]] + + +@rule( + desc="Associate resolves with all Python Distribution targets in project", level=LogLevel.DEBUG +) +async def sort_all_python_distributions_by_resolve( + all_dists: AllPythonDistributionTargets, + python_setup: PythonSetup, +) -> ResolveSortedPythonDistributionTargets: + dists = defaultdict(list) + + if not python_setup.enable_resolves: + resolve = None + return ResolveSortedPythonDistributionTargets( + FrozenDict({resolve: tuple(all_dists.targets)}) + ) + + dist_owned_deps = await MultiGet( + Get(OwnedDependencies, DependencyOwner(ExportedTarget(tgt))) for tgt in all_dists.targets + ) + + for dist, owned_deps in zip(all_dists.targets, dist_owned_deps): + resolve = None + # assumption: all owned deps are in the same resolve + for dep in owned_deps: + if dep.target.has_field(PythonResolveField): + resolve = dep.target[PythonResolveField].normalized_value(python_setup) + break + dists[resolve].append(dist) + return ResolveSortedPythonDistributionTargets( + FrozenDict({resolve: tuple(targets) for resolve, targets in dists.items()}) + ) + + +@dataclass(frozen=True) +class EditableLocalDistsRequest: + """Request to generate PEP660 wheels of local dists in the given resolve. + + The editable wheel files must not be exported or made available to the end-user (according to + PEP 660). Instead, the PEP660 editable wheels serve as intermediate, internal-only, + representation of what should be installed in the exported virtualenv to create the editable + installs of local python_distributions. + """ + + resolve: str | None # None if resolves is not enabled + + +@dataclass(frozen=True) +class EditableLocalDists: + """A Digest populated by editable (PEP660) wheels of local dists. + + According to PEP660, these wheels should not be exported to users and must be discarded + after install. Anything that uses this should ensure that these wheels get installed and + then deleted. + + Installing PEP660 wheels creates an "editable" install such that the sys.path gets + adjusted to include source directories from the build root (not from the sandbox). + This is decidedly not hermetic or portable and should only be used locally. + + PEP660 wheels have .dist-info metadata and the .pth files (or similar) that adjust sys.path. + """ + + optional_digest: Digest | None + + +@rule(desc="Building editable local distributions (PEP 660)") +async def build_editable_local_dists( + request: EditableLocalDistsRequest, + all_dists: ResolveSortedPythonDistributionTargets, + python_setup: PythonSetup, +) -> EditableLocalDists: + resolve = request.resolve if python_setup.enable_resolves else None + resolve_dists = all_dists.targets.get(resolve, ()) + + if not resolve_dists: + return EditableLocalDists(None) + + local_dists_wheels = await MultiGet( + Get( + LocalDistPEP660Wheels, + PythonDistributionFieldSet, + PythonDistributionFieldSet.create(target), + ) + for target in resolve_dists + ) + + wheels: list[str] = [] + wheels_digests = [] + for local_dist_wheels in local_dists_wheels: + wheels.extend(local_dist_wheels.pep660_wheel_paths) + wheels_digests.append(local_dist_wheels.pep660_wheels_digest) + + wheels_digest = await Get(Digest, MergeDigests(wheels_digests)) + + return EditableLocalDists(wheels_digest) + + +def rules(): + return ( + *collect_rules(), + *dists_rules(), + *package_dists.rules(), + *system_binaries.rules(), + ) diff --git a/src/python/pants/backend/python/util_rules/local_dists_pep660_test.py b/src/python/pants/backend/python/util_rules/local_dists_pep660_test.py new file mode 100644 index 00000000000..29667ece0b1 --- /dev/null +++ b/src/python/pants/backend/python/util_rules/local_dists_pep660_test.py @@ -0,0 +1,214 @@ +# Copyright 2023 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +from __future__ import annotations + +import io +import zipfile +from pathlib import PurePath +from textwrap import dedent + +import pytest + +from pants.backend.python import target_types_rules +from pants.backend.python.macros.python_artifact import PythonArtifact +from pants.backend.python.subsystems.setuptools import rules as setuptools_rules +from pants.backend.python.target_types import PythonDistribution, PythonSourcesGeneratorTarget +from pants.backend.python.util_rules import local_dists_pep660, pex_from_targets +from pants.backend.python.util_rules.dists import BuildSystem, DistBuildRequest +from pants.backend.python.util_rules.interpreter_constraints import InterpreterConstraints +from pants.backend.python.util_rules.local_dists_pep660 import ( + EditableLocalDists, + EditableLocalDistsRequest, + PEP660BuildResult, + ResolveSortedPythonDistributionTargets, +) +from pants.backend.python.util_rules.pex_from_targets import InterpreterConstraintsRequest +from pants.backend.python.util_rules.pex_requirements import PexRequirements +from pants.build_graph.address import Address +from pants.engine.fs import CreateDigest, Digest, DigestContents, FileContent +from pants.engine.internals.parametrize import Parametrize +from pants.testutil.python_interpreter_selection import ( + skip_unless_python27_present, + skip_unless_python39_present, +) +from pants.testutil.rule_runner import QueryRule, RuleRunner +from pants.util.frozendict import FrozenDict + + +@pytest.fixture +def rule_runner() -> RuleRunner: + ret = RuleRunner( + rules=[ + *local_dists_pep660.rules(), + *setuptools_rules(), + *target_types_rules.rules(), + *pex_from_targets.rules(), + QueryRule(InterpreterConstraints, (InterpreterConstraintsRequest,)), + QueryRule(ResolveSortedPythonDistributionTargets, ()), + QueryRule(EditableLocalDists, (EditableLocalDistsRequest,)), + QueryRule(PEP660BuildResult, (DistBuildRequest,)), + ], + target_types=[PythonSourcesGeneratorTarget, PythonDistribution], + objects={"python_artifact": PythonArtifact, "parametrize": Parametrize}, + ) + ret.set_options( + [], + env_inherit={"PATH", "PYENV_ROOT", "HOME"}, + ) + return ret + + +def do_test_backend_wrapper(rule_runner: RuleRunner, constraints: str) -> None: + setup_py = "from setuptools import setup; setup(name='foobar', version='1.2.3')" + input_digest = rule_runner.request( + Digest, [CreateDigest([FileContent("setup.py", setup_py.encode())])] + ) + req = DistBuildRequest( + build_system=BuildSystem( + PexRequirements( + # NB: These are the last versions compatible with Python 2.7. + ["setuptools==44.1.1", "wheel==0.37.1"] + ), + "setuptools.build_meta", + ), + interpreter_constraints=InterpreterConstraints([constraints]), + build_wheel=True, + build_sdist=True, + input=input_digest, + working_directory="", + build_time_source_roots=tuple(), + output_path="dist", + wheel_config_settings=FrozenDict({"setting1": ("value1",), "setting2": ("value2",)}), + ) + res = rule_runner.request(PEP660BuildResult, [req]) + + is_py2 = "2.7" in constraints + assert ( + res.editable_wheel_path + == f"dist/foobar-1.2.3-0.editable-{'py2.' if is_py2 else ''}py3-none-any.whl" + ) + + +@skip_unless_python27_present +def test_works_with_python2(rule_runner: RuleRunner) -> None: + do_test_backend_wrapper(rule_runner, constraints="CPython==2.7.*") + + +@skip_unless_python39_present +def test_works_with_python39(rule_runner: RuleRunner) -> None: + do_test_backend_wrapper(rule_runner, constraints="CPython==3.9.*") + + +def test_sort_all_python_distributions_by_resolve(rule_runner: RuleRunner) -> None: + rule_runner.set_options( + [ + "--python-enable-resolves=True", + "--python-resolves={'a': 'lock.txt', 'b': 'lock.txt'}", + # Turn off lockfile validation to make the test simpler. + "--python-invalid-lockfile-behavior=ignore", + # Turn off python synthetic lockfile targets to make the test simpler. + "--no-python-enable-lockfile-targets", + ], + env_inherit={"PATH", "PYENV_ROOT", "HOME"}, + ) + rule_runner.write_files( + { + "foo/BUILD": dedent( + """ + python_sources(resolve=parametrize("a", "b")) + + python_distribution( + name="dist", + dependencies=[":foo@resolve=a"], + provides=python_artifact(name="foo", version="9.8.7"), + sdist=False, + generate_setup=False, + ) + """ + ), + "foo/bar.py": "BAR = 42", + "foo/setup.py": dedent( + """ + from setuptools import setup + + setup( + name="foo", + version="9.8.7", + packages=["foo"], + package_dir={"foo": "."}, + entry_points={"foo.plugins": ["bar = foo.bar.BAR"]}, + ) + """ + ), + "lock.txt": "", + } + ) + dist = rule_runner.get_target(Address("foo", target_name="dist")) + result = rule_runner.request(ResolveSortedPythonDistributionTargets, ()) + assert len(result.targets) == 1 + assert "b" not in result.targets + assert "a" in result.targets + assert len(result.targets["a"]) == 1 + assert dist == result.targets["a"][0] + + +def test_build_editable_local_dists(rule_runner: RuleRunner) -> None: + foo = PurePath("foo") + rule_runner.write_files( + { + foo + / "BUILD": dedent( + """ + python_sources() + + python_distribution( + name="dist", + dependencies=[":foo"], + provides=python_artifact(name="foo", version="9.8.7"), + sdist=False, + generate_setup=False, + ) + """ + ), + foo / "bar.py": "BAR = 42", + foo + / "setup.py": dedent( + """ + from setuptools import setup + + setup( + name="foo", + version="9.8.7", + packages=["foo"], + package_dir={"foo": "."}, + entry_points={"foo.plugins": ["bar = foo.bar.BAR"]}, + ) + """ + ), + } + ) + request = EditableLocalDistsRequest( + resolve=None, # resolves is disabled + ) + result = rule_runner.request(EditableLocalDists, [request]) + + assert result.optional_digest is not None + contents = rule_runner.request(DigestContents, [result.optional_digest]) + assert len(contents) == 1 + whl_content = contents[0] + assert whl_content + assert whl_content.path == "foo-9.8.7-0.editable-py3-none-any.whl" + with io.BytesIO(whl_content.content) as fp: + with zipfile.ZipFile(fp, "r") as whl: + whl_files = whl.namelist() + # Check that sources are not present in editable wheel + assert "foo/bar.py" not in whl_files + assert "foo/qux.py" not in whl_files + # Check that pth and metadata files are present in editable wheel + assert "foo__pants__.pth" in whl_files + assert "foo-9.8.7.dist-info/METADATA" in whl_files + assert "foo-9.8.7.dist-info/RECORD" in whl_files + assert "foo-9.8.7.dist-info/WHEEL" in whl_files + assert "foo-9.8.7.dist-info/direct_url__pants__.json" in whl_files + assert "foo-9.8.7.dist-info/entry_points.txt" in whl_files diff --git a/src/python/pants/backend/python/util_rules/scripts/BUILD b/src/python/pants/backend/python/util_rules/scripts/BUILD new file mode 100644 index 00000000000..752b6d575e9 --- /dev/null +++ b/src/python/pants/backend/python/util_rules/scripts/BUILD @@ -0,0 +1,3 @@ +# Copyright 2023 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). +python_sources() diff --git a/src/python/pants/backend/python/util_rules/scripts/__init__.py b/src/python/pants/backend/python/util_rules/scripts/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/src/python/pants/backend/python/util_rules/scripts/pep660_backend_wrapper.py b/src/python/pants/backend/python/util_rules/scripts/pep660_backend_wrapper.py new file mode 100644 index 00000000000..75b9931adf8 --- /dev/null +++ b/src/python/pants/backend/python/util_rules/scripts/pep660_backend_wrapper.py @@ -0,0 +1,176 @@ +# Copyright 2023 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +import base64 +import errno +import hashlib +import importlib +import json +import os +import shutil +import sys +import zipfile + +_WHEEL_TEMPLATE = """\ +Wheel-Version: 1.0 +Generator: pantsbuild.pants +Root-Is-Purelib: true +Tag: {} +Build: 0.editable +""" + + +def import_build_backend(build_backend): + module_path, _, object_path = build_backend.partition(":") + backend_module = importlib.import_module(module_path) + return getattr(backend_module, object_path) if object_path else backend_module + + +def mkdir(directory): + """Python 2.7 doesn't have the exist_ok arg on os.makedirs().""" + try: + os.makedirs(directory) + except OSError as e: + if e.errno != errno.EEXIST: + raise + + +def prepare_dist_info(backend, build_dir, wheel_config_settings): + """Use PEP 660 or PEP 517 backend methods to create .dist-info directory. + + PEP 660 defines `prepare_metadata_for_build_editable`. If PEP 660 is not supported, we fall back + to PEP 517's `prepare_metadata_for_build_wheel`. PEP 517, however, says that method is optional. + So finally we fall back to using `build_wheel` and then extract the dist-info directory and then + delete the extra wheel file (like one of PEP 517's examples). + """ + prepare_metadata = getattr( + backend, + "prepare_metadata_for_build_editable", # PEP 660 + getattr(backend, "prepare_metadata_for_build_wheel", None), # PEP 517 + ) + if prepare_metadata is not None: + print("prepare_metadata: " + str(prepare_metadata)) + metadata_path = prepare_metadata(build_dir, wheel_config_settings) + else: + # Optional PEP 517 method not defined. Use build_wheel instead. + wheel_path = backend.build_wheel(build_dir, wheel_config_settings) + with zipfile.ZipFile(os.path.join(build_dir, wheel_path), "r") as whl: + dist_info_files = [n for n in whl.namelist() if ".dist-info/" in n] + whl.extractall(build_dir, dist_info_files) + metadata_path = os.path.dirname(dist_info_files[0]) + return standardize_dist_info_path(build_dir, metadata_path) + + +def standardize_dist_info_path(build_dir, metadata_path): + """Make sure dist-info dir is named pkg-version.dist-info. + + Returns the package name, version, and update metadata_path + """ + pkg_version = metadata_path.replace(".dist-info", "") + if "-" in pkg_version: + pkg, version = pkg_version.split("-") + else: + # The wrapped backend does not conform to the latest specs. + pkg = pkg_version + version = "" + with open(os.path.join(build_dir, metadata_path, "METADATA"), "r") as f: + lines = f.readlines() + for line in lines: + if line.startswith("Version: "): + version = line[len("Version: ") :].strip() + break + # Standardize the name of the dist-info directory per Binary distribution format spec. + old_metadata_path = metadata_path + metadata_path = pkg + "-" + version + ".dist-info" + shutil.move( + os.path.join(build_dir, old_metadata_path), os.path.join(build_dir, metadata_path) + ) + return pkg, version, metadata_path + + +def remove_record_files(build_dir, metadata_path): + """Any RECORD* file will be incorrect since we are creating the wheel.""" + for file in os.listdir(os.path.join(build_dir, metadata_path)): + if file == "RECORD" or file.startswith("RECORD."): + os.unlink(os.path.join(build_dir, metadata_path, file)) + + +def write_wheel_file(tags, build_dir, metadata_path): + metadata_wheel_file = os.path.join(build_dir, metadata_path, "WHEEL") + if not os.path.exists(metadata_wheel_file): + with open(metadata_wheel_file, "w") as f: + f.write(_WHEEL_TEMPLATE.format(tags)) + + +def write_direct_url_file(direct_url, build_dir, metadata_path): + """Create a direct_url.json file for later use during wheel install. + + We abuse PEX to get the PEP 660 editable wheels into the virtualenv, and then use pip to + actually install the wheel. But PEX and pip do not know that this is an editable install. We + cannot add direct_url.json directly to the wheel because that must be added by the wheel + installer. So we will rename this file to 'direct_url.json' after pip has installed everything + else. + """ + direct_url_contents = {"url": direct_url, "dir_info": {"editable": True}} + direct_url_file = os.path.join(build_dir, metadata_path, "direct_url__pants__.json") + with open(direct_url_file, "w") as f: + json.dump(direct_url_contents, f) + + +def build_editable_wheel(pkg, build_dir, metadata_path, dist_dir, wheel_path, pth_file_path): + """Build the editable wheel, including .pth and RECORD files.""" + + _record = [] + + def record(file_path, file_arcname): + """Calculate an entry for the RECORD file (required by the wheel spec).""" + with open(file_path, "rb") as f: + file_digest = hashlib.sha256(f.read()).digest() + file_hash = "sha256=" + base64.urlsafe_b64encode(file_digest).decode().rstrip("=") + file_size = str(os.stat(file_path).st_size) + _record.append(",".join([file_arcname, file_hash, file_size])) + + with zipfile.ZipFile(os.path.join(dist_dir, wheel_path), "w") as whl: + pth_file_arcname = pkg + "__pants__.pth" + record(pth_file_path, pth_file_arcname) + whl.write(pth_file_path, pth_file_arcname) + + # The following for loop is loosely based on: + # wheel.wheelfile.WheelFile.write_files (by @argonholm MIT license) + for root, dirnames, filenames in os.walk(os.path.join(build_dir, metadata_path)): + dirnames.sort() + for name in sorted(filenames): + path = os.path.normpath(os.path.join(root, name)) + if os.path.isfile(path): + arcname = os.path.relpath(path, build_dir).replace(os.path.sep, "/") + record(path, arcname) + whl.write(path, arcname) + + record_path = os.path.join(metadata_path, "RECORD") + _record.append(record_path + ",,") + _record.append("") # "" to add newline at eof + whl.writestr(record_path, os.linesep.join(_record)) + + +def main(build_backend, dist_dir, pth_file_path, wheel_config_settings, tags, direct_url): + backend = import_build_backend(build_backend) + + build_dir = "build" + mkdir(dist_dir) + mkdir(build_dir) + + pkg, version, metadata_path = prepare_dist_info(backend, build_dir, wheel_config_settings) + remove_record_files(build_dir, metadata_path) + write_wheel_file(tags, build_dir, metadata_path) + write_direct_url_file(direct_url, build_dir, metadata_path) + + wheel_path = "{}-{}-0.editable-{}.whl".format(pkg, version, tags) + build_editable_wheel(pkg, build_dir, metadata_path, dist_dir, wheel_path, pth_file_path) + print("editable_path: {editable_path}".format(editable_path=wheel_path)) + + +if __name__ == "__main__": + with open(sys.argv[1], "r") as f: + settings = json.load(f) + + main(**settings)