An export goal for exporting data to be read by IDEs.

src/python/pants/backend/python/goals/BUILD

@@ -13,6 +13,8 @@ python_tests(
 
 python_tests(name="coverage_py_test", sources=["coverage_py_test.py"], timeout=20)
 
+python_tests(name="export_integration_test", sources=["export_integration_test.py"])
+
 python_tests(name="lockfile_test", sources=["lockfile_test.py"])
 
 python_tests(

src/python/pants/backend/python/goals/export.py

@@ -0,0 +1,83 @@
+# Copyright 2021 Pants project contributors (see CONTRIBUTORS.md).
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+
+from __future__ import annotations
+
+import os
+from dataclasses import dataclass
+
+from pants.backend.python.subsystems.setup import PythonSetup
+from pants.backend.python.util_rules.interpreter_constraints import InterpreterConstraints
+from pants.backend.python.util_rules.pex import VenvPex, VenvPexProcess
+from pants.backend.python.util_rules.pex_environment import PexEnvironment
+from pants.backend.python.util_rules.pex_from_targets import PexFromTargetsRequest
+from pants.core.goals.export import ExportableData, ExportableDataRequest, ExportError, Symlink
+from pants.engine.internals.selectors import Get
+from pants.engine.process import ProcessResult
+from pants.engine.rules import collect_rules, rule
+from pants.engine.unions import UnionRule
+
+
+@dataclass(frozen=True)
+class ExportedVenvRequest(ExportableDataRequest):
+    pass
+
+
+@rule
+async def export_venv(
+    request: ExportedVenvRequest, python_setup: PythonSetup, pex_env: PexEnvironment
+) -> ExportableData:
+    # Pick a single interpreter for the venv.
+    interpreter_constraints = InterpreterConstraints.create_from_targets(
+        request.targets, python_setup
+    )
+    if not interpreter_constraints:
+        # If there were no targets that defined any constraints, fall back to the global ones.
+        interpreter_constraints = InterpreterConstraints(python_setup.interpreter_constraints)
+    min_interpreter = interpreter_constraints.snap_to_minimum(python_setup.interpreter_universe)
+    if not min_interpreter:
+        raise ExportError(
+            "The following interpreter constraints were computed for all the targets for which "
+            f"export was requested: {interpreter_constraints}. There is no python interpreter "
+            "compatible with these constraints. Please restrict the target set to one that shares "
+            "a compatible interpreter."
+        )
+
+    venv_pex = await Get(
+        VenvPex,
+        PexFromTargetsRequest,
+        PexFromTargetsRequest.for_requirements(
+            (tgt.address for tgt in request.targets),
+            internal_only=True,
+            hardcoded_interpreter_constraints=min_interpreter,
+        ),
+    )
+
+    complete_pex_env = pex_env.in_workspace()
+    venv_abspath = os.path.join(complete_pex_env.pex_root, venv_pex.venv_rel_dir)
+
+    # Run the venv_pex to get the full python version (including patch #), so we
+    # can use it in the symlink name.
+    res = await Get(
+        ProcessResult,
+        VenvPexProcess(
+            venv_pex=venv_pex,
+            description="Create virtualenv",
+            argv=["-c", "import sys; print('.'.join(str(x) for x in sys.version_info[0:3]))"],
+            input_digest=venv_pex.digest,
+        ),
+    )
+    py_version = res.stdout.strip().decode()
+
+    return ExportableData(
+        f"virtualenv for {min_interpreter}",
+        os.path.join("python", "virtualenv"),
+        symlinks=[Symlink(venv_abspath, py_version)],
+    )
+
+
+def rules():
+    return [
+        *collect_rules(),
+        UnionRule(ExportableDataRequest, ExportedVenvRequest),
+    ]

src/python/pants/backend/python/goals/export_integration_test.py

@@ -0,0 +1,50 @@
+# Copyright 2021 Pants project contributors (see CONTRIBUTORS.md).
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+import sys
+
+import pytest
+
+from pants.backend.python import target_types_rules
+from pants.backend.python.goals import export
+from pants.backend.python.goals.export import ExportedVenvRequest
+from pants.backend.python.target_types import PythonRequirementTarget
+from pants.backend.python.util_rules import pex_from_targets
+from pants.base.specs import AddressSpecs, DescendantAddresses
+from pants.core.goals.export import ExportableData
+from pants.engine.rules import QueryRule
+from pants.engine.target import Targets
+from pants.testutil.rule_runner import RuleRunner
+
+
+@pytest.fixture
+def rule_runner() -> RuleRunner:
+    return RuleRunner(
+        rules=[
+            *export.rules(),
+            *pex_from_targets.rules(),
+            *target_types_rules.rules(),
+            QueryRule(Targets, [AddressSpecs]),
+            QueryRule(ExportableData, [ExportedVenvRequest]),
+        ],
+        target_types=[PythonRequirementTarget],
+    )
+
+
+def test_export_venv(rule_runner: RuleRunner) -> None:
+    # We know that the current interpreter exists on the system.
+    vinfo = sys.version_info
+    current_interpreter = f"{vinfo.major}.{vinfo.minor}.{vinfo.micro}"
+
+    rule_runner.set_options(
+        [f"--python-interpreter-constraints=['=={current_interpreter}']"],
+        env_inherit={"PATH", "PYENV_ROOT"},
+    )
+    rule_runner.write_files(
+        {"src/foo/BUILD": "python_requirement(name='req', requirements=['ansicolors==1.1.8'])"}
+    )
+    targets = rule_runner.request(Targets, [AddressSpecs([DescendantAddresses("src/foo")])])
+    data = rule_runner.request(ExportableData, [ExportedVenvRequest(targets)])
+    assert len(data.symlinks) == 1
+    symlink = data.symlinks[0]
+    assert symlink.link_rel_path == current_interpreter
+    assert "named_caches/pex_root/venvs/" in symlink.source_path

src/python/pants/backend/python/register.py

@@ -10,6 +10,7 @@
 from pants.backend.python.dependency_inference import rules as dependency_inference_rules
 from pants.backend.python.goals import (
     coverage_py,
+    export,
     lockfile,
     package_pex_binary,
     pytest_runner,
@@ -62,6 +63,7 @@ def build_file_aliases():
 def rules():
     return (
         *coverage_py.rules(),
+        *export.rules(),
         *lockfile.rules(),
         *tailor.rules(),
         *ancestor_files.rules(),

src/python/pants/core/goals/export.py

@@ -0,0 +1,124 @@
+# Copyright 2021 Pants project contributors (see CONTRIBUTORS.md).
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+
+from __future__ import annotations
+
+import os
+from dataclasses import dataclass
+from typing import Iterable, cast
+
+from pants.base.build_root import BuildRoot
+from pants.core.util_rules.distdir import DistDir
+from pants.engine.console import Console
+from pants.engine.fs import EMPTY_DIGEST, AddPrefix, Digest, MergeDigests, Workspace
+from pants.engine.goal import Goal, GoalSubsystem
+from pants.engine.internals.selectors import Get, MultiGet
+from pants.engine.rules import collect_rules, goal_rule
+from pants.engine.target import Targets
+from pants.engine.unions import UnionMembership, union
+from pants.util.dirutil import absolute_symlink
+from pants.util.meta import frozen_after_init
+
+
+class ExportError(Exception):
+    pass
+
+
+@union
+@dataclass(frozen=True)
+class ExportableDataRequest:
+    """A union for exportable data provided by a backend.
+
+    Subclass and install a member of this type to export data.
+    """
+
+    targets: Targets
+
+
+@dataclass(frozen=True)
+class Symlink:
+    """A symlink from link_rel_path pointing to source_path.
+
+    source_path may be absolute, or relative to the repo root.
+
+    link_rel_path is relative to the enclosing ExportableData's reldir, and will be
+    absolutized when a location for that dir is chosen.
+    """
+
+    source_path: str
+    link_rel_path: str
+
+
+@frozen_after_init
+@dataclass(unsafe_hash=True)
+class ExportableData:
+    description: str
+    # Materialize digests and create symlinks under this reldir.
+    reldir: str
+    # Materialize this digest.
+    digest: Digest
+    # Create these symlinks. Symlinks are created after the digest is materialized,
+    # so may reference files/dirs in the digest.
+    symlinks: tuple[Symlink, ...]
+
+    def __init__(
+        self,
+        description: str,
+        reldir: str,
+        *,
+        digest: Digest = EMPTY_DIGEST,
+        symlinks: Iterable[Symlink] = tuple(),
+    ):
+        self.description = description
+        self.reldir = reldir
+        self.digest = digest
+        self.symlinks = tuple(symlinks)
+
+
+class ExportSubsystem(GoalSubsystem):
+    name = "export"
+    help = "Export Pants data for use in other tools, such as IDEs."
+
+
+class Export(Goal):
+    subsystem_cls = ExportSubsystem
+
+
+@goal_rule
+async def export(
+    console: Console,
+    targets: Targets,
+    export_subsystem: ExportSubsystem,
+    workspace: Workspace,
+    union_membership: UnionMembership,
+    build_root: BuildRoot,
+    dist_dir: DistDir,
+) -> Export:
+    request_types = cast(
+        "Iterable[type[ExportableDataRequest]]", union_membership.get(ExportableDataRequest)
+    )
+    requests = tuple(request_type(targets) for request_type in request_types)
+    exportables = await MultiGet(
+        Get(ExportableData, ExportableDataRequest, request) for request in requests
+    )
+    prefixed_digests = await MultiGet(
+        Get(Digest, AddPrefix(exp.digest, exp.reldir)) for exp in exportables
+    )
+    output_dir = os.path.join(str(dist_dir.relpath), "export")
+    merged_digest = await Get(Digest, MergeDigests(prefixed_digests))
+    dist_digest = await Get(Digest, AddPrefix(merged_digest, output_dir))
+    workspace.write_digest(dist_digest)
+    for exp in exportables:
+        for symlink in exp.symlinks:
+            # Note that if symlink.source_path is an abspath, join returns it unchanged.
+            source_abspath = os.path.join(build_root.path, symlink.source_path)
+            link_abspath = os.path.abspath(
+                os.path.join(output_dir, exp.reldir, symlink.link_rel_path)
+            )
+            absolute_symlink(source_abspath, link_abspath)
+        console.print_stdout(f"Wrote {exp.description} to {os.path.join(output_dir, exp.reldir)}")
+    return Export(exit_code=0)
+
+
+def rules():
+    return collect_rules()

src/python/pants/core/goals/export_test.py

@@ -0,0 +1,114 @@
+# Copyright 2021 Pants project contributors (see CONTRIBUTORS.md).
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+
+from __future__ import annotations
+
+import os
+from pathlib import Path
+from typing import List, Tuple
+
+from pants.base.build_root import BuildRoot
+from pants.core.goals.export import (
+    Export,
+    ExportableData,
+    ExportableDataRequest,
+    ExportSubsystem,
+    Symlink,
+    export,
+)
+from pants.core.util_rules.distdir import DistDir
+from pants.engine.addresses import Address
+from pants.engine.fs import AddPrefix, CreateDigest, Digest, FileContent, MergeDigests, Workspace
+from pants.engine.rules import QueryRule
+from pants.engine.target import Target, Targets
+from pants.engine.unions import UnionMembership, UnionRule
+from pants.testutil.option_util import create_goal_subsystem, create_options_bootstrapper
+from pants.testutil.rule_runner import MockGet, RuleRunner, mock_console, run_rule_with_mocks
+
+
+class MockTarget(Target):
+    alias = "target"
+    core_fields = ()
+
+
+def make_target(path: str, target_name: str) -> Target:
+    return MockTarget({}, Address(path, target_name=target_name))
+
+
+class MockExportableDataRequest(ExportableDataRequest):
+    pass
+
+
+def mock_export(
+    edr: ExportableDataRequest, digest: Digest, symlinks: tuple[Symlink, ...]
+) -> ExportableData:
+    return ExportableData(
+        description=f"mock export for {','.join(t.address.spec for t in edr.targets)}",
+        reldir="mock",
+        digest=digest,
+        symlinks=symlinks,
+    )
+
+
+def run_export_rule(rule_runner: RuleRunner, targets: List[Target]) -> Tuple[int, str]:
+    union_membership = UnionMembership({ExportableDataRequest: [MockExportableDataRequest]})
+    with open(os.path.join(rule_runner.build_root, "somefile"), "wb") as fp:
+        fp.write(b"SOMEFILE")
+    with mock_console(create_options_bootstrapper()) as (console, stdio_reader):
+        digest = rule_runner.request(Digest, [CreateDigest([FileContent("foo/bar", b"BAR")])])
+        result: Export = run_rule_with_mocks(
+            export,
+            rule_args=[
+                console,
+                Targets(targets),
+                create_goal_subsystem(ExportSubsystem),
+                Workspace(rule_runner.scheduler, _enforce_effects=False),
+                union_membership,
+                BuildRoot(),
+                DistDir(relpath=Path("dist")),
+            ],
+            mock_gets=[
+                MockGet(
+                    output_type=ExportableData,
+                    input_type=ExportableDataRequest,
+                    mock=lambda edr: mock_export(
+                        edr, digest, (Symlink("somefile", "link_to_somefile"),)
+                    ),
+                ),
+                MockGet(
+                    output_type=Digest,
+                    input_type=MergeDigests,
+                    mock=lambda md: rule_runner.request(Digest, [md]),
+                ),
+                MockGet(
+                    output_type=Digest,
+                    input_type=AddPrefix,
+                    mock=lambda ap: rule_runner.request(Digest, [ap]),
+                ),
+            ],
+            union_membership=union_membership,
+        )
+        return result.exit_code, stdio_reader.get_stdout()
+
+
+def test_run_export_rule() -> None:
+    rule_runner = RuleRunner(
+        rules=[
+            UnionRule(ExportableDataRequest, MockExportableDataRequest),
+            QueryRule(Digest, [CreateDigest]),
+        ],
+        target_types=[MockTarget],
+    )
+    exit_code, stdout = run_export_rule(rule_runner, [make_target("foo/bar", "baz")])
+    assert exit_code == 0
+    assert "Wrote mock export for foo/bar:baz to dist/export/mock" in stdout
+    expected_dist_path = os.path.join(rule_runner.build_root, "dist/export/mock/foo/bar")
+    assert os.path.isfile(expected_dist_path)
+    with open(expected_dist_path, "rb") as fp:
+        assert fp.read() == b"BAR"
+
+    symlink = "dist/export/mock/link_to_somefile"
+    assert os.path.islink(symlink)
+    assert os.readlink(symlink) == os.path.join(rule_runner.build_root, "somefile")
+    with open(symlink, "rb") as fp:
+        assert fp.read() == b"SOMEFILE"