-
-
Notifications
You must be signed in to change notification settings - Fork 631
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
An export goal for exporting data to be read by IDEs. #13415
Changes from 1 commit
1b72946
2dd03f4
44983f5
c80db6b
7228135
b549f8b
4b8ddf8
0a2477d
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,82 @@ | ||
# Copyright 2021 Pants project contributors (see CONTRIBUTORS.md). | ||
# Licensed under the Apache License, Version 2.0 (see LICENSE). | ||
|
||
from __future__ import annotations | ||
|
||
import os | ||
from dataclasses import dataclass | ||
|
||
from pants.backend.python.subsystems.setup import PythonSetup | ||
from pants.backend.python.util_rules.interpreter_constraints import InterpreterConstraints | ||
from pants.backend.python.util_rules.pex import VenvPex, VenvPexProcess | ||
from pants.backend.python.util_rules.pex_environment import PexEnvironment | ||
from pants.backend.python.util_rules.pex_from_targets import PexFromTargetsRequest | ||
from pants.core.goals.export import ExportableData, ExportableDataRequest, ExportError, Symlink | ||
from pants.engine.internals.selectors import Get | ||
from pants.engine.process import ProcessResult | ||
from pants.engine.rules import collect_rules, rule | ||
from pants.engine.unions import UnionRule | ||
|
||
|
||
@dataclass(frozen=True) | ||
class ExportedVenvRequest(ExportableDataRequest): | ||
pass | ||
|
||
|
||
@rule | ||
async def export_venv( | ||
request: ExportedVenvRequest, python_setup: PythonSetup, pex_env: PexEnvironment | ||
) -> ExportableData: | ||
# Pick a single interpreter for the venv. | ||
interpreter_constraints = InterpreterConstraints.create_from_targets( | ||
request.targets, python_setup | ||
) | ||
min_interpreter = interpreter_constraints.snap_to_minimum(python_setup.interpreter_universe) | ||
if not min_interpreter: | ||
raise ExportError( | ||
"There is no single Python interpreter compatible with all the " | ||
"targets for which export was requested. Please restrict the target set " | ||
"to one that shares a compatible interpreter." | ||
benjyw marked this conversation as resolved.
Show resolved
Hide resolved
|
||
) | ||
|
||
venv_pex = await Get( | ||
VenvPex, | ||
PexFromTargetsRequest, | ||
PexFromTargetsRequest.for_requirements( | ||
(tgt.address for tgt in request.targets), | ||
internal_only=True, | ||
hardcoded_interpreter_constraints=min_interpreter, | ||
), | ||
) | ||
|
||
complete_pex_env = pex_env.in_workspace() | ||
venv_abspath = os.path.join(complete_pex_env.pex_root, venv_pex.venv_rel_dir) | ||
|
||
# Run the venv_pex to ensure that the underlying venv is created if necessary. | ||
benjyw marked this conversation as resolved.
Show resolved
Hide resolved
|
||
# We also use this to get the full python version (including patch #), so we | ||
# can use it in the symlink name (not critical, but nice to have). | ||
res = await Get( | ||
ProcessResult, | ||
VenvPexProcess( | ||
venv_pex=venv_pex, | ||
description="Create virtualenv", | ||
argv=["-c", "import sys; print('.'.join(str(x) for x in sys.version_info[0:3]))"], | ||
input_digest=venv_pex.digest, | ||
# TODO: Is there always a python_configured? | ||
benjyw marked this conversation as resolved.
Show resolved
Hide resolved
|
||
extra_env=complete_pex_env.environment_dict(python_configured=True), | ||
), | ||
) | ||
py_version = res.stdout.strip().decode() | ||
|
||
return ExportableData( | ||
f"virtualenv for {min_interpreter}", | ||
os.path.join("python", "virtualenv"), | ||
symlinks=[Symlink(venv_abspath, py_version)], | ||
) | ||
|
||
|
||
def rules(): | ||
return [ | ||
*collect_rules(), | ||
UnionRule(ExportableDataRequest, ExportedVenvRequest), | ||
] |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,115 @@ | ||
# Copyright 2021 Pants project contributors (see CONTRIBUTORS.md). | ||
# Licensed under the Apache License, Version 2.0 (see LICENSE). | ||
|
||
from __future__ import annotations | ||
|
||
import os | ||
from dataclasses import dataclass | ||
from typing import Iterable, cast | ||
|
||
from pants.core.util_rules.distdir import DistDir | ||
from pants.engine.console import Console | ||
from pants.engine.fs import EMPTY_DIGEST, AddPrefix, Digest, MergeDigests, Workspace | ||
from pants.engine.goal import Goal, GoalSubsystem | ||
from pants.engine.internals.selectors import Get, MultiGet | ||
from pants.engine.rules import collect_rules, goal_rule | ||
from pants.engine.target import Targets | ||
from pants.engine.unions import UnionMembership, union | ||
from pants.util.dirutil import absolute_symlink | ||
from pants.util.meta import frozen_after_init | ||
|
||
|
||
class ExportError(Exception): | ||
pass | ||
|
||
|
||
@union | ||
@dataclass(frozen=True) | ||
class ExportableDataRequest: | ||
"""A union for exportable data provided by a backend. | ||
|
||
Subclass and install a member of this type to export data. | ||
""" | ||
|
||
targets: Targets | ||
|
||
|
||
@dataclass(frozen=True) | ||
class Symlink: | ||
"""A symlink from link_rel_path pointing to source_abs_path. | ||
|
||
link_rel_path is relative to the enclosing ExportableData's reldir, and will be absolutized when | ||
a location for that dir is chosen. | ||
""" | ||
|
||
source_abs_path: str | ||
link_rel_path: str | ||
|
||
|
||
@frozen_after_init | ||
@dataclass(unsafe_hash=True) | ||
class ExportableData: | ||
description: str | ||
# Materialize digests and create symlinks under this reldir. | ||
reldir: str | ||
# Materialize this digest. | ||
digest: Digest | ||
# Create these symlinks. Symlinks are created after the digest is materialized, | ||
# so may reference files/dirs in the digest. | ||
symlinks: tuple[Symlink, ...] | ||
|
||
def __init__( | ||
self, | ||
description: str, | ||
reldir: str, | ||
*, | ||
digest: Digest = EMPTY_DIGEST, | ||
symlinks: Iterable[Symlink] = tuple(), | ||
): | ||
self.description = description | ||
self.reldir = reldir | ||
self.digest = digest | ||
self.symlinks = tuple(symlinks) | ||
|
||
|
||
class ExportSubsystem(GoalSubsystem): | ||
name = "export" | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. What's the story with There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. We can probably get rid of There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I'm interested to see the evolution of I'm interested in possibly adding functionality to the There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. How would this be consumed? That functionality would be awesome, but There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. By humans I suppose. You're probably right, let me switch hats: ... and I've forgotten my other use-case. Ignore me for now 😄 There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I tend to agree, that There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Well, export is more about exporting internal information, for consumption by other tools, such as IDEs. We'd export codegen for that reason only - in normal use codegen is a build byproduct that is not visible to the end user. But since the user may want to manually inspect it, and may want to view it in their IDE, we'd export it. Documentation, though, is a build product, like a PEX file or a distribution or a docker image or whatever. It is an actual formal output of the system that you can request. So I don't think There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Agree with @benjyw on this one... I expect that docs meet the bar for a dedicated goal. Likewise, I think that There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Yeah OK I agree. But now you have me wondering if we should There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Good question! |
||
help = "Export Pants data for use in other tools, such as IDEs." | ||
|
||
|
||
class Export(Goal): | ||
subsystem_cls = ExportSubsystem | ||
|
||
|
||
@goal_rule | ||
async def export( | ||
console: Console, | ||
targets: Targets, | ||
export_subsystem: ExportSubsystem, | ||
workspace: Workspace, | ||
union_membership: UnionMembership, | ||
dist_dir: DistDir, | ||
) -> Export: | ||
request_types = cast( | ||
"Iterable[type[ExportableDataRequest]]", union_membership.get(ExportableDataRequest) | ||
) | ||
requests = tuple(request_type(targets) for request_type in request_types) | ||
exportables = await MultiGet( | ||
Get(ExportableData, ExportableDataRequest, request) for request in requests | ||
) | ||
output_dir = os.path.join(str(dist_dir.relpath), "export") | ||
merged_digest = await Get(Digest, MergeDigests(exp.digest for exp in exportables)) | ||
dist_digest = await Get(Digest, AddPrefix(merged_digest, output_dir)) | ||
workspace.write_digest(dist_digest) | ||
for exp in exportables: | ||
for symlink in exp.symlinks: | ||
link_abspath = os.path.abspath( | ||
os.path.join(output_dir, exp.reldir, symlink.link_rel_path) | ||
) | ||
absolute_symlink(symlink.source_abs_path, link_abspath) | ||
console.print_stdout(f"Wrote {exp.description} to {os.path.join(output_dir, exp.reldir)}") | ||
return Export(exit_code=0) | ||
|
||
|
||
def rules(): | ||
return collect_rules() |
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Rather than creating a virtualenv in the dest, this is symlinking from PEX's private cache of venvs into a destination... that seems maybe a bit dangerous, as someone who activated this venv and then fiddled with it would cause reproducibility issues by mutating PEX's cache.
Should this actually create a brand new venv instead, using
pex-tool
's facilities for that?There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
This is a valid concern. We're walking a perf line here,
Note that the fiddling would have to be editing by hand since these venvs don't get Pip installed into them (unlike
PEX_TOOLS=1 ./my.pex venv here
venvs, which do by default.).There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Mm... that does alleviate the concern quite a bit. Not a blocker then probably.