Move/Rename: goals/setup_py.py -> `{util_rules,goals}/package_dists…

….py` + `subsystems/setup_py_generation.py` (#18702)

In pants v1, there was a [`setup-py`
goal](https://v1.pantsbuild.org/options_reference.html#setup-py). In
pants v2, that is now part of the more generic [`package`
goal](https://www.pantsbuild.org/docs/python-package-goal). So, the name
`setup_py` doesn't make sense any more.

Also, most of the rules in that file are used in contexts other than the
`package` goal. For instance, I'm using them in #18639 in the `export`
goal. And importing something from `goals` in `util_rules` feels rather
odd. So, I would like to see most of these rules under `util_rules`.

This change does the following:
- renames: `pants.backends.python.goals.setup_py` to
`pants.backends.python.util_rules.package_dists`
- adds a minimal: `pants.backends.python.goals.package_dists`
- moves `SetupPyGeneration(Subsystem)` to
`pants.backends.python.subsystems.setup_py_generation`
- adjusts imports to use the new locations as needed.

docs/markdown/Writing Plugins/common-plugin-tasks/plugins-setup-py.md

@@ -31,7 +31,7 @@ Set the class method `is_applicable()` to determine whether your implementation
 In this example, we will always use our custom implementation:
 
 ```python
-from pants.backend.python.goals.setup_py import SetupKwargsRequest
+from pants.backend.python.util_rules.package_dists import SetupKwargsRequest
 from pants.engine.target import Target
 
 class CustomSetupKwargsRequest(SetupKwargsRequest):
@@ -115,7 +115,7 @@ Your rule should return `SetupKwargs`, which takes two arguments: `kwargs: dict[
 For example, this will simply hardcode a kwarg:
 
 ```python
-from pants.backend.python.goals.setup_py import SetupKwargs
+from pants.backend.python.util_rules.package_dists import SetupKwargs
 from pants.engine.rules import rule
 
 @rule
@@ -139,7 +139,7 @@ Then, run `pants package path/to:python_distribution` and inspect the generated
 Often, you will want to read from a file in your project to set kwargs like `version` or `long_description`. Use `await Get(DigestContents, PathGlobs)` to do this (see [File system](doc:rules-api-file-system)):
 
 ```python
-from pants.backend.python.goals.setup_py import SetupKwargs
+from pants.backend.python.util_rules.package_dists import SetupKwargs
 from pants.engine.fs import DigestContents, GlobMatchErrorBehavior, PathGlobs
 from pants.engine.rules import rule
 
@@ -179,7 +179,7 @@ python_distribution(
 ```python
 import os.path
 
-from pants.backend.python.goals.setup_py import SetupKwargs
+from pants.backend.python.util_rules.package_dists import SetupKwargs
 from pants.engine.fs import DigestContents, GlobMatchErrorBehavior, PathGlobs
 from pants.engine.rules import rule
 

pants-plugins/internal_plugins/releases/register.py

@@ -5,7 +5,7 @@
 
 from packaging.version import Version
 
-from pants.backend.python.goals.setup_py import SetupKwargs, SetupKwargsRequest
+from pants.backend.python.util_rules.package_dists import SetupKwargs, SetupKwargsRequest
 from pants.core.util_rules.external_tool import (
     DownloadedExternalTool,
     ExternalTool,

src/python/pants/backend/BUILD

@@ -58,7 +58,7 @@ __dependents_rules__(
         DEFAULT_DEPENDENTS_RULES,
     ),
     (
-        "[/python/goals/setup_py.py]",
+        "[/python/util_rules/package_dists.py]",
         "//pants-plugins/internal_plugins/releases/register.py",
         DEFAULT_DEPENDENTS_RULES,
     ),

src/python/pants/backend/python/goals/BUILD

@@ -30,12 +30,11 @@ python_tests(
         },
         "run_pex_binary_integration_test.py": {"timeout": 600},
         "run_python_source_integration_test.py": {"timeout": 180},
-        "setup_py_integration_test.py": {
+        "package_dists_integration_test.py": {
             "dependencies": ["testprojects/src/python:native_directory"],
             "tags": ["platform_specific_behavior"],
             "timeout": 180,
         },
-        "setup_py_test.py": {"timeout": 150},
     },
 )
 

src/python/pants/backend/python/goals/package_dists.py

@@ -0,0 +1,43 @@
+# Copyright 2019 Pants project contributors (see CONTRIBUTORS.md).
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+
+from __future__ import annotations
+
+from pants.backend.python.subsystems.setup import PythonSetup
+from pants.backend.python.subsystems.setuptools import PythonDistributionFieldSet
+from pants.backend.python.util_rules.dists import DistBuildRequest, DistBuildResult
+from pants.backend.python.util_rules.package_dists import create_dist_build_request
+from pants.backend.python.util_rules.package_dists import rules as package_dists_rules
+from pants.core.goals.package import BuiltPackage, BuiltPackageArtifact, PackageFieldSet
+from pants.engine.fs import Digest, Snapshot
+from pants.engine.rules import Get, collect_rules, rule
+from pants.engine.unions import UnionMembership, UnionRule
+
+
+@rule
+async def package_python_dist(
+    field_set: PythonDistributionFieldSet,
+    python_setup: PythonSetup,
+    union_membership: UnionMembership,
+) -> BuiltPackage:
+    dist_build_request = await create_dist_build_request(
+        field_set=field_set,
+        python_setup=python_setup,
+        union_membership=union_membership,
+        # raises an error if both dist_tgt.wheel and dist_tgt.sdist are False
+        validate_wheel_sdist=True,
+    )
+    setup_py_result = await Get(DistBuildResult, DistBuildRequest, dist_build_request)
+    dist_snapshot = await Get(Snapshot, Digest, setup_py_result.output)
+    return BuiltPackage(
+        setup_py_result.output,
+        tuple(BuiltPackageArtifact(path) for path in dist_snapshot.files),
+    )
+
+
+def rules():
+    return [
+        *package_dists_rules(),
+        *collect_rules(),
+        UnionRule(PackageFieldSet, PythonDistributionFieldSet),
+    ]

src/python/pants/backend/python/goals/package_pex_binary_integration_test.py

@@ -13,7 +13,7 @@
 import pytest
 
 from pants.backend.python import target_types_rules
-from pants.backend.python.goals import package_pex_binary, setup_py
+from pants.backend.python.goals import package_dists, package_pex_binary
 from pants.backend.python.goals.package_pex_binary import PexBinaryFieldSet
 from pants.backend.python.macros.python_artifact import PythonArtifact
 from pants.backend.python.subsystems.setuptools import PythonDistributionFieldSet
@@ -46,7 +46,7 @@ def rule_runner() -> RuleRunner:
             *pex_from_targets.rules(),
             *target_types_rules.rules(),
             *core_target_types_rules(),
-            *setup_py.rules(),
+            *package_dists.rules(),
             QueryRule(BuiltPackage, [PexBinaryFieldSet]),
             QueryRule(BuiltPackage, [PythonDistributionFieldSet]),
         ],

src/python/pants/backend/python/goals/pytest_runner_integration_test.py

@@ -13,7 +13,7 @@
 
 from pants.backend.python import target_types_rules
 from pants.backend.python.dependency_inference import rules as dependency_inference_rules
-from pants.backend.python.goals import package_pex_binary, pytest_runner, setup_py
+from pants.backend.python.goals import package_dists, package_pex_binary, pytest_runner
 from pants.backend.python.goals.coverage_py import create_or_update_coverage_config
 from pants.backend.python.goals.pytest_runner import (
     PytestPluginSetup,
@@ -71,7 +71,7 @@ def rule_runner() -> RuleRunner:
             get_filtered_environment,
             *target_types_rules.rules(),
             *local_dists.rules(),
-            *setup_py.rules(),
+            *package_dists.rules(),
             *package.rules(),
             QueryRule(Partitions, (PyTestRequest.PartitionRequest,)),
             QueryRule(TestResult, (PyTestRequest.Batch,)),

src/python/pants/backend/python/goals/run_python_source_integration_test.py

@@ -22,7 +22,7 @@
 from pants.backend.codegen.protobuf.target_types import rules as protobuf_target_types_rules
 from pants.backend.python import target_types_rules
 from pants.backend.python.dependency_inference import rules as dependency_inference_rules
-from pants.backend.python.goals import setup_py
+from pants.backend.python.goals import package_dists
 from pants.backend.python.goals.run_python_source import PythonSourceFieldSet
 from pants.backend.python.goals.run_python_source import rules as run_rules
 from pants.backend.python.macros.python_artifact import PythonArtifact
@@ -51,7 +51,7 @@ def rule_runner() -> RuleRunner:
             *target_types_rules.rules(),
             *local_dists.rules(),
             *pex_from_targets.rules(),
-            *setup_py.rules(),
+            *package_dists.rules(),
             *protobuf_subsystem_rules(),
             *protobuf_target_types_rules(),
             *protobuf_python_rules(),

src/python/pants/backend/python/register.py

@@ -12,13 +12,13 @@
     coverage_py,
     export,
     lockfile,
+    package_dists,
     package_pex_binary,
     pytest_runner,
     repl,
     run_pex_binary,
     run_python_requirement,
     run_python_source,
-    setup_py,
     tailor,
 )
 from pants.backend.python.macros import (
@@ -80,7 +80,7 @@ def rules():
         *run_pex_binary.rules(),
         *run_python_requirement.rules(),
         *run_python_source.rules(),
-        *setup_py.rules(),
+        *package_dists.rules(),
         *tailor.rules(),
         *local_dists.rules(),
         *export.rules(),

src/python/pants/backend/python/subsystems/setup_py_generation.py

@@ -0,0 +1,57 @@
+# Copyright 2023 Pants project contributors (see CONTRIBUTORS.md).
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+import enum
+
+from pants.option.option_types import BoolOption, EnumOption
+from pants.option.subsystem import Subsystem
+from pants.util.strutil import softwrap
+
+
+@enum.unique
+class FirstPartyDependencyVersionScheme(enum.Enum):
+    EXACT = "exact"  # i.e., ==
+    COMPATIBLE = "compatible"  # i.e., ~=
+    ANY = "any"  # i.e., no specifier
+
+
+class SetupPyGeneration(Subsystem):
+    options_scope = "setup-py-generation"
+    help = "Options to control how setup.py is generated from a `python_distribution` target."
+
+    # Generating setup is the more aggressive thing to do, so we'd prefer that the default
+    # be False. However that would break widespread existing usage, so we'll make that
+    # change in a future deprecation cycle.
+    generate_setup_default = BoolOption(
+        default=True,
+        help=softwrap(
+            """
+            The default value for the `generate_setup` field on `python_distribution` targets.
+            Can be overridden per-target by setting that field explicitly. Set this to False
+            if you mostly rely on handwritten setup files (setup.py, setup.cfg and similar).
+            Leave as True if you mostly rely on Pants generating setup files for you.
+            """
+        ),
+    )
+
+    first_party_dependency_version_scheme = EnumOption(
+        default=FirstPartyDependencyVersionScheme.EXACT,
+        help=softwrap(
+            """
+            What version to set in `install_requires` when a `python_distribution` depends on
+            other `python_distribution`s. If `exact`, will use `==`. If `compatible`, will
+            use `~=`. If `any`, will leave off the version. See
+            https://www.python.org/dev/peps/pep-0440/#version-specifiers.
+            """
+        ),
+    )
+
+    def first_party_dependency_version(self, version: str) -> str:
+        """Return the version string (e.g. '~=4.0') for a first-party dependency.
+
+        If the user specified to use "any" version, then this will return an empty string.
+        """
+        scheme = self.first_party_dependency_version_scheme
+        if scheme == FirstPartyDependencyVersionScheme.ANY:
+            return ""
+        specifier = "==" if scheme == FirstPartyDependencyVersionScheme.EXACT else "~="
+        return f"{specifier}{version}"

src/python/pants/backend/python/target_types_rules.py

@@ -24,7 +24,6 @@
     AmbiguityResolution,
     PythonInferSubsystem,
 )
-from pants.backend.python.goals.setup_py import InvalidEntryPoint
 from pants.backend.python.subsystems.setup import PythonSetup
 from pants.backend.python.target_types import (
     EntryPoint,
@@ -46,6 +45,7 @@
     ResolvePythonDistributionEntryPointsRequest,
 )
 from pants.backend.python.util_rules.interpreter_constraints import interpreter_constraints_contains
+from pants.backend.python.util_rules.package_dists import InvalidEntryPoint
 from pants.engine.addresses import Address, Addresses, UnparsedAddressInputs
 from pants.engine.fs import GlobMatchErrorBehavior, PathGlobs, Paths
 from pants.engine.rules import Get, MultiGet, collect_rules, rule

src/python/pants/backend/python/typecheck/mypy/mypyc.py

@@ -5,14 +5,17 @@
 
 from dataclasses import dataclass
 
-from pants.backend.python.goals.setup_py import DistBuildEnvironment, DistBuildEnvironmentRequest
 from pants.backend.python.target_types import PythonDistribution
 from pants.backend.python.typecheck.mypy.subsystem import (
     MyPy,
     MyPyConfigFile,
     MyPyFirstPartyPlugins,
 )
 from pants.backend.python.util_rules import pex_from_targets
+from pants.backend.python.util_rules.package_dists import (
+    DistBuildEnvironment,
+    DistBuildEnvironmentRequest,
+)
 from pants.backend.python.util_rules.pex import Pex, PexRequest
 from pants.backend.python.util_rules.pex_from_targets import RequirementsPexRequest
 from pants.engine.rules import Get, MultiGet, collect_rules, rule

src/python/pants/backend/python/util_rules/BUILD

@@ -9,5 +9,6 @@ python_tests(
         "local_dists_test.py": {"timeout": 120},
         "pex_from_targets_test.py": {"timeout": 200},
         "pex_test.py": {"timeout": 600},
+        "package_dists_test.py": {"timeout": 150},
     },
 )

src/python/pants/backend/python/util_rules/local_dists_test.py

@@ -11,7 +11,7 @@
 import pytest
 
 from pants.backend.python import target_types_rules
-from pants.backend.python.goals.setup_py import rules as setup_py_rules
+from pants.backend.python.goals import package_dists
 from pants.backend.python.macros.python_artifact import PythonArtifact
 from pants.backend.python.subsystems.setuptools import rules as setuptools_rules
 from pants.backend.python.target_types import PythonDistribution, PythonSourcesGeneratorTarget
@@ -31,7 +31,7 @@ def rule_runner() -> RuleRunner:
     return RuleRunner(
         rules=[
             *local_dists.rules(),
-            *setup_py_rules(),
+            *package_dists.rules(),
             *setuptools_rules(),
             *target_types_rules.rules(),
             *pex_from_targets.rules(),