Re-shard tests.

.github/workflows/ci.yml

@@ -6,10 +6,6 @@ defaults:
 env:
   _PEX_TEST_DEV_ROOT: ${{ github.workspace }}/.pex_dev
   _PEX_TEST_POS_ARGS: "--color --devpi --devpi-timeout 15.0 --shutdown-devpi -vvs"
-  # We use this to skip exposing same-versioned Pythons present on Linux hosts. These otherwise can
-  # collide when attempting to load libpython<major>.<minor><flags>.so and lead to mysterious errors
-  # importing builtins like `fcntl` as outlined in https://github.com/pex-tool/pex/issues/1391.
-  _PEX_TEST_PYENV_VERSIONS: "2.7 3.7 3.10"
   _PEX_PEXPECT_TIMEOUT: 10
   # We have integration tests that exercise `--scie` support and these can trigger downloads from
   # GitHub Releases that needed elevated rate limit quota, which this gives.
@@ -64,36 +60,51 @@ jobs:
   # N.B.: The name of this job key (linux-tests) is depended on by scrips/build_cache_image.py. In
   # particular, the tox-env matrix list is used to ensure the cache covers all Linux CI jobs.
   linux-tests:
-    name: ./dtox.sh -e ${{ matrix.tox-env }}
+    name: ./dtox.sh -e ${{ matrix.tox-env }} ${{ matrix.pex-test-pos-args }}
     needs: org-check
     runs-on: ubuntu-22.04
     strategy:
       matrix:
-        tox-env:
-          - py27-pip20
-          # N.B.: Although we support Python 3.5 with a published universal wheel, Pex does not
-          # build under Python 3.5 or 3.6; so we skip to Python 3.7 for tests which must be able to
-          # build a Pex distribution both as part of the tox setup and in many test cases as well.
-          - py37-pip20
-          - py311-pip20
-          - py311-pip22_3_1
-          - py311-pip23_1_2
-          - py312-pip24_2
-          - py313-pip24_2
-          - pypy310-pip20
-          - pypy310-pip22_3_1
-          - pypy310-pip23_1_2
-          - py27-pip20-integration
-          - py37-pip22_3_1-integration
-          - py37-pip23_1_2-integration
-          - py311-pip20-integration
-          - py311-pip22_3_1-integration
-          - py311-pip23_1_2-integration
-          - py312-pip24_2-integration
-          - py313-pip24_2-integration
-          - pypy310-pip20-integration
-          - pypy310-pip22_3_1-integration
-          - pypy310-pip23_1_2-integration
+        include:
+          # Unit tests:
+          # -----------
+          - tox-env: py27-pip20
+          - tox-env: py37-pip22_3_1
+          - tox-env: py311-pip23_3_2
+          - tox-env: py312-pip24_2
+          - tox-env: py313-pip24_2
+          - tox-env: pypy310-pip24_2
+
+          # Integration tests, split most into two shards:
+          # ----------------------------------------------
+
+          # CPython 2.7 is fast enough not to require sharding.
+          - tox-env: py27-pip20-integration
+
+          - tox-env: py37-pip22_3_1-integration
+            pex-test-pos-args: --shard 1/2
+          - tox-env: py37-pip22_3_1-integration
+            pex-test-pos-args: --shard 2/2
+
+          - tox-env: py311-pip23_3_2-integration
+            pex-test-pos-args: --shard 1/2
+          - tox-env: py311-pip23_3_2-integration
+            pex-test-pos-args: --shard 2/2
+
+          - tox-env: py312-pip24_2-integration
+            pex-test-pos-args: --shard 1/2
+          - tox-env: py312-pip24_2-integration
+            pex-test-pos-args: --shard 2/2
+
+          - tox-env: py313-pip24_2-integration
+            pex-test-pos-args: --shard 1/2
+          - tox-env: py313-pip24_2-integration
+            pex-test-pos-args: --shard 2/2
+
+          - tox-env: pypy310-pip24_2-integration
+            pex-test-pos-args: --shard 1/2
+          - tox-env: pypy310-pip24_2-integration
+            pex-test-pos-args: --shard 2/2
     steps:
       - name: Free Up Disk Space
         uses: jlumbroso/[email protected]
@@ -125,9 +136,10 @@ jobs:
           # This is needed to get pexpect tests working under PyPy running under docker.
           export TERM="xterm"
           BASE_MODE=pull CACHE_MODE=pull \
-            ./dtox.sh -e ${{ matrix.tox-env }} -- ${{ env._PEX_TEST_POS_ARGS }}
+            ./dtox.sh -e ${{ matrix.tox-env }} -- \
+              ${{ env._PEX_TEST_POS_ARGS }} ${{ matrix.pex-test-pos-args }}
   mac-tests:
-    name: tox -e ${{ matrix.tox-env }}
+    name: tox -e ${{ matrix.tox-env }} ${{ matrix.pex-test-pos-args }}
     needs: org-check
     runs-on: macos-12
     strategy:
@@ -139,15 +151,12 @@ jobs:
           - python-version: [ 3, 12 ]
             tox-env: py312-pip24_2-integration
             tox-env-python: python3.11
+            pex-test-pos-args: --shard 1/2
+          - python-version: [ 3, 12 ]
+            tox-env: py312-pip24_2-integration
+            tox-env-python: python3.11
+            pex-test-pos-args: --shard 2/2
     steps:
-      - name: Calculate Pythons to Expose
-        id: calculate-pythons-to-expose
-        run: |
-          skip=""
-          if [[ "$(uname -s)" == "Linux" ]]; then
-            skip="${{ env._PEX_TEST_PYENV_VERSIONS }}"
-          fi
-          echo "skip=${skip}" >> $GITHUB_OUTPUT
       - name: Checkout Pex
         uses: actions/checkout@v4
         with:
@@ -160,8 +169,6 @@ jobs:
           python-version: "${{ join(matrix.python-version, '.') }}"
       - name: Expose Pythons
         uses: pantsbuild/actions/expose-pythons@e63d2d0e3c339bdffbe5e51e7c39550e3bc527bb
-        with:
-          skip: "${{ steps.calculate-pythons-to-expose.outputs.skip }}"
       - name: Restore Cached Pyenv Interpreters
         id: restore-pyenv-interpreters
         uses: actions/cache/restore@v4
@@ -190,7 +197,8 @@ jobs:
         with:
           path: repo/tox.ini
           python: ${{ matrix.tox-env-python }}
-          tox-env: ${{ matrix.tox-env }} -- ${{ env._PEX_TEST_POS_ARGS }}
+          tox-env: >-
+            ${{ matrix.tox-env }} -- ${{ env._PEX_TEST_POS_ARGS }} ${{ matrix.pex-test-pos-args }}
       - name: Cache Pyenv Interpreters
         uses: actions/cache/save@v4
         if: github.ref == 'refs/heads/main'

pex/bin/pex.py

@@ -53,9 +53,8 @@
 from pex.resolve.resolver_configuration import (
     LockRepositoryConfiguration,
     PexRepositoryConfiguration,
-    PreResolvedConfiguration,
+    PipConfiguration,
 )
-from pex.resolve.resolver_options import create_pip_configuration
 from pex.resolve.resolvers import Unsatisfiable, sorted_requirements
 from pex.result import Error, ResultError, catch, try_
 from pex.scie import ScieConfiguration
@@ -1014,20 +1013,11 @@ def build_pex(
                 DependencyConfiguration.from_pex_info(requirements_pex_info)
             )
 
-    if isinstance(resolver_configuration, (LockRepositoryConfiguration, PreResolvedConfiguration)):
-        pip_configuration = resolver_configuration.pip_configuration
-    elif isinstance(resolver_configuration, PexRepositoryConfiguration):
-        # TODO(John Sirois): Consider finding a way to support custom --index and --find-links in
-        #  this case. I.E.: I use a corporate index to build a PEX repository and now I want to
-        #  build a --project PEX whose pyproject.toml build-system.requires should be resolved from
-        #  that corporate index.
-        pip_configuration = try_(
-            finalize_resolve_config(
-                create_pip_configuration(options), targets=targets, context="--project building"
-            )
-        )
-    else:
-        pip_configuration = resolver_configuration
+    pip_configuration = (
+        resolver_configuration
+        if isinstance(resolver_configuration, PipConfiguration)
+        else resolver_configuration.pip_configuration
+    )
 
     project_dependencies = OrderedSet()  # type: OrderedSet[Requirement]
     with TRACER.timed(
@@ -1157,11 +1147,14 @@ def _compatible_with_current_platform(interpreter, platforms):
     return current_platforms.intersection(platforms)
 
 
-def configure_requirements_and_targets(options):
-    # type: (Namespace) -> Union[Tuple[RequirementConfiguration, InterpreterConstraints, Targets], Error]
+def configure_requirements_and_targets(
+    options,  # type: Namespace
+    pip_configuration,  # type: PipConfiguration
+):
+    # type: (...) -> Union[Tuple[RequirementConfiguration, InterpreterConstraints, Targets], Error]
 
     requirement_configuration = requirement_options.configure(options)
-    target_config = target_options.configure(options)
+    target_config = target_options.configure(options, pip_configuration=pip_configuration)
     script_metadata = ScriptMetadata()
 
     if options.executable and options.enable_script_metadata:
@@ -1260,15 +1253,22 @@ def main(args=None):
 
     try:
         with global_environment(options) as env:
-            requirement_configuration, interpreter_constraints, targets = try_(
-                configure_requirements_and_targets(options)
-            )
-
             try:
                 resolver_configuration = resolver_options.configure(options)
             except resolver_options.InvalidConfigurationError as e:
                 die(str(e))
 
+            requirement_configuration, interpreter_constraints, targets = try_(
+                configure_requirements_and_targets(
+                    options,
+                    pip_configuration=(
+                        resolver_configuration
+                        if isinstance(resolver_configuration, PipConfiguration)
+                        else resolver_configuration.pip_configuration
+                    ),
+                )
+            )
+
             resolver_configuration = try_(
                 finalize_resolve_config(resolver_configuration, targets, context="PEX building")
             )

pex/cli/commands/lock.py

@@ -549,7 +549,12 @@ def _add_export_arguments(
         cls.add_output_option(export_parser, entity="lock")
         cls._add_target_options(export_parser)
         resolver_options_parser = cls._create_resolver_options_group(export_parser)
-        resolver_options.register_network_options(resolver_options_parser)
+        resolver_options.register(
+            resolver_options_parser,
+            include_pex_repository=False,
+            include_lock=False,
+            include_pre_resolved=False,
+        )
 
     @classmethod
     def _add_export_subset_arguments(cls, export_subset_parser):
@@ -652,11 +657,12 @@ def _add_update_arguments(cls, update_parser):
         cls.add_json_options(update_parser, entity="lock", include_switch=False)
         cls._add_target_options(update_parser)
         resolver_options_parser = cls._create_resolver_options_group(update_parser)
-        resolver_options.register_repos_options(resolver_options_parser)
-        resolver_options.register_network_options(resolver_options_parser)
-        resolver_options.register_max_jobs_option(resolver_options_parser)
-        resolver_options.register_use_pip_config(resolver_options_parser)
-        resolver_options.register_pip_log(resolver_options_parser)
+        resolver_options.register(
+            resolver_options_parser,
+            include_pex_repository=False,
+            include_lock=False,
+            include_pre_resolved=False,
+        )
 
     @classmethod
     def add_update_lock_options(
@@ -792,7 +798,9 @@ def _resolve_targets(
     ):
         # type: (...) -> Union[Targets, Error]
 
-        target_config = target_configuration or target_options.configure(self.options)
+        target_config = target_configuration or target_options.configure(
+            self.options, pip_configuration=resolver_options.create_pip_configuration(self.options)
+        )
         if style is not LockStyle.UNIVERSAL:
             return target_config.resolve_targets()
 
@@ -812,7 +820,6 @@ def _resolve_targets(
             return Targets(
                 platforms=target_config.platforms,
                 complete_platforms=target_config.complete_platforms,
-                assume_manylinux=target_config.assume_manylinux,
             )
 
         try:
@@ -827,7 +834,6 @@ def _resolve_targets(
             interpreters=(interpreter,),
             platforms=target_config.platforms,
             complete_platforms=target_config.complete_platforms,
-            assume_manylinux=target_config.assume_manylinux,
         )
 
     def _gather_requirements(
@@ -860,7 +866,11 @@ def _gather_requirements(
 
     def _create(self):
         # type: () -> Result
-        target_configuration = target_options.configure(self.options)
+
+        pip_configuration = resolver_options.create_pip_configuration(self.options)
+        target_configuration = target_options.configure(
+            self.options, pip_configuration=pip_configuration
+        )
         if self.options.style == LockStyle.UNIVERSAL:
             lock_configuration = LockConfiguration(
                 style=LockStyle.UNIVERSAL,
@@ -888,7 +898,7 @@ def _create(self):
         )
         pip_configuration = try_(
             finalize_resolve_config(
-                resolver_configuration=resolver_options.create_pip_configuration(self.options),
+                resolver_configuration=pip_configuration,
                 targets=targets,
                 context="lock creation",
             )
@@ -953,19 +963,21 @@ def _export(self, requirement_configuration=RequirementConfiguration()):
             )
 
         lockfile_path, lock_file = self._load_lockfile()
-        targets = target_options.configure(self.options).resolve_targets()
+        pip_configuration = resolver_options.create_pip_configuration(self.options)
+        targets = target_options.configure(
+            self.options, pip_configuration=pip_configuration
+        ).resolve_targets()
         target = targets.require_unique_target(
             purpose="exporting a lock in the {pip!r} format".format(pip=ExportFormat.PIP)
         )
 
-        network_configuration = resolver_options.create_network_configuration(self.options)
         with TRACER.timed("Selecting locks for {target}".format(target=target)):
             subset_result = try_(
                 subset(
                     targets=targets,
                     lock=lock_file,
                     requirement_configuration=requirement_configuration,
-                    network_configuration=network_configuration,
+                    network_configuration=pip_configuration.network_configuration,
                     build_configuration=lock_file.build_configuration(),
                     transitive=lock_file.transitive,
                     include_all_matches=True,
@@ -1079,18 +1091,20 @@ def _create_lock_update_request(
     ):
         # type: (...) -> Union[LockUpdateRequest, Error]
 
-        network_configuration = resolver_options.create_network_configuration(self.options)
+        pip_configuration = resolver_options.create_pip_configuration(self.options)
         lock_updater = LockUpdater.create(
             lock_file=lock_file,
-            repos_configuration=resolver_options.create_repos_configuration(self.options),
-            network_configuration=network_configuration,
-            max_jobs=resolver_options.get_max_jobs_value(self.options),
-            use_pip_config=resolver_options.get_use_pip_config_value(self.options),
+            repos_configuration=pip_configuration.repos_configuration,
+            network_configuration=pip_configuration.network_configuration,
+            max_jobs=pip_configuration.max_jobs,
+            use_pip_config=pip_configuration.use_pip_config,
             dependency_configuration=dependency_config,
             pip_log=resolver_options.get_pip_log(self.options),
         )
 
-        target_configuration = target_options.configure(self.options)
+        target_configuration = target_options.configure(
+            self.options, pip_configuration=pip_configuration
+        )
         targets = try_(
             self._resolve_targets(
                 action="updating", style=lock_file.style, target_configuration=target_configuration
@@ -1146,7 +1160,7 @@ def _create_lock_update_request(
                     subset(
                         targets=targets,
                         lock=lock_file,
-                        network_configuration=network_configuration,
+                        network_configuration=pip_configuration.network_configuration,
                         build_configuration=lock_file.build_configuration(),
                         transitive=lock_file.transitive,
                     )
@@ -1498,7 +1512,9 @@ def _sync(self):
         pip_configuration = resolver_configuration.pip_configuration
         dependency_config = dependency_configuration.configure(self.options)
 
-        target_configuration = target_options.configure(self.options)
+        target_configuration = target_options.configure(
+            self.options, pip_configuration=pip_configuration
+        )
         if self.options.style == LockStyle.UNIVERSAL:
             lock_configuration = LockConfiguration(
                 style=LockStyle.UNIVERSAL,
@@ -1613,7 +1629,9 @@ def _sync(self):
                         else PythonInterpreter.from_env(self.options.venv_python)
                     )
                 else:
-                    targets = target_options.configure(self.options).resolve_targets()
+                    targets = target_options.configure(
+                        self.options, pip_configuration=pip_configuration
+                    ).resolve_targets()
                     interpreters = [
                         target.get_interpreter()
                         for target in targets.unique_targets()