Prepare the 2.20.0 release. (#2539)

Also turn on CI IT sharding for a ~40% wall time reduction, fixing up
some existing test issues this smoked out.

.github/workflows/ci.yml

@@ -6,10 +6,6 @@ defaults:
 env:
   _PEX_TEST_DEV_ROOT: ${{ github.workspace }}/.pex_dev
   _PEX_TEST_POS_ARGS: "--color --devpi --devpi-timeout 15.0 --shutdown-devpi -vvs"
-  # We use this to skip exposing same-versioned Pythons present on Linux hosts. These otherwise can
-  # collide when attempting to load libpython<major>.<minor><flags>.so and lead to mysterious errors
-  # importing builtins like `fcntl` as outlined in https://github.com/pex-tool/pex/issues/1391.
-  _PEX_TEST_PYENV_VERSIONS: "2.7 3.7 3.10"
   _PEX_PEXPECT_TIMEOUT: 10
   # We have integration tests that exercise `--scie` support and these can trigger downloads from
   # GitHub Releases that needed elevated rate limit quota, which this gives.
@@ -64,36 +60,51 @@ jobs:
   # N.B.: The name of this job key (linux-tests) is depended on by scrips/build_cache_image.py. In
   # particular, the tox-env matrix list is used to ensure the cache covers all Linux CI jobs.
   linux-tests:
-    name: ./dtox.sh -e ${{ matrix.tox-env }}
+    name: ./dtox.sh -e ${{ matrix.tox-env }} ${{ matrix.pex-test-pos-args }}
     needs: org-check
     runs-on: ubuntu-22.04
     strategy:
       matrix:
-        tox-env:
-          - py27-pip20
-          # N.B.: Although we support Python 3.5 with a published universal wheel, Pex does not
-          # build under Python 3.5 or 3.6; so we skip to Python 3.7 for tests which must be able to
-          # build a Pex distribution both as part of the tox setup and in many test cases as well.
-          - py37-pip20
-          - py311-pip20
-          - py311-pip22_3_1
-          - py311-pip23_1_2
-          - py312-pip24_2
-          - py313-pip24_2
-          - pypy310-pip20
-          - pypy310-pip22_3_1
-          - pypy310-pip23_1_2
-          - py27-pip20-integration
-          - py37-pip22_3_1-integration
-          - py37-pip23_1_2-integration
-          - py311-pip20-integration
-          - py311-pip22_3_1-integration
-          - py311-pip23_1_2-integration
-          - py312-pip24_2-integration
-          - py313-pip24_2-integration
-          - pypy310-pip20-integration
-          - pypy310-pip22_3_1-integration
-          - pypy310-pip23_1_2-integration
+        include:
+          # Unit tests:
+          # -----------
+          - tox-env: py27-pip20
+          - tox-env: py37-pip22_3_1
+          - tox-env: py311-pip23_3_2
+          - tox-env: py312-pip24_2
+          - tox-env: py313-pip24_2
+          - tox-env: pypy310-pip24_2
+
+          # Integration tests, split most into two shards:
+          # ----------------------------------------------
+
+          # CPython 2.7 is fast enough not to require sharding.
+          - tox-env: py27-pip20-integration
+
+          - tox-env: py37-pip22_3_1-integration
+            pex-test-pos-args: --shard 1/2
+          - tox-env: py37-pip22_3_1-integration
+            pex-test-pos-args: --shard 2/2
+
+          - tox-env: py311-pip23_3_2-integration
+            pex-test-pos-args: --shard 1/2
+          - tox-env: py311-pip23_3_2-integration
+            pex-test-pos-args: --shard 2/2
+
+          - tox-env: py312-pip24_2-integration
+            pex-test-pos-args: --shard 1/2
+          - tox-env: py312-pip24_2-integration
+            pex-test-pos-args: --shard 2/2
+
+          - tox-env: py313-pip24_2-integration
+            pex-test-pos-args: --shard 1/2
+          - tox-env: py313-pip24_2-integration
+            pex-test-pos-args: --shard 2/2
+
+          - tox-env: pypy310-pip24_2-integration
+            pex-test-pos-args: --shard 1/2
+          - tox-env: pypy310-pip24_2-integration
+            pex-test-pos-args: --shard 2/2
     steps:
       - name: Free Up Disk Space
         uses: jlumbroso/[email protected]
@@ -125,9 +136,10 @@ jobs:
           # This is needed to get pexpect tests working under PyPy running under docker.
           export TERM="xterm"
           BASE_MODE=pull CACHE_MODE=pull \
-            ./dtox.sh -e ${{ matrix.tox-env }} -- ${{ env._PEX_TEST_POS_ARGS }}
+            ./dtox.sh -e ${{ matrix.tox-env }} -- \
+              ${{ env._PEX_TEST_POS_ARGS }} ${{ matrix.pex-test-pos-args }}
   mac-tests:
-    name: tox -e ${{ matrix.tox-env }}
+    name: tox -e ${{ matrix.tox-env }} ${{ matrix.pex-test-pos-args }}
     needs: org-check
     runs-on: macos-12
     strategy:
@@ -139,15 +151,12 @@ jobs:
           - python-version: [ 3, 12 ]
             tox-env: py312-pip24_2-integration
             tox-env-python: python3.11
+            pex-test-pos-args: --shard 1/2
+          - python-version: [ 3, 12 ]
+            tox-env: py312-pip24_2-integration
+            tox-env-python: python3.11
+            pex-test-pos-args: --shard 2/2
     steps:
-      - name: Calculate Pythons to Expose
-        id: calculate-pythons-to-expose
-        run: |
-          skip=""
-          if [[ "$(uname -s)" == "Linux" ]]; then
-            skip="${{ env._PEX_TEST_PYENV_VERSIONS }}"
-          fi
-          echo "skip=${skip}" >> $GITHUB_OUTPUT
       - name: Checkout Pex
         uses: actions/checkout@v4
         with:
@@ -160,8 +169,6 @@ jobs:
           python-version: "${{ join(matrix.python-version, '.') }}"
       - name: Expose Pythons
         uses: pantsbuild/actions/expose-pythons@e63d2d0e3c339bdffbe5e51e7c39550e3bc527bb
-        with:
-          skip: "${{ steps.calculate-pythons-to-expose.outputs.skip }}"
       - name: Restore Cached Pyenv Interpreters
         id: restore-pyenv-interpreters
         uses: actions/cache/restore@v4
@@ -190,7 +197,8 @@ jobs:
         with:
           path: repo/tox.ini
           python: ${{ matrix.tox-env-python }}
-          tox-env: ${{ matrix.tox-env }} -- ${{ env._PEX_TEST_POS_ARGS }}
+          tox-env: >-
+            ${{ matrix.tox-env }} -- ${{ env._PEX_TEST_POS_ARGS }} ${{ matrix.pex-test-pos-args }}
       - name: Cache Pyenv Interpreters
         uses: actions/cache/save@v4
         if: github.ref == 'refs/heads/main'

CHANGES.md

@@ -1,5 +1,28 @@
 # Release Notes
 
+## 2.20.0
+
+This release adds the `--pip-log` alias for the existing
+`--preserve-pip-download-log` option as well as the ability to specify
+the log file path. So, to debug a resolve, you can now specify
+`--pip-log log.txt` and Pex will deposit the Pip resolve log to
+`log.txt` in the current directory for easy tailing or post-resolve
+inspection. In addition, the log file itself is more useful in some
+cases. When you specify any abbreviated `--platform` targets, those
+targets calculated wheel compatibility tags are included in the Pip
+log. Also, when multiple targets are specified, their log outputs are
+now merged at the end of the resolve in a serialized fashion with
+prefixes on each log line indicating which target the log line
+corresponds to.
+
+In addition, a race in Pex's PEP-517 implementation that could (rarely)
+lead to spurious metadata generation errors or sdist creation errors is
+fixed.
+
+* Fix intermittent PEP-517 failures. (#2540)
+* Plumb `--pip-version` to Platform tag calculation. (#2538)
+* Add the ability to specify the `--pip-log` path. (#2536)
+
 ## 2.19.1
 
 This release fixes a regression introduced by #2512 in the 2.19.0
@@ -8,7 +31,7 @@ Instead of failing certain builds that used to succeed, Pex now warns
 that the resulting PEX may fail at runtime and that
 `--complete-platform` should be used instead.
 
-# Only warn when `--platform` resolves fail tag checks. (#2533)
+* Only warn when `--platform` resolves fail tag checks. (#2533)
 
 ## 2.19.0
 

pex/version.py

@@ -1,4 +1,4 @@
 # Copyright 2015 Pex project contributors.
 # Licensed under the Apache License, Version 2.0 (see LICENSE).
 
-__version__ = "2.19.1"
+__version__ = "2.20.0"

tests/integration/resolve/test_issue_1907.py

@@ -18,7 +18,7 @@
 from pex.pex import PEX
 from pex.pip.version import PipVersion
 from pex.typing import TYPE_CHECKING
-from testing import PY_VER, data, run_pex_command
+from testing import IS_PYPY, PY_VER, data, run_pex_command
 from testing.cli import run_pex3
 
 if TYPE_CHECKING:
@@ -168,11 +168,15 @@ def test_pre_resolved_dists_offline(
     offline = os.path.join(str(tmpdir), "offline")
     os.mkdir(offline)
 
-    # In order to go offline and still be able to build sdists, we need both the un-vendored Pip and
-    # its basic build requirements.
-    if PipVersion.DEFAULT is not PipVersion.VENDORED:
+    if IS_PYPY or PipVersion.DEFAULT is not PipVersion.VENDORED:
         args = [sys.executable, "-m", "pip", "wheel", "-w", offline]
-        args.extend(str(req) for req in PipVersion.DEFAULT.requirements)
+        if PipVersion.DEFAULT is not PipVersion.VENDORED:
+            # In order to go offline and still be able to build sdists, we need both the un-vendored Pip and
+            # its basic build requirements.
+            args.extend(str(req) for req in PipVersion.DEFAULT.requirements)
+        if IS_PYPY:
+            # For PyPy, we need extra build dependencies for argon2-cffi-bindings.
+            args.append("setuptools_scm")
         subprocess.check_call(args)
 
     for dist in glob.glob(os.path.join(dists, "*")):

tests/integration/test_integration.py

@@ -1979,10 +1979,15 @@ def test_require_hashes(tmpdir):
     run_pex_command(args=["-r", requirements, "-o", requests_pex]).assert_success()
     subprocess.check_call(args=[requests_pex, "-c", "import requests"])
 
-    # The hash checking mode should also work in constraints context.
-    run_pex_command(
-        args=["--constraints", requirements, "requests", "-o", requests_pex]
-    ).assert_success()
+    result = run_pex_command(args=["--constraints", requirements, "requests", "-o", requests_pex])
+    # The hash checking mode should also work in constraints context for Pip prior to 23.2 when
+    # Pip got more strict about the contents of constraints files (just specifiers and markers; no
+    # extras, hashes, etc.).
+    if PipVersion.DEFAULT < PipVersion.v23_2:
+        result.assert_success()
+    else:
+        result.assert_failure()
+
     subprocess.check_call(args=[requests_pex, "-c", "import requests"])
 
     with open(requirements, "w") as fp:
@@ -2007,13 +2012,9 @@ def test_require_hashes(tmpdir):
     as_requirements_result = run_pex_command(args=["-r", requirements])
     as_requirements_result.assert_failure()
 
-    # The hash checking mode should also work in constraints context.
-    as_constraints_result = run_pex_command(args=["--constraints", requirements, "requests"])
-    as_constraints_result.assert_failure()
-
     error_lines = {
         re.sub(r"\s+", " ", line.strip()): index
-        for index, line in enumerate(as_constraints_result.error.splitlines())
+        for index, line in enumerate(as_requirements_result.error.splitlines())
     }
     index = error_lines["pip: Expected sha512 worse"]
     assert (

tests/integration/test_issue_1179.py

@@ -6,16 +6,18 @@
 
 import pytest
 
+from pex.pip.version import PipVersion
 from pex.targets import LocalInterpreter
 from testing import run_pex_command
 
 
 @pytest.mark.skipif(
-    sys.version_info[:2] >= (3, 12),
+    sys.version_info[:2] >= (3, 12) or PipVersion.DEFAULT >= PipVersion.v23_2,
     reason=(
         "There is an indirect urllib3 dependency which embeds six which uses a meta path importer "
         "that only implements the PEP-302 finder spec and not the modern spec. Only the modern "
-        "finder spec is supported by Python 3.12+."
+        "finder spec is supported by Python 3.12+. Also, Pip 23.2 dropped support for the legacy "
+        "resolver, which this test needs."
     ),
 )
 def test_pip_2020_resolver_engaged():

tests/integration/test_issue_940.py

@@ -7,6 +7,7 @@
 
 import pytest
 
+from pex.pip.version import PipVersion
 from pex.typing import TYPE_CHECKING
 from testing import built_wheel, run_pex_command, run_simple_pex
 
@@ -15,8 +16,12 @@
 
 
 @pytest.mark.skipif(
-    sys.version_info[:2] >= (3, 12),
-    reason="We need to use setuptools<66 but Python 3.12+ require greater.",
+    sys.version_info[:2] >= (3, 12) or PipVersion.DEFAULT >= PipVersion.v24_1,
+    reason=(
+        "We need to use setuptools<66 but Python 3.12+ require greater. We also need to avoid "
+        "Pip>=24.1 which upgrades its vendored packaging to a version that rejects invalid "
+        "versions"
+    ),
 )
 def test_resolve_arbitrary_equality(tmpdir):
     # type: (Any) -> None

tests/test_resolver.py

@@ -17,6 +17,7 @@
 from pex.common import safe_copy, safe_mkdtemp, temporary_dir
 from pex.dist_metadata import Distribution, Requirement
 from pex.interpreter import PythonInterpreter
+from pex.pip.version import PipVersion
 from pex.resolve import abbreviated_platforms
 from pex.resolve.configured_resolver import ConfiguredResolver
 from pex.resolve.resolver_configuration import BuildConfiguration, ResolverVersion
@@ -536,8 +537,12 @@ def assert_dist(
 
 
 @pytest.mark.skipif(
-    sys.version_info[:2] >= (3, 12),
-    reason="We need to use setuptools<66 ut Python 3.12+ require greater.",
+    sys.version_info[:2] >= (3, 12) or PipVersion.DEFAULT >= PipVersion.v24_1,
+    reason=(
+        "We need to use setuptools<66, but Python 3.12+ require greater. We also need to avoid "
+        "Pip>=24.1 which upgrades its vendored packaging to a version that rejects invalid "
+        "versions"
+    ),
 )
 def test_resolve_arbitrary_equality_issues_940():
     # type: (...) -> None