Improve publish test script resilience

scripts/publish/test_publish.py

@@ -53,12 +53,13 @@
 import os
 import re
 import shutil
+import sys
 import time
 from argparse import ArgumentParser
 from dataclasses import dataclass
 from pathlib import Path
 from shutil import rmtree
-from subprocess import PIPE, check_call, check_output, run
+from subprocess import PIPE, check_call, run
 from time import sleep
 
 import httpx
@@ -180,12 +181,12 @@ def get_latest_version(project_name: str, client: httpx.Client) -> Version:
                 break
             except httpx.HTTPError as err:
                 error = err
-                print(f"Error getting version, sleeping for 1s: {err}")
+                print(f"Error getting version, sleeping for 1s: {err}", file=sys.stderr)
                 time.sleep(1)
             except InvalidSdistFilename as err:
                 # Sometimes there's a link that says "status page"
                 error = err
-                print(f"Invalid index page, sleeping for 1s: {err}")
+                print(f"Invalid index page, sleeping for 1s: {err}", file=sys.stderr)
                 time.sleep(1)
         else:
             raise RuntimeError(f"Failed to fetch {url}") from error
@@ -292,7 +293,7 @@ def wait_for_index(
     just `get_filenames` fails non-deterministically.
     """
     for _ in range(50):
-        output = check_output(
+        result = run(
             [
                 uv,
                 "pip",
@@ -310,16 +311,31 @@ def wait_for_index(
             ],
             text=True,
             input=f"{project_name}",
+            stdout=PIPE,
         )
-        if f"{project_name}=={version}" in output and output.count("--hash") == 2:
+        # codeberg sometimes times out
+        if result.returncode != 0:
+            print(
+                f"uv pip compile not updated, missing 2 files for {version}, "
+                + f"sleeping for 2s: `{index_url}`:\n",
+                file=sys.stderr,
+            )
+            sleep(2)
+            continue
+
+        if (
+            f"{project_name}=={version}" in result.stdout
+            and result.stdout.count("--hash") == 2
+        ):
             break
 
         print(
             f"uv pip compile not updated, missing 2 files for {version}, "
             + f"sleeping for 2s: `{index_url}`:\n"
             + "```\n"
-            + output.replace("\\\n    ", "")
-            + "```"
+            + result.stdout.replace("\\\n    ", "")
+            + "```",
+            file=sys.stderr,
         )
         sleep(2)
 
@@ -333,33 +349,58 @@ def publish_project(target: str, uv: Path, client: httpx.Client):
     """
     project_name = all_targets[target].project_name
 
-    print(f"\nPublish {project_name} for {target}")
+    # If a version was recently uploaded by another run of this script,
+    # `get_latest_version` may get a cached version and uploading fails. In this case
+    # we wait and try again.
+    retries = 3
+    while True:
+        print(f"\nPublish {project_name} for {target}", file=sys.stderr)
+
+        # The distributions are build to the dist directory of the project.
+        previous_version = get_latest_version(project_name, client)
+        version = get_new_version(previous_version)
+        project_dir = build_project_at_version(target, version, uv)
+
+        # Upload configuration
+        publish_url = all_targets[target].publish_url
+        index_url = all_targets[target].index_url
+        env, extra_args = target_configuration(target)
+        env = {**os.environ, **env}
+        expected_filenames = [
+            path.name for path in project_dir.joinpath("dist").iterdir()
+        ]
+        # Ignore the gitignore file in dist
+        expected_filenames.remove(".gitignore")
+        # Ignore our test file
+        expected_filenames.remove(".DS_Store")
 
-    # The distributions are build to the dist directory of the project.
-    previous_version = get_latest_version(project_name, client)
-    version = get_new_version(previous_version)
-    project_dir = build_project_at_version(target, version, uv)
+        print(
+            f"\n=== 1. Publishing a new version: {project_name} {version} {publish_url} ===",
+            file=sys.stderr,
+        )
 
-    # Upload configuration
-    publish_url = all_targets[target].publish_url
-    index_url = all_targets[target].index_url
-    env, extra_args = target_configuration(target)
-    env = {**os.environ, **env}
-    expected_filenames = [path.name for path in project_dir.joinpath("dist").iterdir()]
-    # Ignore the gitignore file in dist
-    expected_filenames.remove(".gitignore")
-    # Ignore our test file
-    expected_filenames.remove(".DS_Store")
+        args = [uv, "publish", "--publish-url", publish_url, *extra_args]
+        result = run(args, cwd=project_dir, env=env, stderr=PIPE)
+        if result.returncode == 0:
+            # Successful upload
+            break
 
-    print(
-        f"\n=== 1. Publishing a new version: {project_name} {version} {publish_url} ==="
-    )
-    args = [uv, "publish", "--publish-url", publish_url, *extra_args]
-    check_call(args, cwd=project_dir, env=env)
+        retries -= 1
+        if retries > 0:
+            print(
+                f"Publish failed, retrying after 10s: {result.stderr}", file=sys.stderr
+            )
+            sleep(10)
+        else:
+            # Raise the error after three failures
+            result.check_returncode()
 
     if publish_url == TEST_PYPI_PUBLISH_URL:
         # Confirm pypi behaviour: Uploading the same file again is fine.
-        print(f"\n=== 2. Publishing {project_name} {version} again (PyPI) ===")
+        print(
+            f"\n=== 2. Publishing {project_name} {version} again (PyPI) ===",
+            file=sys.stderr,
+        )
         wait_for_index(index_url, project_name, version, uv)
         args = [uv, "publish", "--publish-url", publish_url, *extra_args]
         output = run(
@@ -377,7 +418,10 @@ def publish_project(target: str, uv: Path, client: httpx.Client):
             )
 
     mode = "index" if all_targets[target].index else "check URL"
-    print(f"\n=== 3. Publishing {project_name} {version} again with {mode} ===")
+    print(
+        f"\n=== 3. Publishing {project_name} {version} again with {mode} ===",
+        file=sys.stderr,
+    )
     wait_for_index(index_url, project_name, version, uv)
     # Test twine-style and index-style uploads for different packages.
     if index := all_targets[target].index:
@@ -418,7 +462,8 @@ def publish_project(target: str, uv: Path, client: httpx.Client):
 
     print(
         f"\n=== 4. Publishing modified {project_name} {version} "
-        f"again with skip existing (error test) ==="
+        f"again with skip existing (error test) ===",
+        file=sys.stderr,
     )
     wait_for_index(index_url, project_name, version, uv)
     args = [