ci: retry transient compliance export failures

.github/actions/compliance-extract/action.yml

@@ -130,7 +130,6 @@ runs:
         GIT_SHA: ${{ inputs.git_sha }}
         EPP_IMAGE: ${{ inputs.epp_image }}
       run: |
-        mkdir -p /tmp/compliance
         # Reuse the warm cache the build just populated by matching ALL the
         # cache-key-affecting build-args the build passed. Otherwise the shared
         # stages rebuild here instead of cache-hitting:
@@ -174,7 +173,8 @@ runs:
         # --build-arg (it would clear the ARG and reintroduce the failure).
         EPP_IMAGE_ARG=""
         [ -n "${EPP_IMAGE:-}" ] && EPP_IMAGE_ARG="--build-arg EPP_IMAGE=${EPP_IMAGE}"
-        docker buildx build \
+        bash "${{ github.action_path }}/extract-with-retry.sh" /tmp/compliance \
+            docker buildx build \
             --builder ${{ inputs.builder_name }} \
             --platform ${{ inputs.platform }} \
             --target compliance_artifact \
@@ -183,7 +183,6 @@ runs:
             --build-arg SCCACHE_BUCKET="${SCCACHE_BUCKET}" \
             --build-arg SCCACHE_REGION="${SCCACHE_REGION}" \
             $EPP_IMAGE_ARG \
-            --output type=local,dest=/tmp/compliance \
             --progress=plain \
             -f "${DOCKERFILE_PATH}" \
             $SECRET_ARGS \
@@ -357,7 +356,6 @@ runs:
         GIT_SHA: ${{ inputs.git_sha }}
         EPP_IMAGE: ${{ inputs.epp_image }}
       run: |
-        mkdir -p /tmp/sources
         # ENABLE_SOURCE_ARCHIVAL=true to actually run sources_collect; the other
         # build-args match the build so this shares the warm cache (same reason
         # as the compliance_artifact extract above). Same IRSA secrets too, so a
@@ -385,7 +383,8 @@ runs:
         # resolves the GC'd default staging tag. Empty for non-frontend targets.
         EPP_IMAGE_ARG=""
         [ -n "${EPP_IMAGE:-}" ] && EPP_IMAGE_ARG="--build-arg EPP_IMAGE=${EPP_IMAGE}"
-        docker buildx build \
+        bash "${{ github.action_path }}/extract-with-retry.sh" /tmp/sources \
+            docker buildx build \
             --builder ${{ inputs.builder_name }} \
             --platform ${{ inputs.platform }} \
             --target sources_archive \
@@ -395,7 +394,6 @@ runs:
             --build-arg SCCACHE_REGION="${SCCACHE_REGION}" \
             --build-arg ENABLE_SOURCE_ARCHIVAL=true \
             $EPP_IMAGE_ARG \
-            --output type=local,dest=/tmp/sources \
             --progress=plain \
             -f "${DOCKERFILE_PATH}" \
             $SECRET_ARGS \

.github/actions/compliance-extract/extract-with-retry.sh

@@ -0,0 +1,41 @@
+#!/usr/bin/env bash
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+# Retry only transient BuildKit transport failures, preserving each attempt's log.
+set -euo pipefail
+
+destination=$1
+shift
+attempt_root=$(mktemp -d "${RUNNER_TEMP:-${TMPDIR:-/tmp}}/compliance-extract.XXXXXX")
+echo "Compliance extraction logs: ${attempt_root}"
+
+for attempt in 1 2 3; do
+    output="${attempt_root}/output-${attempt}"
+    log="${attempt_root}/attempt-${attempt}.log"
+    mkdir -p "$output"
+    status=0
+    "$@" --output "type=local,dest=${output}" 2>&1 | tee "$log" || status=$?
+    if [ "$status" -eq 0 ]; then
+        # Publish only a complete export; failed attempts never reach consumers.
+        rm -rf -- "$destination"
+        mv -- "$output" "$destination"
+        exit 0
+    fi
+    rm -rf -- "$output"
+
+    # Inspect the terminal build error, not earlier warnings or command output.
+    error=$(grep '^ERROR:' "$log" | tail -n 1 || true)
+    if printf '%s\n' "$error" | grep -Eiq \
+        '(process .*did not complete successfully|401 Unauthorized|403 Forbidden|denied:|manifest unknown)'; then
+        exit "$status"
+    fi
+    if [ "$attempt" -eq 3 ] || ! printf '%s\n' "$error" | grep -Eiq \
+        '(Unavailable:.*error reading from server: (unexpected )?EOF|connection reset by peer|transport is closing|unexpected status.*(502 Bad Gateway|503 Service Unavailable|504 Gateway Timeout))'; then
+        exit "$status"
+    fi
+    delay=10
+    [ "$attempt" -eq 2 ] && delay=30
+    echo "::warning::Compliance extraction attempt ${attempt}/3 failed with a transient transport error; retrying in ${delay}s"
+    sleep "$delay"
+done

.github/actions/compliance-extract/test_extract_with_retry.py

@@ -0,0 +1,118 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+"""Exercise the real retry wrapper with a fake BuildKit command and sleep."""
+
+import os
+import subprocess
+import tempfile
+import unittest
+from pathlib import Path
+
+SCRIPT = Path(__file__).with_name("extract-with-retry.sh")
+TRANSIENT = "ERROR: failed to build: failed to solve: Unavailable: error reading from server: EOF"
+
+
+class ExtractRetryTest(unittest.TestCase):
+    def run_extract(self, errors):
+        with tempfile.TemporaryDirectory() as temp:
+            root = Path(temp)
+            for index, error in enumerate(errors, 1):
+                (root / f"error-{index}").write_text(error)
+            fake = root / "build"
+            fake.write_text(
+                """#!/usr/bin/env bash
+set -eu
+count=0
+[ ! -f "$STATE/count" ] || count=$(cat "$STATE/count")
+count=$((count + 1))
+echo "$count" > "$STATE/count"
+for arg in "$@"; do
+    case "$arg" in type=local,dest=*) output=${arg#type=local,dest=} ;; esac
+done
+mkdir -p "$output"
+echo "$count" > "$output/attempt-$count"
+error="$STATE/error-$count"
+if [ -s "$error" ]; then
+    cat "$error"
+    exit 7
+fi
+"""
+            )
+            fake.chmod(0o755)
+            sleep = root / "sleep"
+            sleep.write_text('#!/usr/bin/env bash\necho "$1" >> "$STATE/delays"\n')
+            sleep.chmod(0o755)
+            destination = root / "result"
+            env = dict(
+                os.environ,
+                STATE=temp,
+                RUNNER_TEMP=temp,
+                PATH=temp + os.pathsep + os.environ["PATH"],
+            )
+            result = subprocess.run(
+                ["bash", str(SCRIPT), str(destination), str(fake)],
+                env=env,
+                capture_output=True,
+                text=True,
+            )
+            count = int((root / "count").read_text())
+            delays = (
+                (root / "delays").read_text().splitlines()
+                if (root / "delays").exists()
+                else []
+            )
+            files = sorted(p.name for p in destination.glob("*"))
+            logs = list(root.glob("compliance-extract.*/attempt-*.log"))
+            self.assertEqual(len(logs), count)
+            return result.returncode, count, delays, files
+
+    def test_success(self):
+        self.assertEqual(self.run_extract([""]), (0, 1, [], ["attempt-1"]))
+
+    def test_transient_recovers_without_partial_files(self):
+        self.assertEqual(
+            self.run_extract([TRANSIENT, ""]), (0, 2, ["10"], ["attempt-2"])
+        )
+
+    def test_exhaustion_preserves_failure(self):
+        self.assertEqual(self.run_extract([TRANSIENT] * 3), (7, 3, ["10", "30"], []))
+
+    def test_deterministic_errors_do_not_retry(self):
+        for error in [
+            "ERROR: failed to solve: process did not complete: exit code: 1",
+            "ERROR: failed to authorize: 401 Unauthorized",
+            "ERROR: manifest unknown",
+            "No solution found when resolving dependencies",
+        ]:
+            with self.subTest(error=error):
+                self.assertEqual(self.run_extract([error]), (7, 1, [], []))
+
+    def test_earlier_transient_does_not_mask_terminal_failure(self):
+        self.assertEqual(
+            self.run_extract([TRANSIENT + "\nERROR: manifest unknown"]), (7, 1, [], [])
+        )
+
+    def test_other_transport_errors_recover(self):
+        for error in [
+            "ERROR: failed to solve: connection reset by peer",
+            "ERROR: failed to solve: rpc error: transport is closing",
+            "ERROR: failed to solve: unexpected status from HEAD request: 503 Service Unavailable",
+        ]:
+            with self.subTest(error=error):
+                self.assertEqual(
+                    self.run_extract([error, ""]), (0, 2, ["10"], ["attempt-2"])
+                )
+
+    def test_build_command_containing_transport_text_does_not_retry(self):
+        error = 'ERROR: process "echo connection reset by peer" did not complete successfully: exit code: 1'
+        self.assertEqual(self.run_extract([error]), (7, 1, [], []))
+
+    def test_permanent_error_after_retry_stops(self):
+        self.assertEqual(
+            self.run_extract([TRANSIENT, "ERROR: manifest unknown"]), (7, 2, ["10"], [])
+        )
+
+
+if __name__ == "__main__":
+    unittest.main()

.pre-commit-config.yaml

@@ -198,6 +198,12 @@ repos:
       # be a second copy to keep in sync. Full scan is well under 0.1s.
       files: ^\.github/
       pass_filenames: false
+    - id: compliance-extract-selftest
+      name: Test compliance export retries
+      entry: python3 .github/actions/compliance-extract/test_extract_with_retry.py
+      language: python
+      pass_filenames: false
+      files: ^(\.github/actions/compliance-extract/|\.pre-commit-config\.yaml$)
     - id: check-action-pins-selftest
       name: Self-test the action-pin matcher
       entry: python3 scripts/check_action_pins.py --test