From d5896ac8c0d260dd13e79be07e902427e6283af8 Mon Sep 17 00:00:00 2001 From: xianlubird Date: Fri, 11 Sep 2026 11:11:31 +0800 Subject: [PATCH 1/4] ci: retry transient compliance export failures Signed-off-by: xianlubird --- .github/actions/compliance-extract/action.yml | 5 +- .../compliance-extract/extract-with-retry.sh | 41 ++++++ .../test_extract_with_retry.py | 124 ++++++++++++++++++ 3 files changed, 167 insertions(+), 3 deletions(-) create mode 100755 .github/actions/compliance-extract/extract-with-retry.sh create mode 100644 .github/actions/compliance-extract/test_extract_with_retry.py diff --git a/.github/actions/compliance-extract/action.yml b/.github/actions/compliance-extract/action.yml index b470a2fa6935..bdb9d28370ea 100644 --- a/.github/actions/compliance-extract/action.yml +++ b/.github/actions/compliance-extract/action.yml @@ -130,7 +130,6 @@ runs: GIT_SHA: ${{ inputs.git_sha }} EPP_IMAGE: ${{ inputs.epp_image }} run: | - mkdir -p /tmp/compliance # Reuse the warm cache the build just populated by matching ALL the # cache-key-affecting build-args the build passed. Otherwise the shared # stages rebuild here instead of cache-hitting: @@ -174,7 +173,8 @@ runs: # --build-arg (it would clear the ARG and reintroduce the failure). EPP_IMAGE_ARG="" [ -n "${EPP_IMAGE:-}" ] && EPP_IMAGE_ARG="--build-arg EPP_IMAGE=${EPP_IMAGE}" - docker buildx build \ + bash "${{ github.action_path }}/extract-with-retry.sh" /tmp/compliance \ + docker buildx build \ --builder ${{ inputs.builder_name }} \ --platform ${{ inputs.platform }} \ --target compliance_artifact \ @@ -183,7 +183,6 @@ runs: --build-arg SCCACHE_BUCKET="${SCCACHE_BUCKET}" \ --build-arg SCCACHE_REGION="${SCCACHE_REGION}" \ $EPP_IMAGE_ARG \ - --output type=local,dest=/tmp/compliance \ --progress=plain \ -f "${DOCKERFILE_PATH}" \ $SECRET_ARGS \ diff --git a/.github/actions/compliance-extract/extract-with-retry.sh b/.github/actions/compliance-extract/extract-with-retry.sh new file mode 100755 index 000000000000..4feed8b62576 --- /dev/null +++ b/.github/actions/compliance-extract/extract-with-retry.sh @@ -0,0 +1,41 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Retry only transient BuildKit transport failures, preserving each attempt's log. +set -euo pipefail + +destination=$1 +shift +attempt_root=$(mktemp -d "${RUNNER_TEMP:-${TMPDIR:-/tmp}}/compliance-extract.XXXXXX") +echo "Compliance extraction logs: ${attempt_root}" + +for attempt in 1 2 3; do + output="${attempt_root}/output-${attempt}" + log="${attempt_root}/attempt-${attempt}.log" + mkdir -p "$output" + status=0 + "$@" --output "type=local,dest=${output}" 2>&1 | tee "$log" || status=$? + if [ "$status" -eq 0 ]; then + # Publish only a complete export; failed attempts never reach consumers. + rm -rf -- "$destination" + mv -- "$output" "$destination" + exit 0 + fi + rm -rf -- "$output" + + # Inspect the terminal build error, not earlier warnings or command output. + error=$(grep '^ERROR:' "$log" | tail -n 1 || true) + if printf '%s\n' "$error" | grep -Eiq \ + '(process .*did not complete successfully|401 Unauthorized|403 Forbidden|denied:|manifest unknown)'; then + exit "$status" + fi + if [ "$attempt" -eq 3 ] || ! printf '%s\n' "$error" | grep -Eiq \ + '(Unavailable:.*error reading from server: (unexpected )?EOF|connection reset by peer|transport is closing|unexpected status.*(502 Bad Gateway|503 Service Unavailable|504 Gateway Timeout))'; then + exit "$status" + fi + delay=10 + [ "$attempt" -eq 2 ] && delay=30 + echo "::warning::Compliance extraction attempt ${attempt}/3 failed with a transient transport error; retrying in ${delay}s" + sleep "$delay" +done diff --git a/.github/actions/compliance-extract/test_extract_with_retry.py b/.github/actions/compliance-extract/test_extract_with_retry.py new file mode 100644 index 000000000000..15ab536fed28 --- /dev/null +++ b/.github/actions/compliance-extract/test_extract_with_retry.py @@ -0,0 +1,124 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Exercise the real retry wrapper with a fake BuildKit command and sleep.""" + +import os +import subprocess +import tempfile +import unittest +from pathlib import Path + +SCRIPT = Path(__file__).with_name("extract-with-retry.sh") +TRANSIENT = "ERROR: failed to build: failed to solve: Unavailable: error reading from server: EOF" + + +class ExtractRetryTest(unittest.TestCase): + def run_extract(self, errors): + with tempfile.TemporaryDirectory() as temp: + root = Path(temp) + for index, error in enumerate(errors, 1): + (root / f"error-{index}").write_text(error) + fake = root / "build" + fake.write_text( + """#!/usr/bin/env bash +set -eu +count=0 +[ ! -f "$STATE/count" ] || count=$(cat "$STATE/count") +count=$((count + 1)) +echo "$count" > "$STATE/count" +for arg in "$@"; do + case "$arg" in type=local,dest=*) output=${arg#type=local,dest=} ;; esac +done +mkdir -p "$output" +echo "$count" > "$output/attempt-$count" +error="$STATE/error-$count" +if [ -s "$error" ]; then + cat "$error" + exit 7 +fi +""" + ) + fake.chmod(0o755) + sleep = root / "sleep" + sleep.write_text('#!/usr/bin/env bash\necho "$1" >> "$STATE/delays"\n') + sleep.chmod(0o755) + destination = root / "result" + env = dict( + os.environ, + STATE=temp, + RUNNER_TEMP=temp, + PATH=temp + os.pathsep + os.environ["PATH"], + ) + result = subprocess.run( + ["bash", str(SCRIPT), str(destination), str(fake)], + env=env, + capture_output=True, + text=True, + ) + count = int((root / "count").read_text()) + delays = ( + (root / "delays").read_text().splitlines() + if (root / "delays").exists() + else [] + ) + files = sorted(p.name for p in destination.glob("*")) + logs = list(root.glob("compliance-extract.*/attempt-*.log")) + self.assertEqual(len(logs), count) + return result.returncode, count, delays, files + + def test_success(self): + self.assertEqual(self.run_extract([""]), (0, 1, [], ["attempt-1"])) + + def test_transient_recovers_without_partial_files(self): + self.assertEqual( + self.run_extract([TRANSIENT, ""]), (0, 2, ["10"], ["attempt-2"]) + ) + + def test_third_attempt_recovers(self): + self.assertEqual( + self.run_extract([TRANSIENT, TRANSIENT, ""]), + (0, 3, ["10", "30"], ["attempt-3"]), + ) + + def test_exhaustion_preserves_failure(self): + self.assertEqual(self.run_extract([TRANSIENT] * 3), (7, 3, ["10", "30"], [])) + + def test_deterministic_errors_do_not_retry(self): + for error in [ + "ERROR: failed to solve: process did not complete: exit code: 1", + "ERROR: failed to authorize: 401 Unauthorized", + "ERROR: manifest unknown", + "No solution found when resolving dependencies", + ]: + with self.subTest(error=error): + self.assertEqual(self.run_extract([error]), (7, 1, [], [])) + + def test_earlier_transient_does_not_mask_terminal_failure(self): + self.assertEqual( + self.run_extract([TRANSIENT + "\nERROR: manifest unknown"]), (7, 1, [], []) + ) + + def test_other_transport_errors_recover(self): + for error in [ + "ERROR: failed to solve: connection reset by peer", + "ERROR: failed to solve: rpc error: transport is closing", + "ERROR: failed to solve: unexpected status from HEAD request: 503 Service Unavailable", + ]: + with self.subTest(error=error): + self.assertEqual( + self.run_extract([error, ""]), (0, 2, ["10"], ["attempt-2"]) + ) + + def test_build_command_containing_transport_text_does_not_retry(self): + error = 'ERROR: process "echo connection reset by peer" did not complete successfully: exit code: 1' + self.assertEqual(self.run_extract([error]), (7, 1, [], [])) + + def test_permanent_error_after_retry_stops(self): + self.assertEqual( + self.run_extract([TRANSIENT, "ERROR: manifest unknown"]), (7, 2, ["10"], []) + ) + + +if __name__ == "__main__": + unittest.main() From 78a18d5437622df080dc55440eef80e373e3d342 Mon Sep 17 00:00:00 2001 From: xianlubird Date: Fri, 11 Sep 2026 11:36:42 +0800 Subject: [PATCH 2/4] ci: run compliance retry regression tests in pre-commit Signed-off-by: xianlubird --- .../actions/compliance-extract/test_extract_with_retry.py | 1 + .pre-commit-config.yaml | 6 ++++++ 2 files changed, 7 insertions(+) diff --git a/.github/actions/compliance-extract/test_extract_with_retry.py b/.github/actions/compliance-extract/test_extract_with_retry.py index 15ab536fed28..b30fae78f38a 100644 --- a/.github/actions/compliance-extract/test_extract_with_retry.py +++ b/.github/actions/compliance-extract/test_extract_with_retry.py @@ -15,6 +15,7 @@ class ExtractRetryTest(unittest.TestCase): def run_extract(self, errors): + """Run the wrapper with scripted failures and inspect retries and output.""" with tempfile.TemporaryDirectory() as temp: root = Path(temp) for index, error in enumerate(errors, 1): diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 8eea48d678d0..7c3c7b938b1f 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -198,6 +198,12 @@ repos: # be a second copy to keep in sync. Full scan is well under 0.1s. files: ^\.github/ pass_filenames: false + - id: compliance-extract-selftest + name: Test compliance export retries + entry: python3 .github/actions/compliance-extract/test_extract_with_retry.py + language: python + pass_filenames: false + files: ^(\.github/actions/compliance-extract/|\.pre-commit-config\.yaml$) - id: check-action-pins-selftest name: Self-test the action-pin matcher entry: python3 scripts/check_action_pins.py --test From e8b93c48c5d5d14f4329307e9a694121dff1eb12 Mon Sep 17 00:00:00 2001 From: xianlubird Date: Fri, 11 Sep 2026 13:56:49 +0800 Subject: [PATCH 3/4] test(ci): trim redundant compliance retry coverage Signed-off-by: xianlubird --- .../actions/compliance-extract/test_extract_with_retry.py | 7 ------- 1 file changed, 7 deletions(-) diff --git a/.github/actions/compliance-extract/test_extract_with_retry.py b/.github/actions/compliance-extract/test_extract_with_retry.py index b30fae78f38a..616ae9342061 100644 --- a/.github/actions/compliance-extract/test_extract_with_retry.py +++ b/.github/actions/compliance-extract/test_extract_with_retry.py @@ -15,7 +15,6 @@ class ExtractRetryTest(unittest.TestCase): def run_extract(self, errors): - """Run the wrapper with scripted failures and inspect retries and output.""" with tempfile.TemporaryDirectory() as temp: root = Path(temp) for index, error in enumerate(errors, 1): @@ -76,12 +75,6 @@ def test_transient_recovers_without_partial_files(self): self.run_extract([TRANSIENT, ""]), (0, 2, ["10"], ["attempt-2"]) ) - def test_third_attempt_recovers(self): - self.assertEqual( - self.run_extract([TRANSIENT, TRANSIENT, ""]), - (0, 3, ["10", "30"], ["attempt-3"]), - ) - def test_exhaustion_preserves_failure(self): self.assertEqual(self.run_extract([TRANSIENT] * 3), (7, 3, ["10", "30"], [])) From 1a03cf1f55f9299a13f5d162b815cd96f9d035f1 Mon Sep 17 00:00:00 2001 From: xianlubird Date: Fri, 18 Sep 2026 10:04:23 +0800 Subject: [PATCH 4/4] ci: retry source archive export failures Signed-off-by: xianlubird --- .github/actions/compliance-extract/action.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/actions/compliance-extract/action.yml b/.github/actions/compliance-extract/action.yml index bdb9d28370ea..076deaf5420c 100644 --- a/.github/actions/compliance-extract/action.yml +++ b/.github/actions/compliance-extract/action.yml @@ -356,7 +356,6 @@ runs: GIT_SHA: ${{ inputs.git_sha }} EPP_IMAGE: ${{ inputs.epp_image }} run: | - mkdir -p /tmp/sources # ENABLE_SOURCE_ARCHIVAL=true to actually run sources_collect; the other # build-args match the build so this shares the warm cache (same reason # as the compliance_artifact extract above). Same IRSA secrets too, so a @@ -384,7 +383,8 @@ runs: # resolves the GC'd default staging tag. Empty for non-frontend targets. EPP_IMAGE_ARG="" [ -n "${EPP_IMAGE:-}" ] && EPP_IMAGE_ARG="--build-arg EPP_IMAGE=${EPP_IMAGE}" - docker buildx build \ + bash "${{ github.action_path }}/extract-with-retry.sh" /tmp/sources \ + docker buildx build \ --builder ${{ inputs.builder_name }} \ --platform ${{ inputs.platform }} \ --target sources_archive \ @@ -394,7 +394,6 @@ runs: --build-arg SCCACHE_REGION="${SCCACHE_REGION}" \ --build-arg ENABLE_SOURCE_ARCHIVAL=true \ $EPP_IMAGE_ARG \ - --output type=local,dest=/tmp/sources \ --progress=plain \ -f "${DOCKERFILE_PATH}" \ $SECRET_ARGS \