ci(#6462): add retry with backoff to eval setup-fixture.sh

eval/scripts/setup-fixture-test.sh

@@ -0,0 +1,128 @@
+#!/usr/bin/env bash
+# setup-fixture-test.sh — Tests for the retry_cmd function in setup-fixture.sh
+#
+# Run from the repo root:
+#   bash eval/scripts/setup-fixture-test.sh
+
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+FAILURES=0
+TMPDIR="$(mktemp -d)"
+trap 'rm -rf "${TMPDIR}"' EXIT
+
+# Extract retry_cmd from setup-fixture.sh so we can test it in isolation.
+# This avoids sourcing the whole script (which requires env vars and tools).
+sed -n '/^retry_cmd()/,/^}/p' "${SCRIPT_DIR}/setup-fixture.sh" > "${TMPDIR}/retry_cmd.sh"
+
+assert_eq() {
+  local test_name="$1" expected="$2" actual="$3"
+  if [[ "$expected" == "$actual" ]]; then
+    echo "PASS: ${test_name}"
+  else
+    echo "FAIL: ${test_name} — expected '${expected}', got '${actual}'"
+    (( FAILURES++ )) || true
+  fi
+}
+
+# ── Test 1: succeeds on first attempt ──────────────────────────────
+test_succeeds_first_attempt() {
+  local out
+  out=$(
+    source "${TMPDIR}/retry_cmd.sh"
+    retry_cmd true 2>/dev/null
+  )
+  assert_eq "succeeds on first attempt (exit 0)" "0" "$?"
+}
+test_succeeds_first_attempt
+
+# ── Test 2: fails after max attempts ──────────────────────────────
+test_fails_after_max_attempts() {
+  local rc=0
+  source "${TMPDIR}/retry_cmd.sh"
+  retry_cmd false 2>/dev/null || rc=$?
+  assert_eq "fails after max attempts (exit 1)" "1" "$rc"
+}
+test_fails_after_max_attempts
+
+# ── Test 3: retries and eventually succeeds ────────────────────────
+test_retries_then_succeeds() {
+  # Create a command that fails twice then succeeds.
+  local counter_file="${TMPDIR}/attempt_count"
+  echo "0" > "$counter_file"
+
+  cat > "${TMPDIR}/flaky_cmd.sh" <<SCRIPT
+#!/usr/bin/env bash
+count=\$(cat "${counter_file}")
+count=\$((count + 1))
+echo "\$count" > "${counter_file}"
+if [[ \$count -lt 3 ]]; then
+  exit 1
+fi
+exit 0
+SCRIPT
+  chmod +x "${TMPDIR}/flaky_cmd.sh"
+
+  local rc=0
+  source "${TMPDIR}/retry_cmd.sh"
+  retry_cmd "${TMPDIR}/flaky_cmd.sh" 2>/dev/null || rc=$?
+  assert_eq "retries then succeeds (exit 0)" "0" "$rc"
+
+  local attempts
+  attempts=$(cat "$counter_file")
+  assert_eq "ran 3 attempts before success" "3" "$attempts"
+}
+test_retries_then_succeeds
+
+# ── Test 4: diagnostic output goes to stderr, not stdout ──────────
+test_stderr_not_stdout() {
+  local counter_file="${TMPDIR}/attempt_count2"
+  echo "0" > "$counter_file"
+
+  cat > "${TMPDIR}/fail_once.sh" <<SCRIPT
+#!/usr/bin/env bash
+count=\$(cat "${counter_file}")
+count=\$((count + 1))
+echo "\$count" > "${counter_file}"
+if [[ \$count -lt 2 ]]; then
+  exit 1
+fi
+echo "command_output"
+SCRIPT
+  chmod +x "${TMPDIR}/fail_once.sh"
+
+  local stdout stderr
+  source "${TMPDIR}/retry_cmd.sh"
+  stdout=$(retry_cmd "${TMPDIR}/fail_once.sh" 2>"${TMPDIR}/stderr.txt")
+  stderr=$(cat "${TMPDIR}/stderr.txt")
+
+  # stdout must contain only the command output, no retry warnings
+  assert_eq "stdout is clean (no retry warnings)" "command_output" "$stdout"
+
+  # stderr must contain the retry warning
+  if echo "$stderr" | grep -q "WARNING.*attempt.*failed"; then
+    echo "PASS: retry warning goes to stderr"
+  else
+    echo "FAIL: retry warning not found in stderr"
+    (( FAILURES++ )) || true
+  fi
+}
+test_stderr_not_stdout
+
+# ── Test 5: preserves stdout from successful command ──────────────
+test_preserves_stdout() {
+  local out
+  source "${TMPDIR}/retry_cmd.sh"
+  out=$(retry_cmd echo "hello world" 2>/dev/null)
+  assert_eq "preserves command stdout" "hello world" "$out"
+}
+test_preserves_stdout
+
+# ── Summary ────────────────────────────────────────────────────────
+echo ""
+if [[ "$FAILURES" -gt 0 ]]; then
+  echo "FAILED: ${FAILURES} test(s) failed"
+  exit 1
+else
+  echo "ALL TESTS PASSED"
+fi

eval/scripts/setup-fixture.sh

@@ -15,6 +15,35 @@
 #   $CASE_WORKSPACE/.hook-outputs.yaml — env vars for the runner
 set -euo pipefail
 
+# ---------------------------------------------------------------------------
+# retry_cmd — retry a command with exponential backoff.
+#
+# Usage: retry_cmd <command> [args...]
+#
+# Retries up to 3 times with 2s/4s/8s delays between attempts.
+# All diagnostic output goes to stderr to avoid stdout contamination
+# when the caller captures output via command substitution (see
+# docs/contributing/shell-scripting.md).
+# ---------------------------------------------------------------------------
+retry_cmd() {
+  local max_attempts=3
+  local attempt=1
+  local delay=2
+  while true; do
+    if "$@"; then
+      return 0
+    fi
+    if (( attempt >= max_attempts )); then
+      echo "ERROR: command failed after ${max_attempts} attempts: $*" >&2
+      return 1
+    fi
+    echo "WARNING: attempt ${attempt}/${max_attempts} failed, retrying in ${delay}s: $*" >&2
+    sleep "$delay"
+    (( delay *= 2 ))
+    (( attempt++ ))
+  done
+}
+
 CASE_WORKSPACE="${CASE_WORKSPACE:?CASE_WORKSPACE is required}"
 EVAL_ORG="${EVAL_ORG:?EVAL_ORG is required}"
 
@@ -57,13 +86,19 @@ CASE_ID_SAFE=$(basename "$CASE_SOURCE_DIR" | tr '[:upper:]' '[:lower:]' | sed 's
 repo_name="eval-${CASE_ID_SAFE}-${uuid}"
 EPHEMERAL_REPO="${EVAL_ORG}/${repo_name}"
 
-gh repo create "$EPHEMERAL_REPO" --public --description "Ephemeral eval repo (auto-deleted)"
+retry_cmd gh repo create "$EPHEMERAL_REPO" --public --description "Ephemeral eval repo (auto-deleted)"
 echo "Created repo: $EPHEMERAL_REPO"
 
 TARGET_DIR=$(mktemp -d)
 GH_CRED_HELPER='!f(){ echo "password=${GH_TOKEN}"; };f'
-git -c "credential.helper=${GH_CRED_HELPER}" \
-  clone "https://x-access-token@github.com/${EPHEMERAL_REPO}.git" "$TARGET_DIR"
+# Clone with retry. Each attempt removes the target directory first so a
+# partial clone from a previous attempt doesn't cause the next one to fail.
+retry_clone() {
+  rm -rf "$TARGET_DIR"
+  git -c "credential.helper=${GH_CRED_HELPER}" \
+    clone "https://x-access-token@github.com/${EPHEMERAL_REPO}.git" "$TARGET_DIR"
+}
+retry_cmd retry_clone
 git -C "$TARGET_DIR" config credential.helper "${GH_CRED_HELPER}"
 git -C "$TARGET_DIR" config commit.gpgsign false
 
@@ -76,7 +111,7 @@ fi
 git -C "$TARGET_DIR" add -A
 if ! git -C "$TARGET_DIR" diff --cached --quiet; then
   git -C "$TARGET_DIR" commit -m "eval: initial content"
-  git -C "$TARGET_DIR" push origin HEAD
+  retry_cmd git -C "$TARGET_DIR" push origin HEAD
 fi
 
 # --- Create seed issues (if any) ---
@@ -85,7 +120,7 @@ if [[ "$SEED_COUNT" -gt 0 ]]; then
   for i in $(seq 0 $((SEED_COUNT - 1))); do
     seed_title=$(yq -r ".seed_issues[$i].title" "$INPUT")
     seed_body=$(yq -r ".seed_issues[$i].body" "$INPUT")
-    seed_url=$(gh issue create \
+    seed_url=$(retry_cmd gh issue create \
       --repo "$EPHEMERAL_REPO" \
       --title "$seed_title" \
       --body "$seed_body")
@@ -99,7 +134,7 @@ FIXTURE_NUMBER=""
 
 case "${FORGE}:${FIXTURE_TYPE}" in
   github:issue)
-    FIXTURE_URL=$(gh issue create \
+    FIXTURE_URL=$(retry_cmd gh issue create \
       --repo "$EPHEMERAL_REPO" \
       --title "$FIXTURE_TITLE" \
       --body "$FIXTURE_BODY")
@@ -117,8 +152,8 @@ case "${FORGE}:${FIXTURE_TYPE}" in
     done
     git -C "$TARGET_DIR" add -A
     git -C "$TARGET_DIR" commit -m "eval: fixture changes"
-    git -C "$TARGET_DIR" push origin "$PR_BRANCH"
-    FIXTURE_URL=$(gh pr create \
+    retry_cmd git -C "$TARGET_DIR" push origin "$PR_BRANCH"
+    FIXTURE_URL=$(retry_cmd gh pr create \
       --repo "$EPHEMERAL_REPO" \
       --base "$FIXTURE_BASE" \
       --head "$PR_BRANCH" \