fix(ci): improve e2e worker failure diagnostics and cleanup

.github/workflows/e2e-gpu-job.yml

@@ -113,7 +113,7 @@ jobs:
 
       # Run tests
       - name: Run E2E tests
-        timeout-minutes: 20
+        timeout-minutes: 25
         env:
           SHOW_WORKER_LOGS: "0"
           SHOW_ROUTER_LOGS: "1"
@@ -128,16 +128,20 @@ jobs:
             --reruns 2 --reruns-delay 5 \
             -s -vv
 
-      - name: Compute log artifact label
+      - name: Worker failure diagnostics
         if: failure() || cancelled()
-        id: log-label
-        run: echo "label=$(echo '${{ inputs.test_dirs }}' | tr '/' '-')" >> "$GITHUB_OUTPUT"
+        run: |
+          LABEL=$(echo '${{ inputs.test_dirs }}' | tr '/' '-')
+          ARTIFACT="e2e-worker-logs-${{ inputs.engine }}-gpu${{ inputs.gpu_tier }}-${LABEL}"
+          bash scripts/ci_dump_worker_logs.sh e2e-logs "$ARTIFACT"
+          echo "label=${LABEL}" >> "$GITHUB_OUTPUT"
+        id: log-snapshot
 
       - name: Upload worker logs
         if: failure() || cancelled()
         uses: actions/upload-artifact@v7
         with:
-          name: e2e-worker-logs-${{ inputs.engine }}-gpu${{ inputs.gpu_tier }}-${{ steps.log-label.outputs.label }}
+          name: e2e-worker-logs-${{ inputs.engine }}-gpu${{ inputs.gpu_tier }}-${{ steps.log-snapshot.outputs.label }}
           path: e2e-logs/
           retention-days: 7
           if-no-files-found: ignore

.github/workflows/pr-test-rust.yml

@@ -345,7 +345,11 @@ jobs:
           name: genai-bench-results-all-policies
           path: benchmark_**/
 
-      - name: Upload worker logs on failure
+      - name: Worker failure diagnostics
+        if: failure() || cancelled()
+        run: bash scripts/ci_dump_worker_logs.sh benchmark-logs benchmark-worker-logs
+
+      - name: Upload worker logs
         if: failure() || cancelled()
         uses: actions/upload-artifact@v7
         with:

e2e_test/conftest.py

@@ -27,7 +27,6 @@
 from __future__ import annotations
 
 import logging
-import os
 import sys
 from importlib.util import find_spec
 from pathlib import Path
@@ -113,38 +112,6 @@ def pytest_runtest_logstart(nodeid: str, location: tuple) -> None:
 )
 from smg_client import SmgClient
 
-# ---------------------------------------------------------------------------
-# Failure diagnostics
-# ---------------------------------------------------------------------------
-
-
-@pytest.hookimpl(hookwrapper=True)
-def pytest_runtest_makereport(item, call):
-    """Print worker log summary on test failure."""
-    from infra import LOG_SEPARATOR_WIDTH
-
-    outcome = yield
-    report = outcome.get_result()
-
-    if not report.failed:
-        return
-
-    log_dir = os.environ.get("E2E_LOG_DIR")
-    if not log_dir or not Path(log_dir).is_dir():
-        return
-
-    logs = sorted(p.name for p in Path(log_dir).glob("*.log") if p.is_file())
-    if logs:
-        print(f"\n{'!' * LOG_SEPARATOR_WIDTH}")
-        print(f"Worker logs available in CI artifacts ({len(logs)} files)")
-        # Show only the most recent logs to avoid flooding the console
-        for f in logs[-5:]:
-            print(f"  {f}")
-        if len(logs) > 5:
-            print(f"  ... and {len(logs) - 5} more")
-        print(f"{'!' * LOG_SEPARATOR_WIDTH}")
-
-
 # ---------------------------------------------------------------------------
 # Fixtures
 # ---------------------------------------------------------------------------
@@ -181,7 +148,6 @@ def api_client(request, setup_backend):
 __all__ = [
     # Hooks
     "pytest_runtest_logstart",
-    "pytest_runtest_makereport",
     "pytest_runtest_setup",
     "pytest_collection_modifyitems",
     "pytest_configure",

e2e_test/infra/model_specs.py

@@ -97,6 +97,7 @@ def _resolve_model_path(hf_path: str) -> str:
         "model": _resolve_model_path("openai/gpt-oss-120b"),
         "tp": 4,
         "features": ["chat", "streaming", "reasoning", "harmony"],
+        "startup_timeout": 600,
         "vllm_args": [
             "--structured-outputs-config",
             '{"enable_in_reasoning": true}',

e2e_test/infra/worker.py

@@ -298,6 +298,7 @@ def _build_trtllm_cmd(self, model_path: str, tp_size: int, spec: dict) -> list[s
     def _build_env(self) -> dict[str, str]:
         """Build environment variables for the worker process."""
         env = os.environ.copy()
+        env.setdefault("PYTHONUNBUFFERED", "1")
         env["CUDA_VISIBLE_DEVICES"] = ",".join(map(str, self.gpu_ids))
 
         # TRT-LLM multi-GPU needs NCCL tuning for CI compatibility
@@ -457,8 +458,8 @@ def start_workers(
                 logger.info("Staggering launch by %ds", LAUNCH_STAGGER_DELAY)
                 time.sleep(LAUNCH_STAGGER_DELAY)
 
-            worker.start(timeout=timeout, wait_ready=wait_ready)
             workers.append(worker)
+            worker.start(timeout=timeout, wait_ready=wait_ready)
     except Exception:
         stop_workers(workers)
         raise

scripts/ci_dump_worker_logs.sh

@@ -0,0 +1,39 @@
+#!/usr/bin/env bash
+# Dump the last worker log snapshot and list all log files.
+#
+# Usage: ci_dump_worker_logs.sh <log_dir> [artifact_name]
+#
+# Arguments:
+#   log_dir        Directory containing worker log files
+#   artifact_name  (optional) Artifact name to display in the file listing
+
+set -euo pipefail
+
+LOG_DIR="${1:?Usage: ci_dump_worker_logs.sh <log_dir> [artifact_name]}"
+ARTIFACT="${2:-$LOG_DIR}"
+
+if [ ! -d "$LOG_DIR" ]; then exit 0; fi
+
+SEP="============================================================"
+DASH="------------------------------------------------------------"
+
+# Dump last worker log (most recent = last retry of the failed model)
+LAST_LOG=$(ls -t "$LOG_DIR"/worker-*.log 2>/dev/null | head -1 || true)
+if [ -n "$LAST_LOG" ]; then
+    TOTAL=$(wc -l < "$LAST_LOG")
+    SHOW=$(( TOTAL < 200 ? TOTAL : 200 ))
+    echo ""
+    echo "$SEP"
+    echo "Last worker log: $(basename "$LAST_LOG") (last $SHOW of $TOTAL lines)"
+    echo "$DASH"
+    tail -200 "$LAST_LOG"
+    echo "$SEP"
+fi
+
+# List all log files with sizes
+echo ""
+echo "$SEP"
+echo "All worker logs in artifact: $ARTIFACT"
+echo "$DASH"
+ls -lhS "$LOG_DIR"/*.log 2>/dev/null || echo "  (none)"
+echo "$SEP"