[AMD][dsv4] fix 2P1D eval 503 "all circuits open" (readiness canary + faster CB recovery)

benchmarks/multi_node/amd_utils/server_sglang.sh

@@ -506,12 +506,23 @@ if [ "$NODE_RANK" -eq 0 ]; then
     fi
     echo "Congratulations!!! All prefill and decode servers are up . . ."
 
+    # Circuit-breaker recovery tuning (run 28696443568):
+    # With defaults the per-worker circuit stays OPEN for cb-timeout-duration-secs=60
+    # before a half-open retrial. In that run every request 503'd from request #1
+    # ("all circuits open or unhealthy") for ~31s and lm_eval (max_retries=5) then gave
+    # up -- i.e. the circuit was still open when the client budget ran out, so 0 result
+    # files were produced. Shortening the open->half-open window (and letting the router
+    # itself retry a failed worker selection) lets a transient trip re-close INSIDE the
+    # client retry budget instead of nuking the whole eval. The breaker stays fully
+    # ENABLED (thresholds unchanged); this only speeds recovery.
+    ROUTER_CB_ARGS="${ROUTER_CB_ARGS:---cb-timeout-duration-secs 15 --retry-max-retries 3}"
     ROUTER_CMD="python -m sglang_router.launch_router \
         --pd-disaggregation \
         --port 30000 \
         --policy random \
         --prefill-policy random \
         --decode-policy random \
+        ${ROUTER_CB_ARGS} \
         ${PREFILL_ARGS} \
         ${DECODE_ARGS}"
 
@@ -557,6 +568,39 @@ if [ "$NODE_RANK" -eq 0 ]; then
             wait_or_die "$prefill0_pid" bash -c "$HEALTH_BARRIER_CMD" || exit 1
         fi
 
+        # ---- End-to-end router readiness canary (run 28696443568) ----
+        # The /readiness barrier above only proves the router PROCESS is up; it does
+        # NOT prove the router can reach a prefill worker and complete a generation.
+        # In that run the eval fired the instant /readiness passed and EVERY request
+        # 503'd ("No available prefill workers (all circuits open or unhealthy)") from
+        # request #1 -> lm_eval gave up -> 0 result files -> "Verify eval scores" failed.
+        # Gate the benchmark on ONE successful generation THROUGH the router so the eval
+        # never starts against a router whose prefill path is not yet actually serving.
+        if [[ "${ROUTER_READINESS_CANARY:-1}" == "1" ]]; then
+            CANARY_URL="http://${NODE0_ADDR}:30000/v1/chat/completions"
+            CANARY_MODEL="${MODEL_DIR}/${MODEL_NAME}"
+            canary_ok=0
+            canary_deadline=$(( $(date +%s) + ${ROUTER_CANARY_TIMEOUT:-600} ))
+            while [ "$(date +%s)" -lt "$canary_deadline" ]; do
+                canary_code=$(curl -s -o /tmp/router_canary.out -w '%{http_code}' \
+                    -m "${ROUTER_CANARY_REQ_TIMEOUT:-120}" \
+                    -X POST "$CANARY_URL" -H 'Content-Type: application/json' \
+                    -d "{\"model\":\"${CANARY_MODEL}\",\"messages\":[{\"role\":\"user\",\"content\":\"ping\"}],\"max_tokens\":1,\"temperature\":0}" 2>/dev/null)
+                if [ "$canary_code" = "200" ] && \
+                   ! grep -qE "circuits open|server_selection_failed|No available" /tmp/router_canary.out 2>/dev/null; then
+                    canary_ok=1; break
+                fi
+                echo "Router readiness canary not ready yet (http=${canary_code}); retrying in 5s . . ."
+                sleep 5
+            done
+            if [ "$canary_ok" -ne 1 ]; then
+                echo "ERROR: router readiness canary failed after ${ROUTER_CANARY_TIMEOUT:-600}s -- the router cannot complete a generation through a prefill worker (all circuits open/unhealthy). Refusing to start the eval against a non-serving router."
+                head -c 800 /tmp/router_canary.out 2>/dev/null
+                exit 1
+            fi
+            echo "Router readiness canary passed (end-to-end generation OK)"
+        fi
+
         echo "Router is ready for benchmarking"
     fi