From b58d08e7b149cd677602a703dc14c3c6f12730f9 Mon Sep 17 00:00:00 2001 From: arimu1 <19286898+arimu1@users.noreply.github.com> Date: Fri, 24 Jul 2026 14:17:27 +0700 Subject: [PATCH] fix(agent): jittered, interrupt-aware backoff for empty-response retries Empty content retries previously fired back-to-back with no delay, wasting up to 3 rapid API calls, and could not be cancelled mid-wait. Apply the same jittered_backoff() already used for rate-limit and API-error retries, sleeping in small increments so a user interrupt aborts the wait instead of blocking until it elapses. Fixes #35230 --- agent/conversation_loop.py | 38 ++++++++++++++-- tests/run_agent/test_run_agent.py | 74 +++++++++++++++++++++++++++++++ 2 files changed, 109 insertions(+), 3 deletions(-) diff --git a/agent/conversation_loop.py b/agent/conversation_loop.py index a234213e32caf..91c38515a8378 100644 --- a/agent/conversation_loop.py +++ b/agent/conversation_loop.py @@ -6054,15 +6054,47 @@ def _perform_api_call(next_api_kwargs): ) if _truly_empty and (not _has_structured or _prefill_exhausted) and agent._empty_content_retries < 3: agent._empty_content_retries += 1 + wait_time = jittered_backoff( + agent._empty_content_retries, + base_delay=5.0, + max_delay=60.0, + ) logger.warning( "Empty response (no content or reasoning) — " - "retry %d/3 (model=%s)", - agent._empty_content_retries, agent.model, + "retry %d/3 in %.1fs (model=%s)", + agent._empty_content_retries, wait_time, agent.model, ) agent._buffer_status( f"⚠️ Empty response from model — retrying " - f"({agent._empty_content_retries}/3)" + f"({agent._empty_content_retries}/3) in {wait_time:.0f}s" ) + # Sleep in small increments to stay responsive to interrupts + sleep_end = time.time() + wait_time + _backoff_touch_counter = 0 + while time.time() < sleep_end: + if agent._interrupt_requested: + agent._vprint(f"{agent.log_prefix}⚡ Interrupt detected during empty-response retry wait, aborting.", force=True) + _interrupt_text = ( + f"Operation interrupted: retrying empty response from model " + f"(retry {agent._empty_content_retries}/3)." + ) + close_interrupted_tool_sequence(messages, _interrupt_text) + agent._persist_session(messages, conversation_history) + agent.clear_interrupt() + return { + "final_response": _interrupt_text, + "messages": messages, + "api_calls": api_call_count, + "completed": False, + "interrupted": True, + } + time.sleep(0.2) + _backoff_touch_counter += 1 + if _backoff_touch_counter % 150 == 0: # 150 × 0.2s = 30s + agent._touch_activity( + f"empty response retry backoff ({agent._empty_content_retries}/3), " + f"{int(sleep_end - time.time())}s remaining" + ) continue # ── Exhausted retries — try fallback provider ── diff --git a/tests/run_agent/test_run_agent.py b/tests/run_agent/test_run_agent.py index 98f5a779ca026..23a2140a94490 100644 --- a/tests/run_agent/test_run_agent.py +++ b/tests/run_agent/test_run_agent.py @@ -4739,6 +4739,80 @@ def _capture_status(msg): failure_msgs = [m for m in status_messages if "no content" in m.lower() or "no fallback" in m.lower()] assert len(failure_msgs) >= 1, f"Expected at least 1 failure status, got: {status_messages}" + def test_empty_response_retry_backoff_interrupted(self, agent, monkeypatch): + """If an interrupt is requested during the empty response retry wait, we abort.""" + self._setup_agent(agent) + agent.base_url = "http://127.0.0.1:1234/v1" + empty_resp = _mock_response(content=None, finish_reason="stop") + agent.client.chat.completions.create.side_effect = [empty_resp, empty_resp] + + from agent import conversation_loop as _conv_loop + + # Make backoff return 10.0 seconds + monkeypatch.setattr(_conv_loop, "jittered_backoff", lambda *a, **k: 10.0) + + # Trigger the interrupt on the first sleep call inside the wait loop + original_sleep = time.sleep + sleep_called = [] + + def _mock_sleep(seconds): + sleep_called.append(seconds) + if seconds == 0.2: + agent._interrupt_requested = True + else: + original_sleep(seconds) + + monkeypatch.setattr(time, "sleep", _mock_sleep) + + with ( + patch.object(agent, "_persist_session") as mock_persist, + patch.object(agent, "_save_trajectory"), + patch.object(agent, "_cleanup_task_resources"), + ): + result = agent.run_conversation("answer me") + + assert result["interrupted"] is True + assert "Operation interrupted: retrying empty response from model" in result["final_response"] + assert agent._empty_content_retries == 1 + assert 0.2 in sleep_called + assert mock_persist.call_count == 2 + + def test_empty_response_retry_backoff_status(self, agent, monkeypatch): + """Empty response retry wait updates the agent's status with wait time and sleeps.""" + self._setup_agent(agent) + agent.base_url = "http://127.0.0.1:1234/v1" + + # Two responses: first empty, second succeeds so it doesn't run forever + empty_resp = _mock_response(content=None, finish_reason="stop") + ok_resp = _mock_response(content="Final ok response.", finish_reason="stop") + agent.client.chat.completions.create.side_effect = [empty_resp, ok_resp] + + from agent import conversation_loop as _conv_loop + + monkeypatch.setattr(_conv_loop, "jittered_backoff", lambda *a, **k: 7.5) + + sleep_calls = [] + monkeypatch.setattr(time, "sleep", lambda secs: sleep_calls.append(secs)) + + status_messages = [] + monkeypatch.setattr(agent, "_buffer_status", lambda status: status_messages.append(status)) + + with ( + patch.object(agent, "_persist_session"), + patch.object(agent, "_save_trajectory"), + patch.object(agent, "_cleanup_task_resources"), + ): + result = agent.run_conversation("answer me") + + assert result["completed"] is True + assert result["final_response"] == "Final ok response." + + # 7.5s wait, slept in 0.2s increments -> 37.5 -> at least 37 calls + assert len([c for c in sleep_calls if c == 0.2]) >= 37 + + retry_status = [m for m in status_messages if "Empty response from model — retrying (1/3) in 8s" in m] + assert len(retry_status) == 1 + def test_partial_stream_recovery_uses_streamed_content(self, agent): """When streaming fails after partial delivery, recovered partial content becomes final response.""" self._setup_agent(agent)