From f910b62087ff45824c7963129aa95381f21ba4b5 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Mon, 31 Aug 2026 17:04:10 +0900 Subject: [PATCH 1/2] fix: classify structured synthesis provider failures Signed-off-by: Seongho Bae --- contextual_orchestrator/orchestrator.py | 9 +++++ tests/test_model_judge.py | 5 ++- tests/test_openai_passthrough.py | 52 +++++++++++++++++++++++++ 3 files changed, 65 insertions(+), 1 deletion(-) diff --git a/contextual_orchestrator/orchestrator.py b/contextual_orchestrator/orchestrator.py index 7953b968c..cbb8568df 100644 --- a/contextual_orchestrator/orchestrator.py +++ b/contextual_orchestrator/orchestrator.py @@ -4352,6 +4352,15 @@ def send_synthesis( self._record_failure(final_agent.id) if final_agent.group_name and not _is_request_too_large_error(exc): self._group_router.observe_failure(final_agent.id) + if not isinstance( + exc, (ProviderRequestTooLargeError, ProviderUpstreamError) + ): + raise classify_provider_failure( + exc, + agent_id=final_agent.id, + model=final_agent.model, + transport=endpoint, + ) from None raise def provider_output(response: Mapping[str, Any]) -> str: diff --git a/tests/test_model_judge.py b/tests/test_model_judge.py index 224a93819..a7d41d5c4 100644 --- a/tests/test_model_judge.py +++ b/tests/test_model_judge.py @@ -29,6 +29,7 @@ _parse_model_judge_reply, _structured_output_error, ) +from contextual_orchestrator.provider_errors import ProviderUpstreamError # noqa: E402 RISKY_VERIFIER_REPORT = "The plan is sound overall but discusses downtime risks and error handling." @@ -969,7 +970,7 @@ def test_structured_synthesis_failure_updates_provider_health() -> None: "proxy_send", side_effect=RuntimeError("synthetic provider failure"), ), - pytest.raises(RuntimeError, match="synthetic provider failure"), + pytest.raises(ProviderUpstreamError) as exc_info, ): orchestrator.proxy_completion( { @@ -980,6 +981,8 @@ def test_structured_synthesis_failure_updates_provider_health() -> None: single_agent=False, ) + assert exc_info.value.agent_id == "general_agent" + assert "synthetic provider failure" not in str(exc_info.value) assert orchestrator._circuit["general_agent"]["failures"] == 1 diff --git a/tests/test_openai_passthrough.py b/tests/test_openai_passthrough.py index e91bc31f8..be42483ea 100644 --- a/tests/test_openai_passthrough.py +++ b/tests/test_openai_passthrough.py @@ -779,6 +779,58 @@ def proxy_send_once(self, agent, endpoint, payload): assert body["error_message"] == "request body exceeds every eligible provider limit" +def test_http_virtual_structured_synthesis_failure_returns_provider_error() -> None: + """A final provider failure is classified, not an internal server bug.""" + + class FailingSynthesisClient(ModelClient): + def proxy_send_once(self, agent, endpoint, payload): + del agent, endpoint, payload + raise urllib.error.URLError("synthetic provider outage") + + proxy_send = proxy_send_once + + orchestrator = TaskOrchestrator( + [ModelAgent("worker_agent", "worker-model", tags=("response_format",))], + client=FailingSynthesisClient(), # type: ignore[arg-type] + ) + orchestrator.conduct = lambda *args, **kwargs: { # type: ignore[method-assign] + "mode": "conduct", + "answer": "evidence", + "trace": [{ + "id": 0, + "role": "worker", + "agent_id": "worker_agent", + "subtask": "Evidence", + "access": [], + "output": "evidence", + }], + "verification": {"accepted": True, "reason": "test", "verifier_output": ""}, + } + token = "passthrough_token" + server = build_server( + orchestrator, port=0, security=SecurityConfig(auth_token=token) + ) + threading.Thread(target=server.serve_forever, daemon=True).start() + try: + status, body = _post( + f"http://127.0.0.1:{server.server_address[1]}/v1/chat/completions", + { + "model": TaskOrchestrator.AUTO_MODEL, + "messages": [{"role": "user", "content": "return JSON"}], + "response_format": {"type": "json_object"}, + }, + token, + ) + finally: + server.shutdown() + server.server_close() + + assert status == 502 + assert body["error"]["code"] == "provider_connection_error" + assert body["error"]["detail"]["retryable"] is True + assert "synthetic provider outage" not in json.dumps(body) + + def test_http_chat_completions_accepts_response_format_and_passes_through() -> None: server, port, token = _serve() url = f"http://127.0.0.1:{port}/v1/chat/completions" From 5567e0084e80fb00fea65c3c44f0f98c0572cff8 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Mon, 31 Aug 2026 17:38:10 +0900 Subject: [PATCH 2/2] fix: preserve stacked exhaustion contract Signed-off-by: Seongho Bae --- contextual_orchestrator/orchestrator.py | 13 ++++++------- tests/test_model_judge.py | 5 +---- tests/test_openai_passthrough.py | 10 +++++----- tests/test_passthrough_provider_failover.py | 4 +++- 4 files changed, 15 insertions(+), 17 deletions(-) diff --git a/contextual_orchestrator/orchestrator.py b/contextual_orchestrator/orchestrator.py index cbb8568df..d9cc60276 100644 --- a/contextual_orchestrator/orchestrator.py +++ b/contextual_orchestrator/orchestrator.py @@ -4352,14 +4352,13 @@ def send_synthesis( self._record_failure(final_agent.id) if final_agent.group_name and not _is_request_too_large_error(exc): self._group_router.observe_failure(final_agent.id) - if not isinstance( - exc, (ProviderRequestTooLargeError, ProviderUpstreamError) + if ( + virtual_model + and response_format_requested + and not isinstance(exc, ProviderRequestTooLargeError) ): - raise classify_provider_failure( - exc, - agent_id=final_agent.id, - model=final_agent.model, - transport=endpoint, + raise NoViableAgentError( + retry_after_seconds=max(1, math.ceil(self.circuit_reset_seconds)) ) from None raise diff --git a/tests/test_model_judge.py b/tests/test_model_judge.py index a7d41d5c4..224a93819 100644 --- a/tests/test_model_judge.py +++ b/tests/test_model_judge.py @@ -29,7 +29,6 @@ _parse_model_judge_reply, _structured_output_error, ) -from contextual_orchestrator.provider_errors import ProviderUpstreamError # noqa: E402 RISKY_VERIFIER_REPORT = "The plan is sound overall but discusses downtime risks and error handling." @@ -970,7 +969,7 @@ def test_structured_synthesis_failure_updates_provider_health() -> None: "proxy_send", side_effect=RuntimeError("synthetic provider failure"), ), - pytest.raises(ProviderUpstreamError) as exc_info, + pytest.raises(RuntimeError, match="synthetic provider failure"), ): orchestrator.proxy_completion( { @@ -981,8 +980,6 @@ def test_structured_synthesis_failure_updates_provider_health() -> None: single_agent=False, ) - assert exc_info.value.agent_id == "general_agent" - assert "synthetic provider failure" not in str(exc_info.value) assert orchestrator._circuit["general_agent"]["failures"] == 1 diff --git a/tests/test_openai_passthrough.py b/tests/test_openai_passthrough.py index be42483ea..01451df02 100644 --- a/tests/test_openai_passthrough.py +++ b/tests/test_openai_passthrough.py @@ -779,8 +779,8 @@ def proxy_send_once(self, agent, endpoint, payload): assert body["error_message"] == "request body exceeds every eligible provider limit" -def test_http_virtual_structured_synthesis_failure_returns_provider_error() -> None: - """A final provider failure is classified, not an internal server bug.""" +def test_http_virtual_structured_synthesis_failure_returns_retryable_503() -> None: + """A final provider failure is availability, not an internal server bug.""" class FailingSynthesisClient(ModelClient): def proxy_send_once(self, agent, endpoint, payload): @@ -825,9 +825,9 @@ def proxy_send_once(self, agent, endpoint, payload): server.shutdown() server.server_close() - assert status == 502 - assert body["error"]["code"] == "provider_connection_error" - assert body["error"]["detail"]["retryable"] is True + assert status == 503 + assert body["error"]["code"] == "no_viable_agent" + assert body["error"]["detail"]["retry_after_seconds"] == 30 assert "synthetic provider outage" not in json.dumps(body) diff --git a/tests/test_passthrough_provider_failover.py b/tests/test_passthrough_provider_failover.py index 6b33d52a8..5aee05c98 100644 --- a/tests/test_passthrough_provider_failover.py +++ b/tests/test_passthrough_provider_failover.py @@ -19,6 +19,7 @@ ) from contextual_orchestrator.orchestrator import ( ModelClient, + NoViableAgentError, ProviderRequestTooLargeError, ) @@ -506,7 +507,7 @@ def test_structured_synthesis_records_non_413_failure_on_actual_provider() -> No "verification": {"accepted": True, "reason": "test", "verifier_output": ""}, } - with pytest.raises(RuntimeError, match="fallback unavailable"): + with pytest.raises(NoViableAgentError) as exc_info: orchestrator.proxy_completion( { "model": TaskOrchestrator.AUTO_MODEL, @@ -516,6 +517,7 @@ def test_structured_synthesis_records_non_413_failure_on_actual_provider() -> No single_agent=False, ) + assert "fallback unavailable" not in str(exc_info.value) assert orchestrator._circuit["fallback_agent"]["failures"] == 1.0 assert "primary_agent" not in orchestrator._circuit