Merge pull request #26 from asamal4/agenteval-fix-streaming-errorhandling

fix: streaming error handling

Author: tisnik

lsc_agent_eval/src/lsc_agent_eval/core/utils/api_client.py

@@ -1,5 +1,6 @@
 """HTTP client for agent API communication."""
 
+import json
 import logging
 import os
 from typing import Any, Optional
@@ -120,17 +121,42 @@ def streaming_query_agent(
                 json=api_input,
                 timeout=timeout,
             ) as response:
-                response.raise_for_status()
+                # Potential change lsc-stack to provide SSE error message
+                if response.status_code != 200:
+                    error_content = response.read().decode("utf-8")
+                    try:
+                        error_data = json.loads(error_content)
+                        if isinstance(error_data, dict) and "detail" in error_data:
+                            detail = error_data["detail"]
+                            if isinstance(detail, dict):
+                                response_msg = detail.get("response", "")
+                                cause_msg = detail.get("cause", "")
+                                error_msg = (
+                                    f"{response_msg} - {cause_msg}"
+                                    if cause_msg
+                                    else response_msg
+                                )
+                            else:
+                                error_msg = str(detail)
+                        else:
+                            error_msg = error_content
+                    except (json.JSONDecodeError, KeyError, TypeError):
+                        error_msg = error_content
+
+                    raise httpx.HTTPStatusError(
+                        message=f"Agent API error: {response.status_code} - {error_msg}",
+                        request=response.request,
+                        response=response,
+                    )
+
                 return parse_streaming_response(response)
 
         except httpx.TimeoutException as e:
             raise AgentAPIError(
                 f"Agent streaming query timeout after {timeout} seconds"
             ) from e
         except httpx.HTTPStatusError as e:
-            raise AgentAPIError(
-                f"Agent API error: {e.response.status_code} - {e.response.text}"
-            ) from e
+            raise AgentAPIError(str(e)) from e
         except ValueError as e:
             raise AgentAPIError(f"Streaming response validation error: {e}") from e
         except Exception as e:

lsc_agent_eval/src/lsc_agent_eval/core/utils/streaming_parser.py

@@ -33,6 +33,10 @@ def parse_streaming_response(response: httpx.Response) -> dict[str, Any]:
         if event == "start" and "conversation_id" in event_data:
             conversation_id = event_data["conversation_id"].strip()
             logger.debug("Found conversation_id: %s", conversation_id)
+        elif event == "error" and "token" in event_data:
+            error_message = event_data["token"]
+            logger.error("Received error event from streaming API: %s", error_message)
+            raise ValueError(f"Streaming API error: {error_message}")
         elif event == "turn_complete" and "token" in event_data:
             final_response = event_data["token"].strip()
             logger.debug("Found final response (%d characters)", len(final_response))

lsc_agent_eval/tests/core/utils/test_api_client.py

@@ -1,5 +1,6 @@
 """Tests for agent API client."""
 
+import json
 from unittest.mock import MagicMock, Mock, mock_open, patch
 
 import httpx
@@ -218,6 +219,7 @@ def test_client_setup_exception(self):
     def test_streaming_query_agent_success(self):
         """Test successful streaming agent query."""
         mock_response = Mock()
+        mock_response.status_code = 200
         mock_response.raise_for_status.return_value = None
 
         mock_stream_response = MagicMock()
@@ -265,6 +267,7 @@ def test_streaming_query_agent_success(self):
     def test_streaming_query_agent_parser_error(self):
         """Test streaming agent query when parser raises ValueError."""
         mock_response = Mock()
+        mock_response.status_code = 200
         mock_response.raise_for_status.return_value = None
 
         mock_stream_response = MagicMock()
@@ -312,19 +315,32 @@ def test_streaming_query_agent_timeout(self):
 
     def test_streaming_query_agent_http_error(self):
         """Test streaming agent query with HTTP error."""
+        error_response = {
+            "detail": {
+                "response": "Access denied",
+                "cause": "You do not have permission to access this conversation",
+            }
+        }
+        error_json = json.dumps(error_response)
+
         mock_response = Mock()
-        mock_response.status_code = 500
-        mock_response.text = "Internal Server Error"
+        mock_response.status_code = 403
+        mock_response.read.return_value = error_json.encode("utf-8")
+
+        mock_stream_response = MagicMock()
+        mock_stream_response.__enter__.return_value = mock_response
+        mock_stream_response.__exit__.return_value = None
 
         mock_client = Mock()
-        mock_client.stream.side_effect = httpx.HTTPStatusError(
-            "Server error", request=Mock(), response=mock_response
-        )
+        mock_client.stream.return_value = mock_stream_response
 
         with patch("httpx.Client", return_value=mock_client):
             client = AgentHttpClient("http://localhost:8080")
 
             api_input = {"query": "Test query", "provider": "openai", "model": "gpt-4"}
 
-            with pytest.raises(AgentAPIError, match="Agent API error: 500"):
+            with pytest.raises(
+                AgentAPIError,
+                match="Agent API error: 403 - Access denied - You do not have permission to access this conversation",
+            ):
                 client.streaming_query_agent(api_input)

lsc_agent_eval/tests/core/utils/test_streaming_parser.py

@@ -63,6 +63,20 @@ def test_error_conditions(self):
         with pytest.raises(ValueError, match="No Conversation ID found"):
             parse_streaming_response(mock_response)
 
+    def test_error_event_handling(self):
+        """Test error event handling in streaming response."""
+        mock_response = Mock()
+        mock_response.iter_lines.return_value = [
+            'data: {"event": "start", "data": {"conversation_id": "conv-error"}}',
+            'data: {"event": "error", "data": {"id": 1, "token": "Unable to connect to LLama Stack backend: Connection timed out"}}',
+        ]
+
+        with pytest.raises(
+            ValueError,
+            match="Streaming API error: Unable to connect to LLama Stack backend: Connection timed out",
+        ):
+            parse_streaming_response(mock_response)
+
     def test_tool_call_parsing(self):
         """Test tool call parsing functionality."""
         # Valid tool call