feat: add support for Claude agents to OpenInference mapper

src/strands_evals/mappers/constants.py

@@ -8,6 +8,7 @@
 SCOPE_LANGCHAIN_OTEL = "opentelemetry.instrumentation.langchain"
 SCOPE_OPENINFERENCE = "openinference.instrumentation.langchain"
 SCOPE_OPENINFERENCE_SMOLAGENTS = "openinference.instrumentation.smolagents"
+SCOPE_OPENINFERENCE_CLAUDE_AGENT_SDK = "openinference.instrumentation.claude_agent_sdk"
 SCOPE_ADK = "gcp.vertex.agent"
 SCOPE_STRANDS = "strands.telemetry.tracer"
 
@@ -16,6 +17,7 @@
     [
         SCOPE_OPENINFERENCE,
         SCOPE_OPENINFERENCE_SMOLAGENTS,
+        SCOPE_OPENINFERENCE_CLAUDE_AGENT_SDK,
     ]
 )
 

src/strands_evals/mappers/openinference_session_mapper.py

@@ -4,6 +4,7 @@
 Handles traces from any producer in the OpenInference family:
 - openinference.instrumentation.langchain (LangChain / LangGraph)
 - openinference.instrumentation.smolagents (HuggingFace smolagents)
+- openinference.instrumentation.claude_agent_sdk (Claude Agent SDK)
 
 Each producer emits spans following the OpenInference semantic conventions but
 with producer-specific encoding differences (e.g. attribute paths for message
@@ -34,28 +35,47 @@
     Trace,
     UserMessage,
 )
-from .constants import SCOPE_OPENINFERENCE_SMOLAGENTS, SCOPES_OPENINFERENCE_FAMILY
+from .constants import (
+    SCOPE_OPENINFERENCE,
+    SCOPE_OPENINFERENCE_CLAUDE_AGENT_SDK,
+    SCOPE_OPENINFERENCE_SMOLAGENTS,
+    SCOPES_OPENINFERENCE_FAMILY,
+)
 from .session_mapper import SessionMapper
 from .utils import safe_json_parse
 
 logger = logging.getLogger(__name__)
 
+_LLM_METADATA_KEYS = (
+    "llm.model_name",
+    "llm.token_count.prompt",
+    "llm.token_count.completion",
+    "llm.token_count.total",
+)
+
 
 class OpenInferenceSessionMapper(SessionMapper):
     """Maps OpenInference traces to Session format.
 
     This mapper handles traces produced by any library in the OpenInference family:
     - openinference-instrumentation-langchain (LangChain / LangGraph)
     - openinference-instrumentation-smolagents (HuggingFace smolagents)
+    - openinference-instrumentation-claude-agent-sdk (Claude Agent SDK)
 
     Span type identification uses the openinference.span.kind attribute:
     - Inference spans: "LLM"
     - Tool execution spans: "TOOL"
-    - Agent invocation spans: "AGENT" (smolagents CodeAgent.run) or
+    - Agent invocation spans: "AGENT" (smolagents CodeAgent.run, Claude Agent SDK query) or
       "CHAIN" with name="LangGraph" (LangGraph root graph)
 
     Producer-specific encoding differences (e.g. message attribute paths,
     tool argument wrapping) are normalized before shared conversion logic runs.
+
+    Note: Claude Agent SDK instrumentation never emits kind="LLM" spans, so
+    Claude sessions produce only AgentInvocationSpan + ToolExecutionSpan (no
+    InferenceSpans). Claude's attribute layout (`message.content.0`) also
+    differs from what `_extract_assistant_from_live_attrs` reads, so adding
+    LLM extraction for Claude would require a dedicated normalization step.
     """
 
     def __init__(self):
@@ -256,7 +276,8 @@ def _build_trace(self, trace_id: str, spans: list[dict], session_id: str) -> Tra
         # In multi-agent LangGraph systems, each nested sub-graph produces its own
         # LangGraph CHAIN span. Keep only the last one (root graph finishes last).
         agent_spans = [s for s in converted_spans if isinstance(s, AgentInvocationSpan)]
-        if len(agent_spans) > 1:
+        is_langchain = any(self._get_scope_name(s) == SCOPE_OPENINFERENCE for s in spans)
+        if len(agent_spans) > 1 and is_langchain:
             root = agent_spans[-1]
             converted_spans = [s for s in converted_spans if not isinstance(s, AgentInvocationSpan) or s is root]
 
@@ -327,7 +348,9 @@ def _is_agent_invocation_span(self, span: dict) -> bool:
 
         Detection:
         1. Live instrumentation (LangGraph): CHAIN + name=LangGraph
-        2. Live instrumentation (smolagents): AGENT span kind
+        2. Live instrumentation (smolagents/Claude Agent SDK): AGENT span kind
+           from a known scope, with input.value present and either output.value
+           present or status.code == ERROR.
         3. ADOT body: root LangGraph graph node — input has "messages" without
            "remaining_steps" (intermediate nodes always have "remaining_steps"),
            and output has "messages".
@@ -340,19 +363,20 @@ def _is_agent_invocation_span(self, span: dict) -> bool:
         if span_kind == "CHAIN" and span_name == "LangGraph":
             return True
 
-        # smolagents: AGENT span (e.g. CodeAgent.run) — only accept spans from
-        # the smolagents scope to avoid matching LangChain's route_to_agent spans
-        # which also have kind=AGENT with both input and output values.
+        # Only accept AGENT spans from scopes known to produce real agent
+        # invocations. Other scopes (e.g. LangChain) emit kind=AGENT for
+        # routing nodes that aren't true agent invocations — reject those by default.
         if span_kind == "AGENT":
             scope_name = self._get_scope_name(span)
-            if scope_name == SCOPE_OPENINFERENCE_SMOLAGENTS:
+            if scope_name in (SCOPE_OPENINFERENCE_SMOLAGENTS, SCOPE_OPENINFERENCE_CLAUDE_AGENT_SDK):
                 input_val = attrs.get("input.value")
-                output_val = attrs.get("output.value")
-                if input_val and output_val:
-                    return True
-            # LangChain AGENT spans (e.g. route_to_agent) carry kind=AGENT with
-            # input/output but are routing nodes, not agent invocations. Explicitly
-            # reject any non-smolagents AGENT span.
+                if input_val:
+                    output_val = attrs.get("output.value")
+                    if output_val:
+                        return True
+                    span_status = span.get("status") or {}
+                    if isinstance(span_status, dict) and span_status.get("code") == "ERROR":
+                        return True
             return False
 
         # ADOT fallback: root LangGraph node has messages in/out but no remaining_steps.
@@ -431,6 +455,8 @@ def _convert_tool_execution_span(self, span: dict, session_id: str) -> ToolExecu
             if span_name and span_name not in SCOPES_OPENINFERENCE_FAMILY:
                 tool_name = span_name
 
+        tool_call_id = attrs.get("tool.id")
+
         # Get input from attributes
         input_value = attrs.get("input.value")
         if input_value:
@@ -455,9 +481,19 @@ def _convert_tool_execution_span(self, span: dict, session_id: str) -> ToolExecu
                 except json.JSONDecodeError:
                     parsed = None
                 if isinstance(parsed, dict):
-                    tool_output_content = parsed.get("content", str(parsed))
-                    tool_call_id = parsed.get("tool_call_id")
+                    raw_content = parsed.get("content")
+                    if isinstance(raw_content, list):
+                        tool_output_content = self._flatten_content_blocks(raw_content)
+                    elif isinstance(raw_content, str):
+                        tool_output_content = raw_content
+                    else:
+                        tool_output_content = json.dumps(parsed, ensure_ascii=False)
+                    tool_call_id = parsed.get("tool_call_id") or tool_call_id
                     tool_status = parsed.get("status", "success")
+                elif isinstance(parsed, str):
+                    tool_output_content = parsed
+                elif isinstance(parsed, list):
+                    tool_output_content = self._flatten_content_blocks(parsed)
                 else:
                     tool_output_content = output_value
             elif isinstance(output_value, dict):
@@ -499,6 +535,21 @@ def _convert_tool_execution_span(self, span: dict, session_id: str) -> ToolExecu
                         except json.JSONDecodeError:
                             pass
 
+        # For failed tool calls (e.g. Claude Agent SDK sets status=ERROR with no output.value),
+        # preserve the span with an error message so judges see the failure.
+        if tool_output_content is None:
+            span_status = span.get("status") or {}
+            if isinstance(span_status, dict) and span_status.get("code") == "ERROR":
+                raw_error = span_status.get("description") or self._exception_message(span) or "error"
+                # Try to flatten content blocks if the error is a JSON-encoded block list
+                try:
+                    parsed_error = json.loads(raw_error)
+                except (ValueError, TypeError, RecursionError):
+                    parsed_error = None
+                flattened = self._flatten_content_blocks(parsed_error)
+                tool_output_content = flattened or raw_error
+                tool_status = "error"
+
         # Validate required fields
         if not tool_name or tool_parameters is None or tool_output_content is None:
             logger.warning(f"Missing required fields for tool span {span.get('span_id')}")
@@ -507,7 +558,8 @@ def _convert_tool_execution_span(self, span: dict, session_id: str) -> ToolExecu
         tool_call = ToolCall(name=tool_name, arguments=tool_parameters or {}, tool_call_id=tool_call_id)
         tool_result = ToolResult(
             content=tool_output_content or "",
-            error=None if tool_status == "success" else tool_status,
+            # "success" = smolagents/LangChain default; "completed" = Claude Agent SDK
+            error=None if tool_status in ("success", "completed") else tool_status,
             tool_call_id=tool_call_id,
         )
 
@@ -516,9 +568,10 @@ def _convert_tool_execution_span(self, span: dict, session_id: str) -> ToolExecu
     def _convert_agent_invocation_span(self, span: dict, session_id: str) -> AgentInvocationSpan | None:
         """Convert OTEL span to AgentInvocationSpan.
 
-        Handles two formats:
-        - LangGraph: structured messages in span_events body
-        - smolagents: input.value (user task) and output.value (final answer) as plain strings
+        Handles three producer formats via two code paths:
+        - LangGraph: structured messages in span_events body (span_events path)
+        - smolagents: input.value as JSON with "task" key, output.value as final answer (attrs path)
+        - Claude Agent SDK: input.value as plain text prompt, output.value as plain text response (attrs path)
         """
         span_info = self._create_span_info(span, session_id)
         trace_id = span.get("trace_id", "")
@@ -527,22 +580,21 @@ def _convert_agent_invocation_span(self, span: dict, session_id: str) -> AgentIn
         user_prompt: str | None = None
         agent_response: str | None = None
 
-        # smolagents AGENT spans: input.value is a JSON object with "task" field,
-        # output.value is the final response string
         span_kind = attrs.get("openinference.span.kind", "")
         if span_kind == "AGENT":
             input_value = attrs.get("input.value", "")
             output_value = attrs.get("output.value", "")
+
             if isinstance(input_value, str) and input_value:
+                user_prompt = input_value
                 # smolagents wraps user task in: {"task": "...", "stream": ..., ...}
-                try:
-                    parsed_input = json.loads(input_value)
-                    if isinstance(parsed_input, dict) and "task" in parsed_input:
-                        user_prompt = parsed_input["task"]
-                    else:
-                        user_prompt = input_value
-                except (json.JSONDecodeError, TypeError):
-                    user_prompt = input_value
+                if self._get_scope_name(span) == SCOPE_OPENINFERENCE_SMOLAGENTS:
+                    try:
+                        parsed_input = json.loads(input_value)
+                        if isinstance(parsed_input, dict) and "task" in parsed_input:
+                            user_prompt = parsed_input["task"]
+                    except (json.JSONDecodeError, TypeError):
+                        pass
             if isinstance(output_value, str) and output_value:
                 agent_response = output_value
 
@@ -558,6 +610,12 @@ def _convert_agent_invocation_span(self, span: dict, session_id: str) -> AgentIn
             logger.warning(f"No user_prompt for agent span {span.get('span_id')}")
             return None
 
+        # Surface exception message on errored agent spans so judges see the failure.
+        if not agent_response:
+            span_status = span.get("status") or {}
+            if isinstance(span_status, dict) and span_status.get("code") == "ERROR":
+                agent_response = span_status.get("description") or self._exception_message(span) or "error"
+
         if not agent_response:
             logger.warning(f"No agent_response for agent span {span.get('span_id')}")
             return None
@@ -567,18 +625,31 @@ def _convert_agent_invocation_span(self, span: dict, session_id: str) -> AgentIn
             key=lambda t: t.name,
         )
 
+        # Extract token counts, cost, and model name from attributes
+        metadata = self._extract_llm_metadata(attrs)
+
         return AgentInvocationSpan(
             span_info=span_info,
             user_prompt=user_prompt,
             agent_response=agent_response,
             available_tools=available_tools,
-            metadata={},
+            metadata=metadata,
         )
 
     # =========================================================================
     # Helper Methods
     # =========================================================================
 
+    @staticmethod
+    def _extract_llm_metadata(attrs: dict) -> dict:
+        """Extract token counts and model name from span attributes into metadata."""
+        metadata: dict = {}
+        for key in _LLM_METADATA_KEYS:
+            value = attrs.get(key)
+            if value is not None:
+                metadata[key] = value
+        return metadata
+
     @staticmethod
     def _collect_tools_from_spans(
         converted_spans: list[InferenceSpan | ToolExecutionSpan | AgentInvocationSpan],
@@ -597,6 +668,33 @@ def _get_scope_name(self, span: dict) -> str:
         scope = span.get("scope", {})
         return scope.get("name", "") if isinstance(scope, dict) else ""
 
+    @staticmethod
+    def _flatten_content_blocks(raw: object) -> str | None:
+        """Join text from a list of content blocks; None if not a block list."""
+        if not isinstance(raw, list):
+            return None
+        texts = [b["text"] for b in raw if isinstance(b, dict) and isinstance(b.get("text"), str) and b["text"]]
+        if texts:
+            return "\n".join(texts)
+        if all(isinstance(b, dict) and isinstance(b.get("text"), str) for b in raw) and raw:
+            return ""
+        return json.dumps(raw, ensure_ascii=False)
+
+    @staticmethod
+    def _exception_message(span: dict) -> str | None:
+        """Extract the first exception.message from span events, if any."""
+        for event in span.get("span_events") or []:
+            if not isinstance(event, dict):
+                continue
+            if event.get("event_name") == "exception":
+                attributes = event.get("attributes")
+                if not isinstance(attributes, dict):
+                    continue
+                msg = attributes.get("exception.message")
+                if msg:
+                    return str(msg)
+        return None
+
     def _create_span_info(self, span: dict, session_id: str) -> SpanInfo:
         """Create SpanInfo from span dict."""
         start_time = self.parse_timestamp(span.get("start_time"))