diff --git a/src/strands_evals/mappers/constants.py b/src/strands_evals/mappers/constants.py index 13086e34..4bee5893 100644 --- a/src/strands_evals/mappers/constants.py +++ b/src/strands_evals/mappers/constants.py @@ -8,6 +8,7 @@ SCOPE_LANGCHAIN_OTEL = "opentelemetry.instrumentation.langchain" SCOPE_OPENINFERENCE = "openinference.instrumentation.langchain" SCOPE_OPENINFERENCE_SMOLAGENTS = "openinference.instrumentation.smolagents" +SCOPE_OPENINFERENCE_CLAUDE_AGENT_SDK = "openinference.instrumentation.claude_agent_sdk" SCOPE_ADK = "gcp.vertex.agent" SCOPE_STRANDS = "strands.telemetry.tracer" @@ -16,6 +17,7 @@ [ SCOPE_OPENINFERENCE, SCOPE_OPENINFERENCE_SMOLAGENTS, + SCOPE_OPENINFERENCE_CLAUDE_AGENT_SDK, ] ) diff --git a/src/strands_evals/mappers/openinference_session_mapper.py b/src/strands_evals/mappers/openinference_session_mapper.py index 0890612d..783492b2 100644 --- a/src/strands_evals/mappers/openinference_session_mapper.py +++ b/src/strands_evals/mappers/openinference_session_mapper.py @@ -4,6 +4,7 @@ Handles traces from any producer in the OpenInference family: - openinference.instrumentation.langchain (LangChain / LangGraph) - openinference.instrumentation.smolagents (HuggingFace smolagents) +- openinference.instrumentation.claude_agent_sdk (Claude Agent SDK) Each producer emits spans following the OpenInference semantic conventions but with producer-specific encoding differences (e.g. attribute paths for message @@ -34,12 +35,24 @@ Trace, UserMessage, ) -from .constants import SCOPE_OPENINFERENCE_SMOLAGENTS, SCOPES_OPENINFERENCE_FAMILY +from .constants import ( + SCOPE_OPENINFERENCE, + SCOPE_OPENINFERENCE_CLAUDE_AGENT_SDK, + SCOPE_OPENINFERENCE_SMOLAGENTS, + SCOPES_OPENINFERENCE_FAMILY, +) from .session_mapper import SessionMapper from .utils import safe_json_parse logger = logging.getLogger(__name__) +_LLM_METADATA_KEYS = ( + "llm.model_name", + "llm.token_count.prompt", + "llm.token_count.completion", + "llm.token_count.total", +) + class OpenInferenceSessionMapper(SessionMapper): """Maps OpenInference traces to Session format. @@ -47,15 +60,22 @@ class OpenInferenceSessionMapper(SessionMapper): This mapper handles traces produced by any library in the OpenInference family: - openinference-instrumentation-langchain (LangChain / LangGraph) - openinference-instrumentation-smolagents (HuggingFace smolagents) + - openinference-instrumentation-claude-agent-sdk (Claude Agent SDK) Span type identification uses the openinference.span.kind attribute: - Inference spans: "LLM" - Tool execution spans: "TOOL" - - Agent invocation spans: "AGENT" (smolagents CodeAgent.run) or + - Agent invocation spans: "AGENT" (smolagents CodeAgent.run, Claude Agent SDK query) or "CHAIN" with name="LangGraph" (LangGraph root graph) Producer-specific encoding differences (e.g. message attribute paths, tool argument wrapping) are normalized before shared conversion logic runs. + + Note: Claude Agent SDK instrumentation never emits kind="LLM" spans, so + Claude sessions produce only AgentInvocationSpan + ToolExecutionSpan (no + InferenceSpans). Claude's attribute layout (`message.content.0`) also + differs from what `_extract_assistant_from_live_attrs` reads, so adding + LLM extraction for Claude would require a dedicated normalization step. """ def __init__(self): @@ -256,7 +276,8 @@ def _build_trace(self, trace_id: str, spans: list[dict], session_id: str) -> Tra # In multi-agent LangGraph systems, each nested sub-graph produces its own # LangGraph CHAIN span. Keep only the last one (root graph finishes last). agent_spans = [s for s in converted_spans if isinstance(s, AgentInvocationSpan)] - if len(agent_spans) > 1: + is_langchain = any(self._get_scope_name(s) == SCOPE_OPENINFERENCE for s in spans) + if len(agent_spans) > 1 and is_langchain: root = agent_spans[-1] converted_spans = [s for s in converted_spans if not isinstance(s, AgentInvocationSpan) or s is root] @@ -327,7 +348,9 @@ def _is_agent_invocation_span(self, span: dict) -> bool: Detection: 1. Live instrumentation (LangGraph): CHAIN + name=LangGraph - 2. Live instrumentation (smolagents): AGENT span kind + 2. Live instrumentation (smolagents/Claude Agent SDK): AGENT span kind + from a known scope, with input.value present and either output.value + present or status.code == ERROR. 3. ADOT body: root LangGraph graph node — input has "messages" without "remaining_steps" (intermediate nodes always have "remaining_steps"), and output has "messages". @@ -340,19 +363,20 @@ def _is_agent_invocation_span(self, span: dict) -> bool: if span_kind == "CHAIN" and span_name == "LangGraph": return True - # smolagents: AGENT span (e.g. CodeAgent.run) — only accept spans from - # the smolagents scope to avoid matching LangChain's route_to_agent spans - # which also have kind=AGENT with both input and output values. + # Only accept AGENT spans from scopes known to produce real agent + # invocations. Other scopes (e.g. LangChain) emit kind=AGENT for + # routing nodes that aren't true agent invocations — reject those by default. if span_kind == "AGENT": scope_name = self._get_scope_name(span) - if scope_name == SCOPE_OPENINFERENCE_SMOLAGENTS: + if scope_name in (SCOPE_OPENINFERENCE_SMOLAGENTS, SCOPE_OPENINFERENCE_CLAUDE_AGENT_SDK): input_val = attrs.get("input.value") - output_val = attrs.get("output.value") - if input_val and output_val: - return True - # LangChain AGENT spans (e.g. route_to_agent) carry kind=AGENT with - # input/output but are routing nodes, not agent invocations. Explicitly - # reject any non-smolagents AGENT span. + if input_val: + output_val = attrs.get("output.value") + if output_val: + return True + span_status = span.get("status") or {} + if isinstance(span_status, dict) and span_status.get("code") == "ERROR": + return True return False # ADOT fallback: root LangGraph node has messages in/out but no remaining_steps. @@ -431,6 +455,8 @@ def _convert_tool_execution_span(self, span: dict, session_id: str) -> ToolExecu if span_name and span_name not in SCOPES_OPENINFERENCE_FAMILY: tool_name = span_name + tool_call_id = attrs.get("tool.id") + # Get input from attributes input_value = attrs.get("input.value") if input_value: @@ -455,9 +481,19 @@ def _convert_tool_execution_span(self, span: dict, session_id: str) -> ToolExecu except json.JSONDecodeError: parsed = None if isinstance(parsed, dict): - tool_output_content = parsed.get("content", str(parsed)) - tool_call_id = parsed.get("tool_call_id") + raw_content = parsed.get("content") + if isinstance(raw_content, list): + tool_output_content = self._flatten_content_blocks(raw_content) + elif isinstance(raw_content, str): + tool_output_content = raw_content + else: + tool_output_content = json.dumps(parsed, ensure_ascii=False) + tool_call_id = parsed.get("tool_call_id") or tool_call_id tool_status = parsed.get("status", "success") + elif isinstance(parsed, str): + tool_output_content = parsed + elif isinstance(parsed, list): + tool_output_content = self._flatten_content_blocks(parsed) else: tool_output_content = output_value elif isinstance(output_value, dict): @@ -499,6 +535,21 @@ def _convert_tool_execution_span(self, span: dict, session_id: str) -> ToolExecu except json.JSONDecodeError: pass + # For failed tool calls (e.g. Claude Agent SDK sets status=ERROR with no output.value), + # preserve the span with an error message so judges see the failure. + if tool_output_content is None: + span_status = span.get("status") or {} + if isinstance(span_status, dict) and span_status.get("code") == "ERROR": + raw_error = span_status.get("description") or self._exception_message(span) or "error" + # Try to flatten content blocks if the error is a JSON-encoded block list + try: + parsed_error = json.loads(raw_error) + except (ValueError, TypeError, RecursionError): + parsed_error = None + flattened = self._flatten_content_blocks(parsed_error) + tool_output_content = flattened or raw_error + tool_status = "error" + # Validate required fields if not tool_name or tool_parameters is None or tool_output_content is None: logger.warning(f"Missing required fields for tool span {span.get('span_id')}") @@ -507,7 +558,8 @@ def _convert_tool_execution_span(self, span: dict, session_id: str) -> ToolExecu tool_call = ToolCall(name=tool_name, arguments=tool_parameters or {}, tool_call_id=tool_call_id) tool_result = ToolResult( content=tool_output_content or "", - error=None if tool_status == "success" else tool_status, + # "success" = smolagents/LangChain default; "completed" = Claude Agent SDK + error=None if tool_status in ("success", "completed") else tool_status, tool_call_id=tool_call_id, ) @@ -516,9 +568,10 @@ def _convert_tool_execution_span(self, span: dict, session_id: str) -> ToolExecu def _convert_agent_invocation_span(self, span: dict, session_id: str) -> AgentInvocationSpan | None: """Convert OTEL span to AgentInvocationSpan. - Handles two formats: - - LangGraph: structured messages in span_events body - - smolagents: input.value (user task) and output.value (final answer) as plain strings + Handles three producer formats via two code paths: + - LangGraph: structured messages in span_events body (span_events path) + - smolagents: input.value as JSON with "task" key, output.value as final answer (attrs path) + - Claude Agent SDK: input.value as plain text prompt, output.value as plain text response (attrs path) """ span_info = self._create_span_info(span, session_id) trace_id = span.get("trace_id", "") @@ -527,22 +580,21 @@ def _convert_agent_invocation_span(self, span: dict, session_id: str) -> AgentIn user_prompt: str | None = None agent_response: str | None = None - # smolagents AGENT spans: input.value is a JSON object with "task" field, - # output.value is the final response string span_kind = attrs.get("openinference.span.kind", "") if span_kind == "AGENT": input_value = attrs.get("input.value", "") output_value = attrs.get("output.value", "") + if isinstance(input_value, str) and input_value: + user_prompt = input_value # smolagents wraps user task in: {"task": "...", "stream": ..., ...} - try: - parsed_input = json.loads(input_value) - if isinstance(parsed_input, dict) and "task" in parsed_input: - user_prompt = parsed_input["task"] - else: - user_prompt = input_value - except (json.JSONDecodeError, TypeError): - user_prompt = input_value + if self._get_scope_name(span) == SCOPE_OPENINFERENCE_SMOLAGENTS: + try: + parsed_input = json.loads(input_value) + if isinstance(parsed_input, dict) and "task" in parsed_input: + user_prompt = parsed_input["task"] + except (json.JSONDecodeError, TypeError): + pass if isinstance(output_value, str) and output_value: agent_response = output_value @@ -558,6 +610,12 @@ def _convert_agent_invocation_span(self, span: dict, session_id: str) -> AgentIn logger.warning(f"No user_prompt for agent span {span.get('span_id')}") return None + # Surface exception message on errored agent spans so judges see the failure. + if not agent_response: + span_status = span.get("status") or {} + if isinstance(span_status, dict) and span_status.get("code") == "ERROR": + agent_response = span_status.get("description") or self._exception_message(span) or "error" + if not agent_response: logger.warning(f"No agent_response for agent span {span.get('span_id')}") return None @@ -567,18 +625,31 @@ def _convert_agent_invocation_span(self, span: dict, session_id: str) -> AgentIn key=lambda t: t.name, ) + # Extract token counts, cost, and model name from attributes + metadata = self._extract_llm_metadata(attrs) + return AgentInvocationSpan( span_info=span_info, user_prompt=user_prompt, agent_response=agent_response, available_tools=available_tools, - metadata={}, + metadata=metadata, ) # ========================================================================= # Helper Methods # ========================================================================= + @staticmethod + def _extract_llm_metadata(attrs: dict) -> dict: + """Extract token counts and model name from span attributes into metadata.""" + metadata: dict = {} + for key in _LLM_METADATA_KEYS: + value = attrs.get(key) + if value is not None: + metadata[key] = value + return metadata + @staticmethod def _collect_tools_from_spans( converted_spans: list[InferenceSpan | ToolExecutionSpan | AgentInvocationSpan], @@ -597,6 +668,33 @@ def _get_scope_name(self, span: dict) -> str: scope = span.get("scope", {}) return scope.get("name", "") if isinstance(scope, dict) else "" + @staticmethod + def _flatten_content_blocks(raw: object) -> str | None: + """Join text from a list of content blocks; None if not a block list.""" + if not isinstance(raw, list): + return None + texts = [b["text"] for b in raw if isinstance(b, dict) and isinstance(b.get("text"), str) and b["text"]] + if texts: + return "\n".join(texts) + if all(isinstance(b, dict) and isinstance(b.get("text"), str) for b in raw) and raw: + return "" + return json.dumps(raw, ensure_ascii=False) + + @staticmethod + def _exception_message(span: dict) -> str | None: + """Extract the first exception.message from span events, if any.""" + for event in span.get("span_events") or []: + if not isinstance(event, dict): + continue + if event.get("event_name") == "exception": + attributes = event.get("attributes") + if not isinstance(attributes, dict): + continue + msg = attributes.get("exception.message") + if msg: + return str(msg) + return None + def _create_span_info(self, span: dict, session_id: str) -> SpanInfo: """Create SpanInfo from span dict.""" start_time = self.parse_timestamp(span.get("start_time")) diff --git a/tests/strands_evals/mappers/fixtures/claude_adot_spans.json b/tests/strands_evals/mappers/fixtures/claude_adot_spans.json new file mode 100644 index 00000000..611b1d22 --- /dev/null +++ b/tests/strands_evals/mappers/fixtures/claude_adot_spans.json @@ -0,0 +1,281 @@ +{ + "session_id": "e457cade-6652-42ad-b879-182dde7124c5", + "span_count": 8, + "spans": [ + { + "trace_id": "6a74fd19607936851cc91fef1506624f", + "span_id": "9ba74ad8363a1acd", + "parent_span_id": "5c2a984a75d75f78", + "name": "Bash", + "start_time": 1786051888270252114, + "end_time": 1786051888526973038, + "attributes": { + "aws.local.service": "CWExperiment_ClaudeMulti.DEFAULT", + "tool.parameters": "{\"command\": \"echo $((42 * 7))\", \"description\": \"Calculate 42 * 7\"}", + "output.value": "{\"stdout\": \"294\", \"stderr\": \"\", \"interrupted\": false, \"isImage\": false, \"noOutputExpected\": false}", + "aws.genai.span_kind": "TOOL", + "aws.local.environment": "bedrock-agentcore:default", + "input.mime_type": "application/json", + "output.mime_type": "application/json", + "input.value": "{\"command\": \"echo $((42 * 7))\", \"description\": \"Calculate 42 * 7\"}", + "tool.id": "toolu_bdrk_01SW1rX7PUer9zBaVQv9wLUa", + "openinference.span.kind": "TOOL", + "PlatformType": "AWS::BedrockAgentCore", + "session.id": "e457cade-6652-42ad-b879-182dde7124c5", + "tool.name": "Bash" + }, + "scope": { + "name": "openinference.instrumentation.claude_agent_sdk", + "version": "0.1.8" + }, + "status": { + "code": "OK" + }, + "span_events": [] + }, + { + "trace_id": "6a74fd19607936851cc91fef1506624f", + "span_id": "52c63a60a79dc3e3", + "parent_span_id": "c3d1bcc351bf9653", + "name": "Agent", + "start_time": 1786051886396129012, + "end_time": 1786051890339601282, + "attributes": { + "aws.local.service": "CWExperiment_ClaudeMulti.DEFAULT", + "tool.parameters": "{\"description\": \"Calculate 42 * 7\", \"subagent_type\": \"math-agent\", \"prompt\": \"Calculate 42 * 7 and return the result.\", \"run_in_background\": false}", + "output.value": "{\"status\": \"completed\", \"prompt\": \"Calculate 42 * 7 and return the result.\", \"agentId\": \"aa0f69f4dc9119fd0\", \"agentType\": \"math-agent\", \"content\": [{\"type\": \"text\", \"text\": \"42 * 7 = **294**\"}], \"resolvedModel\": \"us.anthropic.claude-sonnet-4-6\", \"totalDurationMs\": 3930, \"totalTokens\": 4068, \"totalToolUseCount\": 1, \"usage\": {\"input_tokens\": 1, \"cache_creation_input_tokens\": 99, \"cache_read_input_tokens\": 3955, \"output_tokens\": 13, \"server_tool_use\": {\"web_search_requests\": 0, \"web_fetch_requests\": 0}, \"service_tier\": \"standard\", \"cache_creation\": {\"ephemeral_1h_input_tokens\": 0, \"ephemeral_5m_input_tokens\": 99}, \"inference_geo\": \"\", \"iterations\": [], \"speed\": \"standard\"}, \"toolStats\": {\"readCount\": 0, \"searchCount\": 0, \"bashCount\": 1, \"editFileCount\": 0, \"linesAdded\": 0, \"linesRemoved\": 0, \"otherToolCount\": 0}}", + "aws.genai.span_kind": "TOOL", + "aws.local.environment": "bedrock-agentcore:default", + "input.mime_type": "application/json", + "output.mime_type": "application/json", + "input.value": "{\"description\": \"Calculate 42 * 7\", \"subagent_type\": \"math-agent\", \"prompt\": \"Calculate 42 * 7 and return the result.\", \"run_in_background\": false}", + "tool.id": "toolu_bdrk_013DnTxpqTDkwezwKkHizcHT", + "openinference.span.kind": "TOOL", + "PlatformType": "AWS::BedrockAgentCore", + "session.id": "e457cade-6652-42ad-b879-182dde7124c5", + "tool.name": "Agent" + }, + "scope": { + "name": "openinference.instrumentation.claude_agent_sdk", + "version": "0.1.8" + }, + "status": { + "code": "OK" + }, + "span_events": [] + }, + { + "trace_id": "6a74fd19607936851cc91fef1506624f", + "span_id": "d0706922917bdcd1", + "parent_span_id": "f120f1e65d898126", + "name": "Bash", + "start_time": 1786051890133570663, + "end_time": 1786051901406971244, + "attributes": { + "aws.local.service": "CWExperiment_ClaudeMulti.DEFAULT", + "tool.parameters": "{\"command\": \"python3 -c 'miles = 294; km = miles / 0.621371; print(f\\\"{miles} miles = {km:.4f} km\\\")'\", \"description\": \"Convert 294 miles to kilometers\"}", + "output.value": "{\"stdout\": \"Configuration of aws_configurator not loaded, configurator already loaded\\n294 miles = 473.1473 km\", \"stderr\": \"\", \"interrupted\": false, \"isImage\": false, \"noOutputExpected\": false}", + "aws.genai.span_kind": "TOOL", + "aws.local.environment": "bedrock-agentcore:default", + "input.mime_type": "application/json", + "output.mime_type": "application/json", + "input.value": "{\"command\": \"python3 -c 'miles = 294; km = miles / 0.621371; print(f\\\"{miles} miles = {km:.4f} km\\\")'\", \"description\": \"Convert 294 miles to kilometers\"}", + "tool.id": "toolu_bdrk_01T4wcJbc3MxsEgsGoQEowaa", + "openinference.span.kind": "TOOL", + "PlatformType": "AWS::BedrockAgentCore", + "session.id": "e457cade-6652-42ad-b879-182dde7124c5", + "tool.name": "Bash" + }, + "scope": { + "name": "openinference.instrumentation.claude_agent_sdk", + "version": "0.1.8" + }, + "status": { + "code": "OK" + }, + "span_events": [] + }, + { + "trace_id": "6a74fd19607936851cc91fef1506624f", + "span_id": "5399ec80b54ac27c", + "parent_span_id": "c3d1bcc351bf9653", + "name": "Agent", + "start_time": 1786051887131417419, + "end_time": 1786051902970334584, + "attributes": { + "aws.local.service": "CWExperiment_ClaudeMulti.DEFAULT", + "tool.parameters": "{\"description\": \"Convert 294 miles to kilometers\", \"subagent_type\": \"conversion-agent\", \"prompt\": \"Convert 294 miles to kilometers and return the result.\", \"run_in_background\": false}", + "output.value": "{\"status\": \"completed\", \"prompt\": \"Convert 294 miles to kilometers and return the result.\", \"agentId\": \"af64dcd2e3ee99758\", \"agentType\": \"conversion-agent\", \"content\": [{\"type\": \"text\", \"text\": \"Here is the result of the conversion:\\n\\n**294 miles = 473.1473 kilometers**\\n\\nFormula used: km = miles / 0.621371\\n\\nSpecifically: 294 / 0.621371 = 473.1473 km\"}], \"resolvedModel\": \"us.anthropic.claude-sonnet-4-6\", \"totalDurationMs\": 15836, \"totalTokens\": 4228, \"totalToolUseCount\": 1, \"usage\": {\"input_tokens\": 1, \"cache_creation_input_tokens\": 153, \"cache_read_input_tokens\": 4014, \"output_tokens\": 60, \"server_tool_use\": {\"web_search_requests\": 0, \"web_fetch_requests\": 0}, \"service_tier\": \"standard\", \"cache_creation\": {\"ephemeral_1h_input_tokens\": 0, \"ephemeral_5m_input_tokens\": 153}, \"inference_geo\": \"\", \"iterations\": [], \"speed\": \"standard\"}, \"toolStats\": {\"readCount\": 0, \"searchCount\": 0, \"bashCount\": 1, \"editFileCount\": 0, \"linesAdded\": 0, \"linesRemoved\": 0, \"otherToolCount\": 0}}", + "aws.genai.span_kind": "TOOL", + "aws.local.environment": "bedrock-agentcore:default", + "input.mime_type": "application/json", + "output.mime_type": "application/json", + "input.value": "{\"description\": \"Convert 294 miles to kilometers\", \"subagent_type\": \"conversion-agent\", \"prompt\": \"Convert 294 miles to kilometers and return the result.\", \"run_in_background\": false}", + "tool.id": "toolu_bdrk_01RQYH5cNSJ2iYbufJ8VbEk4", + "openinference.span.kind": "TOOL", + "PlatformType": "AWS::BedrockAgentCore", + "session.id": "e457cade-6652-42ad-b879-182dde7124c5", + "tool.name": "Agent" + }, + "scope": { + "name": "openinference.instrumentation.claude_agent_sdk", + "version": "0.1.8" + }, + "status": { + "code": "OK" + }, + "span_events": [] + }, + { + "trace_id": "6a74fd19607936851cc91fef1506624f", + "span_id": "f120f1e65d898126", + "parent_span_id": "5399ec80b54ac27c", + "name": "ClaudeAgentSDK.Agent", + "start_time": 1786051887137321949, + "end_time": 1786051904949534800, + "attributes": { + "aws.local.service": "CWExperiment_ClaudeMulti.DEFAULT", + "gen_ai.request.model": "claude-sonnet-4-6", + "gen_ai.agent.name": "Agent", + "agent.name": "Agent", + "aws.genai.span_kind": "AGENT", + "llm.model_name": "claude-sonnet-4-6", + "openinference.span.kind": "AGENT", + "PlatformType": "AWS::BedrockAgentCore", + "session.id": "e457cade-6652-42ad-b879-182dde7124c5", + "aws.local.environment": "bedrock-agentcore:default" + }, + "scope": { + "name": "openinference.instrumentation.claude_agent_sdk", + "version": "0.1.8" + }, + "status": { + "code": "OK" + }, + "span_events": [] + }, + { + "trace_id": "6a74fd19607936851cc91fef1506624f", + "span_id": "c3d1bcc351bf9653", + "parent_span_id": "7549860608387d0b", + "name": "ClaudeAgentSDK.query", + "start_time": 1786051873454427413, + "end_time": 1786051904949557495, + "attributes": { + "aws.local.service": "CWExperiment_ClaudeMulti.DEFAULT", + "llm.output_messages.3.message.tool_calls.0.tool_call.function.name": "Agent", + "llm.token_count.prompt_details.cache_write": 2861, + "llm.output_messages.3.message.tool_calls.0.tool_call.id": "toolu_bdrk_01RQYH5cNSJ2iYbufJ8VbEk4", + "llm.output_messages.0.message.contents.0.message_content.signature": "mock-signature", + "llm.token_count.prompt": 4, + "llm.output_messages.2.message.tool_calls.0.tool_call.function.arguments": "{\"description\": \"Calculate 42 * 7\", \"subagent_type\": \"math-agent\", \"prompt\": \"Calculate 42 * 7 and return the result.\", \"run_in_background\": false}", + "llm.model_name": "us.anthropic.claude-sonnet-4-6", + "aws.local.environment": "bedrock-agentcore:default", + "gen_ai.provider.name": "anthropic", + "llm.output_messages.0.message.contents.0.message_content.type": "reasoning", + "llm.output_messages.1.message.role": "assistant", + "output.mime_type": "text/plain", + "llm.output_messages.2.message.role": "assistant", + "llm.output_messages.4.message.role": "assistant", + "gen_ai.request.model": "us.anthropic.claude-sonnet-4-6", + "llm.output_messages.3.message.role": "assistant", + "openinference.span.kind": "AGENT", + "llm.output_messages.0.message.contents.0.message_content.text": "The user wants two things:\n1. Calculate 42 * 7 (math calculation \u2192 math-agent)\n2. Convert the result to kilometers (unit conversion \u2192 conversion-agent)\n\nHowever, these are dependent - I need the result of 42 * 7 first before I can convert it to kilometers. Wait, actually 42 * 7 = 294, and then I need to convert 294 miles to kilometers.\n\nBut I don't know the result yet (I should delegate to math-agent). However, I can reason: these are sequential - first get the math result, then do the conversion. But actually, I know that 42 * 7 = 294, so I could run both in parallel by pre-computing... but no, I should delegate properly.\nI'm going back and forth on whether to run these sequentially or in parallel. Since the conversion depends on the math result, the proper approach is to delegate to math-agent first, then pass that result to conversion-agent. I could theoretically run them in parallel since I can compute 42 * 7 = 294 myself, but I should follow the delegation pattern and execute them in order.", + "llm.output_messages.2.message.tool_calls.0.tool_call.id": "toolu_bdrk_013DnTxpqTDkwezwKkHizcHT", + "llm.system": "anthropic", + "gen_ai.usage.output_tokens": 767, + "output.value": "Here are your results:\n\n1. **42 \u00d7 7 = 294**\n2. **294 miles = 473.15 kilometers**\n\nSo the answer to your multiplication is **294**, and that distance in kilometers is approximately **473.15 km**.", + "llm.output_messages.2.message.tool_calls.0.tool_call.function.name": "Agent", + "aws.genai.span_kind": "AGENT", + "llm.token_count.completion": 767, + "llm.output_messages.0.message.role": "assistant", + "llm.token_count.total": 771, + "llm.cost.total": 0.0712653, + "input.mime_type": "text/plain", + "llm.token_count.prompt_details.cache_read": 29447, + "gen_ai.usage.input_tokens": 4, + "aws.genai.token_count_total": 771, + "input.value": "What is 42 * 7, and how many miles is that in kilometers?", + "llm.output_messages.4.message.content.0": "Here are your results:\n\n1. **42 \u00d7 7 = 294**\n2. **294 miles = 473.15 kilometers**\n\nSo the answer to your multiplication is **294**, and that distance in kilometers is approximately **473.15 km**.", + "PlatformType": "AWS::BedrockAgentCore", + "session.id": "e457cade-6652-42ad-b879-182dde7124c5", + "llm.output_messages.1.message.content.0": "I'll delegate both tasks simultaneously \u2014 the math to **math-agent** and the unit conversion to **conversion-agent** (since 42 \u00d7 7 = 294, I can kick off both in parallel)!", + "llm.output_messages.3.message.tool_calls.0.tool_call.function.arguments": "{\"description\": \"Convert 294 miles to kilometers\", \"subagent_type\": \"conversion-agent\", \"prompt\": \"Convert 294 miles to kilometers and return the result.\", \"run_in_background\": false}" + }, + "scope": { + "name": "openinference.instrumentation.claude_agent_sdk", + "version": "0.1.8" + }, + "status": { + "code": "OK" + }, + "span_events": [] + }, + { + "trace_id": "6a74fd19607936851cc91fef1506624f", + "span_id": "5c2a984a75d75f78", + "parent_span_id": "52c63a60a79dc3e3", + "name": "ClaudeAgentSDK.Agent", + "start_time": 1786051886415461604, + "end_time": 1786051904949477781, + "attributes": { + "aws.local.service": "CWExperiment_ClaudeMulti.DEFAULT", + "gen_ai.request.model": "claude-sonnet-4-6", + "gen_ai.agent.name": "Agent", + "agent.name": "Agent", + "aws.genai.span_kind": "AGENT", + "llm.model_name": "claude-sonnet-4-6", + "openinference.span.kind": "AGENT", + "PlatformType": "AWS::BedrockAgentCore", + "session.id": "e457cade-6652-42ad-b879-182dde7124c5", + "aws.local.environment": "bedrock-agentcore:default" + }, + "scope": { + "name": "openinference.instrumentation.claude_agent_sdk", + "version": "0.1.8" + }, + "status": { + "code": "OK" + }, + "span_events": [] + }, + { + "trace_id": "6a74fd19607936851cc91fef1506624f", + "span_id": "7549860608387d0b", + "parent_span_id": "0ddd1d7a4995eb19", + "name": "POST /invocations", + "start_time": 1786051873332529993, + "end_time": 1786051904950868523, + "attributes": { + "aws.local.service": "CWExperiment_ClaudeMulti.DEFAULT", + "net.peer.port": 55494, + "telemetry.extended": "true", + "http.target": "/invocations", + "http.flavor": "1.1", + "http.url": "http://cell01.us-east-1.prod.arp.kepler-analytics.aws.dev/invocations", + "net.peer.ip": "127.0.0.1", + "http.host": "127.0.0.1:8080", + "aws.local.environment": "bedrock-agentcore:default", + "http.status_code": 200, + "aws.local.operation": "POST /invocations", + "aws.span.kind": "SERVER", + "http.server_name": "cell01.us-east-1.prod.arp.kepler-analytics.aws.dev", + "net.host.port": 8080, + "http.route": "/invocations", + "PlatformType": "AWS::BedrockAgentCore", + "http.method": "POST", + "http.response.status_code": 200, + "session.id": "e457cade-6652-42ad-b879-182dde7124c5", + "http.scheme": "http" + }, + "scope": { + "name": "opentelemetry.instrumentation.starlette", + "version": "0.65b0" + }, + "status": { + "code": "UNSET" + }, + "span_events": [] + } + ] +} \ No newline at end of file diff --git a/tests/strands_evals/mappers/fixtures/claude_live_spans.json b/tests/strands_evals/mappers/fixtures/claude_live_spans.json new file mode 100644 index 00000000..d86a5d21 --- /dev/null +++ b/tests/strands_evals/mappers/fixtures/claude_live_spans.json @@ -0,0 +1,268 @@ +[ + { + "trace_id": "4b0b961d81a720593197cd65f829ad3f", + "span_id": "1caa50d4f4973488", + "parent_span_id": "053c311b792d91c6", + "name": "Bash", + "start_time": 1785532828134675000, + "end_time": 1785532830265870000, + "attributes": { + "tool.id": "toolu_mock_ny_weather", + "tool.name": "Bash", + "tool.parameters": "{\"command\": \"echo \\\"Location: New York City, NY\\nTemperature: 89°F\\nConditions: Partly cloudy\\\"\"}", + "input.value": "{\"command\": \"echo \\\"Location: New York City, NY\\nTemperature: 89°F\\nConditions: Partly cloudy\\\"\"}", + "input.mime_type": "application/json", + "output.value": "{\"stdout\": \"Location: New York City, NY\\nTemperature: 89°F\\nConditions: Partly cloudy\", \"stderr\": \"\", \"interrupted\": false}", + "output.mime_type": "application/json", + "openinference.span.kind": "TOOL" + }, + "scope": { + "name": "openinference.instrumentation.claude_agent_sdk", + "version": "0.1.7" + }, + "status": { + "code": "OK" + }, + "span_events": [] + }, + { + "trace_id": "4b0b961d81a720593197cd65f829ad3f", + "span_id": "0030c035f039e139", + "parent_span_id": "a6e166c4a06cc015", + "name": "Bash", + "start_time": 1785532830345624000, + "end_time": 1785532830379943000, + "attributes": { + "tool.id": "toolu_mock_sea_weather", + "tool.name": "Bash", + "tool.parameters": "{\"command\": \"echo \\\"Location: Seattle, WA\\nTemperature: 72°F\\nConditions: Partly cloudy\\\"\"}", + "input.value": "{\"command\": \"echo \\\"Location: Seattle, WA\\nTemperature: 72°F\\nConditions: Partly cloudy\\\"\"}", + "input.mime_type": "application/json", + "output.value": "{\"stdout\": \"Location: Seattle, WA\\nTemperature: 72°F\\nConditions: Partly cloudy\", \"stderr\": \"\", \"interrupted\": false}", + "output.mime_type": "application/json", + "openinference.span.kind": "TOOL" + }, + "scope": { + "name": "openinference.instrumentation.claude_agent_sdk", + "version": "0.1.7" + }, + "status": { + "code": "OK" + }, + "span_events": [] + }, + { + "trace_id": "4b0b961d81a720593197cd65f829ad3f", + "span_id": "1c51a9dac98b8f20", + "parent_span_id": "ebdd7630e6537a19", + "name": "Agent", + "start_time": 1785532825592072000, + "end_time": 1785532831236592000, + "attributes": { + "tool.id": "toolu_mock_agent_seattle", + "tool.name": "Agent", + "tool.parameters": "{\"description\": \"Get current weather in Seattle\", \"subagent_type\": \"research-specialist\", \"prompt\": \"What is the current temperature in Seattle, Washington?\", \"run_in_background\": false}", + "input.value": "{\"description\": \"Get current weather in Seattle\", \"subagent_type\": \"research-specialist\", \"prompt\": \"What is the current temperature in Seattle, Washington?\", \"run_in_background\": false}", + "input.mime_type": "application/json", + "output.value": "{\"status\": \"completed\", \"prompt\": \"What is the current temperature in Seattle, Washington?\", \"agentId\": \"mock-agent-001\", \"agentType\": \"research-specialist\", \"content\": [{\"type\": \"text\", \"text\": \"72°F, partly cloudy\"}], \"resolvedModel\": \"us.anthropic.claude-sonnet-4-6\", \"totalDurationMs\": 5637, \"totalTokens\": 4542, \"totalToolUseCount\": 1}", + "output.mime_type": "application/json", + "openinference.span.kind": "TOOL" + }, + "scope": { + "name": "openinference.instrumentation.claude_agent_sdk", + "version": "0.1.7" + }, + "status": { + "code": "OK" + }, + "span_events": [] + }, + { + "trace_id": "4b0b961d81a720593197cd65f829ad3f", + "span_id": "51e6bd914525fee4", + "parent_span_id": "ebdd7630e6537a19", + "name": "Agent", + "start_time": 1785532824627161000, + "end_time": 1785532831501258000, + "attributes": { + "tool.id": "toolu_mock_agent_ny", + "tool.name": "Agent", + "tool.parameters": "{\"description\": \"Get current weather in New York\", \"subagent_type\": \"research-specialist\", \"prompt\": \"What is the current temperature in New York City?\", \"run_in_background\": false}", + "input.value": "{\"description\": \"Get current weather in New York\", \"subagent_type\": \"research-specialist\", \"prompt\": \"What is the current temperature in New York City?\", \"run_in_background\": false}", + "input.mime_type": "application/json", + "output.value": "{\"status\": \"completed\", \"prompt\": \"What is the current temperature in New York City?\", \"agentId\": \"mock-agent-002\", \"agentType\": \"research-specialist\", \"content\": [{\"type\": \"text\", \"text\": \"89°F, partly cloudy with high humidity\"}], \"resolvedModel\": \"us.anthropic.claude-sonnet-4-6\", \"totalDurationMs\": 6860, \"totalTokens\": 4472, \"totalToolUseCount\": 1}", + "output.mime_type": "application/json", + "openinference.span.kind": "TOOL" + }, + "scope": { + "name": "openinference.instrumentation.claude_agent_sdk", + "version": "0.1.7" + }, + "status": { + "code": "OK" + }, + "span_events": [] + }, + { + "trace_id": "4b0b961d81a720593197cd65f829ad3f", + "span_id": "b29670c3fb945a41", + "parent_span_id": "f028c72828771e95", + "name": "Bash", + "start_time": 1785532836023907000, + "end_time": 1785532836140075000, + "attributes": { + "tool.id": "toolu_mock_calc", + "tool.name": "Bash", + "tool.parameters": "{\"command\": \"python3 -c 'print(89 - 72)'\", \"description\": \"Calculate temperature difference\"}", + "input.value": "{\"command\": \"python3 -c 'print(89 - 72)'\", \"description\": \"Calculate temperature difference\"}", + "input.mime_type": "application/json", + "output.value": "{\"stdout\": \"17\", \"stderr\": \"\", \"interrupted\": false}", + "output.mime_type": "application/json", + "openinference.span.kind": "TOOL" + }, + "scope": { + "name": "openinference.instrumentation.claude_agent_sdk", + "version": "0.1.7" + }, + "status": { + "code": "OK" + }, + "span_events": [] + }, + { + "trace_id": "4b0b961d81a720593197cd65f829ad3f", + "span_id": "a6e405461983ccf4", + "parent_span_id": "ebdd7630e6537a19", + "name": "Agent", + "start_time": 1785532834085541000, + "end_time": 1785532837059862000, + "attributes": { + "tool.id": "toolu_mock_agent_math", + "tool.name": "Agent", + "tool.parameters": "{\"description\": \"Calculate temperature difference\", \"subagent_type\": \"math-specialist\", \"prompt\": \"Calculate 89 minus 72.\", \"run_in_background\": false}", + "input.value": "{\"description\": \"Calculate temperature difference\", \"subagent_type\": \"math-specialist\", \"prompt\": \"Calculate 89 minus 72.\", \"run_in_background\": false}", + "input.mime_type": "application/json", + "output.value": "{\"status\": \"completed\", \"prompt\": \"Calculate 89 minus 72.\", \"agentId\": \"mock-agent-003\", \"agentType\": \"math-specialist\", \"content\": [{\"type\": \"text\", \"text\": \"17\"}], \"resolvedModel\": \"us.anthropic.claude-sonnet-4-6\", \"totalDurationMs\": 2970, \"totalTokens\": 4639, \"totalToolUseCount\": 1}", + "output.mime_type": "application/json", + "openinference.span.kind": "TOOL" + }, + "scope": { + "name": "openinference.instrumentation.claude_agent_sdk", + "version": "0.1.7" + }, + "status": { + "code": "OK" + }, + "span_events": [] + }, + { + "trace_id": "4b0b961d81a720593197cd65f829ad3f", + "span_id": "053c311b792d91c6", + "parent_span_id": "51e6bd914525fee4", + "name": "ClaudeAgentSDK.Agent", + "start_time": 1785532824645170000, + "end_time": 1785532844762479000, + "attributes": { + "agent.name": "Agent", + "llm.model_name": "claude-sonnet-4-6", + "openinference.span.kind": "AGENT" + }, + "scope": { + "name": "openinference.instrumentation.claude_agent_sdk", + "version": "0.1.7" + }, + "status": { + "code": "OK" + }, + "span_events": [] + }, + { + "trace_id": "4b0b961d81a720593197cd65f829ad3f", + "span_id": "a6e166c4a06cc015", + "parent_span_id": "1c51a9dac98b8f20", + "name": "ClaudeAgentSDK.Agent", + "start_time": 1785532825602031000, + "end_time": 1785532844762515000, + "attributes": { + "agent.name": "Agent", + "llm.model_name": "claude-sonnet-4-6", + "openinference.span.kind": "AGENT" + }, + "scope": { + "name": "openinference.instrumentation.claude_agent_sdk", + "version": "0.1.7" + }, + "status": { + "code": "OK" + }, + "span_events": [] + }, + { + "trace_id": "4b0b961d81a720593197cd65f829ad3f", + "span_id": "f028c72828771e95", + "parent_span_id": "a6e405461983ccf4", + "name": "ClaudeAgentSDK.Agent", + "start_time": 1785532834092860000, + "end_time": 1785532844762527000, + "attributes": { + "agent.name": "Agent", + "llm.model_name": "claude-sonnet-4-6", + "openinference.span.kind": "AGENT" + }, + "scope": { + "name": "openinference.instrumentation.claude_agent_sdk", + "version": "0.1.7" + }, + "status": { + "code": "OK" + }, + "span_events": [] + }, + { + "trace_id": "4b0b961d81a720593197cd65f829ad3f", + "span_id": "ebdd7630e6537a19", + "parent_span_id": "9be1bbbdd3211df5", + "name": "ClaudeAgentSDK.query", + "start_time": 1785532815901349000, + "end_time": 1785532844762539000, + "attributes": { + "llm.system": "anthropic", + "input.value": "Look up the weather in New York and Seattle, then calculate the temperature difference.", + "input.mime_type": "text/plain", + "llm.output_messages.0.message.role": "assistant", + "llm.output_messages.0.message.content.0": "I'll look up the weather in both cities and then calculate the difference.", + "llm.output_messages.1.message.tool_calls.0.tool_call.id": "toolu_mock_agent_ny", + "llm.output_messages.1.message.tool_calls.0.tool_call.function.name": "Agent", + "llm.output_messages.1.message.tool_calls.0.tool_call.function.arguments": "{\"description\": \"Get current weather in New York\", \"subagent_type\": \"research-specialist\", \"prompt\": \"What is the current temperature in New York City?\", \"run_in_background\": false}", + "llm.output_messages.1.message.role": "assistant", + "llm.output_messages.2.message.tool_calls.0.tool_call.id": "toolu_mock_agent_seattle", + "llm.output_messages.2.message.tool_calls.0.tool_call.function.name": "Agent", + "llm.output_messages.2.message.tool_calls.0.tool_call.function.arguments": "{\"description\": \"Get current weather in Seattle\", \"subagent_type\": \"research-specialist\", \"prompt\": \"What is the current temperature in Seattle, Washington?\", \"run_in_background\": false}", + "llm.output_messages.2.message.role": "assistant", + "llm.output_messages.3.message.content.0": "Got both temperatures! Now let me calculate the difference.", + "llm.output_messages.3.message.role": "assistant", + "llm.output_messages.4.message.tool_calls.0.tool_call.id": "toolu_mock_agent_math", + "llm.output_messages.4.message.tool_calls.0.tool_call.function.name": "Agent", + "llm.output_messages.4.message.tool_calls.0.tool_call.function.arguments": "{\"description\": \"Calculate temperature difference\", \"subagent_type\": \"math-specialist\", \"prompt\": \"Calculate 89 minus 72.\", \"run_in_background\": false}", + "llm.output_messages.4.message.role": "assistant", + "llm.output_messages.5.message.content.0": "New York is 89°F and Seattle is 72°F. The temperature difference is 17°F — New York is warmer.", + "llm.output_messages.5.message.role": "assistant", + "output.value": "New York is 89°F and Seattle is 72°F. The temperature difference is 17°F — New York is warmer.", + "output.mime_type": "text/plain", + "llm.model_name": "us.anthropic.claude-sonnet-4-6", + "llm.token_count.prompt": 132, + "llm.token_count.completion": 850, + "llm.token_count.total": 982, + "session.id": "mock-session-001", + "openinference.span.kind": "AGENT" + }, + "scope": { + "name": "openinference.instrumentation.claude_agent_sdk", + "version": "0.1.7" + }, + "status": { + "code": "OK" + }, + "span_events": [] + } +] diff --git a/tests/strands_evals/mappers/test_openinference_session_mapper.py b/tests/strands_evals/mappers/test_openinference_session_mapper.py index 36ae6978..659ddc58 100644 --- a/tests/strands_evals/mappers/test_openinference_session_mapper.py +++ b/tests/strands_evals/mappers/test_openinference_session_mapper.py @@ -18,9 +18,12 @@ _LIVE_SPANS_FILE = _FIXTURES_DIR / "openinference_live_spans.json" _ADOT_SPANS_FILE = _FIXTURES_DIR / "openinference_adot_spans.json" _SMOLAGENTS_SPANS_FILE = _FIXTURES_DIR / "smolagents_live_spans.json" +_CLAUDE_SPANS_FILE = _FIXTURES_DIR / "claude_live_spans.json" +_CLAUDE_ADOT_FILE = _FIXTURES_DIR / "claude_adot_spans.json" SCOPE_NAME = "openinference.instrumentation.langchain" SMOLAGENTS_SCOPE_NAME = "openinference.instrumentation.smolagents" +CLAUDE_SDK_SCOPE_NAME = "openinference.instrumentation.claude_agent_sdk" def make_span( @@ -171,22 +174,35 @@ def make_adot_span( def _load_live_spans(): """Load real live (in-memory) spans from fixture file.""" - with open(_LIVE_SPANS_FILE) as f: + with open(_LIVE_SPANS_FILE, encoding="utf-8") as f: return json.load(f) def _load_adot_spans(): """Load ADOT/CloudWatch spans from fixture file.""" - with open(_ADOT_SPANS_FILE) as f: + with open(_ADOT_SPANS_FILE, encoding="utf-8") as f: return json.load(f) def _load_smolagents_spans(): """Load real smolagents (openinference-instrumentation-smolagents) spans from fixture file.""" - with open(_SMOLAGENTS_SPANS_FILE) as f: + with open(_SMOLAGENTS_SPANS_FILE, encoding="utf-8") as f: return json.load(f) +def _load_claude_spans(): + """Load real Claude Agent SDK (openinference-instrumentation-claude-agent-sdk) spans from fixture file.""" + with open(_CLAUDE_SPANS_FILE, encoding="utf-8") as f: + return json.load(f) + + +def _load_claude_adot_spans(): + """Load Claude Agent SDK spans captured from AgentCore (session.id on all spans).""" + with open(_CLAUDE_ADOT_FILE, encoding="utf-8") as f: + data = json.load(f) + return data["session_id"], data["spans"] + + class TestSpanTypeDetection: def setup_method(self): self.mapper = OpenInferenceSessionMapper() @@ -1407,7 +1423,7 @@ def test_smolagents_llm_span(self): @pytest.mark.parametrize( "output_value,expected_content", - [("42", "42"), ("[1, 2, 3]", "[1, 2, 3]"), ('"quoted"', '"quoted"')], + [("42", "42"), ("[1, 2, 3]", "[1, 2, 3]"), ('"quoted"', "quoted")], ids=["number", "list", "quoted-string"], ) def test_non_dict_json_output_not_crash(self, output_value, expected_content): @@ -1602,3 +1618,477 @@ def test_no_empty_response_inference_spans(self, smolagents_session): text_only = all(hasattr(c, "text") for c in assistant_content) if text_only: assert any(c.text for c in assistant_content) + + +class TestClaudeAgentSdkScopeSupport: + """Claude Agent SDK-scoped spans: acceptance and conversion.""" + + def setup_method(self): + self.mapper = OpenInferenceSessionMapper() + + def test_claude_agent_span_with_plain_text_input_detected(self): + """Root AGENT span with plain-text input/output → AgentInvocationSpan.""" + spans = _load_claude_spans() + # Root span has both input.value and output.value populated + root_span = next( + s + for s in spans + if s["attributes"].get("openinference.span.kind") == "AGENT" + and s["attributes"].get("input.value") + and s["attributes"].get("output.value") + ) + session = self.mapper.map_to_session([root_span], "sess-1") + + agent_spans = [s for t in session.traces for s in t.spans if isinstance(s, AgentInvocationSpan)] + assert len(agent_spans) == 1 + assert agent_spans[0].user_prompt == ( + "Look up the weather in New York and Seattle, then calculate the temperature difference." + ) + assert agent_spans[0].agent_response == ( + "New York is 89°F and Seattle is 72°F. The temperature difference is 17°F — New York is warmer." + ) + + def test_nested_claude_agent_span_without_input_output_rejected(self): + """Nested ClaudeAgentSDK.Agent span (no input/output) is not an agent invocation.""" + spans = _load_claude_spans() + # Nested AGENT spans have kind=AGENT but no input.value/output.value + nested_span = next( + s + for s in spans + if s["attributes"].get("openinference.span.kind") == "AGENT" and not s["attributes"].get("input.value") + ) + session = self.mapper.map_to_session([nested_span], "sess-1") + + all_spans = [s for t in session.traces for s in t.spans] + assert not any(isinstance(s, AgentInvocationSpan) for s in all_spans) + + def test_claude_tool_span_json_input_parsed(self): + """TOOL span with JSON input.value extracts tool_call.arguments correctly.""" + spans = _load_claude_spans() + # Pick an Agent tool span (subagent delegation) + agent_tool_span = next( + s + for s in spans + if s["attributes"].get("openinference.span.kind") == "TOOL" and s["attributes"].get("tool.name") == "Agent" + ) + session = self.mapper.map_to_session([agent_tool_span], "sess-1") + + tool_spans = [s for t in session.traces for s in t.spans if isinstance(s, ToolExecutionSpan)] + assert len(tool_spans) == 1 + tool = tool_spans[0] + assert tool.tool_call.name == "Agent" + assert "subagent_type" in tool.tool_call.arguments + assert "prompt" in tool.tool_call.arguments + assert tool.tool_call.tool_call_id is not None + + def test_claude_tool_span_content_blocks_output(self): + """TOOL output with content as list of blocks joins to newline-separated text.""" + span = make_span( + name="Agent", + scope_name=CLAUDE_SDK_SCOPE_NAME, + attributes={ + "openinference.span.kind": "TOOL", + "tool.id": "toolu_bdrk_xyz789", + "tool.name": "Agent", + "input.value": json.dumps( + { + "description": "Research task", + "subagent_type": "research-specialist", + "prompt": "Look up weather in NYC.", + } + ), + "output.value": json.dumps( + { + "status": "completed", + "content": [ + {"type": "text", "text": "Temperature: 89°F"}, + {"type": "text", "text": "Conditions: Partly cloudy"}, + ], + } + ), + }, + ) + session = self.mapper.map_to_session([span], "sess-1") + + tool_spans = [s for t in session.traces for s in t.spans if isinstance(s, ToolExecutionSpan)] + assert len(tool_spans) == 1 + assert tool_spans[0].tool_result.content == "Temperature: 89°F\nConditions: Partly cloudy" + assert tool_spans[0].tool_result.error is None + # Non-ASCII must be preserved literally, not escaped + assert "\\u" not in tool_spans[0].tool_result.content + + @pytest.mark.parametrize( + "status,span_events,expected_error", + [ + pytest.param( + {"code": "ERROR", "description": "EACCES: permission denied"}, + [], + "EACCES: permission denied", + id="description", + ), + pytest.param( + {"code": "ERROR"}, + [ + { + "event_name": "exception", + "timestamp": 1700000000500000000, + "attributes": {"exception.message": "Permission denied: /etc/shadow"}, + } + ], + "Permission denied: /etc/shadow", + id="exception_message", + ), + pytest.param({"code": "ERROR"}, [], "error", id="bare_fallback"), + pytest.param( + {"code": "ERROR"}, + [ + { + "event_name": "exception", + "timestamp": 1700000000500000000, + "attributes": {"exception.message": '[{"type":"text","text":""}]'}, + } + ], + '[{"type":"text","text":""}]', + id="empty_text_block_falls_back_to_raw", + ), + ], + ) + def test_claude_failed_tool_span_no_output_preserved(self, status, span_events, expected_error): + """status=ERROR with no output.value is preserved via description/exception/fallback.""" + span = make_span( + name="Bash", + scope_name=CLAUDE_SDK_SCOPE_NAME, + attributes={ + "openinference.span.kind": "TOOL", + "tool.id": "toolu_failed", + "tool.name": "Bash", + "input.value": json.dumps({"command": "false"}), + }, + span_events=span_events, + ) + span["status"] = status + + session = self.mapper.map_to_session([span], "sess-1") + + tool_spans = [s for t in session.traces for s in t.spans if isinstance(s, ToolExecutionSpan)] + assert len(tool_spans) == 1 + assert tool_spans[0].tool_result.error == "error" + assert tool_spans[0].tool_result.content == expected_error + + def test_empty_text_block_among_non_text_blocks_preserves_siblings(self): + """An empty text block among non-text blocks should preserve siblings as JSON, not discard.""" + non_text_block = {"type": "resource", "resource": {"text": "IMPORTANT DATA"}} + span = make_span( + name="ReadFile", + scope_name=CLAUDE_SDK_SCOPE_NAME, + attributes={ + "openinference.span.kind": "TOOL", + "tool.id": "toolu_mixed", + "tool.name": "ReadFile", + "input.value": json.dumps({"path": "/tmp/data"}), + "output.value": json.dumps( + {"status": "completed", "content": [{"type": "text", "text": ""}, non_text_block]} + ), + }, + ) + + session = self.mapper.map_to_session([span], "sess-1") + + tool_spans = [s for t in session.traces for s in t.spans if isinstance(s, ToolExecutionSpan)] + assert len(tool_spans) == 1 + # Non-text sibling must be preserved (as JSON), not silently discarded + assert "IMPORTANT DATA" in tool_spans[0].tool_result.content + + def test_exception_message_returns_first_not_last(self): + """_exception_message returns the first exception event, not the last.""" + span = make_span( + name="Bash", + scope_name=CLAUDE_SDK_SCOPE_NAME, + attributes={ + "openinference.span.kind": "TOOL", + "tool.id": "toolu_multi_exc", + "tool.name": "Bash", + "input.value": json.dumps({"command": "fail"}), + }, + span_events=[ + { + "event_name": "exception", + "timestamp": 1700000000100000000, + "attributes": {"exception.message": "first error"}, + }, + { + "event_name": "exception", + "timestamp": 1700000000200000000, + "attributes": {"exception.message": "second error"}, + }, + ], + ) + span["status"] = {"code": "ERROR"} + + session = self.mapper.map_to_session([span], "sess-1") + + tool_spans = [s for t in session.traces for s in t.spans if isinstance(s, ToolExecutionSpan)] + assert len(tool_spans) == 1 + assert tool_spans[0].tool_result.content == "first error" + + @pytest.mark.parametrize( + "output_value,status,span_events,expected_content", + [ + pytest.param( + json.dumps([{"type": "text", "text": "EACCES: permission denied"}]), + {"code": "OK"}, + [], + "EACCES: permission denied", + id="bare_list_success", + ), + pytest.param( + None, + {"code": "ERROR"}, + [ + { + "event_name": "exception", + "timestamp": 1700000000500000000, + "attributes": { + "exception.message": json.dumps( + [{"type": "text", "text": "EACCES: permission denied, open '/etc/shadow'"}] + ) + }, + } + ], + "EACCES: permission denied, open '/etc/shadow'", + id="error_path_block_list", + ), + ], + ) + def test_content_block_flattening_beyond_dict_envelope(self, output_value, status, span_events, expected_content): + """Content blocks are flattened on both the bare-list success path and the ERROR path.""" + attrs = { + "openinference.span.kind": "TOOL", + "tool.id": "toolu_flatten", + "tool.name": "Bash", + "input.value": json.dumps({"command": "ls"}), + } + if output_value is not None: + attrs["output.value"] = output_value + span = make_span( + name="Bash", + scope_name=CLAUDE_SDK_SCOPE_NAME, + attributes=attrs, + span_events=span_events, + ) + span["status"] = status + + session = self.mapper.map_to_session([span], "sess-1") + + tool_spans = [s for t in session.traces for s in t.spans if isinstance(s, ToolExecutionSpan)] + assert len(tool_spans) == 1 + assert tool_spans[0].tool_result.content == expected_content + + def test_ensure_ascii_false_on_json_dumps_fallback(self): + """Non-text blocks with unicode hit json.dumps and must not produce escape sequences.""" + span = make_span( + name="ImageGen", + scope_name=CLAUDE_SDK_SCOPE_NAME, + attributes={ + "openinference.span.kind": "TOOL", + "tool.id": "toolu_img", + "tool.name": "ImageGen", + "input.value": json.dumps({"prompt": "weather"}), + "output.value": json.dumps({"status": "completed", "content": [{"type": "image", "alt": "25°C 東京"}]}), + }, + ) + + session = self.mapper.map_to_session([span], "sess-1") + + tool_spans = [s for t in session.traces for s in t.spans if isinstance(s, ToolExecutionSpan)] + assert len(tool_spans) == 1 + assert "25°C 東京" in tool_spans[0].tool_result.content + assert "\\u" not in tool_spans[0].tool_result.content + + def test_non_str_text_value_not_blanked(self): + """A block with text: 500 (non-str) must preserve the block as JSON, not return ''.""" + span = make_span( + name="Bash", + scope_name=CLAUDE_SDK_SCOPE_NAME, + attributes={ + "openinference.span.kind": "TOOL", + "tool.id": "toolu_nonstr", + "tool.name": "Bash", + "input.value": json.dumps({"command": "echo 500"}), + "output.value": json.dumps({"status": "completed", "content": [{"type": "text", "text": 500}]}), + }, + ) + + session = self.mapper.map_to_session([span], "sess-1") + + tool_spans = [s for t in session.traces for s in t.spans if isinstance(s, ToolExecutionSpan)] + assert len(tool_spans) == 1 + assert tool_spans[0].tool_result.content != "" + + def test_errored_agent_span_detected(self): + """AGENT span with input.value + ERROR status (no output.value) is detected.""" + span = make_span( + name="query", + scope_name=CLAUDE_SDK_SCOPE_NAME, + attributes={ + "openinference.span.kind": "AGENT", + "input.value": "What is the weather?", + }, + ) + span["status"] = {"code": "ERROR"} + + assert self.mapper._is_agent_invocation_span(span) is True + + def test_errored_agent_span_without_input_not_detected(self): + """AGENT span with ERROR status but no input.value is not detected.""" + span = make_span( + name="query", + scope_name=CLAUDE_SDK_SCOPE_NAME, + attributes={ + "openinference.span.kind": "AGENT", + }, + ) + span["status"] = {"code": "ERROR"} + + assert self.mapper._is_agent_invocation_span(span) is False + + @pytest.mark.parametrize( + "status,span_events,expected_response", + [ + pytest.param( + {"code": "ERROR", "description": "context deadline exceeded"}, + [{"event_name": "exception", "timestamp": 0, "attributes": {"exception.message": "ignored"}}], + "context deadline exceeded", + id="prefers_status_description", + ), + pytest.param( + {"code": "ERROR"}, + [{"event_name": "exception", "timestamp": 0, "attributes": {"exception.message": "rate limit"}}], + "rate limit", + id="falls_back_to_exception_message", + ), + pytest.param( + {"code": "ERROR"}, + [], + "error", + id="falls_back_to_generic_error", + ), + ], + ) + def test_errored_agent_span_response_fallback(self, status, span_events, expected_response): + """Errored agent span surfaces error via: status.description > exception > 'error'.""" + span = make_span( + name="query", + scope_name=CLAUDE_SDK_SCOPE_NAME, + attributes={ + "openinference.span.kind": "AGENT", + "input.value": "What is the weather?", + }, + span_events=span_events, + ) + span["status"] = status + + session = self.mapper.map_to_session([span], "sess-1") + + agent_spans = [s for t in session.traces for s in t.spans if isinstance(s, AgentInvocationSpan)] + assert len(agent_spans) == 1 + assert agent_spans[0].user_prompt == "What is the weather?" + assert agent_spans[0].agent_response == expected_response + + def test_claude_agent_span_preserves_json_prompt_with_task_key(self): + """Claude prompt that is a JSON object with 'task' key is NOT unwrapped.""" + json_prompt = json.dumps({"task": "summarize", "context": "some data"}) + span = make_span( + name="query", + scope_name=CLAUDE_SDK_SCOPE_NAME, + attributes={ + "openinference.span.kind": "AGENT", + "input.value": json_prompt, + "output.value": "Here is the summary.", + }, + ) + + session = self.mapper.map_to_session([span], "sess-1") + + agent_spans = [s for t in session.traces for s in t.spans if isinstance(s, AgentInvocationSpan)] + assert len(agent_spans) == 1 + assert agent_spans[0].user_prompt == json_prompt + + +@pytest.fixture() +def claude_session(): + """Map real Claude Agent SDK spans to a Session.""" + spans = _load_claude_spans() + mapper = OpenInferenceSessionMapper() + return mapper.map_to_session(spans, "claude-sess") + + +class TestClaudeFixtureIntegration: + """Integration tests using real Claude Agent SDK trace (all 10 spans fed together).""" + + def test_span_counts_and_types(self, claude_session): + """All 10 fixture spans map to 6 ToolExecutionSpans + 1 AgentInvocationSpan. + + The 3 nested AGENT spans (no input/output) are rejected. + """ + assert len(claude_session.traces) >= 1 + all_spans = [s for t in claude_session.traces for s in t.spans] + assert len(all_spans) == 7 + + tool_spans = [s for s in all_spans if isinstance(s, ToolExecutionSpan)] + agent_spans = [s for s in all_spans if isinstance(s, AgentInvocationSpan)] + assert len(tool_spans) == 6 + assert len(agent_spans) == 1 + + def test_agent_span_has_prompt_and_response(self, claude_session): + """Root agent span has user_prompt and agent_response populated.""" + all_spans = [s for t in claude_session.traces for s in t.spans] + agent = next(s for s in all_spans if isinstance(s, AgentInvocationSpan)) + assert agent.user_prompt + assert agent.agent_response + + def test_agent_span_metadata_has_token_counts(self, claude_session): + """Root agent span metadata contains model name and token counts.""" + all_spans = [s for t in claude_session.traces for s in t.spans] + agent = next(s for s in all_spans if isinstance(s, AgentInvocationSpan)) + assert agent.metadata["llm.model_name"] == "us.anthropic.claude-sonnet-4-6" + assert agent.metadata["llm.token_count.total"] > 0 + + def test_tool_spans_have_call_ids_and_content(self, claude_session): + """All tool spans have tool_call_id populated and non-empty content.""" + all_spans = [s for t in claude_session.traces for s in t.spans] + tool_spans = [s for s in all_spans if isinstance(s, ToolExecutionSpan)] + for tool_span in tool_spans: + assert tool_span.tool_call.tool_call_id is not None + assert tool_span.tool_result.content + + +@pytest.fixture() +def claude_adot_session(): + """Map real Claude AgentCore spans (session.id on all spans) to a Session.""" + session_id, spans = _load_claude_adot_spans() + mapper = OpenInferenceSessionMapper() + return mapper.map_to_session(spans, session_id) + + +class TestClaudeAgentCoreFixtureIntegration: + """Integration tests using Claude Agent SDK spans from AgentCore deployment. + + AgentCore's span processor stamps session.id on all spans, so the full set + survives a CloudWatch Logs Insights query filtered by session.id. + """ + + def test_span_counts(self, claude_adot_session): + """AgentCore trace produces 4 tool spans + 1 agent span.""" + all_spans = [s for t in claude_adot_session.traces for s in t.spans] + tool_spans = [s for s in all_spans if isinstance(s, ToolExecutionSpan)] + agent_spans = [s for s in all_spans if isinstance(s, AgentInvocationSpan)] + assert len(tool_spans) == 4 + assert len(agent_spans) == 1 + + def test_all_spans_have_session_id_attribute(self, claude_adot_session): + """Verify the fixture has session.id on all spans (AgentCore behavior).""" + session_id, spans = _load_claude_adot_spans() + for span in spans: + attrs = span.get("attributes", {}) + assert attrs.get("session.id") == session_id diff --git a/tests/strands_evals/mappers/test_utils.py b/tests/strands_evals/mappers/test_utils.py index fe87ad90..778dc98f 100644 --- a/tests/strands_evals/mappers/test_utils.py +++ b/tests/strands_evals/mappers/test_utils.py @@ -263,6 +263,12 @@ def test_unrecognized_scope_without_gen_ai_attrs_defaults_to_strands(self): mapper = detect_otel_mapper(spans) assert isinstance(mapper, StrandsInMemorySessionMapper) + def test_detects_claude_agent_sdk_openinference_scope(self): + """Detects OpenInferenceSessionMapper for Claude Agent SDK openinference scope.""" + spans = [make_span_dict(scope_name="openinference.instrumentation.claude_agent_sdk")] + mapper = detect_otel_mapper(spans) + assert isinstance(mapper, OpenInferenceSessionMapper) + class TestReadableSpansToDicts: def test_converts_readable_spans(self):