diff --git a/src/strands_evals/extractors/trace_extractor.py b/src/strands_evals/extractors/trace_extractor.py index 2a4e996c..7384de6b 100644 --- a/src/strands_evals/extractors/trace_extractor.py +++ b/src/strands_evals/extractors/trace_extractor.py @@ -1,4 +1,5 @@ import logging +from datetime import datetime, timezone from ..types.trace import ( AgentInvocationSpan, @@ -20,6 +21,13 @@ logger = logging.getLogger(__name__) +def _to_aware_utc(dt: datetime) -> datetime: + """Normalize a datetime to timezone-aware UTC.""" + if dt.tzinfo is None: + return dt.replace(tzinfo=timezone.utc) + return dt + + class TraceExtractor: """Extracts structured evaluation inputs from Session traces.""" @@ -85,7 +93,11 @@ def _extract_trace_level(self, session: Session) -> list[TraceLevelInput]: return evaluation_inputs def _extract_tool_level(self, session: Session) -> list[ToolLevelInput]: - """Extract tool-level inputs with session and tool context.""" + """Extract tool-level inputs with session and tool context. + + Note: spans are not scoped by parent_span_id, so nested agent-as-tool traces + may leak child-agent internals into the parent's history. + """ evaluator_inputs: list[ToolLevelInput] = [] session_history: list[UserMessage | list[ToolExecution] | AssistantMessage] = [] available_tools: list[ToolConfig] = [] @@ -100,20 +112,34 @@ def _extract_tool_level(self, session: Session) -> list[ToolLevelInput]: if agent_span and agent_span.user_prompt: session_history.append(UserMessage(content=[TextContent(text=agent_span.user_prompt)])) - for tool_span in tool_spans: + tool_executions = [ + ToolExecution(tool_call=span.tool_call, tool_result=span.tool_result) for span in tool_spans + ] + tool_end_times = [ + max(_to_aware_utc(span.span_info.end_time), _to_aware_utc(span.span_info.start_time)) + for span in tool_spans + ] + + for index, tool_span in enumerate(tool_spans): + target_start = _to_aware_utc(tool_span.span_info.start_time) + prior_executions = [ + tool_executions[position] + for position in range(len(tool_spans)) + if position != index + and tool_end_times[position] <= target_start + and (tool_end_times[position] < target_start or position < index) + ] + evaluator_inputs.append( ToolLevelInput( span_info=tool_span.span_info, available_tools=available_tools, tool_execution_details=tool_span, - session_history=list(session_history), + session_history=list(session_history) + ([prior_executions] if prior_executions else []), ) ) if tool_spans: - tool_executions = [ - ToolExecution(tool_call=span.tool_call, tool_result=span.tool_result) for span in tool_spans - ] session_history.append(tool_executions) if agent_span and agent_span.agent_response: diff --git a/src/strands_evals/types/trace.py b/src/strands_evals/types/trace.py index 0b4a93db..c78d11e0 100644 --- a/src/strands_evals/types/trace.py +++ b/src/strands_evals/types/trace.py @@ -167,7 +167,12 @@ class TraceLevelInput(BaseEvaluationInput): class ToolLevelInput(BaseEvaluationInput): - """Input for tool-level evaluators""" + """Input for tool-level evaluators. + + `session_history` never includes the call under evaluation. Within the same trace, only tool + executions that completed before this tool started (`end_time <= start_time`) are included, + with list position as tiebreaker for equal timestamps. Cross-trace history is unfiltered. + """ available_tools: list[ToolConfig] tool_execution_details: ToolExecutionSpan diff --git a/tests/strands_evals/extractors/test_trace_extractor.py b/tests/strands_evals/extractors/test_trace_extractor.py index cafb6243..9f3521ed 100644 --- a/tests/strands_evals/extractors/test_trace_extractor.py +++ b/tests/strands_evals/extractors/test_trace_extractor.py @@ -157,3 +157,160 @@ def test_extract_empty_session_tool_level(): assert isinstance(result, list) assert len(result) == 0 + + +def test_extract_tool_level_incremental_session_history(): + """Each tool span sees exactly its causally-completed predecessors in session_history.""" + from datetime import timedelta, timezone + + base = datetime(2024, 1, 1, 12, 0, 0, tzinfo=timezone.utc) + + agent_span = AgentInvocationSpan( + span_info=SpanInfo(session_id="test", span_id="a0", start_time=base, end_time=base + timedelta(seconds=10)), + user_prompt="What is the square root of 1764, multiplied by 3, then add 1?", + agent_response="127", + available_tools=[ + ToolConfig(name="square_root"), + ToolConfig(name="multiply_numbers"), + ToolConfig(name="add_numbers"), + ], + ) + tool_span_1 = ToolExecutionSpan( + span_info=SpanInfo( + session_id="test", + span_id="t1", + parent_span_id="a0", + start_time=base, + end_time=base + timedelta(seconds=1), + ), + tool_call=ToolCall(name="square_root", arguments={"n": 1764}), + tool_result=ToolResult(content="42.0"), + ) + # Finishes at t=2.5: AFTER tool_3 starts (t=2) but BEFORE tool_3 ends (t=3) + # This discriminates start_time vs end_time comparison targets + tool_span_2 = ToolExecutionSpan( + span_info=SpanInfo( + session_id="test", + span_id="t2", + parent_span_id="a0", + start_time=base + timedelta(seconds=1), + end_time=base + timedelta(milliseconds=2500), + ), + tool_call=ToolCall(name="multiply_numbers", arguments={"a": 42, "b": 3}), + tool_result=ToolResult(content="126"), + ) + tool_span_3 = ToolExecutionSpan( + span_info=SpanInfo( + session_id="test", + span_id="t3", + parent_span_id="a0", + start_time=base + timedelta(seconds=2), + end_time=base + timedelta(seconds=3), + ), + tool_call=ToolCall(name="add_numbers", arguments={"a": 126, "b": 1}), + tool_result=ToolResult(content="127"), + ) + + trace = Trace(spans=[agent_span, tool_span_1, tool_span_2, tool_span_3], trace_id="trace1", session_id="test") + session = Session(traces=[trace], session_id="test") + + extractor = TraceExtractor(EvaluationLevel.TOOL_LEVEL) + result = extractor.extract(session) + + assert len(result) == 3, f"expected 3 tool-level inputs, got {len(result)}" + + # First tool: sees only the user prompt, no prior executions + assert len(result[0].session_history) == 1, ( + f"first tool should only see user prompt, got {len(result[0].session_history)} entries" + ) + assert result[0].tool_execution_details.tool_call.name == "square_root" + + # Second tool: sees user prompt + first tool's execution (square_root finished at t=1 <= t=1) + assert len(result[1].session_history) == 2, ( + f"second tool should see user prompt + 1 prior execution, got {len(result[1].session_history)} entries" + ) + assert isinstance(result[1].session_history[1], list) + assert result[1].session_history[1][0].tool_call.name == "square_root" + assert result[1].session_history[1][0].tool_result.content == "42.0" + + # Third tool: only square_root is a valid prior + assert len(result[2].session_history) == 2, ( + f"expected 2 entries (user + prior tools), got {len(result[2].session_history)}" + ) + prior_names = [entry.tool_call.name for entry in result[2].session_history[1]] + assert prior_names == ["square_root"], ( + f"expected only [square_root] (multiply_numbers still running at tool_3.start_time), got {prior_names}" + ) + + +def test_extract_tool_level_mixed_tz_timestamps(): + """The causality filter handles mixed naive/aware timestamps without raising TypeError.""" + from datetime import timedelta, timezone + + base_naive = datetime(2024, 1, 1, 12, 0, 0) # no tzinfo + base_aware = datetime(2024, 1, 1, 12, 0, 2, tzinfo=timezone.utc) # aware, starts at +2s + + agent_span = AgentInvocationSpan( + span_info=SpanInfo( + session_id="test", span_id="a0", start_time=base_naive, end_time=base_aware + timedelta(seconds=10) + ), + user_prompt="Mixed tz test", + agent_response="Done", + available_tools=[ + ToolConfig(name="tool_x"), + ToolConfig(name="tool_long"), + ToolConfig(name="tool_y"), + ], + ) + # tool_x: ends at 12:00:01 (before tool_y starts) + tool_x = ToolExecutionSpan( + span_info=SpanInfo( + session_id="test", + span_id="tx", + parent_span_id="a0", + start_time=base_naive, + end_time=base_naive + timedelta(seconds=1), + ), + tool_call=ToolCall(name="tool_x", arguments={}), + tool_result=ToolResult(content="x_result"), + ) + # tool_long: ends at 12:00:05 (after tool_y starts at 12:00:02) + tool_long = ToolExecutionSpan( + span_info=SpanInfo( + session_id="test", + span_id="tl", + parent_span_id="a0", + start_time=base_naive, + end_time=base_naive + timedelta(seconds=5), + ), + tool_call=ToolCall(name="tool_long", arguments={}), + tool_result=ToolResult(content="long_result"), + ) + # tool_y: starts at 12:00:02 UTC + tool_y = ToolExecutionSpan( + span_info=SpanInfo( + session_id="test", + span_id="ty", + parent_span_id="a0", + start_time=base_aware, + end_time=base_aware + timedelta(seconds=1), + ), + tool_call=ToolCall(name="tool_y", arguments={}), + tool_result=ToolResult(content="y_result"), + ) + + trace = Trace(spans=[agent_span, tool_x, tool_long, tool_y], trace_id="trace1", session_id="test") + session = Session(traces=[trace], session_id="test") + + extractor = TraceExtractor(EvaluationLevel.TOOL_LEVEL) + # Should not raise TypeError from mixed tz comparison + result = extractor.extract(session) + + assert len(result) == 3 + # tool_y should see ONLY tool_x as a prior + assert result[2].tool_execution_details.tool_call.name == "tool_y" + assert len(result[2].session_history) == 2 + prior_names = [e.tool_call.name for e in result[2].session_history[1]] + assert prior_names == ["tool_x"], ( + f"expected only ['tool_x'] as prior for tool_y (tool_long still running), got {prior_names}" + )