From d72732a5b82f8e6d7b84e18b84378bd2eb096a17 Mon Sep 17 00:00:00 2001
From: liramon2 <liramon@amazon.com>
Date: Thu, 30 Jul 2026 13:29:18 -0400
Subject: [PATCH 1/5] fix: move tool results into session history

---
 .../extractors/trace_extractor.py             |  9 ++---
 .../extractors/test_trace_extractor.py        | 38 +++++++++++++++++++
 2 files changed, 42 insertions(+), 5 deletions(-)

diff --git a/src/strands_evals/extractors/trace_extractor.py b/src/strands_evals/extractors/trace_extractor.py
index 2a4e996c..eece4b95 100644
--- a/src/strands_evals/extractors/trace_extractor.py
+++ b/src/strands_evals/extractors/trace_extractor.py
@@ -110,11 +110,10 @@ def _extract_tool_level(self, session: Session) -> list[ToolLevelInput]:
                     )
                 )
 
-            if tool_spans:
-                tool_executions = [
-                    ToolExecution(tool_call=span.tool_call, tool_result=span.tool_result) for span in tool_spans
-                ]
-                session_history.append(tool_executions)
+                # Accumulate this tool's execution so subsequent tool spans see it
+                session_history.append(
+                    [ToolExecution(tool_call=tool_span.tool_call, tool_result=tool_span.tool_result)]
+                )
 
             if agent_span and agent_span.agent_response:
                 session_history.append(AssistantMessage(content=[TextContent(text=agent_span.agent_response)]))
diff --git a/tests/strands_evals/extractors/test_trace_extractor.py b/tests/strands_evals/extractors/test_trace_extractor.py
index cafb6243..83d9f8cc 100644
--- a/tests/strands_evals/extractors/test_trace_extractor.py
+++ b/tests/strands_evals/extractors/test_trace_extractor.py
@@ -157,3 +157,41 @@ def test_extract_empty_session_tool_level():
 
     assert isinstance(result, list)
     assert len(result) == 0
+
+
+def test_extract_tool_level_incremental_session_history():
+    """Test that each tool span sees prior tool results in session_history."""
+    now = datetime.now()
+    agent_span = AgentInvocationSpan(
+        span_info=SpanInfo(session_id="test", span_id="a0", start_time=now, end_time=now),
+        user_prompt="What is the square root of 1764, multiplied by 3?",
+        agent_response="126",
+        available_tools=[ToolConfig(name="square_root"), ToolConfig(name="multiply_numbers")],
+    )
+    tool_span_1 = ToolExecutionSpan(
+        span_info=SpanInfo(session_id="test", span_id="t1", parent_span_id="a0", start_time=now, end_time=now),
+        tool_call=ToolCall(name="square_root", arguments={"n": 1764}),
+        tool_result=ToolResult(content="42.0"),
+    )
+    tool_span_2 = ToolExecutionSpan(
+        span_info=SpanInfo(session_id="test", span_id="t2", parent_span_id="a0", start_time=now, end_time=now),
+        tool_call=ToolCall(name="multiply_numbers", arguments={"a": 42, "b": 3}),
+        tool_result=ToolResult(content="126"),
+    )
+
+    trace = Trace(spans=[agent_span, tool_span_1, tool_span_2], trace_id="trace1", session_id="test")
+    session = Session(traces=[trace], session_id="test")
+
+    extractor = TraceExtractor(EvaluationLevel.TOOL_LEVEL)
+    result = extractor.extract(session)
+
+    assert len(result) == 2
+
+    # First tool: sees only the user prompt
+    assert len(result[0].session_history) == 1
+
+    # Second tool: sees user prompt + first tool's execution
+    assert len(result[1].session_history) == 2
+    assert isinstance(result[1].session_history[1], list)
+    assert result[1].session_history[1][0].tool_call.name == "square_root"
+    assert result[1].session_history[1][0].tool_result.content == "42.0"

From 8f0abaa03470a65705ddb04410ff22353f7750fb Mon Sep 17 00:00:00 2001
From: liramon2 <liramon@amazon.com>
Date: Thu, 30 Jul 2026 15:28:04 -0400
Subject: [PATCH 2/5] fix: address strandly concerns

---
 src/strands_evals/evaluators/evaluator.py     | 10 ++++-
 .../extractors/trace_extractor.py             | 25 +++++++----
 src/strands_evals/types/trace.py              |  7 +++-
 .../extractors/test_trace_extractor.py        | 41 +++++++++++++++----
 4 files changed, 64 insertions(+), 19 deletions(-)

diff --git a/src/strands_evals/evaluators/evaluator.py b/src/strands_evals/evaluators/evaluator.py
index e5b2fb7d..a8acbbc4 100644
--- a/src/strands_evals/evaluators/evaluator.py
+++ b/src/strands_evals/evaluators/evaluator.py
@@ -207,7 +207,10 @@ def _format_tool_level_prompt(self, tool_input: ToolLevelInput) -> str:
                     # Handle tool execution lists
                     for tool_exec in msg:
                         history_lines.append(f"Tool call: {tool_exec.tool_call.name}({tool_exec.tool_call.arguments})")
-                        history_lines.append(f"Tool result: {tool_exec.tool_result.content}")
+                        if tool_exec.tool_result.error:
+                            history_lines.append(f"Tool result: ERROR - {tool_exec.tool_result.error}")
+                        else:
+                            history_lines.append(f"Tool result: {tool_exec.tool_result.content}")
                 else:
                     text = msg.content[0].text if msg.content and hasattr(msg.content[0], "text") else ""
                     history_lines.append(f"{msg.role.value.capitalize()}: {text}")
@@ -232,7 +235,10 @@ def _format_trace_level_prompt(self, parsed_input: TraceLevelInput) -> str:
                     # Handle tool execution lists
                     for tool_exec in msg:
                         history_lines.append(f"Tool call: {tool_exec.tool_call.name}({tool_exec.tool_call.arguments})")
-                        history_lines.append(f"Tool result: {tool_exec.tool_result.content}")
+                        if tool_exec.tool_result.error:
+                            history_lines.append(f"Tool result: ERROR - {tool_exec.tool_result.error}")
+                        else:
+                            history_lines.append(f"Tool result: {tool_exec.tool_result.content}")
                 else:
                     text = msg.content[0].text if msg.content and hasattr(msg.content[0], "text") else ""
                     history_lines.append(f"{msg.role.value.capitalize()}: {text}")
diff --git a/src/strands_evals/extractors/trace_extractor.py b/src/strands_evals/extractors/trace_extractor.py
index eece4b95..015b4883 100644
--- a/src/strands_evals/extractors/trace_extractor.py
+++ b/src/strands_evals/extractors/trace_extractor.py
@@ -85,7 +85,11 @@ def _extract_trace_level(self, session: Session) -> list[TraceLevelInput]:
         return evaluation_inputs
 
     def _extract_tool_level(self, session: Session) -> list[ToolLevelInput]:
-        """Extract tool-level inputs with session and tool context."""
+        """Extract tool-level inputs with session and tool context.
+
+        Note: spans are not scoped by parent_span_id, so nested agent-as-tool traces
+        may leak child-agent internals into the parent's history.
+        """
         evaluator_inputs: list[ToolLevelInput] = []
         session_history: list[UserMessage | list[ToolExecution] | AssistantMessage] = []
         available_tools: list[ToolConfig] = []
@@ -100,20 +104,27 @@ def _extract_tool_level(self, session: Session) -> list[ToolLevelInput]:
             if agent_span and agent_span.user_prompt:
                 session_history.append(UserMessage(content=[TextContent(text=agent_span.user_prompt)]))
 
-            for tool_span in tool_spans:
+            for index, tool_span in enumerate(tool_spans):
+                prior_executions = [
+                    ToolExecution(tool_call=span.tool_call, tool_result=span.tool_result)
+                    for span in tool_spans[:index]
+                    if span.span_info.end_time <= tool_span.span_info.start_time
+                ]
+
                 evaluator_inputs.append(
                     ToolLevelInput(
                         span_info=tool_span.span_info,
                         available_tools=available_tools,
                         tool_execution_details=tool_span,
-                        session_history=list(session_history),
+                        session_history=list(session_history) + ([prior_executions] if prior_executions else []),
                     )
                 )
 
-                # Accumulate this tool's execution so subsequent tool spans see it
-                session_history.append(
-                    [ToolExecution(tool_call=tool_span.tool_call, tool_result=tool_span.tool_result)]
-                )
+            if tool_spans:
+                tool_executions = [
+                    ToolExecution(tool_call=span.tool_call, tool_result=span.tool_result) for span in tool_spans
+                ]
+                session_history.append(tool_executions)
 
             if agent_span and agent_span.agent_response:
                 session_history.append(AssistantMessage(content=[TextContent(text=agent_span.agent_response)]))
diff --git a/src/strands_evals/types/trace.py b/src/strands_evals/types/trace.py
index 0b4a93db..82d25dff 100644
--- a/src/strands_evals/types/trace.py
+++ b/src/strands_evals/types/trace.py
@@ -167,7 +167,12 @@ class TraceLevelInput(BaseEvaluationInput):
 
 
 class ToolLevelInput(BaseEvaluationInput):
-    """Input for tool-level evaluators"""
+    """Input for tool-level evaluators.
+
+    `session_history` is the context preceding `tool_execution_details` and never includes the
+    call under evaluation. Unlike `TraceLevelInput`, where each `list[ToolExecution]` entry is one
+    turn's whole batch, here each entry holds exactly one execution.
+    """
 
     available_tools: list[ToolConfig]
     tool_execution_details: ToolExecutionSpan
diff --git a/tests/strands_evals/extractors/test_trace_extractor.py b/tests/strands_evals/extractors/test_trace_extractor.py
index 83d9f8cc..b86b14cd 100644
--- a/tests/strands_evals/extractors/test_trace_extractor.py
+++ b/tests/strands_evals/extractors/test_trace_extractor.py
@@ -160,13 +160,17 @@ def test_extract_empty_session_tool_level():
 
 
 def test_extract_tool_level_incremental_session_history():
-    """Test that each tool span sees prior tool results in session_history."""
+    """Each tool span sees exactly its true predecessors in session_history, in order."""
     now = datetime.now()
     agent_span = AgentInvocationSpan(
         span_info=SpanInfo(session_id="test", span_id="a0", start_time=now, end_time=now),
-        user_prompt="What is the square root of 1764, multiplied by 3?",
-        agent_response="126",
-        available_tools=[ToolConfig(name="square_root"), ToolConfig(name="multiply_numbers")],
+        user_prompt="What is the square root of 1764, multiplied by 3, then add 1?",
+        agent_response="127",
+        available_tools=[
+            ToolConfig(name="square_root"),
+            ToolConfig(name="multiply_numbers"),
+            ToolConfig(name="add_numbers"),
+        ],
     )
     tool_span_1 = ToolExecutionSpan(
         span_info=SpanInfo(session_id="test", span_id="t1", parent_span_id="a0", start_time=now, end_time=now),
@@ -178,20 +182,39 @@ def test_extract_tool_level_incremental_session_history():
         tool_call=ToolCall(name="multiply_numbers", arguments={"a": 42, "b": 3}),
         tool_result=ToolResult(content="126"),
     )
+    tool_span_3 = ToolExecutionSpan(
+        span_info=SpanInfo(session_id="test", span_id="t3", parent_span_id="a0", start_time=now, end_time=now),
+        tool_call=ToolCall(name="add_numbers", arguments={"a": 126, "b": 1}),
+        tool_result=ToolResult(content="127"),
+    )
 
-    trace = Trace(spans=[agent_span, tool_span_1, tool_span_2], trace_id="trace1", session_id="test")
+    trace = Trace(spans=[agent_span, tool_span_1, tool_span_2, tool_span_3], trace_id="trace1", session_id="test")
     session = Session(traces=[trace], session_id="test")
 
     extractor = TraceExtractor(EvaluationLevel.TOOL_LEVEL)
     result = extractor.extract(session)
 
-    assert len(result) == 2
+    assert len(result) == 3, f"expected 3 tool-level inputs, got {len(result)}"
 
-    # First tool: sees only the user prompt
-    assert len(result[0].session_history) == 1
+    # First tool: sees only the user prompt, no prior executions
+    assert len(result[0].session_history) == 1, (
+        f"first tool should only see user prompt, got {len(result[0].session_history)} entries"
+    )
+    assert result[0].tool_execution_details.tool_call.name == "square_root"
 
     # Second tool: sees user prompt + first tool's execution
-    assert len(result[1].session_history) == 2
+    assert len(result[1].session_history) == 2, (
+        f"second tool should see user prompt + 1 prior execution, got {len(result[1].session_history)} entries"
+    )
     assert isinstance(result[1].session_history[1], list)
     assert result[1].session_history[1][0].tool_call.name == "square_root"
     assert result[1].session_history[1][0].tool_result.content == "42.0"
+
+    # Third tool: sees user prompt + [square_root, multiply_numbers] in order
+    assert len(result[2].session_history) == 2, (
+        f"expected 2 entries (user + prior tools), got {len(result[2].session_history)}"
+    )
+    prior_names = [entry.tool_call.name for entry in result[2].session_history[1]]
+    assert prior_names == ["square_root", "multiply_numbers"], (
+        f"expected [square_root, multiply_numbers] in order, got {prior_names}"
+    )

From c0407b7441036310d0dcc7419dbf10a2f4cded1e Mon Sep 17 00:00:00 2001
From: liramon2 <liramon@amazon.com>
Date: Thu, 30 Jul 2026 16:43:32 -0400
Subject: [PATCH 3/5] fix: address strandly suggestions

---
 src/strands_evals/evaluators/evaluator.py     |  10 +-
 .../extractors/trace_extractor.py             |  10 +-
 src/strands_evals/types/trace.py              |   4 +-
 .../extractors/test_trace_extractor.py        | 103 ++++++++++++++++--
 4 files changed, 106 insertions(+), 21 deletions(-)

diff --git a/src/strands_evals/evaluators/evaluator.py b/src/strands_evals/evaluators/evaluator.py
index a8acbbc4..e5b2fb7d 100644
--- a/src/strands_evals/evaluators/evaluator.py
+++ b/src/strands_evals/evaluators/evaluator.py
@@ -207,10 +207,7 @@ def _format_tool_level_prompt(self, tool_input: ToolLevelInput) -> str:
                     # Handle tool execution lists
                     for tool_exec in msg:
                         history_lines.append(f"Tool call: {tool_exec.tool_call.name}({tool_exec.tool_call.arguments})")
-                        if tool_exec.tool_result.error:
-                            history_lines.append(f"Tool result: ERROR - {tool_exec.tool_result.error}")
-                        else:
-                            history_lines.append(f"Tool result: {tool_exec.tool_result.content}")
+                        history_lines.append(f"Tool result: {tool_exec.tool_result.content}")
                 else:
                     text = msg.content[0].text if msg.content and hasattr(msg.content[0], "text") else ""
                     history_lines.append(f"{msg.role.value.capitalize()}: {text}")
@@ -235,10 +232,7 @@ def _format_trace_level_prompt(self, parsed_input: TraceLevelInput) -> str:
                     # Handle tool execution lists
                     for tool_exec in msg:
                         history_lines.append(f"Tool call: {tool_exec.tool_call.name}({tool_exec.tool_call.arguments})")
-                        if tool_exec.tool_result.error:
-                            history_lines.append(f"Tool result: ERROR - {tool_exec.tool_result.error}")
-                        else:
-                            history_lines.append(f"Tool result: {tool_exec.tool_result.content}")
+                        history_lines.append(f"Tool result: {tool_exec.tool_result.content}")
                 else:
                     text = msg.content[0].text if msg.content and hasattr(msg.content[0], "text") else ""
                     history_lines.append(f"{msg.role.value.capitalize()}: {text}")
diff --git a/src/strands_evals/extractors/trace_extractor.py b/src/strands_evals/extractors/trace_extractor.py
index 015b4883..4edc0fe5 100644
--- a/src/strands_evals/extractors/trace_extractor.py
+++ b/src/strands_evals/extractors/trace_extractor.py
@@ -1,4 +1,5 @@
 import logging
+from datetime import datetime, timezone
 
 from ..types.trace import (
     AgentInvocationSpan,
@@ -20,6 +21,13 @@
 logger = logging.getLogger(__name__)
 
 
+def _to_aware_utc(dt: datetime) -> datetime:
+    """Normalize a datetime to timezone-aware UTC."""
+    if dt.tzinfo is None:
+        return dt.replace(tzinfo=timezone.utc)
+    return dt
+
+
 class TraceExtractor:
     """Extracts structured evaluation inputs from Session traces."""
 
@@ -108,7 +116,7 @@ def _extract_tool_level(self, session: Session) -> list[ToolLevelInput]:
                 prior_executions = [
                     ToolExecution(tool_call=span.tool_call, tool_result=span.tool_result)
                     for span in tool_spans[:index]
-                    if span.span_info.end_time <= tool_span.span_info.start_time
+                    if _to_aware_utc(span.span_info.end_time) <= _to_aware_utc(tool_span.span_info.start_time)
                 ]
 
                 evaluator_inputs.append(
diff --git a/src/strands_evals/types/trace.py b/src/strands_evals/types/trace.py
index 82d25dff..727fee96 100644
--- a/src/strands_evals/types/trace.py
+++ b/src/strands_evals/types/trace.py
@@ -170,8 +170,8 @@ class ToolLevelInput(BaseEvaluationInput):
     """Input for tool-level evaluators.
 
     `session_history` is the context preceding `tool_execution_details` and never includes the
-    call under evaluation. Unlike `TraceLevelInput`, where each `list[ToolExecution]` entry is one
-    turn's whole batch, here each entry holds exactly one execution.
+    call under evaluation. Tool executions that precede it within the same trace are appended as a
+    single `list[ToolExecution]` entry, in span order.
     """
 
     available_tools: list[ToolConfig]
diff --git a/tests/strands_evals/extractors/test_trace_extractor.py b/tests/strands_evals/extractors/test_trace_extractor.py
index b86b14cd..a2517041 100644
--- a/tests/strands_evals/extractors/test_trace_extractor.py
+++ b/tests/strands_evals/extractors/test_trace_extractor.py
@@ -160,10 +160,19 @@ def test_extract_empty_session_tool_level():
 
 
 def test_extract_tool_level_incremental_session_history():
-    """Each tool span sees exactly its true predecessors in session_history, in order."""
-    now = datetime.now()
+    """Each tool span sees exactly its causally-completed predecessors in session_history.
+
+    Uses distinct timestamps so the causality filter is exercised:
+    - tool_1 (t=0→1) finishes before tool_2 starts → prior for tool_2 and tool_3
+    - tool_2 (t=1→4) is long-running and still in-flight when tool_3 starts at t=2 → NOT a prior for tool_3
+    - tool_3 (t=2→3) starts while tool_2 is still running
+    """
+    from datetime import timedelta, timezone
+
+    base = datetime(2024, 1, 1, 12, 0, 0, tzinfo=timezone.utc)
+
     agent_span = AgentInvocationSpan(
-        span_info=SpanInfo(session_id="test", span_id="a0", start_time=now, end_time=now),
+        span_info=SpanInfo(session_id="test", span_id="a0", start_time=base, end_time=base + timedelta(seconds=10)),
         user_prompt="What is the square root of 1764, multiplied by 3, then add 1?",
         agent_response="127",
         available_tools=[
@@ -173,17 +182,36 @@ def test_extract_tool_level_incremental_session_history():
         ],
     )
     tool_span_1 = ToolExecutionSpan(
-        span_info=SpanInfo(session_id="test", span_id="t1", parent_span_id="a0", start_time=now, end_time=now),
+        span_info=SpanInfo(
+            session_id="test",
+            span_id="t1",
+            parent_span_id="a0",
+            start_time=base,
+            end_time=base + timedelta(seconds=1),
+        ),
         tool_call=ToolCall(name="square_root", arguments={"n": 1764}),
         tool_result=ToolResult(content="42.0"),
     )
+    # Long-running: finishes at t=4, AFTER tool_3 starts at t=2
     tool_span_2 = ToolExecutionSpan(
-        span_info=SpanInfo(session_id="test", span_id="t2", parent_span_id="a0", start_time=now, end_time=now),
+        span_info=SpanInfo(
+            session_id="test",
+            span_id="t2",
+            parent_span_id="a0",
+            start_time=base + timedelta(seconds=1),
+            end_time=base + timedelta(seconds=4),
+        ),
         tool_call=ToolCall(name="multiply_numbers", arguments={"a": 42, "b": 3}),
         tool_result=ToolResult(content="126"),
     )
     tool_span_3 = ToolExecutionSpan(
-        span_info=SpanInfo(session_id="test", span_id="t3", parent_span_id="a0", start_time=now, end_time=now),
+        span_info=SpanInfo(
+            session_id="test",
+            span_id="t3",
+            parent_span_id="a0",
+            start_time=base + timedelta(seconds=2),
+            end_time=base + timedelta(seconds=3),
+        ),
         tool_call=ToolCall(name="add_numbers", arguments={"a": 126, "b": 1}),
         tool_result=ToolResult(content="127"),
     )
@@ -202,7 +230,7 @@ def test_extract_tool_level_incremental_session_history():
     )
     assert result[0].tool_execution_details.tool_call.name == "square_root"
 
-    # Second tool: sees user prompt + first tool's execution
+    # Second tool: sees user prompt + first tool's execution (square_root finished at t=1 <= t=1)
     assert len(result[1].session_history) == 2, (
         f"second tool should see user prompt + 1 prior execution, got {len(result[1].session_history)} entries"
     )
@@ -210,11 +238,66 @@ def test_extract_tool_level_incremental_session_history():
     assert result[1].session_history[1][0].tool_call.name == "square_root"
     assert result[1].session_history[1][0].tool_result.content == "42.0"
 
-    # Third tool: sees user prompt + [square_root, multiply_numbers] in order
+    # Third tool: only square_root is a valid prior (multiply_numbers ends at t=4 > t=2 start)
     assert len(result[2].session_history) == 2, (
         f"expected 2 entries (user + prior tools), got {len(result[2].session_history)}"
     )
     prior_names = [entry.tool_call.name for entry in result[2].session_history[1]]
-    assert prior_names == ["square_root", "multiply_numbers"], (
-        f"expected [square_root, multiply_numbers] in order, got {prior_names}"
+    assert prior_names == ["square_root"], (
+        f"expected only [square_root] (multiply_numbers still running), got {prior_names}"
+    )
+
+
+def test_extract_tool_level_mixed_tz_timestamps():
+    """The causality filter handles mixed naive/aware timestamps without raising TypeError."""
+    from datetime import timedelta, timezone
+
+    base_naive = datetime(2024, 1, 1, 12, 0, 0)  # no tzinfo
+    base_aware = datetime(2024, 1, 1, 12, 0, 1, tzinfo=timezone.utc)  # aware
+
+    agent_span = AgentInvocationSpan(
+        span_info=SpanInfo(
+            session_id="test", span_id="a0", start_time=base_naive, end_time=base_aware + timedelta(seconds=5)
+        ),
+        user_prompt="Mixed tz test",
+        agent_response="Done",
+        available_tools=[ToolConfig(name="tool_x"), ToolConfig(name="tool_y")],
     )
+    # tool_x: naive timestamps
+    tool_x = ToolExecutionSpan(
+        span_info=SpanInfo(
+            session_id="test",
+            span_id="tx",
+            parent_span_id="a0",
+            start_time=base_naive,
+            end_time=base_naive + timedelta(seconds=1),
+        ),
+        tool_call=ToolCall(name="tool_x", arguments={}),
+        tool_result=ToolResult(content="x_result"),
+    )
+    # tool_y: aware timestamps — starts after tool_x ends
+    tool_y = ToolExecutionSpan(
+        span_info=SpanInfo(
+            session_id="test",
+            span_id="ty",
+            parent_span_id="a0",
+            start_time=base_aware,
+            end_time=base_aware + timedelta(seconds=1),
+        ),
+        tool_call=ToolCall(name="tool_y", arguments={}),
+        tool_result=ToolResult(content="y_result"),
+    )
+
+    trace = Trace(spans=[agent_span, tool_x, tool_y], trace_id="trace1", session_id="test")
+    session = Session(traces=[trace], session_id="test")
+
+    extractor = TraceExtractor(EvaluationLevel.TOOL_LEVEL)
+    # Should not raise TypeError
+    result = extractor.extract(session)
+
+    assert len(result) == 2
+    # tool_y should see tool_x as a prior (naive 12:00:01 <= aware 12:00:01 UTC)
+    assert result[1].tool_execution_details.tool_call.name == "tool_y"
+    assert len(result[1].session_history) == 2
+    prior_names = [e.tool_call.name for e in result[1].session_history[1]]
+    assert prior_names == ["tool_x"]

From 2664f4fab9c34787ad9ff64aa43bb7b44fc42373 Mon Sep 17 00:00:00 2001
From: liramon2 <liramon@amazon.com>
Date: Thu, 30 Jul 2026 20:19:47 -0400
Subject: [PATCH 4/5] fix: address strandly concerns

---
 .../extractors/trace_extractor.py             | 17 ++++--
 .../extractors/test_trace_extractor.py        | 61 +++++++++++--------
 2 files changed, 48 insertions(+), 30 deletions(-)

diff --git a/src/strands_evals/extractors/trace_extractor.py b/src/strands_evals/extractors/trace_extractor.py
index 4edc0fe5..776bda16 100644
--- a/src/strands_evals/extractors/trace_extractor.py
+++ b/src/strands_evals/extractors/trace_extractor.py
@@ -112,11 +112,19 @@ def _extract_tool_level(self, session: Session) -> list[ToolLevelInput]:
             if agent_span and agent_span.user_prompt:
                 session_history.append(UserMessage(content=[TextContent(text=agent_span.user_prompt)]))
 
+            tool_executions = [
+                ToolExecution(tool_call=span.tool_call, tool_result=span.tool_result) for span in tool_spans
+            ]
+            tool_end_times = [_to_aware_utc(span.span_info.end_time) for span in tool_spans]
+
             for index, tool_span in enumerate(tool_spans):
+                target_start = _to_aware_utc(tool_span.span_info.start_time)
                 prior_executions = [
-                    ToolExecution(tool_call=span.tool_call, tool_result=span.tool_result)
-                    for span in tool_spans[:index]
-                    if _to_aware_utc(span.span_info.end_time) <= _to_aware_utc(tool_span.span_info.start_time)
+                    tool_executions[position]
+                    for position in range(len(tool_spans))
+                    if position != index
+                    and tool_end_times[position] <= target_start
+                    and (tool_end_times[position] < target_start or position < index)
                 ]
 
                 evaluator_inputs.append(
@@ -129,9 +137,6 @@ def _extract_tool_level(self, session: Session) -> list[ToolLevelInput]:
                 )
 
             if tool_spans:
-                tool_executions = [
-                    ToolExecution(tool_call=span.tool_call, tool_result=span.tool_result) for span in tool_spans
-                ]
                 session_history.append(tool_executions)
 
             if agent_span and agent_span.agent_response:
diff --git a/tests/strands_evals/extractors/test_trace_extractor.py b/tests/strands_evals/extractors/test_trace_extractor.py
index a2517041..9f3521ed 100644
--- a/tests/strands_evals/extractors/test_trace_extractor.py
+++ b/tests/strands_evals/extractors/test_trace_extractor.py
@@ -160,13 +160,7 @@ def test_extract_empty_session_tool_level():
 
 
 def test_extract_tool_level_incremental_session_history():
-    """Each tool span sees exactly its causally-completed predecessors in session_history.
-
-    Uses distinct timestamps so the causality filter is exercised:
-    - tool_1 (t=0→1) finishes before tool_2 starts → prior for tool_2 and tool_3
-    - tool_2 (t=1→4) is long-running and still in-flight when tool_3 starts at t=2 → NOT a prior for tool_3
-    - tool_3 (t=2→3) starts while tool_2 is still running
-    """
+    """Each tool span sees exactly its causally-completed predecessors in session_history."""
     from datetime import timedelta, timezone
 
     base = datetime(2024, 1, 1, 12, 0, 0, tzinfo=timezone.utc)
@@ -192,14 +186,15 @@ def test_extract_tool_level_incremental_session_history():
         tool_call=ToolCall(name="square_root", arguments={"n": 1764}),
         tool_result=ToolResult(content="42.0"),
     )
-    # Long-running: finishes at t=4, AFTER tool_3 starts at t=2
+    # Finishes at t=2.5: AFTER tool_3 starts (t=2) but BEFORE tool_3 ends (t=3)
+    # This discriminates start_time vs end_time comparison targets
     tool_span_2 = ToolExecutionSpan(
         span_info=SpanInfo(
             session_id="test",
             span_id="t2",
             parent_span_id="a0",
             start_time=base + timedelta(seconds=1),
-            end_time=base + timedelta(seconds=4),
+            end_time=base + timedelta(milliseconds=2500),
         ),
         tool_call=ToolCall(name="multiply_numbers", arguments={"a": 42, "b": 3}),
         tool_result=ToolResult(content="126"),
@@ -238,13 +233,13 @@ def test_extract_tool_level_incremental_session_history():
     assert result[1].session_history[1][0].tool_call.name == "square_root"
     assert result[1].session_history[1][0].tool_result.content == "42.0"
 
-    # Third tool: only square_root is a valid prior (multiply_numbers ends at t=4 > t=2 start)
+    # Third tool: only square_root is a valid prior
     assert len(result[2].session_history) == 2, (
         f"expected 2 entries (user + prior tools), got {len(result[2].session_history)}"
     )
     prior_names = [entry.tool_call.name for entry in result[2].session_history[1]]
     assert prior_names == ["square_root"], (
-        f"expected only [square_root] (multiply_numbers still running), got {prior_names}"
+        f"expected only [square_root] (multiply_numbers still running at tool_3.start_time), got {prior_names}"
     )
 
 
@@ -253,17 +248,21 @@ def test_extract_tool_level_mixed_tz_timestamps():
     from datetime import timedelta, timezone
 
     base_naive = datetime(2024, 1, 1, 12, 0, 0)  # no tzinfo
-    base_aware = datetime(2024, 1, 1, 12, 0, 1, tzinfo=timezone.utc)  # aware
+    base_aware = datetime(2024, 1, 1, 12, 0, 2, tzinfo=timezone.utc)  # aware, starts at +2s
 
     agent_span = AgentInvocationSpan(
         span_info=SpanInfo(
-            session_id="test", span_id="a0", start_time=base_naive, end_time=base_aware + timedelta(seconds=5)
+            session_id="test", span_id="a0", start_time=base_naive, end_time=base_aware + timedelta(seconds=10)
         ),
         user_prompt="Mixed tz test",
         agent_response="Done",
-        available_tools=[ToolConfig(name="tool_x"), ToolConfig(name="tool_y")],
+        available_tools=[
+            ToolConfig(name="tool_x"),
+            ToolConfig(name="tool_long"),
+            ToolConfig(name="tool_y"),
+        ],
     )
-    # tool_x: naive timestamps
+    # tool_x: ends at 12:00:01 (before tool_y starts)
     tool_x = ToolExecutionSpan(
         span_info=SpanInfo(
             session_id="test",
@@ -275,7 +274,19 @@ def test_extract_tool_level_mixed_tz_timestamps():
         tool_call=ToolCall(name="tool_x", arguments={}),
         tool_result=ToolResult(content="x_result"),
     )
-    # tool_y: aware timestamps — starts after tool_x ends
+    # tool_long: ends at 12:00:05 (after tool_y starts at 12:00:02)
+    tool_long = ToolExecutionSpan(
+        span_info=SpanInfo(
+            session_id="test",
+            span_id="tl",
+            parent_span_id="a0",
+            start_time=base_naive,
+            end_time=base_naive + timedelta(seconds=5),
+        ),
+        tool_call=ToolCall(name="tool_long", arguments={}),
+        tool_result=ToolResult(content="long_result"),
+    )
+    # tool_y: starts at 12:00:02 UTC
     tool_y = ToolExecutionSpan(
         span_info=SpanInfo(
             session_id="test",
@@ -288,16 +299,18 @@ def test_extract_tool_level_mixed_tz_timestamps():
         tool_result=ToolResult(content="y_result"),
     )
 
-    trace = Trace(spans=[agent_span, tool_x, tool_y], trace_id="trace1", session_id="test")
+    trace = Trace(spans=[agent_span, tool_x, tool_long, tool_y], trace_id="trace1", session_id="test")
     session = Session(traces=[trace], session_id="test")
 
     extractor = TraceExtractor(EvaluationLevel.TOOL_LEVEL)
-    # Should not raise TypeError
+    # Should not raise TypeError from mixed tz comparison
     result = extractor.extract(session)
 
-    assert len(result) == 2
-    # tool_y should see tool_x as a prior (naive 12:00:01 <= aware 12:00:01 UTC)
-    assert result[1].tool_execution_details.tool_call.name == "tool_y"
-    assert len(result[1].session_history) == 2
-    prior_names = [e.tool_call.name for e in result[1].session_history[1]]
-    assert prior_names == ["tool_x"]
+    assert len(result) == 3
+    # tool_y should see ONLY tool_x as a prior
+    assert result[2].tool_execution_details.tool_call.name == "tool_y"
+    assert len(result[2].session_history) == 2
+    prior_names = [e.tool_call.name for e in result[2].session_history[1]]
+    assert prior_names == ["tool_x"], (
+        f"expected only ['tool_x'] as prior for tool_y (tool_long still running), got {prior_names}"
+    )

From baa56eecafbb884d9def73ba811276bd23d8d9ee Mon Sep 17 00:00:00 2001
From: liramon2 <liramon@amazon.com>
Date: Fri, 31 Jul 2026 09:28:10 -0400
Subject: [PATCH 5/5] fix: address strandly concerns

---
 src/strands_evals/extractors/trace_extractor.py | 5 ++++-
 src/strands_evals/types/trace.py                | 6 +++---
 2 files changed, 7 insertions(+), 4 deletions(-)

diff --git a/src/strands_evals/extractors/trace_extractor.py b/src/strands_evals/extractors/trace_extractor.py
index 776bda16..7384de6b 100644
--- a/src/strands_evals/extractors/trace_extractor.py
+++ b/src/strands_evals/extractors/trace_extractor.py
@@ -115,7 +115,10 @@ def _extract_tool_level(self, session: Session) -> list[ToolLevelInput]:
             tool_executions = [
                 ToolExecution(tool_call=span.tool_call, tool_result=span.tool_result) for span in tool_spans
             ]
-            tool_end_times = [_to_aware_utc(span.span_info.end_time) for span in tool_spans]
+            tool_end_times = [
+                max(_to_aware_utc(span.span_info.end_time), _to_aware_utc(span.span_info.start_time))
+                for span in tool_spans
+            ]
 
             for index, tool_span in enumerate(tool_spans):
                 target_start = _to_aware_utc(tool_span.span_info.start_time)
diff --git a/src/strands_evals/types/trace.py b/src/strands_evals/types/trace.py
index 727fee96..c78d11e0 100644
--- a/src/strands_evals/types/trace.py
+++ b/src/strands_evals/types/trace.py
@@ -169,9 +169,9 @@ class TraceLevelInput(BaseEvaluationInput):
 class ToolLevelInput(BaseEvaluationInput):
     """Input for tool-level evaluators.
 
-    `session_history` is the context preceding `tool_execution_details` and never includes the
-    call under evaluation. Tool executions that precede it within the same trace are appended as a
-    single `list[ToolExecution]` entry, in span order.
+    `session_history` never includes the call under evaluation. Within the same trace, only tool
+    executions that completed before this tool started (`end_time <= start_time`) are included,
+    with list position as tiebreaker for equal timestamps. Cross-trace history is unfiltered.
     """
 
     available_tools: list[ToolConfig]