feat(evaluators): allow custom tools on judge-based evaluators (Trajectory, Output, Multimodal)

src/strands_evals/evaluators/output_evaluator.py

@@ -1,4 +1,4 @@
-from typing import cast
+from typing import Any, cast
 
 from strands import Agent
 from strands.models.model import Model
@@ -20,6 +20,8 @@ class OutputEvaluator(Evaluator[InputT, OutputT]):
         system_prompt: System prompt to guide model behavior.
                     If None, the evaluator will use one of the default template.
         include_inputs: Whether to include inputs to the task in the evaluation or not.
+        tools: Optional tools for the evaluator agent (e.g., domain-specific verification
+                    functions the judge can call). Defaults to None (no tools).
     """
 
     def __init__(
@@ -29,6 +31,7 @@ def __init__(
         system_prompt: str = SYSTEM_PROMPT,
         include_inputs: bool = True,
         uses_environment_state: bool = False,
+        tools: list[Any] | None = None,
         name: str | None = None,
     ):
         super().__init__(name=name)
@@ -37,6 +40,7 @@ def __init__(
         self.include_inputs = include_inputs
         self.system_prompt = system_prompt
         self.uses_environment_state = uses_environment_state
+        self._tools = tools
 
     def _build_prompt(self, evaluation_case: EvaluationData[InputT, OutputT]) -> str | list:
         """Build the evaluation prompt for a test case.
@@ -66,11 +70,18 @@ def evaluate(self, evaluation_case: EvaluationData[InputT, OutputT]) -> list[Eva
         Returns:
             The results of the evaluation as EvaluationOutput.
         """
-        evaluator_agent = Agent(model=self.model, system_prompt=self.system_prompt, callback_handler=None)
+        evaluator_agent = self._create_evaluator_agent()
         evaluation_prompt = self._build_prompt(evaluation_case)
         result = evaluator_agent(evaluation_prompt, structured_output_model=EvaluationOutput)
         return [cast(EvaluationOutput, result.structured_output)]
 
+    def _create_evaluator_agent(self) -> Agent:
+        """Create the judge agent, including custom tools when provided."""
+        kwargs: dict[str, Any] = {"model": self.model, "system_prompt": self.system_prompt, "callback_handler": None}
+        if self._tools:
+            kwargs["tools"] = self._tools
+        return Agent(**kwargs)
+
     async def evaluate_async(self, evaluation_case: EvaluationData[InputT, OutputT]) -> list[EvaluationOutput]:
         """
         Evaluate the performance of the task on the given test cases asynchronously.
@@ -81,7 +92,7 @@ async def evaluate_async(self, evaluation_case: EvaluationData[InputT, OutputT])
         Returns:
             The results of the evaluation as EvaluationOutput.
         """
-        evaluator_agent = Agent(model=self.model, system_prompt=self.system_prompt, callback_handler=None)
+        evaluator_agent = self._create_evaluator_agent()
         evaluation_prompt = self._build_prompt(evaluation_case)
         result = await evaluator_agent.invoke_async(evaluation_prompt, structured_output_model=EvaluationOutput)
         return [cast(EvaluationOutput, result.structured_output)]

src/strands_evals/evaluators/trajectory_evaluator.py

@@ -23,6 +23,8 @@ class TrajectoryEvaluator(Evaluator[InputT, OutputT]):
         system_prompt: System prompt to guide model behavior.
                     If None, the evaluator will use one of the default template.
         include_inputs: Whether to include inputs to the task in the evaluation or not.
+        tools: Optional additional tools for the evaluator agent. Merged with the
+                    default trajectory scoring tools (exact/in-order/any-order match).
     """
 
     def __init__(
@@ -32,14 +34,15 @@ def __init__(
         model: Model | str | None = None,
         system_prompt: str = SYSTEM_PROMPT,
         include_inputs: bool = True,
+        tools: list[Any] | None = None,
         name: str | None = None,
     ):
         super().__init__(name=name)
         self.rubric = rubric
         self.trajectory_description = trajectory_description
         self.model = model
         self.include_inputs = include_inputs
-        self._tools: list[str | dict[str, str] | Any] | None = [
+        self._tools: list[str | dict[str, str] | Any] | None = list(tools or []) + [
             exact_match_scorer,
             in_order_match_scorer,
             any_order_match_scorer,

tests/strands_evals/evaluators/test_output_evaluator.py

@@ -259,3 +259,71 @@ async def test_output_evaluator_evaluate_async_includes_environment_state(mock_a
 
     assert len(result) == 1
     assert result[0].test_pass is True
+
+
+def test_output_evaluator_init_with_tools():
+    """Test OutputEvaluator initialization with custom tools"""
+
+    def verify_claim(claim: str) -> str:
+        return "verified"
+
+    evaluator = OutputEvaluator(rubric="Test rubric", tools=[verify_claim])
+
+    assert evaluator._tools == [verify_claim]
+
+
+def test_output_evaluator_init_without_tools_defaults_to_none():
+    """Test OutputEvaluator has no tools by default (current behavior preserved)"""
+    evaluator = OutputEvaluator(rubric="Test rubric")
+
+    assert evaluator._tools is None
+
+
+@patch("strands_evals.evaluators.output_evaluator.Agent")
+def test_output_evaluator_evaluate_passes_tools_to_agent(mock_agent_class, evaluation_data, mock_agent):
+    """Test that custom tools are passed to the evaluator agent"""
+    mock_agent_class.return_value = mock_agent
+
+    def verify_claim(claim: str) -> str:
+        return "verified"
+
+    evaluator = OutputEvaluator(rubric="Test rubric", tools=[verify_claim])
+
+    result = evaluator.evaluate(evaluation_data)
+
+    mock_agent_class.assert_called_once_with(
+        model=None, system_prompt=evaluator.system_prompt, callback_handler=None, tools=[verify_claim]
+    )
+    assert result[0].score == 0.8
+
+
+@patch("strands_evals.evaluators.output_evaluator.Agent")
+def test_output_evaluator_evaluate_without_tools_omits_tools_kwarg(mock_agent_class, evaluation_data, mock_agent):
+    """Test that no tools kwarg is passed when tools are not provided (backward compatible)"""
+    mock_agent_class.return_value = mock_agent
+    evaluator = OutputEvaluator(rubric="Test rubric")
+
+    evaluator.evaluate(evaluation_data)
+
+    mock_agent_class.assert_called_once_with(model=None, system_prompt=evaluator.system_prompt, callback_handler=None)
+
+
+@pytest.mark.asyncio
+@patch("strands_evals.evaluators.output_evaluator.Agent")
+async def test_output_evaluator_evaluate_async_passes_tools_to_agent(
+    mock_agent_class, evaluation_data, mock_async_agent
+):
+    """Test that custom tools are passed to the evaluator agent in async path"""
+    mock_agent_class.return_value = mock_async_agent
+
+    def verify_claim(claim: str) -> str:
+        return "verified"
+
+    evaluator = OutputEvaluator(rubric="Test rubric", tools=[verify_claim])
+
+    result = await evaluator.evaluate_async(evaluation_data)
+
+    mock_agent_class.assert_called_once_with(
+        model=None, system_prompt=evaluator.system_prompt, callback_handler=None, tools=[verify_claim]
+    )
+    assert result[0].score == 0.8

tests/strands_evals/evaluators/test_trajectory_evaluator.py

@@ -255,3 +255,51 @@ def test_trajectory_evaluator_update_trajectory_description():
     evaluator.update_trajectory_description(new_description)
 
     assert evaluator.trajectory_description == new_description
+
+
+def test_trajectory_evaluator_init_with_tools_merges_with_defaults():
+    """Test that custom tools are merged with the default scoring tools"""
+    from strands_evals.tools.evaluation_tools import (
+        any_order_match_scorer,
+        exact_match_scorer,
+        in_order_match_scorer,
+    )
+
+    def verify_step(step: str) -> str:
+        return "valid"
+
+    evaluator = TrajectoryEvaluator(rubric="Test rubric", tools=[verify_step])
+
+    assert evaluator._tools == [verify_step, exact_match_scorer, in_order_match_scorer, any_order_match_scorer]
+
+
+def test_trajectory_evaluator_init_without_tools_keeps_default_scorers():
+    """Test that default scoring tools are unchanged when no custom tools are provided"""
+    from strands_evals.tools.evaluation_tools import (
+        any_order_match_scorer,
+        exact_match_scorer,
+        in_order_match_scorer,
+    )
+
+    evaluator = TrajectoryEvaluator(rubric="Test rubric")
+
+    assert evaluator._tools == [exact_match_scorer, in_order_match_scorer, any_order_match_scorer]
+
+
+@patch("strands_evals.evaluators.trajectory_evaluator.Agent")
+def test_trajectory_evaluator_evaluate_passes_custom_tools_to_agent(mock_agent_class, evaluation_data, mock_agent):
+    """Test that merged tools (custom + defaults) reach the evaluator agent"""
+    mock_agent_class.return_value = mock_agent
+
+    def verify_step(step: str) -> str:
+        return "valid"
+
+    evaluator = TrajectoryEvaluator(rubric="Test rubric", tools=[verify_step])
+
+    result = evaluator.evaluate(evaluation_data)
+
+    mock_agent_class.assert_called_once_with(
+        model=None, system_prompt=evaluator.system_prompt, tools=evaluator._tools, callback_handler=None
+    )
+    assert verify_step in mock_agent_class.call_args[1]["tools"]
+    assert result[0].score == 0.9