feat(evaluators): allow custom tools on judge-based evaluators (Trajectory, Output, Multimodal)

src/strands_evals/evaluators/multimodal_correctness_evaluator.py

@@ -25,6 +25,7 @@ def __init__(
         reference_suffix: str | None = None,
         uses_environment_state: bool = False,
         name: str | None = None,
+        tools: list | None = None,
     ):
         super().__init__(
             rubric=rubric if rubric is not None else CORRECTNESS_RUBRIC_V0,
@@ -34,4 +35,5 @@ def __init__(
             reference_suffix=reference_suffix,
             uses_environment_state=uses_environment_state,
             name=name,
+            tools=tools,
         )

src/strands_evals/evaluators/multimodal_faithfulness_evaluator.py

@@ -24,6 +24,7 @@ def __init__(
         reference_suffix: str | None = None,
         uses_environment_state: bool = False,
         name: str | None = None,
+        tools: list | None = None,
     ):
         super().__init__(
             rubric=rubric if rubric is not None else FAITHFULNESS_RUBRIC_V0,
@@ -33,4 +34,5 @@ def __init__(
             reference_suffix=reference_suffix,
             uses_environment_state=uses_environment_state,
             name=name,
+            tools=tools,
         )

src/strands_evals/evaluators/multimodal_instruction_following_evaluator.py

@@ -24,6 +24,7 @@ def __init__(
         reference_suffix: str | None = None,
         uses_environment_state: bool = False,
         name: str | None = None,
+        tools: list | None = None,
     ):
         super().__init__(
             rubric=rubric if rubric is not None else INSTRUCTION_FOLLOWING_RUBRIC_V0,
@@ -33,4 +34,5 @@ def __init__(
             reference_suffix=reference_suffix,
             uses_environment_state=uses_environment_state,
             name=name,
+            tools=tools,
         )

src/strands_evals/evaluators/multimodal_output_evaluator.py

@@ -46,6 +46,7 @@ def __init__(
         reference_suffix: str | None = None,
         uses_environment_state: bool = False,
         name: str | None = None,
+        tools: list | None = None,
     ):
         super().__init__(
             rubric=rubric,
@@ -54,6 +55,7 @@ def __init__(
             include_inputs=include_inputs,
             uses_environment_state=uses_environment_state,
             name=name,
+            tools=tools,
         )
         self.reference_suffix = reference_suffix if reference_suffix is not None else self.DEFAULT_REFERENCE_SUFFIX
 

src/strands_evals/evaluators/multimodal_overall_quality_evaluator.py

@@ -35,6 +35,7 @@ def __init__(
         reference_suffix: str | None = None,
         uses_environment_state: bool = False,
         name: str | None = None,
+        tools: list | None = None,
     ):
         super().__init__(
             rubric=rubric if rubric is not None else OVERALL_QUALITY_RUBRIC_V0,
@@ -44,4 +45,5 @@ def __init__(
             reference_suffix=reference_suffix if reference_suffix is not None else _OVERALL_QUALITY_REFERENCE_SUFFIX,
             uses_environment_state=uses_environment_state,
             name=name,
+            tools=tools,
         )

src/strands_evals/evaluators/output_evaluator.py

@@ -1,4 +1,4 @@
-from typing import cast
+from typing import Any, cast
 
 from strands import Agent
 from strands.models.model import Model
@@ -20,6 +20,8 @@ class OutputEvaluator(Evaluator[InputT, OutputT]):
         system_prompt: System prompt to guide model behavior.
                     If None, the evaluator will use one of the default template.
         include_inputs: Whether to include inputs to the task in the evaluation or not.
+        tools: Optional tools for the evaluator agent (e.g., domain-specific verification
+                    functions the judge can call). Defaults to None (no tools).
     """
 
     def __init__(
@@ -30,13 +32,15 @@ def __init__(
         include_inputs: bool = True,
         uses_environment_state: bool = False,
         name: str | None = None,
+        tools: list[Any] | None = None,
     ):
         super().__init__(name=name)
         self.rubric = rubric
         self.model = model
         self.include_inputs = include_inputs
         self.system_prompt = system_prompt
         self.uses_environment_state = uses_environment_state
+        self.tools = tools
 
     def _build_prompt(self, evaluation_case: EvaluationData[InputT, OutputT]) -> str | list:
         """Build the evaluation prompt for a test case.
@@ -66,7 +70,9 @@ def evaluate(self, evaluation_case: EvaluationData[InputT, OutputT]) -> list[Eva
         Returns:
             The results of the evaluation as EvaluationOutput.
         """
-        evaluator_agent = Agent(model=self.model, system_prompt=self.system_prompt, callback_handler=None)
+        evaluator_agent = Agent(
+            model=self.model, tools=self.tools, system_prompt=self.system_prompt, callback_handler=None
+        )
         evaluation_prompt = self._build_prompt(evaluation_case)
         result = evaluator_agent(evaluation_prompt, structured_output_model=EvaluationOutput)
         return [cast(EvaluationOutput, result.structured_output)]
@@ -81,7 +87,9 @@ async def evaluate_async(self, evaluation_case: EvaluationData[InputT, OutputT])
         Returns:
             The results of the evaluation as EvaluationOutput.
         """
-        evaluator_agent = Agent(model=self.model, system_prompt=self.system_prompt, callback_handler=None)
+        evaluator_agent = Agent(
+            model=self.model, tools=self.tools, system_prompt=self.system_prompt, callback_handler=None
+        )
         evaluation_prompt = self._build_prompt(evaluation_case)
         result = await evaluator_agent.invoke_async(evaluation_prompt, structured_output_model=EvaluationOutput)
         return [cast(EvaluationOutput, result.structured_output)]

src/strands_evals/evaluators/trajectory_evaluator.py

@@ -23,6 +23,8 @@ class TrajectoryEvaluator(Evaluator[InputT, OutputT]):
         system_prompt: System prompt to guide model behavior.
                     If None, the evaluator will use one of the default template.
         include_inputs: Whether to include inputs to the task in the evaluation or not.
+        tools: Optional additional tools for the evaluator agent. Merged with the
+                    default trajectory scoring tools (exact/in-order/any-order match).
     """
 
     def __init__(
@@ -33,16 +35,18 @@ def __init__(
         system_prompt: str = SYSTEM_PROMPT,
         include_inputs: bool = True,
         name: str | None = None,
+        tools: list[Any] | None = None,
     ):
         super().__init__(name=name)
         self.rubric = rubric
         self.trajectory_description = trajectory_description
         self.model = model
         self.include_inputs = include_inputs
-        self._tools: list[str | dict[str, str] | Any] | None = [
+        self._tools: list[str | dict[str, str] | Any] = [
             exact_match_scorer,
             in_order_match_scorer,
             any_order_match_scorer,
+            *(tools or []),
         ]
         self.system_prompt = system_prompt
 

tests/strands_evals/evaluators/test_output_evaluator.py

@@ -70,7 +70,9 @@ def test_output_evaluator_evaluate_with_inputs(mock_agent_class, evaluation_data
     result = evaluator.evaluate(evaluation_data)
 
     # Verify Agent was created with correct parameters
-    mock_agent_class.assert_called_once_with(model=None, system_prompt=evaluator.system_prompt, callback_handler=None)
+    mock_agent_class.assert_called_once_with(
+        model=None, tools=None, system_prompt=evaluator.system_prompt, callback_handler=None
+    )
 
     # Verify agent was called
     mock_agent.assert_called_once()
@@ -148,7 +150,9 @@ async def test_output_evaluator_evaluate_async_with_inputs(mock_agent_class, eva
     result = await evaluator.evaluate_async(evaluation_data)
 
     # Verify Agent was created with correct parameters
-    mock_agent_class.assert_called_once_with(model=None, system_prompt=evaluator.system_prompt, callback_handler=None)
+    mock_agent_class.assert_called_once_with(
+        model=None, tools=None, system_prompt=evaluator.system_prompt, callback_handler=None
+    )
 
     assert len(result) == 1
     assert result[0].score == 0.8
@@ -259,3 +263,60 @@ async def test_output_evaluator_evaluate_async_includes_environment_state(mock_a
 
     assert len(result) == 1
     assert result[0].test_pass is True
+
+
+def test_output_evaluator_init_with_tools():
+    """Test OutputEvaluator initialization with custom tools"""
+
+    def verify_claim(claim: str) -> str:
+        return "verified"
+
+    evaluator = OutputEvaluator(rubric="Test rubric", tools=[verify_claim])
+
+    assert evaluator.tools == [verify_claim]
+
+
+def test_output_evaluator_init_without_tools_defaults_to_none():
+    """Test OutputEvaluator has no tools by default (current behavior preserved)"""
+    evaluator = OutputEvaluator(rubric="Test rubric")
+
+    assert evaluator.tools is None
+
+
+@patch("strands_evals.evaluators.output_evaluator.Agent")
+def test_output_evaluator_evaluate_passes_tools_to_agent(mock_agent_class, evaluation_data, mock_agent):
+    """Test that custom tools are passed to the evaluator agent"""
+    mock_agent_class.return_value = mock_agent
+
+    def verify_claim(claim: str) -> str:
+        return "verified"
+
+    evaluator = OutputEvaluator(rubric="Test rubric", tools=[verify_claim])
+
+    result = evaluator.evaluate(evaluation_data)
+
+    mock_agent_class.assert_called_once_with(
+        model=None, tools=[verify_claim], system_prompt=evaluator.system_prompt, callback_handler=None
+    )
+    assert result[0].score == 0.8
+
+
+@pytest.mark.asyncio
+@patch("strands_evals.evaluators.output_evaluator.Agent")
+async def test_output_evaluator_evaluate_async_passes_tools_to_agent(
+    mock_agent_class, evaluation_data, mock_async_agent
+):
+    """Test that custom tools are passed to the evaluator agent in async path"""
+    mock_agent_class.return_value = mock_async_agent
+
+    def verify_claim(claim: str) -> str:
+        return "verified"
+
+    evaluator = OutputEvaluator(rubric="Test rubric", tools=[verify_claim])
+
+    result = await evaluator.evaluate_async(evaluation_data)
+
+    mock_agent_class.assert_called_once_with(
+        model=None, tools=[verify_claim], system_prompt=evaluator.system_prompt, callback_handler=None
+    )
+    assert result[0].score == 0.8

tests/strands_evals/evaluators/test_trajectory_evaluator.py

@@ -255,3 +255,59 @@ def test_trajectory_evaluator_update_trajectory_description():
     evaluator.update_trajectory_description(new_description)
 
     assert evaluator.trajectory_description == new_description
+
+
+def test_trajectory_evaluator_init_with_tools_merges_with_defaults():
+    """Test that custom tools are appended after the default scoring tools"""
+    from strands_evals.tools.evaluation_tools import (
+        any_order_match_scorer,
+        exact_match_scorer,
+        in_order_match_scorer,
+    )
+
+    def verify_step(step: str) -> str:
+        return "valid"
+
+    evaluator = TrajectoryEvaluator(rubric="Test rubric", tools=[verify_step])
+
+    assert evaluator._tools == [exact_match_scorer, in_order_match_scorer, any_order_match_scorer, verify_step]
+
+
+def test_trajectory_evaluator_init_without_tools_keeps_default_scorers():
+    """Test that default scoring tools are unchanged when no custom tools are provided"""
+    from strands_evals.tools.evaluation_tools import (
+        any_order_match_scorer,
+        exact_match_scorer,
+        in_order_match_scorer,
+    )
+
+    evaluator = TrajectoryEvaluator(rubric="Test rubric")
+
+    assert evaluator._tools == [exact_match_scorer, in_order_match_scorer, any_order_match_scorer]
+
+
+@patch("strands_evals.evaluators.trajectory_evaluator.Agent")
+def test_trajectory_evaluator_evaluate_passes_custom_tools_to_agent(mock_agent_class, evaluation_data, mock_agent):
+    """Test that merged tools (defaults + custom) reach the evaluator agent"""
+    from strands_evals.tools.evaluation_tools import (
+        any_order_match_scorer,
+        exact_match_scorer,
+        in_order_match_scorer,
+    )
+
+    mock_agent_class.return_value = mock_agent
+
+    def verify_step(step: str) -> str:
+        return "valid"
+
+    evaluator = TrajectoryEvaluator(rubric="Test rubric", tools=[verify_step])
+
+    result = evaluator.evaluate(evaluation_data)
+
+    mock_agent_class.assert_called_once_with(
+        model=None,
+        system_prompt=evaluator.system_prompt,
+        tools=[exact_match_scorer, in_order_match_scorer, any_order_match_scorer, verify_step],
+        callback_handler=None,
+    )
+    assert result[0].score == 0.9