From 653fbef2515d28af86de5dfb650410bca145ca12 Mon Sep 17 00:00:00 2001 From: DEBJYOTI PAUL Date: Thu, 23 Jul 2026 08:27:49 -0700 Subject: [PATCH 1/2] feat(evaluators): allow custom tools on TrajectoryEvaluator and OutputEvaluator Adds an optional tools= constructor parameter so the judge agent can call domain-specific verification functions during evaluation. - TrajectoryEvaluator: user tools are merged ahead of the default trajectory scoring tools (exact/in-order/any-order match), which are always retained. - OutputEvaluator: tools default to None, preserving the current no-tools judge. When provided, they are passed to the evaluator Agent in both sync and async paths (and inherited by MultimodalOutputEvaluator subclasses via _create_evaluator_agent). Non-breaking: omitting tools= keeps existing behavior exactly. --- .../evaluators/output_evaluator.py | 17 ++++- .../evaluators/trajectory_evaluator.py | 5 +- .../evaluators/test_output_evaluator.py | 68 +++++++++++++++++++ .../evaluators/test_trajectory_evaluator.py | 48 +++++++++++++ 4 files changed, 134 insertions(+), 4 deletions(-) diff --git a/src/strands_evals/evaluators/output_evaluator.py b/src/strands_evals/evaluators/output_evaluator.py index f732e9d3..ab4d1899 100644 --- a/src/strands_evals/evaluators/output_evaluator.py +++ b/src/strands_evals/evaluators/output_evaluator.py @@ -1,4 +1,4 @@ -from typing import cast +from typing import Any, cast from strands import Agent from strands.models.model import Model @@ -20,6 +20,8 @@ class OutputEvaluator(Evaluator[InputT, OutputT]): system_prompt: System prompt to guide model behavior. If None, the evaluator will use one of the default template. include_inputs: Whether to include inputs to the task in the evaluation or not. + tools: Optional tools for the evaluator agent (e.g., domain-specific verification + functions the judge can call). Defaults to None (no tools). """ def __init__( @@ -29,6 +31,7 @@ def __init__( system_prompt: str = SYSTEM_PROMPT, include_inputs: bool = True, uses_environment_state: bool = False, + tools: list[Any] | None = None, name: str | None = None, ): super().__init__(name=name) @@ -37,6 +40,7 @@ def __init__( self.include_inputs = include_inputs self.system_prompt = system_prompt self.uses_environment_state = uses_environment_state + self._tools = tools def _build_prompt(self, evaluation_case: EvaluationData[InputT, OutputT]) -> str | list: """Build the evaluation prompt for a test case. @@ -66,11 +70,18 @@ def evaluate(self, evaluation_case: EvaluationData[InputT, OutputT]) -> list[Eva Returns: The results of the evaluation as EvaluationOutput. """ - evaluator_agent = Agent(model=self.model, system_prompt=self.system_prompt, callback_handler=None) + evaluator_agent = self._create_evaluator_agent() evaluation_prompt = self._build_prompt(evaluation_case) result = evaluator_agent(evaluation_prompt, structured_output_model=EvaluationOutput) return [cast(EvaluationOutput, result.structured_output)] + def _create_evaluator_agent(self) -> Agent: + """Create the judge agent, including custom tools when provided.""" + kwargs: dict[str, Any] = {"model": self.model, "system_prompt": self.system_prompt, "callback_handler": None} + if self._tools: + kwargs["tools"] = self._tools + return Agent(**kwargs) + async def evaluate_async(self, evaluation_case: EvaluationData[InputT, OutputT]) -> list[EvaluationOutput]: """ Evaluate the performance of the task on the given test cases asynchronously. @@ -81,7 +92,7 @@ async def evaluate_async(self, evaluation_case: EvaluationData[InputT, OutputT]) Returns: The results of the evaluation as EvaluationOutput. """ - evaluator_agent = Agent(model=self.model, system_prompt=self.system_prompt, callback_handler=None) + evaluator_agent = self._create_evaluator_agent() evaluation_prompt = self._build_prompt(evaluation_case) result = await evaluator_agent.invoke_async(evaluation_prompt, structured_output_model=EvaluationOutput) return [cast(EvaluationOutput, result.structured_output)] diff --git a/src/strands_evals/evaluators/trajectory_evaluator.py b/src/strands_evals/evaluators/trajectory_evaluator.py index 298ddeed..d1494349 100644 --- a/src/strands_evals/evaluators/trajectory_evaluator.py +++ b/src/strands_evals/evaluators/trajectory_evaluator.py @@ -23,6 +23,8 @@ class TrajectoryEvaluator(Evaluator[InputT, OutputT]): system_prompt: System prompt to guide model behavior. If None, the evaluator will use one of the default template. include_inputs: Whether to include inputs to the task in the evaluation or not. + tools: Optional additional tools for the evaluator agent. Merged with the + default trajectory scoring tools (exact/in-order/any-order match). """ def __init__( @@ -32,6 +34,7 @@ def __init__( model: Model | str | None = None, system_prompt: str = SYSTEM_PROMPT, include_inputs: bool = True, + tools: list[Any] | None = None, name: str | None = None, ): super().__init__(name=name) @@ -39,7 +42,7 @@ def __init__( self.trajectory_description = trajectory_description self.model = model self.include_inputs = include_inputs - self._tools: list[str | dict[str, str] | Any] | None = [ + self._tools: list[str | dict[str, str] | Any] | None = list(tools or []) + [ exact_match_scorer, in_order_match_scorer, any_order_match_scorer, diff --git a/tests/strands_evals/evaluators/test_output_evaluator.py b/tests/strands_evals/evaluators/test_output_evaluator.py index 6d51990e..dd45ba69 100644 --- a/tests/strands_evals/evaluators/test_output_evaluator.py +++ b/tests/strands_evals/evaluators/test_output_evaluator.py @@ -259,3 +259,71 @@ async def test_output_evaluator_evaluate_async_includes_environment_state(mock_a assert len(result) == 1 assert result[0].test_pass is True + + +def test_output_evaluator_init_with_tools(): + """Test OutputEvaluator initialization with custom tools""" + + def verify_claim(claim: str) -> str: + return "verified" + + evaluator = OutputEvaluator(rubric="Test rubric", tools=[verify_claim]) + + assert evaluator._tools == [verify_claim] + + +def test_output_evaluator_init_without_tools_defaults_to_none(): + """Test OutputEvaluator has no tools by default (current behavior preserved)""" + evaluator = OutputEvaluator(rubric="Test rubric") + + assert evaluator._tools is None + + +@patch("strands_evals.evaluators.output_evaluator.Agent") +def test_output_evaluator_evaluate_passes_tools_to_agent(mock_agent_class, evaluation_data, mock_agent): + """Test that custom tools are passed to the evaluator agent""" + mock_agent_class.return_value = mock_agent + + def verify_claim(claim: str) -> str: + return "verified" + + evaluator = OutputEvaluator(rubric="Test rubric", tools=[verify_claim]) + + result = evaluator.evaluate(evaluation_data) + + mock_agent_class.assert_called_once_with( + model=None, system_prompt=evaluator.system_prompt, callback_handler=None, tools=[verify_claim] + ) + assert result[0].score == 0.8 + + +@patch("strands_evals.evaluators.output_evaluator.Agent") +def test_output_evaluator_evaluate_without_tools_omits_tools_kwarg(mock_agent_class, evaluation_data, mock_agent): + """Test that no tools kwarg is passed when tools are not provided (backward compatible)""" + mock_agent_class.return_value = mock_agent + evaluator = OutputEvaluator(rubric="Test rubric") + + evaluator.evaluate(evaluation_data) + + mock_agent_class.assert_called_once_with(model=None, system_prompt=evaluator.system_prompt, callback_handler=None) + + +@pytest.mark.asyncio +@patch("strands_evals.evaluators.output_evaluator.Agent") +async def test_output_evaluator_evaluate_async_passes_tools_to_agent( + mock_agent_class, evaluation_data, mock_async_agent +): + """Test that custom tools are passed to the evaluator agent in async path""" + mock_agent_class.return_value = mock_async_agent + + def verify_claim(claim: str) -> str: + return "verified" + + evaluator = OutputEvaluator(rubric="Test rubric", tools=[verify_claim]) + + result = await evaluator.evaluate_async(evaluation_data) + + mock_agent_class.assert_called_once_with( + model=None, system_prompt=evaluator.system_prompt, callback_handler=None, tools=[verify_claim] + ) + assert result[0].score == 0.8 diff --git a/tests/strands_evals/evaluators/test_trajectory_evaluator.py b/tests/strands_evals/evaluators/test_trajectory_evaluator.py index 7adcd058..deaff747 100644 --- a/tests/strands_evals/evaluators/test_trajectory_evaluator.py +++ b/tests/strands_evals/evaluators/test_trajectory_evaluator.py @@ -255,3 +255,51 @@ def test_trajectory_evaluator_update_trajectory_description(): evaluator.update_trajectory_description(new_description) assert evaluator.trajectory_description == new_description + + +def test_trajectory_evaluator_init_with_tools_merges_with_defaults(): + """Test that custom tools are merged with the default scoring tools""" + from strands_evals.tools.evaluation_tools import ( + any_order_match_scorer, + exact_match_scorer, + in_order_match_scorer, + ) + + def verify_step(step: str) -> str: + return "valid" + + evaluator = TrajectoryEvaluator(rubric="Test rubric", tools=[verify_step]) + + assert evaluator._tools == [verify_step, exact_match_scorer, in_order_match_scorer, any_order_match_scorer] + + +def test_trajectory_evaluator_init_without_tools_keeps_default_scorers(): + """Test that default scoring tools are unchanged when no custom tools are provided""" + from strands_evals.tools.evaluation_tools import ( + any_order_match_scorer, + exact_match_scorer, + in_order_match_scorer, + ) + + evaluator = TrajectoryEvaluator(rubric="Test rubric") + + assert evaluator._tools == [exact_match_scorer, in_order_match_scorer, any_order_match_scorer] + + +@patch("strands_evals.evaluators.trajectory_evaluator.Agent") +def test_trajectory_evaluator_evaluate_passes_custom_tools_to_agent(mock_agent_class, evaluation_data, mock_agent): + """Test that merged tools (custom + defaults) reach the evaluator agent""" + mock_agent_class.return_value = mock_agent + + def verify_step(step: str) -> str: + return "valid" + + evaluator = TrajectoryEvaluator(rubric="Test rubric", tools=[verify_step]) + + result = evaluator.evaluate(evaluation_data) + + mock_agent_class.assert_called_once_with( + model=None, system_prompt=evaluator.system_prompt, tools=evaluator._tools, callback_handler=None + ) + assert verify_step in mock_agent_class.call_args[1]["tools"] + assert result[0].score == 0.9 From 36a4ceeb44bcadc70c7271deee109ae8920d57a4 Mon Sep 17 00:00:00 2001 From: DEBJYOTI PAUL Date: Fri, 24 Jul 2026 14:22:08 -0700 Subject: [PATCH 2/2] fix: address review feedback on tools parameter - Move tools after name in both signatures so existing positional callers are unaffected - OutputEvaluator: use public self.tools and pass tools=self.tools directly to Agent (Agent accepts None); drop the _create_evaluator_agent helper - TrajectoryEvaluator: build the merged list with defaults first and custom tools spread after - Forward tools through MultimodalOutputEvaluator and its specialized subclasses so the param is available there too - Tests: assert the exact expected merged tools list instead of the circular evaluator._tools reference; drop the redundant membership assertion --- .../multimodal_correctness_evaluator.py | 2 ++ .../multimodal_faithfulness_evaluator.py | 2 ++ ...timodal_instruction_following_evaluator.py | 2 ++ .../evaluators/multimodal_output_evaluator.py | 2 ++ .../multimodal_overall_quality_evaluator.py | 2 ++ .../evaluators/output_evaluator.py | 19 ++++++------- .../evaluators/trajectory_evaluator.py | 5 ++-- .../evaluators/test_output_evaluator.py | 27 +++++++------------ .../evaluators/test_trajectory_evaluator.py | 18 +++++++++---- 9 files changed, 44 insertions(+), 35 deletions(-) diff --git a/src/strands_evals/evaluators/multimodal_correctness_evaluator.py b/src/strands_evals/evaluators/multimodal_correctness_evaluator.py index 557b8a17..21fd23a3 100644 --- a/src/strands_evals/evaluators/multimodal_correctness_evaluator.py +++ b/src/strands_evals/evaluators/multimodal_correctness_evaluator.py @@ -25,6 +25,7 @@ def __init__( reference_suffix: str | None = None, uses_environment_state: bool = False, name: str | None = None, + tools: list | None = None, ): super().__init__( rubric=rubric if rubric is not None else CORRECTNESS_RUBRIC_V0, @@ -34,4 +35,5 @@ def __init__( reference_suffix=reference_suffix, uses_environment_state=uses_environment_state, name=name, + tools=tools, ) diff --git a/src/strands_evals/evaluators/multimodal_faithfulness_evaluator.py b/src/strands_evals/evaluators/multimodal_faithfulness_evaluator.py index 319950b2..d10252d7 100644 --- a/src/strands_evals/evaluators/multimodal_faithfulness_evaluator.py +++ b/src/strands_evals/evaluators/multimodal_faithfulness_evaluator.py @@ -24,6 +24,7 @@ def __init__( reference_suffix: str | None = None, uses_environment_state: bool = False, name: str | None = None, + tools: list | None = None, ): super().__init__( rubric=rubric if rubric is not None else FAITHFULNESS_RUBRIC_V0, @@ -33,4 +34,5 @@ def __init__( reference_suffix=reference_suffix, uses_environment_state=uses_environment_state, name=name, + tools=tools, ) diff --git a/src/strands_evals/evaluators/multimodal_instruction_following_evaluator.py b/src/strands_evals/evaluators/multimodal_instruction_following_evaluator.py index 57782ba4..a9629bf9 100644 --- a/src/strands_evals/evaluators/multimodal_instruction_following_evaluator.py +++ b/src/strands_evals/evaluators/multimodal_instruction_following_evaluator.py @@ -24,6 +24,7 @@ def __init__( reference_suffix: str | None = None, uses_environment_state: bool = False, name: str | None = None, + tools: list | None = None, ): super().__init__( rubric=rubric if rubric is not None else INSTRUCTION_FOLLOWING_RUBRIC_V0, @@ -33,4 +34,5 @@ def __init__( reference_suffix=reference_suffix, uses_environment_state=uses_environment_state, name=name, + tools=tools, ) diff --git a/src/strands_evals/evaluators/multimodal_output_evaluator.py b/src/strands_evals/evaluators/multimodal_output_evaluator.py index 827bbd5f..e26fcb56 100644 --- a/src/strands_evals/evaluators/multimodal_output_evaluator.py +++ b/src/strands_evals/evaluators/multimodal_output_evaluator.py @@ -46,6 +46,7 @@ def __init__( reference_suffix: str | None = None, uses_environment_state: bool = False, name: str | None = None, + tools: list | None = None, ): super().__init__( rubric=rubric, @@ -54,6 +55,7 @@ def __init__( include_inputs=include_inputs, uses_environment_state=uses_environment_state, name=name, + tools=tools, ) self.reference_suffix = reference_suffix if reference_suffix is not None else self.DEFAULT_REFERENCE_SUFFIX diff --git a/src/strands_evals/evaluators/multimodal_overall_quality_evaluator.py b/src/strands_evals/evaluators/multimodal_overall_quality_evaluator.py index e246f020..459e859a 100644 --- a/src/strands_evals/evaluators/multimodal_overall_quality_evaluator.py +++ b/src/strands_evals/evaluators/multimodal_overall_quality_evaluator.py @@ -35,6 +35,7 @@ def __init__( reference_suffix: str | None = None, uses_environment_state: bool = False, name: str | None = None, + tools: list | None = None, ): super().__init__( rubric=rubric if rubric is not None else OVERALL_QUALITY_RUBRIC_V0, @@ -44,4 +45,5 @@ def __init__( reference_suffix=reference_suffix if reference_suffix is not None else _OVERALL_QUALITY_REFERENCE_SUFFIX, uses_environment_state=uses_environment_state, name=name, + tools=tools, ) diff --git a/src/strands_evals/evaluators/output_evaluator.py b/src/strands_evals/evaluators/output_evaluator.py index ab4d1899..b03e2193 100644 --- a/src/strands_evals/evaluators/output_evaluator.py +++ b/src/strands_evals/evaluators/output_evaluator.py @@ -31,8 +31,8 @@ def __init__( system_prompt: str = SYSTEM_PROMPT, include_inputs: bool = True, uses_environment_state: bool = False, - tools: list[Any] | None = None, name: str | None = None, + tools: list[Any] | None = None, ): super().__init__(name=name) self.rubric = rubric @@ -40,7 +40,7 @@ def __init__( self.include_inputs = include_inputs self.system_prompt = system_prompt self.uses_environment_state = uses_environment_state - self._tools = tools + self.tools = tools def _build_prompt(self, evaluation_case: EvaluationData[InputT, OutputT]) -> str | list: """Build the evaluation prompt for a test case. @@ -70,18 +70,13 @@ def evaluate(self, evaluation_case: EvaluationData[InputT, OutputT]) -> list[Eva Returns: The results of the evaluation as EvaluationOutput. """ - evaluator_agent = self._create_evaluator_agent() + evaluator_agent = Agent( + model=self.model, tools=self.tools, system_prompt=self.system_prompt, callback_handler=None + ) evaluation_prompt = self._build_prompt(evaluation_case) result = evaluator_agent(evaluation_prompt, structured_output_model=EvaluationOutput) return [cast(EvaluationOutput, result.structured_output)] - def _create_evaluator_agent(self) -> Agent: - """Create the judge agent, including custom tools when provided.""" - kwargs: dict[str, Any] = {"model": self.model, "system_prompt": self.system_prompt, "callback_handler": None} - if self._tools: - kwargs["tools"] = self._tools - return Agent(**kwargs) - async def evaluate_async(self, evaluation_case: EvaluationData[InputT, OutputT]) -> list[EvaluationOutput]: """ Evaluate the performance of the task on the given test cases asynchronously. @@ -92,7 +87,9 @@ async def evaluate_async(self, evaluation_case: EvaluationData[InputT, OutputT]) Returns: The results of the evaluation as EvaluationOutput. """ - evaluator_agent = self._create_evaluator_agent() + evaluator_agent = Agent( + model=self.model, tools=self.tools, system_prompt=self.system_prompt, callback_handler=None + ) evaluation_prompt = self._build_prompt(evaluation_case) result = await evaluator_agent.invoke_async(evaluation_prompt, structured_output_model=EvaluationOutput) return [cast(EvaluationOutput, result.structured_output)] diff --git a/src/strands_evals/evaluators/trajectory_evaluator.py b/src/strands_evals/evaluators/trajectory_evaluator.py index d1494349..d46c5870 100644 --- a/src/strands_evals/evaluators/trajectory_evaluator.py +++ b/src/strands_evals/evaluators/trajectory_evaluator.py @@ -34,18 +34,19 @@ def __init__( model: Model | str | None = None, system_prompt: str = SYSTEM_PROMPT, include_inputs: bool = True, - tools: list[Any] | None = None, name: str | None = None, + tools: list[Any] | None = None, ): super().__init__(name=name) self.rubric = rubric self.trajectory_description = trajectory_description self.model = model self.include_inputs = include_inputs - self._tools: list[str | dict[str, str] | Any] | None = list(tools or []) + [ + self._tools: list[str | dict[str, str] | Any] = [ exact_match_scorer, in_order_match_scorer, any_order_match_scorer, + *(tools or []), ] self.system_prompt = system_prompt diff --git a/tests/strands_evals/evaluators/test_output_evaluator.py b/tests/strands_evals/evaluators/test_output_evaluator.py index dd45ba69..8dd5540e 100644 --- a/tests/strands_evals/evaluators/test_output_evaluator.py +++ b/tests/strands_evals/evaluators/test_output_evaluator.py @@ -70,7 +70,9 @@ def test_output_evaluator_evaluate_with_inputs(mock_agent_class, evaluation_data result = evaluator.evaluate(evaluation_data) # Verify Agent was created with correct parameters - mock_agent_class.assert_called_once_with(model=None, system_prompt=evaluator.system_prompt, callback_handler=None) + mock_agent_class.assert_called_once_with( + model=None, tools=None, system_prompt=evaluator.system_prompt, callback_handler=None + ) # Verify agent was called mock_agent.assert_called_once() @@ -148,7 +150,9 @@ async def test_output_evaluator_evaluate_async_with_inputs(mock_agent_class, eva result = await evaluator.evaluate_async(evaluation_data) # Verify Agent was created with correct parameters - mock_agent_class.assert_called_once_with(model=None, system_prompt=evaluator.system_prompt, callback_handler=None) + mock_agent_class.assert_called_once_with( + model=None, tools=None, system_prompt=evaluator.system_prompt, callback_handler=None + ) assert len(result) == 1 assert result[0].score == 0.8 @@ -269,14 +273,14 @@ def verify_claim(claim: str) -> str: evaluator = OutputEvaluator(rubric="Test rubric", tools=[verify_claim]) - assert evaluator._tools == [verify_claim] + assert evaluator.tools == [verify_claim] def test_output_evaluator_init_without_tools_defaults_to_none(): """Test OutputEvaluator has no tools by default (current behavior preserved)""" evaluator = OutputEvaluator(rubric="Test rubric") - assert evaluator._tools is None + assert evaluator.tools is None @patch("strands_evals.evaluators.output_evaluator.Agent") @@ -292,22 +296,11 @@ def verify_claim(claim: str) -> str: result = evaluator.evaluate(evaluation_data) mock_agent_class.assert_called_once_with( - model=None, system_prompt=evaluator.system_prompt, callback_handler=None, tools=[verify_claim] + model=None, tools=[verify_claim], system_prompt=evaluator.system_prompt, callback_handler=None ) assert result[0].score == 0.8 -@patch("strands_evals.evaluators.output_evaluator.Agent") -def test_output_evaluator_evaluate_without_tools_omits_tools_kwarg(mock_agent_class, evaluation_data, mock_agent): - """Test that no tools kwarg is passed when tools are not provided (backward compatible)""" - mock_agent_class.return_value = mock_agent - evaluator = OutputEvaluator(rubric="Test rubric") - - evaluator.evaluate(evaluation_data) - - mock_agent_class.assert_called_once_with(model=None, system_prompt=evaluator.system_prompt, callback_handler=None) - - @pytest.mark.asyncio @patch("strands_evals.evaluators.output_evaluator.Agent") async def test_output_evaluator_evaluate_async_passes_tools_to_agent( @@ -324,6 +317,6 @@ def verify_claim(claim: str) -> str: result = await evaluator.evaluate_async(evaluation_data) mock_agent_class.assert_called_once_with( - model=None, system_prompt=evaluator.system_prompt, callback_handler=None, tools=[verify_claim] + model=None, tools=[verify_claim], system_prompt=evaluator.system_prompt, callback_handler=None ) assert result[0].score == 0.8 diff --git a/tests/strands_evals/evaluators/test_trajectory_evaluator.py b/tests/strands_evals/evaluators/test_trajectory_evaluator.py index deaff747..afbe616f 100644 --- a/tests/strands_evals/evaluators/test_trajectory_evaluator.py +++ b/tests/strands_evals/evaluators/test_trajectory_evaluator.py @@ -258,7 +258,7 @@ def test_trajectory_evaluator_update_trajectory_description(): def test_trajectory_evaluator_init_with_tools_merges_with_defaults(): - """Test that custom tools are merged with the default scoring tools""" + """Test that custom tools are appended after the default scoring tools""" from strands_evals.tools.evaluation_tools import ( any_order_match_scorer, exact_match_scorer, @@ -270,7 +270,7 @@ def verify_step(step: str) -> str: evaluator = TrajectoryEvaluator(rubric="Test rubric", tools=[verify_step]) - assert evaluator._tools == [verify_step, exact_match_scorer, in_order_match_scorer, any_order_match_scorer] + assert evaluator._tools == [exact_match_scorer, in_order_match_scorer, any_order_match_scorer, verify_step] def test_trajectory_evaluator_init_without_tools_keeps_default_scorers(): @@ -288,7 +288,13 @@ def test_trajectory_evaluator_init_without_tools_keeps_default_scorers(): @patch("strands_evals.evaluators.trajectory_evaluator.Agent") def test_trajectory_evaluator_evaluate_passes_custom_tools_to_agent(mock_agent_class, evaluation_data, mock_agent): - """Test that merged tools (custom + defaults) reach the evaluator agent""" + """Test that merged tools (defaults + custom) reach the evaluator agent""" + from strands_evals.tools.evaluation_tools import ( + any_order_match_scorer, + exact_match_scorer, + in_order_match_scorer, + ) + mock_agent_class.return_value = mock_agent def verify_step(step: str) -> str: @@ -299,7 +305,9 @@ def verify_step(step: str) -> str: result = evaluator.evaluate(evaluation_data) mock_agent_class.assert_called_once_with( - model=None, system_prompt=evaluator.system_prompt, tools=evaluator._tools, callback_handler=None + model=None, + system_prompt=evaluator.system_prompt, + tools=[exact_match_scorer, in_order_match_scorer, any_order_match_scorer, verify_step], + callback_handler=None, ) - assert verify_step in mock_agent_class.call_args[1]["tools"] assert result[0].score == 0.9