From e707964c6a404c18890c69250681363e9709e2d8 Mon Sep 17 00:00:00 2001 From: Max Rattray Date: Mon, 10 Aug 2026 17:03:33 +0000 Subject: [PATCH 1/3] feat: add evaluator metadata method and types Add a metadata() instance method to the Evaluator base class that lets evaluators declare what they check, how they work, and their tier in the evaluation hierarchy. New types in strands_evals.types: - EvaluatorMetadata TypedDict (checks, method, threshold, tier, description) - MethodInfo TypedDict (category, summary) - MethodCategory Literal type (7 categories) - Tier Literal type (guardrail, quality, diagnostic) - validate_metadata() function for runtime validation The base class returns None by default so existing evaluators are not broken. Built-in evaluators that declare metadata: - Contains, Equals, StartsWith (deterministic_string) - ToolCalled, StateEquals (deterministic_extraction) - FaithfulnessEvaluator, HarmfulnessEvaluator (llm_judge_output, guardrail) - CorrectnessEvaluator (llm_judge_output, quality) - ToolSelectionAccuracyEvaluator, ToolParameterAccuracyEvaluator (llm_judge_trajectory) - GoalSuccessRateEvaluator (llm_judge_trajectory) --- .../evaluators/correctness_evaluator.py | 15 + .../deterministic/environment_state.py | 12 + .../evaluators/deterministic/output.py | 36 ++ .../evaluators/deterministic/trajectory.py | 12 + src/strands_evals/evaluators/evaluator.py | 13 + .../evaluators/faithfulness_evaluator.py | 15 + .../evaluators/goal_success_rate_evaluator.py | 15 + .../evaluators/harmfulness_evaluator.py | 14 + .../tool_parameter_accuracy_evaluator.py | 15 + .../tool_selection_accuracy_evaluator.py | 15 + src/strands_evals/types/__init__.py | 12 + src/strands_evals/types/evaluator_metadata.py | 137 ++++++ .../evaluators/test_evaluator_metadata.py | 449 ++++++++++++++++++ 13 files changed, 760 insertions(+) create mode 100644 src/strands_evals/types/evaluator_metadata.py create mode 100644 tests/strands_evals/evaluators/test_evaluator_metadata.py diff --git a/src/strands_evals/evaluators/correctness_evaluator.py b/src/strands_evals/evaluators/correctness_evaluator.py index c7bbcacd..fee5402a 100644 --- a/src/strands_evals/evaluators/correctness_evaluator.py +++ b/src/strands_evals/evaluators/correctness_evaluator.py @@ -6,6 +6,7 @@ from strands.models.model import Model from ..types.evaluation import EvaluationData, EvaluationOutput, InputT, OutputT +from ..types.evaluator_metadata import EvaluatorMetadata from ..types.trace import EvaluationLevel, TraceLevelInput from .evaluator import Evaluator from .prompt_templates.correctness import get_reference_template, get_template @@ -87,6 +88,20 @@ def __init__( self.version = version self.model = model + def metadata(self) -> EvaluatorMetadata: + return { + "checks": "Whether the agent's response is factually correct", + "method": { + "category": "llm_judge_output", + "summary": ( + "An LLM judge evaluates correctness of the response using either a " + "3-level rubric or a binary reference comparison." + ), + }, + "threshold": "score >= 1.0 (basic) or CORRECT verdict (reference)", + "tier": "quality", + } + def _has_reference(self, evaluation_case: EvaluationData[InputT, OutputT]) -> bool: """Check if the evaluation case contains an expected_assertion for reference-based evaluation.""" return bool(evaluation_case.expected_assertion) diff --git a/src/strands_evals/evaluators/deterministic/environment_state.py b/src/strands_evals/evaluators/deterministic/environment_state.py index f70391c1..80684129 100644 --- a/src/strands_evals/evaluators/deterministic/environment_state.py +++ b/src/strands_evals/evaluators/deterministic/environment_state.py @@ -1,6 +1,7 @@ from typing_extensions import Any from ...types.evaluation import EnvironmentState, EvaluationData, EvaluationOutput, InputT, OutputT +from ...types.evaluator_metadata import EvaluatorMetadata from ..evaluator import Evaluator @@ -27,6 +28,17 @@ def __init__(self, name: str, value: Any | None = None): super().__init__(name=name) self.value = value + def metadata(self) -> EvaluatorMetadata: + return { + "checks": f"Whether environment state '{self.name}' matches the expected value", + "method": { + "category": "deterministic_extraction", + "summary": "Exact equality comparison of a named environment state against an expected value.", + }, + "threshold": "exact match", + "tier": "quality", + } + def evaluate(self, evaluation_case: EvaluationData[InputT, OutputT]) -> list[EvaluationOutput]: if not evaluation_case.actual_environment_state: return [ diff --git a/src/strands_evals/evaluators/deterministic/output.py b/src/strands_evals/evaluators/deterministic/output.py index 174628a9..79126d0e 100644 --- a/src/strands_evals/evaluators/deterministic/output.py +++ b/src/strands_evals/evaluators/deterministic/output.py @@ -1,6 +1,7 @@ from typing_extensions import Any from ...types.evaluation import EvaluationData, EvaluationOutput, InputT, OutputT +from ...types.evaluator_metadata import EvaluatorMetadata from ..evaluator import Evaluator @@ -11,6 +12,17 @@ def __init__(self, value: Any | None = None, name: str | None = None): super().__init__(name=name) self.value = value + def metadata(self) -> EvaluatorMetadata: + return { + "checks": "Whether actual_output exactly equals an expected value", + "method": { + "category": "deterministic_string", + "summary": "Exact equality comparison between actual_output and expected value.", + }, + "threshold": "exact match", + "tier": "quality", + } + def evaluate(self, evaluation_case: EvaluationData[InputT, OutputT]) -> list[EvaluationOutput]: expected = self.value if self.value is not None else evaluation_case.expected_output match = evaluation_case.actual_output == expected @@ -34,6 +46,18 @@ def __init__(self, value: str, case_sensitive: bool = True, name: str | None = N self.value = value self.case_sensitive = case_sensitive + def metadata(self) -> EvaluatorMetadata: + sensitivity = "Case-sensitive" if self.case_sensitive else "Case-insensitive" + return { + "checks": "Whether actual_output contains a required substring", + "method": { + "category": "deterministic_string", + "summary": f"{sensitivity} substring search on actual_output.", + }, + "threshold": "substring present", + "tier": "quality", + } + def evaluate(self, evaluation_case: EvaluationData[InputT, OutputT]) -> list[EvaluationOutput]: actual = str(evaluation_case.actual_output) target = self.value @@ -61,6 +85,18 @@ def __init__(self, value: str, case_sensitive: bool = True, name: str | None = N self.value = value self.case_sensitive = case_sensitive + def metadata(self) -> EvaluatorMetadata: + sensitivity = "Case-sensitive" if self.case_sensitive else "Case-insensitive" + return { + "checks": "Whether actual_output starts with a required prefix", + "method": { + "category": "deterministic_string", + "summary": f"{sensitivity} prefix check on actual_output.", + }, + "threshold": "prefix present", + "tier": "quality", + } + def evaluate(self, evaluation_case: EvaluationData[InputT, OutputT]) -> list[EvaluationOutput]: actual = str(evaluation_case.actual_output) target = self.value diff --git a/src/strands_evals/evaluators/deterministic/trajectory.py b/src/strands_evals/evaluators/deterministic/trajectory.py index 0995ee29..ade86c09 100644 --- a/src/strands_evals/evaluators/deterministic/trajectory.py +++ b/src/strands_evals/evaluators/deterministic/trajectory.py @@ -1,4 +1,5 @@ from ...types.evaluation import EvaluationData, EvaluationOutput, InputT, OutputT +from ...types.evaluator_metadata import EvaluatorMetadata from ...types.trace import Session, ToolExecutionSpan from ..evaluator import Evaluator @@ -10,6 +11,17 @@ def __init__(self, tool_name: str, name: str | None = None): super().__init__(name=name) self.tool_name = tool_name + def metadata(self) -> EvaluatorMetadata: + return { + "checks": f"Whether the tool '{self.tool_name}' was called during execution", + "method": { + "category": "deterministic_extraction", + "summary": "Searches the trajectory for a tool execution span matching the target tool name.", + }, + "threshold": "tool called at least once", + "tier": "quality", + } + def evaluate(self, evaluation_case: EvaluationData[InputT, OutputT]) -> list[EvaluationOutput]: trajectory = evaluation_case.actual_trajectory if trajectory is None: diff --git a/src/strands_evals/evaluators/evaluator.py b/src/strands_evals/evaluators/evaluator.py index 4da8c479..6cfc20b8 100644 --- a/src/strands_evals/evaluators/evaluator.py +++ b/src/strands_evals/evaluators/evaluator.py @@ -7,6 +7,7 @@ from ..extractors import TraceExtractor from ..types.evaluation import EvaluationData, EvaluationOutput, InputT, OutputT +from ..types.evaluator_metadata import EvaluatorMetadata from ..types.trace import ( AssistantMessage, Context, @@ -57,6 +58,18 @@ class name when unset. elif self.evaluation_level: self._trace_extractor = TraceExtractor(self.evaluation_level) + def metadata(self) -> EvaluatorMetadata | None: + """Declare what this evaluator checks and how it works. + + Subclasses override this method to return a typed dict describing + themselves. The base implementation returns None, which signals + that the evaluator has not declared metadata. + + Returns: + An EvaluatorMetadata dict, or None if not declared. + """ + return None + def _get_model_id(self, model: Model | str | None) -> str: """Extract model_id from a Model instance or string for serialization. diff --git a/src/strands_evals/evaluators/faithfulness_evaluator.py b/src/strands_evals/evaluators/faithfulness_evaluator.py index 5c5f05e9..e356fa9a 100644 --- a/src/strands_evals/evaluators/faithfulness_evaluator.py +++ b/src/strands_evals/evaluators/faithfulness_evaluator.py @@ -6,6 +6,7 @@ from strands.models.model import Model from ..types.evaluation import EvaluationData, EvaluationOutput, InputT, OutputT +from ..types.evaluator_metadata import EvaluatorMetadata from ..types.trace import EvaluationLevel from .evaluator import Evaluator from .prompt_templates.faithfulness import get_template @@ -53,6 +54,20 @@ def __init__( self.version = version self.model = model + def metadata(self) -> EvaluatorMetadata: + return { + "checks": "Whether the agent's response is grounded in the conversation history", + "method": { + "category": "llm_judge_output", + "summary": ( + "An LLM judge compares the agent's last response against prior " + "tool outputs and conversation for unsupported claims." + ), + }, + "threshold": "score >= 0.50", + "tier": "guardrail", + } + def evaluate(self, evaluation_case: EvaluationData[InputT, OutputT]) -> list[EvaluationOutput]: parsed_input = self._get_last_turn(evaluation_case) prompt = self._format_trace_level_prompt(parsed_input) diff --git a/src/strands_evals/evaluators/goal_success_rate_evaluator.py b/src/strands_evals/evaluators/goal_success_rate_evaluator.py index 8c33061d..7fa4f4a5 100644 --- a/src/strands_evals/evaluators/goal_success_rate_evaluator.py +++ b/src/strands_evals/evaluators/goal_success_rate_evaluator.py @@ -6,6 +6,7 @@ from strands.models.model import Model from ..types.evaluation import EvaluationData, EvaluationOutput, InputT, OutputT +from ..types.evaluator_metadata import EvaluatorMetadata from ..types.trace import EvaluationLevel, SessionLevelInput from .evaluator import Evaluator from .prompt_templates.goal_success_rate import get_assertion_template, get_template @@ -84,6 +85,20 @@ def __init__( self.version = version self.model = model + def metadata(self) -> EvaluatorMetadata: + return { + "checks": "Whether all user goals were successfully achieved in the conversation", + "method": { + "category": "llm_judge_trajectory", + "summary": ( + "An LLM judge analyzes the full session to determine if user goals " + "were met, either by inferring goals or checking explicit assertions." + ), + }, + "threshold": "score == 1.0 (Yes or SUCCESS)", + "tier": "quality", + } + def _has_assertion(self, evaluation_case: EvaluationData[InputT, OutputT]) -> bool: """Check if the evaluation case contains expected_assertion for assertion mode.""" return bool(evaluation_case.expected_assertion) diff --git a/src/strands_evals/evaluators/harmfulness_evaluator.py b/src/strands_evals/evaluators/harmfulness_evaluator.py index edd0b463..f30fb27e 100644 --- a/src/strands_evals/evaluators/harmfulness_evaluator.py +++ b/src/strands_evals/evaluators/harmfulness_evaluator.py @@ -6,6 +6,7 @@ from strands.models.model import Model from ..types.evaluation import EvaluationData, EvaluationOutput, InputT, OutputT +from ..types.evaluator_metadata import EvaluatorMetadata from ..types.trace import EvaluationLevel from .evaluator import Evaluator from .prompt_templates.harmfulness import get_template @@ -47,6 +48,19 @@ def __init__( self.version = version self.model = model + def metadata(self) -> EvaluatorMetadata: + return { + "checks": "Whether the agent's response contains harmful content", + "method": { + "category": "llm_judge_output", + "summary": ( + "An LLM judge examines the agent's response for harmful, dangerous, or inappropriate content." + ), + }, + "threshold": "score == 1.0 (not harmful)", + "tier": "guardrail", + } + def evaluate(self, evaluation_case: EvaluationData[InputT, OutputT]) -> list[EvaluationOutput]: parsed_input = self._get_last_turn(evaluation_case) prompt = self._format_trace_level_prompt(parsed_input) diff --git a/src/strands_evals/evaluators/tool_parameter_accuracy_evaluator.py b/src/strands_evals/evaluators/tool_parameter_accuracy_evaluator.py index 40ee7f1e..b2fabe20 100644 --- a/src/strands_evals/evaluators/tool_parameter_accuracy_evaluator.py +++ b/src/strands_evals/evaluators/tool_parameter_accuracy_evaluator.py @@ -6,6 +6,7 @@ from strands.models.model import Model from ..types.evaluation import EvaluationData, EvaluationOutput, InputT, OutputT +from ..types.evaluator_metadata import EvaluatorMetadata from ..types.trace import EvaluationLevel from .evaluator import Evaluator from .prompt_templates.tool_parameter_accuracy import get_template @@ -47,6 +48,20 @@ def __init__( self.version = version self.model = model + def metadata(self) -> EvaluatorMetadata: + return { + "checks": "Whether tool call parameters faithfully use information from the conversation context", + "method": { + "category": "llm_judge_trajectory", + "summary": ( + "An LLM judge evaluates each tool call's parameters to verify they " + "accurately reflect information from the preceding conversation and tool results." + ), + }, + "threshold": "all tool calls scored Yes", + "tier": "quality", + } + def evaluate(self, evaluation_case: EvaluationData[InputT, OutputT]) -> list[EvaluationOutput]: tool_inputs = self._parse_trajectory(evaluation_case) results = [] diff --git a/src/strands_evals/evaluators/tool_selection_accuracy_evaluator.py b/src/strands_evals/evaluators/tool_selection_accuracy_evaluator.py index ced8756c..af8805b7 100644 --- a/src/strands_evals/evaluators/tool_selection_accuracy_evaluator.py +++ b/src/strands_evals/evaluators/tool_selection_accuracy_evaluator.py @@ -6,6 +6,7 @@ from strands.models.model import Model from ..types.evaluation import EvaluationData, EvaluationOutput, InputT, OutputT +from ..types.evaluator_metadata import EvaluatorMetadata from ..types.trace import EvaluationLevel from .evaluator import Evaluator from .prompt_templates.tool_selection_accuracy import get_template @@ -47,6 +48,20 @@ def __init__( self.version = version self.model = model + def metadata(self) -> EvaluatorMetadata: + return { + "checks": "Whether each tool call is justified given the conversation context", + "method": { + "category": "llm_judge_trajectory", + "summary": ( + "An LLM judge evaluates each tool call in the trajectory to determine " + "if it was appropriate given the available tools and conversation context." + ), + }, + "threshold": "all tool calls scored Yes", + "tier": "quality", + } + def evaluate(self, evaluation_case: EvaluationData[InputT, OutputT]) -> list[EvaluationOutput]: tool_inputs = self._parse_trajectory(evaluation_case) results = [] diff --git a/src/strands_evals/types/__init__.py b/src/strands_evals/types/__init__.py index 8d38173a..12133341 100644 --- a/src/strands_evals/types/__init__.py +++ b/src/strands_evals/types/__init__.py @@ -8,13 +8,24 @@ RCAStructuredOutput, ) from .evaluation import EnvironmentState, EvaluationData, EvaluationOutput, InputT, Interaction, OutputT, TaskOutput +from .evaluator_metadata import ( + EvaluatorMetadata, + MethodCategory, + MethodInfo, + Tier, + validate_metadata, +) from .multimodal import AnyMediaData, ImageData, MultimodalInput, resolve_image_bytes from .simulation import ActorProfile, ActorResponse __all__ = [ "EnvironmentState", + "EvaluatorMetadata", "Interaction", + "MethodCategory", + "MethodInfo", "TaskOutput", + "Tier", "EvaluationData", "EvaluationOutput", "ActorProfile", @@ -32,4 +43,5 @@ "RCAItem", "RCAOutput", "RCAStructuredOutput", + "validate_metadata", ] diff --git a/src/strands_evals/types/evaluator_metadata.py b/src/strands_evals/types/evaluator_metadata.py new file mode 100644 index 00000000..520db4a3 --- /dev/null +++ b/src/strands_evals/types/evaluator_metadata.py @@ -0,0 +1,137 @@ +"""Types for evaluator metadata declarations. + +Evaluator metadata lets evaluators declare what they check and how they work, +enabling downstream systems to aggregate results by tier, render informative +reports, and route models based on evaluation method. +""" + +from typing_extensions import Literal, TypedDict + +MethodCategory = Literal[ + "llm_judge_output", + "llm_judge_trajectory", + "deterministic_string", + "deterministic_extraction", + "threshold_comparison", + "composite", + "custom", +] +"""Category describing the evaluation method. + +- llm_judge_output: An LLM judge evaluates the final output. +- llm_judge_trajectory: An LLM judge evaluates the full trajectory. +- deterministic_string: A deterministic string comparison (contains, equals, etc.). +- deterministic_extraction: Deterministic extraction and comparison of structured data. +- threshold_comparison: A numeric threshold comparison. +- composite: Combines multiple evaluation methods. +- custom: A custom evaluation method not covered by other categories. +""" + +Tier = Literal["guardrail", "quality", "diagnostic"] +"""Tier describing how this evaluator's result affects the overall verdict. + +- guardrail: Any failure overrides the overall verdict to fail. Non-negotiable rules. +- quality: The primary correctness signal. Feeds the headline score and gates pass/fail. +- diagnostic: Surfaced in reports but does not gate pass/fail. +""" + +VALID_METHOD_CATEGORIES: set[str] = { + "llm_judge_output", + "llm_judge_trajectory", + "deterministic_string", + "deterministic_extraction", + "threshold_comparison", + "composite", + "custom", +} + +VALID_TIERS: set[str] = {"guardrail", "quality", "diagnostic"} + + +class MethodInfo(TypedDict): + """Describes how an evaluator works. + + Attributes: + category: The evaluation method category. + summary: One to two sentences explaining how the evaluator works. + """ + + category: MethodCategory + summary: str + + +class EvaluatorMetadata(TypedDict, total=False): + """Metadata that an evaluator declares about itself. + + Required keys: checks, method, threshold. + Optional keys: tier, description. + + Attributes: + checks: One sentence describing what is measured. + method: How the evaluator works. + threshold: The pass condition (e.g. "score >= 0.50"). + tier: How this result affects the overall verdict. Defaults to "quality". + description: A longer explanation of the evaluator. + """ + + checks: str + method: MethodInfo + threshold: str + tier: Tier + description: str + + +REQUIRED_METADATA_KEYS: set[str] = {"checks", "method", "threshold"} + + +def validate_metadata(metadata: EvaluatorMetadata, evaluator_name: str) -> None: + """Validate that evaluator metadata has all required keys and valid values. + + Args: + metadata: The metadata dict returned by an evaluator's metadata() method. + evaluator_name: The evaluator name, used in error messages. + + Raises: + ValueError: If required keys are missing or values are invalid. + """ + # Check required keys + missing_keys = REQUIRED_METADATA_KEYS - set(metadata.keys()) + if missing_keys: + raise ValueError(f"Evaluator '{evaluator_name}' metadata is missing required keys: {sorted(missing_keys)}") + + # Validate checks is a non-empty string + checks = metadata.get("checks", "") + if not isinstance(checks, str) or not checks.strip(): + raise ValueError(f"Evaluator '{evaluator_name}' metadata 'checks' must be a non-empty string") + + # Validate method + method = metadata.get("method") + if not isinstance(method, dict): + raise ValueError(f"Evaluator '{evaluator_name}' metadata 'method' must be a MethodInfo dict") + + if "category" not in method: + raise ValueError(f"Evaluator '{evaluator_name}' metadata 'method' is missing required key: 'category'") + + if method["category"] not in VALID_METHOD_CATEGORIES: + raise ValueError( + f"Evaluator '{evaluator_name}' metadata method.category '{method['category']}' " + f"is not valid. Must be one of: {sorted(VALID_METHOD_CATEGORIES)}" + ) + + if "summary" not in method: + raise ValueError(f"Evaluator '{evaluator_name}' metadata 'method' is missing required key: 'summary'") + + if not isinstance(method["summary"], str) or not method["summary"].strip(): + raise ValueError(f"Evaluator '{evaluator_name}' metadata method.summary must be a non-empty string") + + # Validate threshold is a non-empty string + threshold = metadata.get("threshold", "") + if not isinstance(threshold, str) or not threshold.strip(): + raise ValueError(f"Evaluator '{evaluator_name}' metadata 'threshold' must be a non-empty string") + + # Validate tier if present + tier = metadata.get("tier") + if tier is not None and tier not in VALID_TIERS: + raise ValueError( + f"Evaluator '{evaluator_name}' metadata tier '{tier}' is not valid. Must be one of: {sorted(VALID_TIERS)}" + ) diff --git a/tests/strands_evals/evaluators/test_evaluator_metadata.py b/tests/strands_evals/evaluators/test_evaluator_metadata.py new file mode 100644 index 00000000..b160ddb1 --- /dev/null +++ b/tests/strands_evals/evaluators/test_evaluator_metadata.py @@ -0,0 +1,449 @@ +"""Tests for evaluator metadata types and the metadata() method on evaluators.""" + +import pytest + +from strands_evals.evaluators import ( + Contains, + CorrectnessEvaluator, + Equals, + Evaluator, + FaithfulnessEvaluator, + GoalSuccessRateEvaluator, + HarmfulnessEvaluator, + StartsWith, + StateEquals, + ToolCalled, + ToolParameterAccuracyEvaluator, + ToolSelectionAccuracyEvaluator, +) +from strands_evals.types import EvaluatorMetadata, MethodInfo, validate_metadata +from strands_evals.types.evaluator_metadata import REQUIRED_METADATA_KEYS, VALID_METHOD_CATEGORIES, VALID_TIERS + + +class TestEvaluatorMetadataTypes: + """Tests for the metadata type definitions.""" + + def test_method_info_can_be_constructed(self): + """MethodInfo accepts category and summary keys.""" + info: MethodInfo = { + "category": "deterministic_string", + "summary": "A simple substring check.", + } + assert info["category"] == "deterministic_string" + assert info["summary"] == "A simple substring check." + + def test_evaluator_metadata_all_fields(self): + """EvaluatorMetadata accepts all defined fields.""" + meta: EvaluatorMetadata = { + "checks": "Whether output contains keyword", + "method": { + "category": "deterministic_string", + "summary": "Substring search.", + }, + "threshold": "substring present", + "tier": "quality", + "description": "A longer explanation.", + } + assert meta["checks"] == "Whether output contains keyword" + assert meta["tier"] == "quality" + assert meta["description"] == "A longer explanation." + + def test_evaluator_metadata_required_keys_only(self): + """EvaluatorMetadata works with only required keys (total=False).""" + meta: EvaluatorMetadata = { + "checks": "Something", + "method": {"category": "custom", "summary": "Custom method."}, + "threshold": "passes", + } + assert "tier" not in meta + assert "description" not in meta + + def test_valid_method_categories_match_literal(self): + """VALID_METHOD_CATEGORIES contains all expected values.""" + expected = { + "llm_judge_output", + "llm_judge_trajectory", + "deterministic_string", + "deterministic_extraction", + "threshold_comparison", + "composite", + "custom", + } + assert VALID_METHOD_CATEGORIES == expected + + def test_valid_tiers_match_literal(self): + """VALID_TIERS contains all expected values.""" + expected = {"guardrail", "quality", "diagnostic"} + assert VALID_TIERS == expected + + def test_required_metadata_keys(self): + """REQUIRED_METADATA_KEYS lists the mandatory fields.""" + assert REQUIRED_METADATA_KEYS == {"checks", "method", "threshold"} + + +class TestValidateMetadata: + """Tests for the validate_metadata function.""" + + def _valid_metadata(self) -> EvaluatorMetadata: + return { + "checks": "Whether output is correct", + "method": { + "category": "llm_judge_output", + "summary": "An LLM judge evaluates correctness.", + }, + "threshold": "score >= 0.5", + "tier": "quality", + } + + def test_valid_metadata_passes(self): + """Valid metadata does not raise.""" + validate_metadata(self._valid_metadata(), "TestEvaluator") + + def test_valid_metadata_without_optional_fields(self): + """Metadata without optional fields (tier, description) is valid.""" + meta: EvaluatorMetadata = { + "checks": "Check something", + "method": {"category": "custom", "summary": "Custom check."}, + "threshold": "always passes", + } + validate_metadata(meta, "TestEvaluator") + + def test_missing_checks_raises(self): + """Missing 'checks' key raises ValueError.""" + meta = self._valid_metadata() + del meta["checks"] # type: ignore[misc] + with pytest.raises(ValueError, match="missing required keys.*checks"): + validate_metadata(meta, "MyEvaluator") + + def test_missing_method_raises(self): + """Missing 'method' key raises ValueError.""" + meta = self._valid_metadata() + del meta["method"] # type: ignore[misc] + with pytest.raises(ValueError, match="missing required keys.*method"): + validate_metadata(meta, "MyEvaluator") + + def test_missing_threshold_raises(self): + """Missing 'threshold' key raises ValueError.""" + meta = self._valid_metadata() + del meta["threshold"] # type: ignore[misc] + with pytest.raises(ValueError, match="missing required keys.*threshold"): + validate_metadata(meta, "MyEvaluator") + + def test_empty_checks_raises(self): + """Empty string for 'checks' raises ValueError.""" + meta = self._valid_metadata() + meta["checks"] = " " + with pytest.raises(ValueError, match="'checks' must be a non-empty string"): + validate_metadata(meta, "MyEvaluator") + + def test_invalid_method_type_raises(self): + """Non-dict 'method' raises ValueError.""" + meta = self._valid_metadata() + meta["method"] = "not a dict" # type: ignore[typeddict-item] + with pytest.raises(ValueError, match="'method' must be a MethodInfo dict"): + validate_metadata(meta, "MyEvaluator") + + def test_method_missing_category_raises(self): + """Method dict missing 'category' raises ValueError.""" + meta = self._valid_metadata() + meta["method"] = {"summary": "Some summary."} # type: ignore[typeddict-item] + with pytest.raises(ValueError, match="'method' is missing required key: 'category'"): + validate_metadata(meta, "MyEvaluator") + + def test_method_invalid_category_raises(self): + """Invalid method category raises ValueError.""" + meta = self._valid_metadata() + meta["method"] = {"category": "invalid_category", "summary": "Some summary."} # type: ignore[typeddict-item] + with pytest.raises(ValueError, match="method.category 'invalid_category' is not valid"): + validate_metadata(meta, "MyEvaluator") + + def test_method_missing_summary_raises(self): + """Method dict missing 'summary' raises ValueError.""" + meta = self._valid_metadata() + meta["method"] = {"category": "custom"} # type: ignore[typeddict-item] + with pytest.raises(ValueError, match="'method' is missing required key: 'summary'"): + validate_metadata(meta, "MyEvaluator") + + def test_method_empty_summary_raises(self): + """Empty method summary raises ValueError.""" + meta = self._valid_metadata() + meta["method"] = {"category": "custom", "summary": " "} + with pytest.raises(ValueError, match="method.summary must be a non-empty string"): + validate_metadata(meta, "MyEvaluator") + + def test_empty_threshold_raises(self): + """Empty threshold raises ValueError.""" + meta = self._valid_metadata() + meta["threshold"] = "" + with pytest.raises(ValueError, match="'threshold' must be a non-empty string"): + validate_metadata(meta, "MyEvaluator") + + def test_invalid_tier_raises(self): + """Invalid tier value raises ValueError.""" + meta = self._valid_metadata() + meta["tier"] = "critical" # type: ignore[typeddict-item] + with pytest.raises(ValueError, match="tier 'critical' is not valid"): + validate_metadata(meta, "MyEvaluator") + + def test_all_valid_tiers_pass(self): + """Each valid tier value passes validation.""" + for tier in VALID_TIERS: + meta = self._valid_metadata() + meta["tier"] = tier # type: ignore[typeddict-item] + validate_metadata(meta, "TestEvaluator") + + def test_all_valid_method_categories_pass(self): + """Each valid method category passes validation.""" + for category in VALID_METHOD_CATEGORIES: + meta = self._valid_metadata() + meta["method"] = {"category": category, "summary": "Valid."} # type: ignore[typeddict-item] + validate_metadata(meta, "TestEvaluator") + + def test_error_message_includes_evaluator_name(self): + """Error messages include the evaluator name for debugging.""" + meta = self._valid_metadata() + del meta["checks"] # type: ignore[misc] + with pytest.raises(ValueError, match="'SpecificEvaluatorName'"): + validate_metadata(meta, "SpecificEvaluatorName") + + +class TestBaseEvaluatorMetadata: + """Tests for the metadata() method on the base Evaluator class.""" + + def test_base_evaluator_returns_none(self): + """Base Evaluator.metadata() returns None by default.""" + evaluator = Evaluator() + assert evaluator.metadata() is None + + def test_subclass_without_override_returns_none(self): + """Subclass that does not override metadata() returns None.""" + + class MyEvaluator(Evaluator[str, str]): + def evaluate(self, evaluation_case): + return [] + + evaluator = MyEvaluator() + assert evaluator.metadata() is None + + def test_subclass_can_override_metadata(self): + """Subclass can override metadata() to return valid metadata.""" + + class MyEvaluator(Evaluator[str, str]): + def evaluate(self, evaluation_case): + return [] + + def metadata(self) -> EvaluatorMetadata: + return { + "checks": "Something custom", + "method": {"category": "custom", "summary": "Custom check."}, + "threshold": "passes", + "tier": "diagnostic", + } + + evaluator = MyEvaluator() + meta = evaluator.metadata() + assert meta is not None + assert meta["checks"] == "Something custom" + assert meta["tier"] == "diagnostic" + + +class TestDeterministicEvaluatorMetadata: + """Tests for metadata() on deterministic evaluators.""" + + def test_contains_metadata(self): + """Contains returns valid metadata with case sensitivity info.""" + evaluator = Contains(value="hello") + meta = evaluator.metadata() + assert meta is not None + assert meta["checks"] == "Whether actual_output contains a required substring" + assert meta["method"]["category"] == "deterministic_string" + assert "Case-sensitive" in meta["method"]["summary"] + assert meta["threshold"] == "substring present" + assert meta["tier"] == "quality" + validate_metadata(meta, evaluator.get_name()) + + def test_contains_case_insensitive_metadata(self): + """Contains with case_sensitive=False reports case-insensitive in summary.""" + evaluator = Contains(value="hello", case_sensitive=False) + meta = evaluator.metadata() + assert meta is not None + assert "Case-insensitive" in meta["method"]["summary"] + + def test_equals_metadata(self): + """Equals returns valid metadata.""" + evaluator = Equals(value="expected") + meta = evaluator.metadata() + assert meta is not None + assert meta["checks"] == "Whether actual_output exactly equals an expected value" + assert meta["method"]["category"] == "deterministic_string" + assert meta["tier"] == "quality" + validate_metadata(meta, evaluator.get_name()) + + def test_starts_with_metadata(self): + """StartsWith returns valid metadata with case sensitivity info.""" + evaluator = StartsWith(value="prefix") + meta = evaluator.metadata() + assert meta is not None + assert meta["checks"] == "Whether actual_output starts with a required prefix" + assert meta["method"]["category"] == "deterministic_string" + assert "Case-sensitive" in meta["method"]["summary"] + assert meta["threshold"] == "prefix present" + validate_metadata(meta, evaluator.get_name()) + + def test_starts_with_case_insensitive_metadata(self): + """StartsWith with case_sensitive=False reports correctly.""" + evaluator = StartsWith(value="prefix", case_sensitive=False) + meta = evaluator.metadata() + assert meta is not None + assert "Case-insensitive" in meta["method"]["summary"] + + def test_tool_called_metadata(self): + """ToolCalled returns valid metadata including the tool name.""" + evaluator = ToolCalled(tool_name="search_web") + meta = evaluator.metadata() + assert meta is not None + assert "search_web" in meta["checks"] + assert meta["method"]["category"] == "deterministic_extraction" + assert meta["tier"] == "quality" + validate_metadata(meta, evaluator.get_name()) + + def test_state_equals_metadata(self): + """StateEquals returns valid metadata including the state name.""" + evaluator = StateEquals(name="cart") + meta = evaluator.metadata() + assert meta is not None + assert "cart" in meta["checks"] + assert meta["method"]["category"] == "deterministic_extraction" + assert meta["tier"] == "quality" + validate_metadata(meta, evaluator.get_name()) + + +class TestLLMEvaluatorMetadata: + """Tests for metadata() on LLM-judge evaluators.""" + + def test_faithfulness_metadata(self): + """FaithfulnessEvaluator returns valid guardrail metadata.""" + evaluator = FaithfulnessEvaluator() + meta = evaluator.metadata() + assert meta is not None + assert meta["method"]["category"] == "llm_judge_output" + assert meta["tier"] == "guardrail" + assert "grounded" in meta["checks"] + validate_metadata(meta, evaluator.get_name()) + + def test_harmfulness_metadata(self): + """HarmfulnessEvaluator returns valid guardrail metadata.""" + evaluator = HarmfulnessEvaluator() + meta = evaluator.metadata() + assert meta is not None + assert meta["method"]["category"] == "llm_judge_output" + assert meta["tier"] == "guardrail" + assert "harmful" in meta["checks"] + validate_metadata(meta, evaluator.get_name()) + + def test_correctness_metadata(self): + """CorrectnessEvaluator returns valid quality metadata.""" + evaluator = CorrectnessEvaluator() + meta = evaluator.metadata() + assert meta is not None + assert meta["method"]["category"] == "llm_judge_output" + assert meta["tier"] == "quality" + assert "correct" in meta["checks"] + validate_metadata(meta, evaluator.get_name()) + + def test_tool_selection_accuracy_metadata(self): + """ToolSelectionAccuracyEvaluator returns valid metadata.""" + evaluator = ToolSelectionAccuracyEvaluator() + meta = evaluator.metadata() + assert meta is not None + assert meta["method"]["category"] == "llm_judge_trajectory" + assert meta["tier"] == "quality" + validate_metadata(meta, evaluator.get_name()) + + def test_tool_parameter_accuracy_metadata(self): + """ToolParameterAccuracyEvaluator returns valid metadata.""" + evaluator = ToolParameterAccuracyEvaluator() + meta = evaluator.metadata() + assert meta is not None + assert meta["method"]["category"] == "llm_judge_trajectory" + assert meta["tier"] == "quality" + validate_metadata(meta, evaluator.get_name()) + + def test_goal_success_rate_metadata(self): + """GoalSuccessRateEvaluator returns valid metadata.""" + evaluator = GoalSuccessRateEvaluator() + meta = evaluator.metadata() + assert meta is not None + assert meta["method"]["category"] == "llm_judge_trajectory" + assert meta["tier"] == "quality" + assert "goals" in meta["checks"].lower() + validate_metadata(meta, evaluator.get_name()) + + +class TestMetadataInstanceDependence: + """Tests that metadata can depend on instance state.""" + + def test_contains_metadata_reflects_case_sensitivity(self): + """Two Contains instances produce different metadata based on case_sensitive.""" + sensitive = Contains(value="hello", case_sensitive=True) + insensitive = Contains(value="hello", case_sensitive=False) + + meta_s = sensitive.metadata() + meta_i = insensitive.metadata() + + assert meta_s is not None + assert meta_i is not None + assert meta_s["method"]["summary"] != meta_i["method"]["summary"] + + def test_tool_called_metadata_reflects_tool_name(self): + """Two ToolCalled instances produce different metadata based on tool_name.""" + eval_a = ToolCalled(tool_name="tool_a") + eval_b = ToolCalled(tool_name="tool_b") + + meta_a = eval_a.metadata() + meta_b = eval_b.metadata() + + assert meta_a is not None + assert meta_b is not None + assert "tool_a" in meta_a["checks"] + assert "tool_b" in meta_b["checks"] + + def test_state_equals_metadata_reflects_state_name(self): + """Two StateEquals instances produce different metadata based on name.""" + eval_cart = StateEquals(name="cart") + eval_balance = StateEquals(name="balance") + + meta_cart = eval_cart.metadata() + meta_balance = eval_balance.metadata() + + assert meta_cart is not None + assert meta_balance is not None + assert "cart" in meta_cart["checks"] + assert "balance" in meta_balance["checks"] + + +class TestMetadataImportAccessibility: + """Tests that metadata types are importable from expected locations.""" + + def test_import_from_types(self): + """Types are importable from strands_evals.types.""" + import strands_evals.types as types_mod + + assert hasattr(types_mod, "EvaluatorMetadata") + assert hasattr(types_mod, "MethodInfo") + assert hasattr(types_mod, "MethodCategory") + assert hasattr(types_mod, "Tier") + assert hasattr(types_mod, "validate_metadata") + + def test_import_from_types_evaluator_metadata_module(self): + """Types are importable from the evaluator_metadata module directly.""" + import strands_evals.types.evaluator_metadata as meta_mod + + assert hasattr(meta_mod, "EvaluatorMetadata") + assert hasattr(meta_mod, "MethodInfo") + assert hasattr(meta_mod, "MethodCategory") + assert hasattr(meta_mod, "Tier") + assert hasattr(meta_mod, "validate_metadata") + assert hasattr(meta_mod, "REQUIRED_METADATA_KEYS") + assert hasattr(meta_mod, "VALID_METHOD_CATEGORIES") + assert hasattr(meta_mod, "VALID_TIERS") From 2ce9e304752f78e54bc32812534e15c2d8a33e38 Mon Sep 17 00:00:00 2001 From: Max Rattray Date: Mon, 10 Aug 2026 18:11:39 +0000 Subject: [PATCH 2/3] fix: address review feedback --- src/strands_evals/types/evaluator_metadata.py | 20 +- .../evaluators/test_evaluator_metadata.py | 187 +++++++++++++----- 2 files changed, 147 insertions(+), 60 deletions(-) diff --git a/src/strands_evals/types/evaluator_metadata.py b/src/strands_evals/types/evaluator_metadata.py index 520db4a3..19b66ee3 100644 --- a/src/strands_evals/types/evaluator_metadata.py +++ b/src/strands_evals/types/evaluator_metadata.py @@ -5,6 +5,8 @@ reports, and route models based on evaluation method. """ +from typing import get_args + from typing_extensions import Literal, TypedDict MethodCategory = Literal[ @@ -35,17 +37,9 @@ - diagnostic: Surfaced in reports but does not gate pass/fail. """ -VALID_METHOD_CATEGORIES: set[str] = { - "llm_judge_output", - "llm_judge_trajectory", - "deterministic_string", - "deterministic_extraction", - "threshold_comparison", - "composite", - "custom", -} +VALID_METHOD_CATEGORIES: set[str] = set(get_args(MethodCategory)) -VALID_TIERS: set[str] = {"guardrail", "quality", "diagnostic"} +VALID_TIERS: set[str] = set(get_args(Tier)) class MethodInfo(TypedDict): @@ -84,16 +78,20 @@ class EvaluatorMetadata(TypedDict, total=False): REQUIRED_METADATA_KEYS: set[str] = {"checks", "method", "threshold"} -def validate_metadata(metadata: EvaluatorMetadata, evaluator_name: str) -> None: +def validate_metadata(metadata: EvaluatorMetadata | None, evaluator_name: str) -> None: """Validate that evaluator metadata has all required keys and valid values. Args: metadata: The metadata dict returned by an evaluator's metadata() method. + If None, the evaluator has not declared metadata and validation is skipped. evaluator_name: The evaluator name, used in error messages. Raises: ValueError: If required keys are missing or values are invalid. """ + if metadata is None: + return + # Check required keys missing_keys = REQUIRED_METADATA_KEYS - set(metadata.keys()) if missing_keys: diff --git a/tests/strands_evals/evaluators/test_evaluator_metadata.py b/tests/strands_evals/evaluators/test_evaluator_metadata.py index b160ddb1..460cd8bb 100644 --- a/tests/strands_evals/evaluators/test_evaluator_metadata.py +++ b/tests/strands_evals/evaluators/test_evaluator_metadata.py @@ -99,6 +99,10 @@ def test_valid_metadata_passes(self): """Valid metadata does not raise.""" validate_metadata(self._valid_metadata(), "TestEvaluator") + def test_none_metadata_passes(self): + """None metadata does not raise (evaluator has not declared metadata).""" + validate_metadata(None, "TestEvaluator") + def test_valid_metadata_without_optional_fields(self): """Metadata without optional fields (tier, description) is valid.""" meta: EvaluatorMetadata = { @@ -254,67 +258,103 @@ def test_contains_metadata(self): """Contains returns valid metadata with case sensitivity info.""" evaluator = Contains(value="hello") meta = evaluator.metadata() - assert meta is not None - assert meta["checks"] == "Whether actual_output contains a required substring" - assert meta["method"]["category"] == "deterministic_string" - assert "Case-sensitive" in meta["method"]["summary"] - assert meta["threshold"] == "substring present" - assert meta["tier"] == "quality" + assert meta == { + "checks": "Whether actual_output contains a required substring", + "method": { + "category": "deterministic_string", + "summary": "Case-sensitive substring search on actual_output.", + }, + "threshold": "substring present", + "tier": "quality", + } validate_metadata(meta, evaluator.get_name()) def test_contains_case_insensitive_metadata(self): """Contains with case_sensitive=False reports case-insensitive in summary.""" evaluator = Contains(value="hello", case_sensitive=False) meta = evaluator.metadata() - assert meta is not None - assert "Case-insensitive" in meta["method"]["summary"] + assert meta == { + "checks": "Whether actual_output contains a required substring", + "method": { + "category": "deterministic_string", + "summary": "Case-insensitive substring search on actual_output.", + }, + "threshold": "substring present", + "tier": "quality", + } def test_equals_metadata(self): """Equals returns valid metadata.""" evaluator = Equals(value="expected") meta = evaluator.metadata() - assert meta is not None - assert meta["checks"] == "Whether actual_output exactly equals an expected value" - assert meta["method"]["category"] == "deterministic_string" - assert meta["tier"] == "quality" + assert meta == { + "checks": "Whether actual_output exactly equals an expected value", + "method": { + "category": "deterministic_string", + "summary": "Exact equality comparison between actual_output and expected value.", + }, + "threshold": "exact match", + "tier": "quality", + } validate_metadata(meta, evaluator.get_name()) def test_starts_with_metadata(self): """StartsWith returns valid metadata with case sensitivity info.""" evaluator = StartsWith(value="prefix") meta = evaluator.metadata() - assert meta is not None - assert meta["checks"] == "Whether actual_output starts with a required prefix" - assert meta["method"]["category"] == "deterministic_string" - assert "Case-sensitive" in meta["method"]["summary"] - assert meta["threshold"] == "prefix present" + assert meta == { + "checks": "Whether actual_output starts with a required prefix", + "method": { + "category": "deterministic_string", + "summary": "Case-sensitive prefix check on actual_output.", + }, + "threshold": "prefix present", + "tier": "quality", + } validate_metadata(meta, evaluator.get_name()) def test_starts_with_case_insensitive_metadata(self): """StartsWith with case_sensitive=False reports correctly.""" evaluator = StartsWith(value="prefix", case_sensitive=False) meta = evaluator.metadata() - assert meta is not None - assert "Case-insensitive" in meta["method"]["summary"] + assert meta == { + "checks": "Whether actual_output starts with a required prefix", + "method": { + "category": "deterministic_string", + "summary": "Case-insensitive prefix check on actual_output.", + }, + "threshold": "prefix present", + "tier": "quality", + } def test_tool_called_metadata(self): """ToolCalled returns valid metadata including the tool name.""" evaluator = ToolCalled(tool_name="search_web") meta = evaluator.metadata() - assert meta is not None - assert "search_web" in meta["checks"] - assert meta["method"]["category"] == "deterministic_extraction" - assert meta["tier"] == "quality" + assert meta == { + "checks": "Whether the tool 'search_web' was called during execution", + "method": { + "category": "deterministic_extraction", + "summary": "Searches the trajectory for a tool execution span matching the target tool name.", + }, + "threshold": "tool called at least once", + "tier": "quality", + } validate_metadata(meta, evaluator.get_name()) def test_state_equals_metadata(self): """StateEquals returns valid metadata including the state name.""" evaluator = StateEquals(name="cart") meta = evaluator.metadata() - assert meta is not None - assert "cart" in meta["checks"] - assert meta["method"]["category"] == "deterministic_extraction" - assert meta["tier"] == "quality" + assert meta == { + "checks": "Whether environment state 'cart' matches the expected value", + "method": { + "category": "deterministic_extraction", + "summary": "Exact equality comparison of a named environment state against an expected value.", + }, + "threshold": "exact match", + "tier": "quality", + } validate_metadata(meta, evaluator.get_name()) @@ -325,58 +365,107 @@ def test_faithfulness_metadata(self): """FaithfulnessEvaluator returns valid guardrail metadata.""" evaluator = FaithfulnessEvaluator() meta = evaluator.metadata() - assert meta is not None - assert meta["method"]["category"] == "llm_judge_output" - assert meta["tier"] == "guardrail" - assert "grounded" in meta["checks"] + assert meta == { + "checks": "Whether the agent's response is grounded in the conversation history", + "method": { + "category": "llm_judge_output", + "summary": ( + "An LLM judge compares the agent's last response against prior tool outputs" + " and conversation for unsupported claims." + ), + }, + "threshold": "score >= 0.50", + "tier": "guardrail", + } validate_metadata(meta, evaluator.get_name()) def test_harmfulness_metadata(self): """HarmfulnessEvaluator returns valid guardrail metadata.""" evaluator = HarmfulnessEvaluator() meta = evaluator.metadata() - assert meta is not None - assert meta["method"]["category"] == "llm_judge_output" - assert meta["tier"] == "guardrail" - assert "harmful" in meta["checks"] + assert meta == { + "checks": "Whether the agent's response contains harmful content", + "method": { + "category": "llm_judge_output", + "summary": ( + "An LLM judge examines the agent's response for harmful, dangerous, or inappropriate content." + ), + }, + "threshold": "score == 1.0 (not harmful)", + "tier": "guardrail", + } validate_metadata(meta, evaluator.get_name()) def test_correctness_metadata(self): """CorrectnessEvaluator returns valid quality metadata.""" evaluator = CorrectnessEvaluator() meta = evaluator.metadata() - assert meta is not None - assert meta["method"]["category"] == "llm_judge_output" - assert meta["tier"] == "quality" - assert "correct" in meta["checks"] + assert meta == { + "checks": "Whether the agent's response is factually correct", + "method": { + "category": "llm_judge_output", + "summary": ( + "An LLM judge evaluates correctness of the response using either" + " a 3-level rubric or a binary reference comparison." + ), + }, + "threshold": "score >= 1.0 (basic) or CORRECT verdict (reference)", + "tier": "quality", + } validate_metadata(meta, evaluator.get_name()) def test_tool_selection_accuracy_metadata(self): """ToolSelectionAccuracyEvaluator returns valid metadata.""" evaluator = ToolSelectionAccuracyEvaluator() meta = evaluator.metadata() - assert meta is not None - assert meta["method"]["category"] == "llm_judge_trajectory" - assert meta["tier"] == "quality" + assert meta == { + "checks": "Whether each tool call is justified given the conversation context", + "method": { + "category": "llm_judge_trajectory", + "summary": ( + "An LLM judge evaluates each tool call in the trajectory to determine if it was" + " appropriate given the available tools and conversation context." + ), + }, + "threshold": "all tool calls scored Yes", + "tier": "quality", + } validate_metadata(meta, evaluator.get_name()) def test_tool_parameter_accuracy_metadata(self): """ToolParameterAccuracyEvaluator returns valid metadata.""" evaluator = ToolParameterAccuracyEvaluator() meta = evaluator.metadata() - assert meta is not None - assert meta["method"]["category"] == "llm_judge_trajectory" - assert meta["tier"] == "quality" + assert meta == { + "checks": "Whether tool call parameters faithfully use information from the conversation context", + "method": { + "category": "llm_judge_trajectory", + "summary": ( + "An LLM judge evaluates each tool call's parameters to verify they accurately" + " reflect information from the preceding conversation and tool results." + ), + }, + "threshold": "all tool calls scored Yes", + "tier": "quality", + } validate_metadata(meta, evaluator.get_name()) def test_goal_success_rate_metadata(self): """GoalSuccessRateEvaluator returns valid metadata.""" evaluator = GoalSuccessRateEvaluator() meta = evaluator.metadata() - assert meta is not None - assert meta["method"]["category"] == "llm_judge_trajectory" - assert meta["tier"] == "quality" - assert "goals" in meta["checks"].lower() + assert meta == { + "checks": "Whether all user goals were successfully achieved in the conversation", + "method": { + "category": "llm_judge_trajectory", + "summary": ( + "An LLM judge analyzes the full session to determine if user goals were met," + " either by inferring goals or checking explicit assertions." + ), + }, + "threshold": "score == 1.0 (Yes or SUCCESS)", + "tier": "quality", + } validate_metadata(meta, evaluator.get_name()) From 74ae44280d70e735e41bbe6d1b4625d15230d225 Mon Sep 17 00:00:00 2001 From: Max Rattray Date: Mon, 10 Aug 2026 18:22:09 +0000 Subject: [PATCH 3/3] feat: wire validate_metadata into Experiment.run_evaluations Metadata is now validated at the start of run_evaluations for all evaluators that declare it. Evaluators returning None from metadata() are silently skipped. Adds three tests covering rejection, acceptance, and the None-skip path. --- src/strands_evals/experiment.py | 15 +++++ .../evaluators/test_evaluator_metadata.py | 63 +++++++++++++++++++ 2 files changed, 78 insertions(+) diff --git a/src/strands_evals/experiment.py b/src/strands_evals/experiment.py index de42b984..a33047cf 100644 --- a/src/strands_evals/experiment.py +++ b/src/strands_evals/experiment.py @@ -47,6 +47,7 @@ from .types.detector import DiagnosisConfig from .types.evaluation import EvaluationData, InputT, OutputT from .types.evaluation_report import EvaluationReport +from .types.evaluator_metadata import validate_metadata from .types.trace import Session from .utils import is_throttling_error @@ -192,6 +193,19 @@ def _validate_evaluator_names(self) -> None: f"multiple instances of the same Evaluator subclass." ) + def _validate_evaluator_metadata(self) -> None: + """Validate metadata for all evaluators that declare it. + + Iterates over evaluators and calls validate_metadata on any that + return non-None from metadata(). Evaluators without metadata are + skipped silently. + + Raises: + ValueError: If any evaluator's metadata has invalid structure. + """ + for evaluator in self._evaluators: + validate_metadata(evaluator.metadata(), evaluator.get_name()) + def _validate_case_names(self) -> None: """Validate that all cases have unique, non-None names. @@ -627,6 +641,7 @@ async def run_evaluations_async( an `evaluator` key naming which evaluator produced it. """ self._validate_evaluator_names() + self._validate_evaluator_metadata() if evaluation_data_store is not None: self._validate_case_names() diff --git a/tests/strands_evals/evaluators/test_evaluator_metadata.py b/tests/strands_evals/evaluators/test_evaluator_metadata.py index 460cd8bb..c6ca2ea3 100644 --- a/tests/strands_evals/evaluators/test_evaluator_metadata.py +++ b/tests/strands_evals/evaluators/test_evaluator_metadata.py @@ -536,3 +536,66 @@ def test_import_from_types_evaluator_metadata_module(self): assert hasattr(meta_mod, "REQUIRED_METADATA_KEYS") assert hasattr(meta_mod, "VALID_METHOD_CATEGORIES") assert hasattr(meta_mod, "VALID_TIERS") + + +class TestExperimentMetadataValidation: + """Tests that Experiment validates evaluator metadata at construction time.""" + + def test_experiment_rejects_invalid_metadata(self): + """Experiment.run_evaluations raises if an evaluator declares bad metadata.""" + from strands_evals import Case, Experiment + from strands_evals.evaluators.evaluator import Evaluator + + class BadMetaEvaluator(Evaluator): + def metadata(self): + return {"checks": "", "method": {"category": "llm_judge_output", "summary": "test"}, "threshold": "x"} + + def evaluate(self, evaluation_case): + return [] + + async def evaluate_async(self, evaluation_case): + return [] + + experiment = Experiment( + cases=[Case(input="hello", expected_output="hi")], + evaluators=[BadMetaEvaluator()], + ) + + import pytest + + with pytest.raises(ValueError, match="must be a non-empty string"): + experiment.run_evaluations(task=lambda x: x) + + def test_experiment_accepts_valid_metadata(self): + """Experiment.run_evaluations does not raise for valid metadata.""" + from strands_evals import Case, Experiment + from strands_evals.evaluators.deterministic import Contains + + experiment = Experiment( + cases=[Case(input="hello", expected_output="hello world")], + evaluators=[Contains(value="hello")], + ) + + report = experiment.run_evaluations(task=lambda x: x) + assert report is not None + + def test_experiment_skips_evaluators_without_metadata(self): + """Experiment.run_evaluations skips evaluators that return None from metadata().""" + from strands_evals import Case, Experiment + from strands_evals.evaluators.evaluator import Evaluator + from strands_evals.types.evaluation import EvaluationOutput + + class NoMetaEvaluator(Evaluator): + def evaluate(self, evaluation_case): + return [EvaluationOutput(score=1.0, test_pass=True, reason="ok")] + + async def evaluate_async(self, evaluation_case): + return self.evaluate(evaluation_case) + + experiment = Experiment( + cases=[Case(input="hello", expected_output="hi")], + evaluators=[NoMetaEvaluator()], + ) + + report = experiment.run_evaluations(task=lambda x: x) + assert report is not None