diff --git a/src/strands_evals/cli/commands/validate.py b/src/strands_evals/cli/commands/validate.py index 4488506d..44fdd726 100644 --- a/src/strands_evals/cli/commands/validate.py +++ b/src/strands_evals/cli/commands/validate.py @@ -16,7 +16,7 @@ def _run(args: argparse.Namespace) -> int: experiment: Experiment = Experiment.from_file(args.experiment_file, custom_evaluators=custom_evaluators) case_count = len(experiment.cases) - evaluator_names = [evaluator.get_type_name() for evaluator in experiment.evaluators] + evaluator_names = [evaluator.get_name() for evaluator in experiment.evaluators] fmt = resolve_format(args) if fmt == "json": diff --git a/src/strands_evals/evaluators/chaos/failure_communication_evaluator.py b/src/strands_evals/evaluators/chaos/failure_communication_evaluator.py index 021c0e78..f4fce300 100644 --- a/src/strands_evals/evaluators/chaos/failure_communication_evaluator.py +++ b/src/strands_evals/evaluators/chaos/failure_communication_evaluator.py @@ -46,8 +46,9 @@ def __init__( version: str = "v0", model: Model | str | None = None, system_prompt: str | None = None, + name: str | None = None, ): - super().__init__() + super().__init__(name=name) self.version = version default_prompt = get_template(version).SYSTEM_PROMPT self.system_prompt = system_prompt if system_prompt is not None else default_prompt diff --git a/src/strands_evals/evaluators/chaos/partial_completion_evaluator.py b/src/strands_evals/evaluators/chaos/partial_completion_evaluator.py index 1254fd62..ac6ba9ed 100644 --- a/src/strands_evals/evaluators/chaos/partial_completion_evaluator.py +++ b/src/strands_evals/evaluators/chaos/partial_completion_evaluator.py @@ -27,8 +27,9 @@ def __init__( version: str = "v0", model: Model | str | None = None, system_prompt: str | None = None, + name: str | None = None, ): - super().__init__() + super().__init__(name=name) self.version = version default_prompt = get_template(version).SYSTEM_PROMPT self.system_prompt = system_prompt if system_prompt is not None else default_prompt diff --git a/src/strands_evals/evaluators/chaos/recovery_strategy_evaluator.py b/src/strands_evals/evaluators/chaos/recovery_strategy_evaluator.py index 3e044452..21ccf888 100644 --- a/src/strands_evals/evaluators/chaos/recovery_strategy_evaluator.py +++ b/src/strands_evals/evaluators/chaos/recovery_strategy_evaluator.py @@ -46,8 +46,9 @@ def __init__( version: str = "v0", model: Model | str | None = None, system_prompt: str | None = None, + name: str | None = None, ): - super().__init__() + super().__init__(name=name) self.version = version default_prompt = get_template(version).SYSTEM_PROMPT self.system_prompt = system_prompt if system_prompt is not None else default_prompt diff --git a/src/strands_evals/evaluators/coherence_evaluator.py b/src/strands_evals/evaluators/coherence_evaluator.py index c85d14fe..dc2e085c 100644 --- a/src/strands_evals/evaluators/coherence_evaluator.py +++ b/src/strands_evals/evaluators/coherence_evaluator.py @@ -59,8 +59,9 @@ def __init__( model: Model | str | None = None, system_prompt: str | None = None, include_inputs: bool = True, + name: str | None = None, ): - super().__init__() + super().__init__(name=name) self.system_prompt = system_prompt or get_template(version).SYSTEM_PROMPT self.version = version self.model = model diff --git a/src/strands_evals/evaluators/conciseness_evaluator.py b/src/strands_evals/evaluators/conciseness_evaluator.py index 2005accb..8bea4b9a 100644 --- a/src/strands_evals/evaluators/conciseness_evaluator.py +++ b/src/strands_evals/evaluators/conciseness_evaluator.py @@ -43,8 +43,9 @@ def __init__( model: Model | str | None = None, system_prompt: str | None = None, include_inputs: bool = True, + name: str | None = None, ): - super().__init__() + super().__init__(name=name) self.system_prompt = system_prompt or get_template(version).SYSTEM_PROMPT self.version = version self.model = model diff --git a/src/strands_evals/evaluators/correctness_evaluator.py b/src/strands_evals/evaluators/correctness_evaluator.py index b0c8f4f9..c7bbcacd 100644 --- a/src/strands_evals/evaluators/correctness_evaluator.py +++ b/src/strands_evals/evaluators/correctness_evaluator.py @@ -75,8 +75,9 @@ def __init__( model: Model | str | None = None, system_prompt: str | None = None, reference_system_prompt: str | None = None, + name: str | None = None, ): - super().__init__() + super().__init__(name=name) self.system_prompt = system_prompt if system_prompt is not None else get_template(version).SYSTEM_PROMPT self.reference_system_prompt = ( reference_system_prompt diff --git a/src/strands_evals/evaluators/deterministic/environment_state.py b/src/strands_evals/evaluators/deterministic/environment_state.py index 23673906..f70391c1 100644 --- a/src/strands_evals/evaluators/deterministic/environment_state.py +++ b/src/strands_evals/evaluators/deterministic/environment_state.py @@ -13,11 +13,18 @@ def _find_state_by_name(states: list[EnvironmentState], name: str) -> Environmen class StateEquals(Evaluator[InputT, OutputT]): - """Checks if a named environment state matches an expected value.""" + """Checks if a named environment state matches an expected value. + + The `name` arg doubles as the state key in `EnvironmentState` and the + instance-level evaluator name surfaced through `get_name()`, so two + `StateEquals(name="cart")` / `StateEquals(name="balance")` instances + are naturally distinguishable in `EvaluationReport.cases`. + """ + + name: str # required, not Optional — narrows the base class annotation def __init__(self, name: str, value: Any | None = None): - super().__init__() - self.name = name + super().__init__(name=name) self.value = value def evaluate(self, evaluation_case: EvaluationData[InputT, OutputT]) -> list[EvaluationOutput]: diff --git a/src/strands_evals/evaluators/deterministic/output.py b/src/strands_evals/evaluators/deterministic/output.py index b97445fa..174628a9 100644 --- a/src/strands_evals/evaluators/deterministic/output.py +++ b/src/strands_evals/evaluators/deterministic/output.py @@ -7,8 +7,8 @@ class Equals(Evaluator[InputT, OutputT]): """Checks if actual_output equals an expected value.""" - def __init__(self, value: Any | None = None): - super().__init__() + def __init__(self, value: Any | None = None, name: str | None = None): + super().__init__(name=name) self.value = value def evaluate(self, evaluation_case: EvaluationData[InputT, OutputT]) -> list[EvaluationOutput]: @@ -29,8 +29,8 @@ async def evaluate_async(self, evaluation_case: EvaluationData[InputT, OutputT]) class Contains(Evaluator[InputT, OutputT]): """Checks if actual_output contains a substring.""" - def __init__(self, value: str, case_sensitive: bool = True): - super().__init__() + def __init__(self, value: str, case_sensitive: bool = True, name: str | None = None): + super().__init__(name=name) self.value = value self.case_sensitive = case_sensitive @@ -56,8 +56,8 @@ async def evaluate_async(self, evaluation_case: EvaluationData[InputT, OutputT]) class StartsWith(Evaluator[InputT, OutputT]): """Checks if actual_output starts with a prefix.""" - def __init__(self, value: str, case_sensitive: bool = True): - super().__init__() + def __init__(self, value: str, case_sensitive: bool = True, name: str | None = None): + super().__init__(name=name) self.value = value self.case_sensitive = case_sensitive diff --git a/src/strands_evals/evaluators/deterministic/trajectory.py b/src/strands_evals/evaluators/deterministic/trajectory.py index 0c129bf6..0995ee29 100644 --- a/src/strands_evals/evaluators/deterministic/trajectory.py +++ b/src/strands_evals/evaluators/deterministic/trajectory.py @@ -6,8 +6,8 @@ class ToolCalled(Evaluator[InputT, OutputT]): """Checks if a specific tool was called in the trajectory.""" - def __init__(self, tool_name: str): - super().__init__() + def __init__(self, tool_name: str, name: str | None = None): + super().__init__(name=name) self.tool_name = tool_name def evaluate(self, evaluation_case: EvaluationData[InputT, OutputT]) -> list[EvaluationOutput]: diff --git a/src/strands_evals/evaluators/evaluator.py b/src/strands_evals/evaluators/evaluator.py index bb70af73..75b50924 100644 --- a/src/strands_evals/evaluators/evaluator.py +++ b/src/strands_evals/evaluators/evaluator.py @@ -36,14 +36,22 @@ class Evaluator(Generic[InputT, OutputT]): evaluation_level: EvaluationLevel | None = None _trace_extractor: TraceExtractor | None = None - def __init__(self, trace_extractor: TraceExtractor | None = None): + def __init__(self, trace_extractor: TraceExtractor | None = None, name: str | None = None): """Initialize evaluator with optional custom trace extractor. Args: trace_extractor: Custom trace extractor. If None and evaluation_level is set, a default TraceExtractor will be created. + name: Instance-level identifier used as the evaluator tag in + `EvaluationReport.cases[i]["evaluator"]` and as + `gen_ai.evaluation.name` on emitted spans/logs. When two + instances of the same class run in one experiment (e.g., + `Contains(value="x")` and `Contains(value="y")`), distinct + names keep their results from colliding. Defaults to the + class name when unset. """ self.aggregator = self._default_aggregator + self.name = name if trace_extractor: self._trace_extractor = trace_extractor elif self.evaluation_level: @@ -264,6 +272,18 @@ def get_type_name(cls) -> str: """ return cls.__name__ + def get_name(self) -> str: + """Get the instance-level evaluator name, falling back to the class name. + + Used for the per-row `evaluator` tag in `EvaluationReport` and the + `gen_ai.evaluation.name` OTel attribute. `get_type_name()` is still + used for class-keyed lookups such as `from_dict` registry resolution. + + Returns: + str: The instance name if set, otherwise the class name. + """ + return self.name or self.get_type_name() + def to_dict(self) -> dict: """ Convert the evaluator into a dictionary. diff --git a/src/strands_evals/evaluators/faithfulness_evaluator.py b/src/strands_evals/evaluators/faithfulness_evaluator.py index 36762b46..5c5f05e9 100644 --- a/src/strands_evals/evaluators/faithfulness_evaluator.py +++ b/src/strands_evals/evaluators/faithfulness_evaluator.py @@ -46,8 +46,9 @@ def __init__( version: str = "v0", model: Model | str | None = None, system_prompt: str | None = None, + name: str | None = None, ): - super().__init__() + super().__init__(name=name) self.system_prompt = system_prompt if system_prompt is not None else get_template(version).SYSTEM_PROMPT self.version = version self.model = model diff --git a/src/strands_evals/evaluators/goal_success_rate_evaluator.py b/src/strands_evals/evaluators/goal_success_rate_evaluator.py index edcd661e..8c33061d 100644 --- a/src/strands_evals/evaluators/goal_success_rate_evaluator.py +++ b/src/strands_evals/evaluators/goal_success_rate_evaluator.py @@ -72,8 +72,9 @@ def __init__( model: Model | str | None = None, system_prompt: str | None = None, assertion_system_prompt: str | None = None, + name: str | None = None, ): - super().__init__() + super().__init__(name=name) self.system_prompt = system_prompt if system_prompt is not None else get_template(version).SYSTEM_PROMPT self.assertion_system_prompt = ( assertion_system_prompt diff --git a/src/strands_evals/evaluators/harmfulness_evaluator.py b/src/strands_evals/evaluators/harmfulness_evaluator.py index 08a777de..edd0b463 100644 --- a/src/strands_evals/evaluators/harmfulness_evaluator.py +++ b/src/strands_evals/evaluators/harmfulness_evaluator.py @@ -40,8 +40,9 @@ def __init__( version: str = "v0", model: Model | str | None = None, system_prompt: str | None = None, + name: str | None = None, ): - super().__init__() + super().__init__(name=name) self.system_prompt = system_prompt if system_prompt is not None else get_template(version).SYSTEM_PROMPT self.version = version self.model = model diff --git a/src/strands_evals/evaluators/helpfulness_evaluator.py b/src/strands_evals/evaluators/helpfulness_evaluator.py index 8a29de0f..f5b99f53 100644 --- a/src/strands_evals/evaluators/helpfulness_evaluator.py +++ b/src/strands_evals/evaluators/helpfulness_evaluator.py @@ -51,8 +51,9 @@ def __init__( model: Model | str | None = None, system_prompt: str | None = None, include_inputs: bool = True, + name: str | None = None, ): - super().__init__() + super().__init__(name=name) self.system_prompt = system_prompt if system_prompt is not None else get_template(version).SYSTEM_PROMPT self.version = version self.model = model diff --git a/src/strands_evals/evaluators/instruction_following_evaluator.py b/src/strands_evals/evaluators/instruction_following_evaluator.py index c395a102..bc1f2acd 100644 --- a/src/strands_evals/evaluators/instruction_following_evaluator.py +++ b/src/strands_evals/evaluators/instruction_following_evaluator.py @@ -40,8 +40,9 @@ def __init__( version: str = "v0", model: Model | str | None = None, system_prompt: str | None = None, + name: str | None = None, ): - super().__init__() + super().__init__(name=name) self.system_prompt = system_prompt if system_prompt is not None else get_template(version).SYSTEM_PROMPT self.version = version self.model = model diff --git a/src/strands_evals/evaluators/interactions_evaluator.py b/src/strands_evals/evaluators/interactions_evaluator.py index 09781ce1..ea7be7f6 100644 --- a/src/strands_evals/evaluators/interactions_evaluator.py +++ b/src/strands_evals/evaluators/interactions_evaluator.py @@ -32,8 +32,9 @@ def __init__( model: Model | str | None = None, system_prompt: str = SYSTEM_PROMPT, include_inputs: bool = True, + name: str | None = None, ): - super().__init__() + super().__init__(name=name) self.rubric = rubric self.interaction_description = interaction_description self.model = model diff --git a/src/strands_evals/evaluators/multimodal_correctness_evaluator.py b/src/strands_evals/evaluators/multimodal_correctness_evaluator.py index 04549f8a..557b8a17 100644 --- a/src/strands_evals/evaluators/multimodal_correctness_evaluator.py +++ b/src/strands_evals/evaluators/multimodal_correctness_evaluator.py @@ -24,6 +24,7 @@ def __init__( system_prompt: str | None = None, reference_suffix: str | None = None, uses_environment_state: bool = False, + name: str | None = None, ): super().__init__( rubric=rubric if rubric is not None else CORRECTNESS_RUBRIC_V0, @@ -32,4 +33,5 @@ def __init__( system_prompt=system_prompt, reference_suffix=reference_suffix, uses_environment_state=uses_environment_state, + name=name, ) diff --git a/src/strands_evals/evaluators/multimodal_faithfulness_evaluator.py b/src/strands_evals/evaluators/multimodal_faithfulness_evaluator.py index 9b09ccc1..319950b2 100644 --- a/src/strands_evals/evaluators/multimodal_faithfulness_evaluator.py +++ b/src/strands_evals/evaluators/multimodal_faithfulness_evaluator.py @@ -23,6 +23,7 @@ def __init__( system_prompt: str | None = None, reference_suffix: str | None = None, uses_environment_state: bool = False, + name: str | None = None, ): super().__init__( rubric=rubric if rubric is not None else FAITHFULNESS_RUBRIC_V0, @@ -31,4 +32,5 @@ def __init__( system_prompt=system_prompt, reference_suffix=reference_suffix, uses_environment_state=uses_environment_state, + name=name, ) diff --git a/src/strands_evals/evaluators/multimodal_instruction_following_evaluator.py b/src/strands_evals/evaluators/multimodal_instruction_following_evaluator.py index dcea0b88..57782ba4 100644 --- a/src/strands_evals/evaluators/multimodal_instruction_following_evaluator.py +++ b/src/strands_evals/evaluators/multimodal_instruction_following_evaluator.py @@ -23,6 +23,7 @@ def __init__( system_prompt: str | None = None, reference_suffix: str | None = None, uses_environment_state: bool = False, + name: str | None = None, ): super().__init__( rubric=rubric if rubric is not None else INSTRUCTION_FOLLOWING_RUBRIC_V0, @@ -31,4 +32,5 @@ def __init__( system_prompt=system_prompt, reference_suffix=reference_suffix, uses_environment_state=uses_environment_state, + name=name, ) diff --git a/src/strands_evals/evaluators/multimodal_output_evaluator.py b/src/strands_evals/evaluators/multimodal_output_evaluator.py index 232e9553..827bbd5f 100644 --- a/src/strands_evals/evaluators/multimodal_output_evaluator.py +++ b/src/strands_evals/evaluators/multimodal_output_evaluator.py @@ -45,6 +45,7 @@ def __init__( system_prompt: str | None = None, reference_suffix: str | None = None, uses_environment_state: bool = False, + name: str | None = None, ): super().__init__( rubric=rubric, @@ -52,6 +53,7 @@ def __init__( system_prompt=system_prompt if system_prompt is not None else MLLM_JUDGE_SYSTEM_PROMPT, include_inputs=include_inputs, uses_environment_state=uses_environment_state, + name=name, ) self.reference_suffix = reference_suffix if reference_suffix is not None else self.DEFAULT_REFERENCE_SUFFIX diff --git a/src/strands_evals/evaluators/multimodal_overall_quality_evaluator.py b/src/strands_evals/evaluators/multimodal_overall_quality_evaluator.py index acdd0fec..e246f020 100644 --- a/src/strands_evals/evaluators/multimodal_overall_quality_evaluator.py +++ b/src/strands_evals/evaluators/multimodal_overall_quality_evaluator.py @@ -34,6 +34,7 @@ def __init__( system_prompt: str | None = None, reference_suffix: str | None = None, uses_environment_state: bool = False, + name: str | None = None, ): super().__init__( rubric=rubric if rubric is not None else OVERALL_QUALITY_RUBRIC_V0, @@ -42,4 +43,5 @@ def __init__( system_prompt=system_prompt, reference_suffix=reference_suffix if reference_suffix is not None else _OVERALL_QUALITY_REFERENCE_SUFFIX, uses_environment_state=uses_environment_state, + name=name, ) diff --git a/src/strands_evals/evaluators/output_evaluator.py b/src/strands_evals/evaluators/output_evaluator.py index 24148adb..f732e9d3 100644 --- a/src/strands_evals/evaluators/output_evaluator.py +++ b/src/strands_evals/evaluators/output_evaluator.py @@ -29,8 +29,9 @@ def __init__( system_prompt: str = SYSTEM_PROMPT, include_inputs: bool = True, uses_environment_state: bool = False, + name: str | None = None, ): - super().__init__() + super().__init__(name=name) self.rubric = rubric self.model = model self.include_inputs = include_inputs diff --git a/src/strands_evals/evaluators/refusal_evaluator.py b/src/strands_evals/evaluators/refusal_evaluator.py index 401f0ea1..6e00285d 100644 --- a/src/strands_evals/evaluators/refusal_evaluator.py +++ b/src/strands_evals/evaluators/refusal_evaluator.py @@ -40,8 +40,9 @@ def __init__( version: str = "v0", model: Model | str | None = None, system_prompt: str | None = None, + name: str | None = None, ): - super().__init__() + super().__init__(name=name) self.system_prompt = system_prompt if system_prompt is not None else get_template(version).SYSTEM_PROMPT self.version = version self.model = model diff --git a/src/strands_evals/evaluators/response_relevance_evaluator.py b/src/strands_evals/evaluators/response_relevance_evaluator.py index 27288ea4..c9277223 100644 --- a/src/strands_evals/evaluators/response_relevance_evaluator.py +++ b/src/strands_evals/evaluators/response_relevance_evaluator.py @@ -48,8 +48,9 @@ def __init__( model: Model | str | None = None, system_prompt: str | None = None, include_inputs: bool = True, + name: str | None = None, ): - super().__init__() + super().__init__(name=name) self.system_prompt = system_prompt or get_template(version).SYSTEM_PROMPT self.version = version self.model = model diff --git a/src/strands_evals/evaluators/stereotyping_evaluator.py b/src/strands_evals/evaluators/stereotyping_evaluator.py index a63afc17..99f46295 100644 --- a/src/strands_evals/evaluators/stereotyping_evaluator.py +++ b/src/strands_evals/evaluators/stereotyping_evaluator.py @@ -40,8 +40,9 @@ def __init__( version: str = "v0", model: Model | str | None = None, system_prompt: str | None = None, + name: str | None = None, ): - super().__init__() + super().__init__(name=name) self.system_prompt = system_prompt if system_prompt is not None else get_template(version).SYSTEM_PROMPT self.version = version self.model = model diff --git a/src/strands_evals/evaluators/tool_parameter_accuracy_evaluator.py b/src/strands_evals/evaluators/tool_parameter_accuracy_evaluator.py index 1be12e08..40ee7f1e 100644 --- a/src/strands_evals/evaluators/tool_parameter_accuracy_evaluator.py +++ b/src/strands_evals/evaluators/tool_parameter_accuracy_evaluator.py @@ -40,8 +40,9 @@ def __init__( version: str = "v0", model: Model | str | None = None, system_prompt: str | None = None, + name: str | None = None, ): - super().__init__() + super().__init__(name=name) self.system_prompt = system_prompt if system_prompt is not None else get_template(version).SYSTEM_PROMPT self.version = version self.model = model diff --git a/src/strands_evals/evaluators/tool_selection_accuracy_evaluator.py b/src/strands_evals/evaluators/tool_selection_accuracy_evaluator.py index 76e29e6c..ced8756c 100644 --- a/src/strands_evals/evaluators/tool_selection_accuracy_evaluator.py +++ b/src/strands_evals/evaluators/tool_selection_accuracy_evaluator.py @@ -40,8 +40,9 @@ def __init__( version: str = "v0", model: Model | str | None = None, system_prompt: str | None = None, + name: str | None = None, ): - super().__init__() + super().__init__(name=name) self.system_prompt = system_prompt if system_prompt is not None else get_template(version).SYSTEM_PROMPT self.version = version self.model = model diff --git a/src/strands_evals/evaluators/trajectory_evaluator.py b/src/strands_evals/evaluators/trajectory_evaluator.py index acc5fa36..298ddeed 100644 --- a/src/strands_evals/evaluators/trajectory_evaluator.py +++ b/src/strands_evals/evaluators/trajectory_evaluator.py @@ -32,8 +32,9 @@ def __init__( model: Model | str | None = None, system_prompt: str = SYSTEM_PROMPT, include_inputs: bool = True, + name: str | None = None, ): - super().__init__() + super().__init__(name=name) self.rubric = rubric self.trajectory_description = trajectory_description self.model = model diff --git a/src/strands_evals/experiment.py b/src/strands_evals/experiment.py index 75afc494..de42b984 100644 --- a/src/strands_evals/experiment.py +++ b/src/strands_evals/experiment.py @@ -174,6 +174,24 @@ def evaluators(self, new_evaluators: list[Evaluator[InputT, OutputT]]): """ self._evaluators = new_evaluators + def _validate_evaluator_names(self) -> None: + """Validate that all evaluators expose unique names. + + Two instances of the same Evaluator subclass collide on the default + class-name fallback; pass `name="..."` to disambiguate. + + Raises: + ValueError: If two evaluators resolve to the same `get_name()`. + """ + names = [evaluator.get_name() for evaluator in self._evaluators] + duplicates = sorted({n for n in names if names.count(n) > 1}) + if duplicates: + raise ValueError( + f"Evaluator names must be unique within an experiment. " + f"Duplicates: {duplicates}. Pass `name=...` when constructing " + f"multiple instances of the same Evaluator subclass." + ) + def _validate_case_names(self) -> None: """Validate that all cases have unique, non-None names. @@ -334,9 +352,9 @@ async def _evaluate_with_retry(evaluator=evaluator, evaluation_context=evaluatio try: with self._tracer.start_as_current_span( - f"evaluator {evaluator.get_type_name()}", + f"evaluator {evaluator.get_name()}", attributes={ - "gen_ai.evaluation.name": evaluator.get_type_name(), + "gen_ai.evaluation.name": evaluator.get_name(), "gen_ai.evaluation.case.name": case_name, }, ) as eval_span: @@ -363,7 +381,7 @@ async def _evaluate_with_retry(evaluator=evaluator, evaluation_context=evaluatio # CloudWatch logging for this evaluator try: - evaluator_full_name = f"Custom.{evaluator.get_type_name()}" + evaluator_full_name = f"Custom.{evaluator.get_name()}" region = os.environ.get("AWS_REGION", "us-east-1") _config_arn = f"arn:aws:strands:{region}::strands-evaluation-empty-config/{self._config_id}" _evaluator_arn = f"arn:aws:strands-evals:::evaluator/{evaluator_full_name}" @@ -398,7 +416,8 @@ async def _evaluate_with_retry(evaluator=evaluator, evaluation_context=evaluatio logger.debug(f"Skipping CloudWatch logging: {str(e)}") return { - "evaluator_name": evaluator.get_type_name(), + "evaluator_name": evaluator.get_name(), + "evaluator_type": evaluator.get_type_name(), "test_pass": aggregate_pass, "score": aggregate_score, "reason": aggregate_reason or "", @@ -410,15 +429,16 @@ async def _evaluate_with_retry(evaluator=evaluator, evaluation_context=evaluatio original_exception = e.last_attempt.exception() if original_exception is None: original_exception = Exception( - f"Evaluator {evaluator.get_type_name()} failed after {_MAX_RETRY_ATTEMPTS} retries" + f"Evaluator {evaluator.get_name()} failed after {_MAX_RETRY_ATTEMPTS} retries" ) logger.error( f"Max retry attempts ({_MAX_RETRY_ATTEMPTS}) exceeded for evaluator " - f"{evaluator.get_type_name()} on case {case_name}. " + f"{evaluator.get_name()} on case {case_name}. " f"Last error: {str(original_exception)}" ) return { - "evaluator_name": evaluator.get_type_name(), + "evaluator_name": evaluator.get_name(), + "evaluator_type": evaluator.get_type_name(), "test_pass": False, "score": 0, "reason": f"Evaluator error: {str(original_exception)}", @@ -427,7 +447,8 @@ async def _evaluate_with_retry(evaluator=evaluator, evaluation_context=evaluatio except Exception as e: # Catch non-throttling errors and record as failure (error isolation) return { - "evaluator_name": evaluator.get_type_name(), + "evaluator_name": evaluator.get_name(), + "evaluator_type": evaluator.get_type_name(), "test_pass": False, "score": 0, "reason": f"Evaluator error: {str(e)}", @@ -542,7 +563,8 @@ async def _worker( for evaluator in self._evaluators: evaluator_results.append( { - "evaluator_name": evaluator.get_type_name(), + "evaluator_name": evaluator.get_name(), + "evaluator_type": evaluator.get_type_name(), "test_pass": False, "score": 0, "reason": f"An error occurred: {str(e)}", @@ -604,6 +626,8 @@ async def run_evaluations_async( A single EvaluationReport flattened across every evaluator. Each row in `cases` carries an `evaluator` key naming which evaluator produced it. """ + self._validate_evaluator_names() + if evaluation_data_store is not None: self._validate_case_names() @@ -624,9 +648,10 @@ async def run_evaluations_async( worker.cancel() await asyncio.gather(*workers, return_exceptions=True) - # Organize results by evaluator + # Organize results by evaluator (keyed on instance name so two instances + # of the same class with distinct `name=` kwargs do not collide). evaluator_data: dict[str, dict[str, list]] = { - evaluator.get_type_name(): { + evaluator.get_name(): { "scores": [], "test_passes": [], "cases": [], @@ -644,7 +669,9 @@ async def run_evaluations_async( recommendation = result.get("recommendation") for eval_result in result["evaluator_results"]: eval_name = eval_result["evaluator_name"] - evaluator_data[eval_name]["cases"].append({**case_data, "evaluator": eval_name}) + evaluator_data[eval_name]["cases"].append( + {**case_data, "evaluator": eval_name, "evaluator_type": eval_result["evaluator_type"]} + ) evaluator_data[eval_name]["scores"].append(eval_result["score"]) evaluator_data[eval_name]["test_passes"].append(eval_result["test_pass"]) evaluator_data[eval_name]["reasons"].append(eval_result["reason"]) @@ -654,7 +681,7 @@ async def run_evaluations_async( reports = [] for evaluator in self._evaluators: - eval_name = evaluator.get_type_name() + eval_name = evaluator.get_name() data = evaluator_data[eval_name] scores = data["scores"] report = EvaluationReport( diff --git a/src/strands_evals/experimental/redteam/evaluators/attack_success_evaluator.py b/src/strands_evals/experimental/redteam/evaluators/attack_success_evaluator.py index d2f2905d..4e6a97b7 100644 --- a/src/strands_evals/experimental/redteam/evaluators/attack_success_evaluator.py +++ b/src/strands_evals/experimental/redteam/evaluators/attack_success_evaluator.py @@ -33,8 +33,9 @@ def __init__( model: Model | str | None = None, system_prompt: str | None = None, pass_threshold: float = 0.3, + name: str | None = None, ): - super().__init__() + super().__init__(name=name) template = get_template(version) self.version = version self.model = model diff --git a/tests/strands_evals/evaluators/test_evaluator.py b/tests/strands_evals/evaluators/test_evaluator.py index 09f9c65d..4fe8c561 100644 --- a/tests/strands_evals/evaluators/test_evaluator.py +++ b/tests/strands_evals/evaluators/test_evaluator.py @@ -288,3 +288,18 @@ def test_extract_text_content_user_message_with_tool_result(): result = evaluator._extract_text_content(msg) assert result == "Here's the result" + + +def test_get_name_defaults_to_class_name(): + """Without a name kwarg, get_name() falls back to the class name.""" + evaluator = SimpleEvaluator() + assert evaluator.get_name() == "SimpleEvaluator" + assert evaluator.get_name() == evaluator.get_type_name() + + +def test_get_name_uses_explicit_name(): + """An explicit name kwarg overrides the class-name fallback.""" + evaluator = SimpleEvaluator(name="my_simple_check") + assert evaluator.get_name() == "my_simple_check" + # get_type_name() still reports the class for from_dict registry lookups. + assert evaluator.get_type_name() == "SimpleEvaluator" diff --git a/tests/strands_evals/test_experiment.py b/tests/strands_evals/test_experiment.py index c1867133..b8be325e 100644 --- a/tests/strands_evals/test_experiment.py +++ b/tests/strands_evals/test_experiment.py @@ -2052,3 +2052,74 @@ def task_with_session(c): assert report.diagnoses == [None] assert report.recommendations == [None] + + +def test_run_evaluations_two_same_class_evaluators_with_distinct_names(): + """Two instances of the same class with `name=...` produce distinct rows.""" + cases = [ + Case(name="france", input="france", expected_output="paris"), + Case(name="japan", input="japan", expected_output="tokyo"), + ] + experiment = Experiment( + cases=cases, + evaluators=[ + Contains(value="paris", name="contains_paris"), + Contains(value="tokyo", name="contains_tokyo"), + ], + ) + + def task(case): + return {"france": "paris", "japan": "tokyo"}[case.input] + + report = experiment.run_evaluations(task) + + tags = [row["evaluator"] for row in report.cases] + assert sorted(tags) == ["contains_paris", "contains_paris", "contains_tokyo", "contains_tokyo"] + + # Both instances share the class name, so consumers can group/aggregate by type + # even when instance names diverge. + assert {row["evaluator_type"] for row in report.cases} == {"Contains"} + + rows = list(zip(report.cases, report.test_passes, strict=True)) + paris_by_case = {row["name"]: passed for row, passed in rows if row["evaluator"] == "contains_paris"} + tokyo_by_case = {row["name"]: passed for row, passed in rows if row["evaluator"] == "contains_tokyo"} + assert paris_by_case == {"france": True, "japan": False} + assert tokyo_by_case == {"france": False, "japan": True} + + +def test_run_evaluations_rejects_duplicate_evaluator_names(): + """Two evaluators with the same effective name raise before running.""" + experiment = Experiment( + cases=[Case(name="c1", input="x")], + evaluators=[Contains(value="a"), Contains(value="b")], + ) + + with pytest.raises(ValueError, match="Evaluator names must be unique"): + experiment.run_evaluations(lambda case: case.input) + + +def test_evaluator_name_round_trips_through_to_dict_from_dict(): + """to_dict emits `name` for explicitly-named evaluators, from_dict restores it.""" + experiment = Experiment( + cases=[Case(name="c1", input="x")], + evaluators=[Contains(value="hello", name="contains_hello")], + ) + + payload = experiment.to_dict() + assert payload["evaluators"][0]["evaluator_type"] == "Contains" + assert payload["evaluators"][0]["name"] == "contains_hello" + + restored = Experiment.from_dict(payload) + assert restored.evaluators[0].get_name() == "contains_hello" + assert restored.evaluators[0].get_type_name() == "Contains" + + +def test_evaluator_name_default_omitted_from_to_dict(): + """When no name is set, to_dict omits the field (matches existing default-stripping).""" + experiment = Experiment( + cases=[Case(name="c1", input="x")], + evaluators=[Contains(value="hello")], + ) + + payload = experiment.to_dict() + assert "name" not in payload["evaluators"][0]