diff --git a/packages/nemo_evaluator_sdk/examples/hermes/example.py b/packages/nemo_evaluator_sdk/examples/hermes/example.py index eee0476b39..e216130706 100644 --- a/packages/nemo_evaluator_sdk/examples/hermes/example.py +++ b/packages/nemo_evaluator_sdk/examples/hermes/example.py @@ -219,7 +219,7 @@ async def main() -> None: result = await evaluate() answer = result.trials[0].output.output_text if result.trials[0].output else None - aggregate = next(score for score in result.summary.scores.scores if score.name == "keyword_match.score") + aggregate = result.summary.score("keyword_match.score") print(f"response: {answer}") print(f"{aggregate.name}: {aggregate.mean}") diff --git a/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/results.py b/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/results.py index b31d1b5078..211610da55 100644 --- a/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/results.py +++ b/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/results.py @@ -6,7 +6,7 @@ from __future__ import annotations import math -from collections.abc import Sequence +from collections.abc import Mapping, Sequence from datetime import datetime from pathlib import Path @@ -63,6 +63,19 @@ class AgentEvalSummary(BaseModel): trial_count: int = Field(default=0, description="Number of distinct trials scored.") score_count: int = Field(default=0, description="Total number of metric scores.") + @property + def scores_by_name(self) -> Mapping[str, AggregateScore]: + """Aggregates keyed by name — see :attr:`AggregatedMetricResult.scores_by_name`.""" + return self.scores.scores_by_name + + def score(self, name: str) -> AggregateScore: + """Return the aggregate named ``name`` — see :meth:`AggregatedMetricResult.score`. + + Exists so callers needn't know the aggregates sit one level down, behind a field whose name + differs from the summary's own accessor by a single character. + """ + return self.scores.score(name) + @staticmethod def from_scores( scores: Sequence[AgentEvalTaskScore], diff --git a/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/values/results.py b/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/values/results.py index c52c31d71d..02eef50977 100644 --- a/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/values/results.py +++ b/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/values/results.py @@ -7,6 +7,8 @@ import json import math +from collections.abc import Mapping +from difflib import get_close_matches from typing import Any, Literal, Self import pyarrow as pa @@ -417,12 +419,84 @@ def _serialize(self, handler): AggregateScore = AggregateRangeScore | AggregateRubricScore | AggregateScalarScore +#: How many names a lookup-miss message lists when nothing resembles what was asked for. A judgement +#: call rather than a measured optimum -- enough to show the naming convention, few enough to stay +#: readable, since a run with several metrics times pass@k can carry dozens. Only this fallback is +#: truncated; a near-miss is surfaced by similarity, so finding the name you meant never depends on +#: where it happens to fall alphabetically. +_MISS_NAME_LIMIT = 10 + + class AggregatedMetricResult(BaseModel): """Result of aggregating metric scores with full statistics.""" model_config = ConfigDict(extra="forbid") scores: list[AggregateScore] = Field(description="The list of aggregated scores.") + @property + def scores_by_name(self) -> Mapping[str, AggregateScore]: + """Aggregates keyed by :attr:`AggregateScoreBase.name`, for ``in``, ``.get()``, and iteration. + + Reach for this when a score's absence is a legitimate outcome ("did this metric run?"); use + :meth:`score` when it isn't. Names are expected unique, but runner-contributed extras are + appended as-is, so a collision is possible: the first wins, matching the ``next(...)`` scans + this replaces. + """ + by_name: dict[str, AggregateScore] = {} + for score in self.scores: + by_name.setdefault(score.name, score) + return by_name + + def score(self, name: str) -> AggregateScore: + """Return the aggregate named ``name``, raising :class:`KeyError` if there isn't one. + + Raises rather than returning ``None`` because an unknown name is nearly always a typo or a + metric that didn't run. Both are bugs worth surfacing at the lookup, where the name is in + hand, instead of as an ``AttributeError`` on ``.mean`` further downstream. When absence is a + real possibility, use ``scores_by_name.get(...)``. + """ + # A direct scan rather than a lookup into `scores_by_name`: building the whole mapping to + # return one element allocates a dict per call, and the score list is short enough that the + # scan wins outright. + for score in self.scores: + if score.name == name: + return score + raise KeyError(self._unknown_score_message(name)) + + def _unknown_score_message(self, name: str) -> str: + """Explain a lookup miss, leading with near-misses when the name looks like a typo. + + "Close" is :func:`difflib.get_close_matches`: SequenceMatcher (Ratcliff/Obershelp) similarity + of at least 0.6, best three first. That is a subsequence-overlap ratio, not an edit distance. + + Listing every name alphabetically would be simpler, and is the better answer if the list is + left whole. Truncating one is what breaks it -- the name a caller meant is not reliably in + the first :data:`_MISS_NAME_LIMIT`, since a typo'd ``view.solved`` sits behind a page of + ``gym_reward.*`` in a run carrying pass@1..8 for two metrics. + """ + # Deduplicated: names are expected unique, but runner-contributed extras are appended as-is, + # and a repeat would otherwise be suggested twice, listed twice, and counted twice in the + # "N other aggregates" tally -- making a collision look like two distinct near-misses. + available = sorted({score.name for score in self.scores}) + if not available: + return f"no aggregate score named {name!r}: this result has no aggregates at all" + close = get_close_matches(name, available, n=3) + if close: + suggestions = ", ".join(repr(match) for match in close) + message = f"no aggregate score named {name!r}; did you mean {suggestions}?" + # Say how many others there are, so a wrong guess isn't a dead end: without this the + # caller can't tell whether the suggestions are the whole set or three of forty. + others = len(available) - len(close) + if others == 0: + return message + noun = "aggregate" if others == 1 else "aggregates" + return f"{message} ({others} other {noun} in this result)" + shown = ", ".join(repr(score_name) for score_name in available[:_MISS_NAME_LIMIT]) + remainder = len(available) - _MISS_NAME_LIMIT + if remainder > 0: + shown = f"{shown}, ... ({remainder} more)" + return f"no aggregate score named {name!r}; available: {shown}" + class RowScore(BaseModel): """Normalized row-level score payload for metric/benchmark job results.""" diff --git a/packages/nemo_evaluator_sdk/tests/agent_eval/test_evaluator.py b/packages/nemo_evaluator_sdk/tests/agent_eval/test_evaluator.py index 94d4efec56..8274ddb20d 100644 --- a/packages/nemo_evaluator_sdk/tests/agent_eval/test_evaluator.py +++ b/packages/nemo_evaluator_sdk/tests/agent_eval/test_evaluator.py @@ -47,7 +47,6 @@ RunConfigOnlineModel, ) from nemo_evaluator_sdk.values.evidence import CandidateEvidence, EvidenceDescriptor -from nemo_evaluator_sdk.values.results import AggregateScore def test_trial_from_sample_falls_back_to_reasoning_content() -> None: @@ -147,13 +146,6 @@ def test_metric_row_exposes_reference_but_task_row_hides_it() -> None: assert "reference" not in task_row -def _score(summary: AgentEvalSummary, name: str) -> AggregateScore: - for aggregate in summary.scores.scores: - if aggregate.name == name: - return aggregate - raise KeyError(name) - - class _ConstantMetric: @property def type(self) -> str: @@ -327,7 +319,7 @@ async def test_scores_imported_trials_with_metric_and_persists_bundle(tmp_path: # run() no longer writes anything; persisting is the caller's call and defaults to the work_dir. location = result.persist() - assert _score(result.summary, "constant_metric.score").mean == 0.75 + assert result.summary.score("constant_metric.score").mean == 0.75 assert location.output_dir == tmp_path assert location.dashboard_path == tmp_path / "report.html" assert (tmp_path / "run.json").exists() @@ -371,7 +363,7 @@ async def test_scores_partial_trials() -> None: ], ) - assert _score(result.summary, "constant_metric.score").mean == 0.75 + assert result.summary.score("constant_metric.score").mean == 0.75 @pytest.mark.asyncio @@ -385,7 +377,7 @@ async def test_target_runtime_produces_trials_before_scoring() -> None: assert result.trials[0].id == "task-1:runtime" assert runtime.config is not None assert runtime.config.run_id == result.run_id - assert _score(result.summary, "constant_metric.score").mean == 0.75 + assert result.summary.score("constant_metric.score").mean == 0.75 @pytest.mark.asyncio @@ -411,7 +403,7 @@ async def fake_model_inference( assert result.trials[0].metadata["model_id"] == "target-model" assert result.trials[0].output is not None assert result.trials[0].output.output_text == "Generated model answer" - assert _score(result.summary, "constant_metric.score").mean == 0.75 + assert result.summary.score("constant_metric.score").mean == 0.75 @pytest.mark.asyncio @@ -524,9 +516,9 @@ def test_summary_reports_coverage_and_merges_views_into_scores() -> None: summary = AgentEvalSummary.from_scores(scores, tasks=[task]) - assert _score(summary, "constant_metric.score").mean == 1.0 - assert _score(summary, "other_metric.quality").mean == 0.0 - assert _score(summary, "view.outcome_correctness").mean == 0.5 + assert summary.score("constant_metric.score").mean == 1.0 + assert summary.score("other_metric.quality").mean == 0.0 + assert summary.score("view.outcome_correctness").mean == 0.5 assert summary.metric_coverage["constant_metric"]["score"].total == 1 assert summary.metric_coverage["constant_metric"]["score"].scored == 1 diff --git a/packages/nemo_evaluator_sdk/tests/agent_eval/test_gym_aggregate_scores.py b/packages/nemo_evaluator_sdk/tests/agent_eval/test_gym_aggregate_scores.py index df5c60166a..cbbbf63ebe 100644 --- a/packages/nemo_evaluator_sdk/tests/agent_eval/test_gym_aggregate_scores.py +++ b/packages/nemo_evaluator_sdk/tests/agent_eval/test_gym_aggregate_scores.py @@ -223,7 +223,7 @@ def test_median_matches_p50_wherever_both_are_reported() -> None: ] summary = AgentEvalSummary.from_scores(scores) - aggregate = next(s for s in summary.scores.scores if s.name == "m.score") + aggregate = summary.score("m.score") assert isinstance(aggregate, AggregateRangeScore) assert aggregate.percentiles is not None diff --git a/packages/nemo_evaluator_sdk/tests/agent_eval/test_hermes_example.py b/packages/nemo_evaluator_sdk/tests/agent_eval/test_hermes_example.py index 5e2df89c61..12e04f1b45 100644 --- a/packages/nemo_evaluator_sdk/tests/agent_eval/test_hermes_example.py +++ b/packages/nemo_evaluator_sdk/tests/agent_eval/test_hermes_example.py @@ -76,7 +76,7 @@ def handle(request: httpx.Request) -> httpx.Response: assert trial.output is not None assert trial.output.output_text == "streaming works" - aggregate = next(score for score in result.summary.scores.scores if score.name == "keyword_match.score") + aggregate = result.summary.score("keyword_match.score") assert aggregate.mean == 1.0 assert trial.evidence is not None diff --git a/packages/nemo_evaluator_sdk/tests/agent_eval/test_pass_at_k.py b/packages/nemo_evaluator_sdk/tests/agent_eval/test_pass_at_k.py index 30df07e1c0..00d3bf2625 100644 --- a/packages/nemo_evaluator_sdk/tests/agent_eval/test_pass_at_k.py +++ b/packages/nemo_evaluator_sdk/tests/agent_eval/test_pass_at_k.py @@ -136,9 +136,7 @@ def test_population_and_sample_stats_are_both_reported() -> None: values = [1.0, 1.0, 0.0, 0.0, 1.0, 1.0, 1.0, 1.0, 0.0, 0.0] # n=10, mean=0.6, sum_sq_dev=2.4 scores = [_score(f"t{1 + index % 2}", f"a{index}", "reward", "reward", value) for index, value in enumerate(values)] - aggregate = next( - s for s in AgentEvalSummary.from_scores(scores, tasks=tasks).scores.scores if s.name == "reward.reward" - ) + aggregate = AgentEvalSummary.from_scores(scores, tasks=tasks).score("reward.reward") assert aggregate.count == 10 assert aggregate.mean == pytest.approx(0.6) @@ -153,9 +151,7 @@ def test_sample_stats_undefined_for_a_single_value() -> None: tasks = [_task("t1", _ScoreMetric("reward"))] scores = [_score("t1", "a0", "reward", "reward", 1.0)] - aggregate = next( - s for s in AgentEvalSummary.from_scores(scores, tasks=tasks).scores.scores if s.name == "reward.reward" - ) + aggregate = AgentEvalSummary.from_scores(scores, tasks=tasks).score("reward.reward") assert aggregate.std_dev == 0.0 # population is well-defined for one value assert aggregate.sample_std_dev is None diff --git a/packages/nemo_evaluator_sdk/tests/agent_eval/test_summary_accessors.py b/packages/nemo_evaluator_sdk/tests/agent_eval/test_summary_accessors.py new file mode 100644 index 0000000000..4a0e47c48e --- /dev/null +++ b/packages/nemo_evaluator_sdk/tests/agent_eval/test_summary_accessors.py @@ -0,0 +1,146 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""By-name aggregate lookup on AgentEvalSummary and the AggregatedMetricResult it delegates to.""" + +from __future__ import annotations + +import pytest +from nemo_evaluator_sdk.agent_eval.results import AgentEvalSummary +from nemo_evaluator_sdk.values.results import AggregatedMetricResult, AggregateRangeScore + + +def _aggregates(*names: str) -> AggregatedMetricResult: + return AggregatedMetricResult( + scores=[AggregateRangeScore(name=name, count=2, nan_count=0, mean=0.5) for name in names] + ) + + +def _summary(*names: str) -> AgentEvalSummary: + return AgentEvalSummary(scores=_aggregates(*names)) + + +def test_score_returns_the_aggregate_with_that_name() -> None: + summary = _summary("gym_reward.reward", "gym_reward.reward.pass@2") + + assert summary.score("gym_reward.reward.pass@2").mean == 0.5 + + +def test_score_finds_a_name_that_is_not_the_first_in_the_list() -> None: + # Guards the scan itself: returning self.scores[0] regardless of name would satisfy a + # single-aggregate test but is plainly wrong. + summary = _summary("a.first", "b.second", "c.third") + + assert summary.score("c.third").name == "c.third" + + +def test_score_suggests_the_intended_name_when_the_lookup_looks_like_a_typo() -> None: + summary = _summary("gym_reward.reward", "view.solved") + + with pytest.raises(KeyError, match="did you mean") as excinfo: + summary.score("gym_reward.rewrad") + + # The suggestion is the whole point: a transposed name should be fixable from the message alone, + # without going back to the aggregation code to find out what was produced. + assert "gym_reward.reward" in str(excinfo.value) + + +@pytest.mark.parametrize( + ("extra_names", "expected"), + [ + # A suggestion that misses shouldn't be a dead end: the count tells the caller there is more + # to look at, rather than implying the offered names are the whole set. + (20, "(20 other aggregates in this result)"), + (1, "(1 other aggregate in this result)"), + (0, None), + ], +) +def test_score_reports_how_many_other_names_exist_alongside_a_suggestion( + extra_names: int, expected: str | None +) -> None: + aggregates = _aggregates("reward.reward", *(f"metric_{index:02d}.zzz" for index in range(extra_names))) + + with pytest.raises(KeyError) as excinfo: + aggregates.score("reward.rewrad") + + message = str(excinfo.value) + assert "'reward.reward'" in message + if expected is None: + assert "in this result" not in message + else: + assert expected in message + + +def test_miss_message_names_a_repeated_aggregate_only_once() -> None: + # A duplicate name is one name, not two near-misses: suggesting it twice, listing it twice, or + # counting it twice in the "other aggregates" tally all misrepresent what the result holds. + duplicated = _aggregates("reward.reward", "reward.reward", "view.solved") + + with pytest.raises(KeyError) as excinfo: + duplicated.score("reward.rewrad") + + message = str(excinfo.value) + assert message.count("'reward.reward'") == 1 + assert "(1 other aggregate in this result)" in message + + +def test_miss_message_does_not_repeat_a_duplicate_in_the_fallback_listing() -> None: + duplicated = _aggregates("alpha.one", "alpha.one", "beta.two") + + with pytest.raises(KeyError) as excinfo: + duplicated.score("zzzzzz") + + message = str(excinfo.value) + assert message.count("'alpha.one'") == 1 + + +def test_score_lists_available_names_when_nothing_is_close() -> None: + summary = _summary("gym_reward.reward", "view.solved") + + with pytest.raises(KeyError) as excinfo: + summary.score("totally_unrelated") + + message = str(excinfo.value) + assert "did you mean" not in message + assert "gym_reward.reward" in message + assert "view.solved" in message + + +def test_score_truncates_a_long_name_list_rather_than_dumping_all_of_them() -> None: + # A real run carries several metrics times pass@k values; an untruncated dump buries the answer. + aggregates = _aggregates(*(f"metric_{index:02d}.zzz" for index in range(25))) + + with pytest.raises(KeyError) as excinfo: + aggregates.score("qqq") + + message = str(excinfo.value) + assert "(15 more)" in message + assert "metric_00.zzz" in message + assert "metric_24.zzz" not in message + + +def test_score_says_so_when_the_result_has_no_aggregates_at_all() -> None: + # Distinct from a typo: nothing was produced, so no name would have worked. + with pytest.raises(KeyError, match="no aggregates at all"): + _summary().score("anything") + + +def test_scores_by_name_supports_membership_and_get_for_optional_aggregates() -> None: + summary = _summary("gym_reward.reward") + + assert "gym_reward.reward" in summary.scores_by_name + assert summary.scores_by_name.get("never_ran") is None + + +def test_scores_by_name_keeps_the_first_of_a_repeated_name() -> None: + # Names are expected unique, but runner-contributed extras are appended as-is. First-wins matches + # the `next(...)` scans this replaced, so a collision behaves as it did before. + duplicated = AggregatedMetricResult( + scores=[ + AggregateRangeScore(name="m.score", count=2, nan_count=0, mean=0.25), + AggregateRangeScore(name="m.score", count=2, nan_count=0, mean=0.75), + ] + ) + + assert duplicated.score("m.score").mean == 0.25 + assert duplicated.scores_by_name["m.score"].mean == 0.25 diff --git a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/results.py b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/results.py index cddf78e2ef..20c38c695d 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/results.py +++ b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/results.py @@ -6,7 +6,7 @@ from __future__ import annotations import math -from collections.abc import Sequence +from collections.abc import Mapping, Sequence from datetime import datetime from pathlib import Path @@ -63,6 +63,19 @@ class AgentEvalSummary(BaseModel): trial_count: int = Field(default=0, description="Number of distinct trials scored.") score_count: int = Field(default=0, description="Total number of metric scores.") + @property + def scores_by_name(self) -> Mapping[str, AggregateScore]: + """Aggregates keyed by name — see :attr:`AggregatedMetricResult.scores_by_name`.""" + return self.scores.scores_by_name + + def score(self, name: str) -> AggregateScore: + """Return the aggregate named ``name`` — see :meth:`AggregatedMetricResult.score`. + + Exists so callers needn't know the aggregates sit one level down, behind a field whose name + differs from the summary's own accessor by a single character. + """ + return self.scores.score(name) + @staticmethod def from_scores( scores: Sequence[AgentEvalTaskScore], diff --git a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/values/results.py b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/values/results.py index 9cf87e2674..52551f88c8 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/values/results.py +++ b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/values/results.py @@ -7,6 +7,8 @@ import json import math +from collections.abc import Mapping +from difflib import get_close_matches from typing import Any, Literal, Self import pyarrow as pa @@ -417,12 +419,84 @@ def _serialize(self, handler): AggregateScore = AggregateRangeScore | AggregateRubricScore | AggregateScalarScore +#: How many names a lookup-miss message lists when nothing resembles what was asked for. A judgement +#: call rather than a measured optimum -- enough to show the naming convention, few enough to stay +#: readable, since a run with several metrics times pass@k can carry dozens. Only this fallback is +#: truncated; a near-miss is surfaced by similarity, so finding the name you meant never depends on +#: where it happens to fall alphabetically. +_MISS_NAME_LIMIT = 10 + + class AggregatedMetricResult(BaseModel): """Result of aggregating metric scores with full statistics.""" model_config = ConfigDict(extra="forbid") scores: list[AggregateScore] = Field(description="The list of aggregated scores.") + @property + def scores_by_name(self) -> Mapping[str, AggregateScore]: + """Aggregates keyed by :attr:`AggregateScoreBase.name`, for ``in``, ``.get()``, and iteration. + + Reach for this when a score's absence is a legitimate outcome ("did this metric run?"); use + :meth:`score` when it isn't. Names are expected unique, but runner-contributed extras are + appended as-is, so a collision is possible: the first wins, matching the ``next(...)`` scans + this replaces. + """ + by_name: dict[str, AggregateScore] = {} + for score in self.scores: + by_name.setdefault(score.name, score) + return by_name + + def score(self, name: str) -> AggregateScore: + """Return the aggregate named ``name``, raising :class:`KeyError` if there isn't one. + + Raises rather than returning ``None`` because an unknown name is nearly always a typo or a + metric that didn't run. Both are bugs worth surfacing at the lookup, where the name is in + hand, instead of as an ``AttributeError`` on ``.mean`` further downstream. When absence is a + real possibility, use ``scores_by_name.get(...)``. + """ + # A direct scan rather than a lookup into `scores_by_name`: building the whole mapping to + # return one element allocates a dict per call, and the score list is short enough that the + # scan wins outright. + for score in self.scores: + if score.name == name: + return score + raise KeyError(self._unknown_score_message(name)) + + def _unknown_score_message(self, name: str) -> str: + """Explain a lookup miss, leading with near-misses when the name looks like a typo. + + "Close" is :func:`difflib.get_close_matches`: SequenceMatcher (Ratcliff/Obershelp) similarity + of at least 0.6, best three first. That is a subsequence-overlap ratio, not an edit distance. + + Listing every name alphabetically would be simpler, and is the better answer if the list is + left whole. Truncating one is what breaks it -- the name a caller meant is not reliably in + the first :data:`_MISS_NAME_LIMIT`, since a typo'd ``view.solved`` sits behind a page of + ``gym_reward.*`` in a run carrying pass@1..8 for two metrics. + """ + # Deduplicated: names are expected unique, but runner-contributed extras are appended as-is, + # and a repeat would otherwise be suggested twice, listed twice, and counted twice in the + # "N other aggregates" tally -- making a collision look like two distinct near-misses. + available = sorted({score.name for score in self.scores}) + if not available: + return f"no aggregate score named {name!r}: this result has no aggregates at all" + close = get_close_matches(name, available, n=3) + if close: + suggestions = ", ".join(repr(match) for match in close) + message = f"no aggregate score named {name!r}; did you mean {suggestions}?" + # Say how many others there are, so a wrong guess isn't a dead end: without this the + # caller can't tell whether the suggestions are the whole set or three of forty. + others = len(available) - len(close) + if others == 0: + return message + noun = "aggregate" if others == 1 else "aggregates" + return f"{message} ({others} other {noun} in this result)" + shown = ", ".join(repr(score_name) for score_name in available[:_MISS_NAME_LIMIT]) + remainder = len(available) - _MISS_NAME_LIMIT + if remainder > 0: + shown = f"{shown}, ... ({remainder} more)" + return f"no aggregate score named {name!r}; available: {shown}" + class RowScore(BaseModel): """Normalized row-level score payload for metric/benchmark job results."""