feat(evaluator): add by-name aggregate lookup to AgentEvalSummary

packages/nemo_evaluator_sdk/examples/hermes/example.py

@@ -219,7 +219,7 @@ async def main() -> None:
     result = await evaluate()
 
     answer = result.trials[0].output.output_text if result.trials[0].output else None
-    aggregate = next(score for score in result.summary.scores.scores if score.name == "keyword_match.score")
+    aggregate = result.summary.score("keyword_match.score")
 
     print(f"response: {answer}")
     print(f"{aggregate.name}: {aggregate.mean}")

packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/results.py

@@ -6,7 +6,7 @@
 from __future__ import annotations
 
 import math
-from collections.abc import Sequence
+from collections.abc import Mapping, Sequence
 from datetime import datetime
 from pathlib import Path
 
@@ -63,6 +63,19 @@ class AgentEvalSummary(BaseModel):
     trial_count: int = Field(default=0, description="Number of distinct trials scored.")
     score_count: int = Field(default=0, description="Total number of metric scores.")
 
+    @property
+    def scores_by_name(self) -> Mapping[str, AggregateScore]:
+        """Aggregates keyed by name — see :attr:`AggregatedMetricResult.scores_by_name`."""
+        return self.scores.scores_by_name
+
+    def score(self, name: str) -> AggregateScore:
+        """Return the aggregate named ``name`` — see :meth:`AggregatedMetricResult.score`.
+
+        Exists so callers needn't know the aggregates sit one level down, behind a field whose name
+        differs from the summary's own accessor by a single character.
+        """
+        return self.scores.score(name)
+
     @staticmethod
     def from_scores(
         scores: Sequence[AgentEvalTaskScore],

packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/values/results.py

@@ -7,6 +7,8 @@
 
 import json
 import math
+from collections.abc import Mapping
+from difflib import get_close_matches
 from typing import Any, Literal, Self
 
 import pyarrow as pa
@@ -417,12 +419,84 @@ def _serialize(self, handler):
 AggregateScore = AggregateRangeScore | AggregateRubricScore | AggregateScalarScore
 
 
+#: How many names a lookup-miss message lists when nothing resembles what was asked for. A judgement
+#: call rather than a measured optimum -- enough to show the naming convention, few enough to stay
+#: readable, since a run with several metrics times pass@k can carry dozens. Only this fallback is
+#: truncated; a near-miss is surfaced by similarity, so finding the name you meant never depends on
+#: where it happens to fall alphabetically.
+_MISS_NAME_LIMIT = 10
+
+
 class AggregatedMetricResult(BaseModel):
     """Result of aggregating metric scores with full statistics."""
 
     model_config = ConfigDict(extra="forbid")
     scores: list[AggregateScore] = Field(description="The list of aggregated scores.")
 
+    @property
+    def scores_by_name(self) -> Mapping[str, AggregateScore]:
+        """Aggregates keyed by :attr:`AggregateScoreBase.name`, for ``in``, ``.get()``, and iteration.
+
+        Reach for this when a score's absence is a legitimate outcome ("did this metric run?"); use
+        :meth:`score` when it isn't. Names are expected unique, but runner-contributed extras are
+        appended as-is, so a collision is possible: the first wins, matching the ``next(...)`` scans
+        this replaces.
+        """
+        by_name: dict[str, AggregateScore] = {}
+        for score in self.scores:
+            by_name.setdefault(score.name, score)
+        return by_name
+
+    def score(self, name: str) -> AggregateScore:
+        """Return the aggregate named ``name``, raising :class:`KeyError` if there isn't one.
+
+        Raises rather than returning ``None`` because an unknown name is nearly always a typo or a
+        metric that didn't run. Both are bugs worth surfacing at the lookup, where the name is in
+        hand, instead of as an ``AttributeError`` on ``.mean`` further downstream. When absence is a
+        real possibility, use ``scores_by_name.get(...)``.
+        """
+        # A direct scan rather than a lookup into `scores_by_name`: building the whole mapping to
+        # return one element allocates a dict per call, and the score list is short enough that the
+        # scan wins outright.
+        for score in self.scores:
+            if score.name == name:
+                return score
+        raise KeyError(self._unknown_score_message(name))
+
+    def _unknown_score_message(self, name: str) -> str:
+        """Explain a lookup miss, leading with near-misses when the name looks like a typo.
+
+        "Close" is :func:`difflib.get_close_matches`: SequenceMatcher (Ratcliff/Obershelp) similarity
+        of at least 0.6, best three first. That is a subsequence-overlap ratio, not an edit distance.
+
+        Listing every name alphabetically would be simpler, and is the better answer if the list is
+        left whole. Truncating one is what breaks it -- the name a caller meant is not reliably in
+        the first :data:`_MISS_NAME_LIMIT`, since a typo'd ``view.solved`` sits behind a page of
+        ``gym_reward.*`` in a run carrying pass@1..8 for two metrics.
+        """
+        # Deduplicated: names are expected unique, but runner-contributed extras are appended as-is,
+        # and a repeat would otherwise be suggested twice, listed twice, and counted twice in the
+        # "N other aggregates" tally -- making a collision look like two distinct near-misses.
+        available = sorted({score.name for score in self.scores})
+        if not available:
+            return f"no aggregate score named {name!r}: this result has no aggregates at all"
+        close = get_close_matches(name, available, n=3)
+        if close:
+            suggestions = ", ".join(repr(match) for match in close)
+            message = f"no aggregate score named {name!r}; did you mean {suggestions}?"
+            # Say how many others there are, so a wrong guess isn't a dead end: without this the
+            # caller can't tell whether the suggestions are the whole set or three of forty.
+            others = len(available) - len(close)
+            if others == 0:
+                return message
+            noun = "aggregate" if others == 1 else "aggregates"
+            return f"{message} ({others} other {noun} in this result)"
+        shown = ", ".join(repr(score_name) for score_name in available[:_MISS_NAME_LIMIT])
+        remainder = len(available) - _MISS_NAME_LIMIT
+        if remainder > 0:
+            shown = f"{shown}, ... ({remainder} more)"
+        return f"no aggregate score named {name!r}; available: {shown}"
+
 
 class RowScore(BaseModel):
     """Normalized row-level score payload for metric/benchmark job results."""

packages/nemo_evaluator_sdk/tests/agent_eval/test_evaluator.py

@@ -47,7 +47,6 @@
     RunConfigOnlineModel,
 )
 from nemo_evaluator_sdk.values.evidence import CandidateEvidence, EvidenceDescriptor
-from nemo_evaluator_sdk.values.results import AggregateScore
 
 
 def test_trial_from_sample_falls_back_to_reasoning_content() -> None:
@@ -147,13 +146,6 @@ def test_metric_row_exposes_reference_but_task_row_hides_it() -> None:
     assert "reference" not in task_row
 
 
-def _score(summary: AgentEvalSummary, name: str) -> AggregateScore:
-    for aggregate in summary.scores.scores:
-        if aggregate.name == name:
-            return aggregate
-    raise KeyError(name)
-
-
 class _ConstantMetric:
     @property
     def type(self) -> str:
@@ -327,7 +319,7 @@ async def test_scores_imported_trials_with_metric_and_persists_bundle(tmp_path:
     # run() no longer writes anything; persisting is the caller's call and defaults to the work_dir.
     location = result.persist()
 
-    assert _score(result.summary, "constant_metric.score").mean == 0.75
+    assert result.summary.score("constant_metric.score").mean == 0.75
     assert location.output_dir == tmp_path
     assert location.dashboard_path == tmp_path / "report.html"
     assert (tmp_path / "run.json").exists()
@@ -371,7 +363,7 @@ async def test_scores_partial_trials() -> None:
         ],
     )
 
-    assert _score(result.summary, "constant_metric.score").mean == 0.75
+    assert result.summary.score("constant_metric.score").mean == 0.75
 
 
 @pytest.mark.asyncio
@@ -385,7 +377,7 @@ async def test_target_runtime_produces_trials_before_scoring() -> None:
     assert result.trials[0].id == "task-1:runtime"
     assert runtime.config is not None
     assert runtime.config.run_id == result.run_id
-    assert _score(result.summary, "constant_metric.score").mean == 0.75
+    assert result.summary.score("constant_metric.score").mean == 0.75
 
 
 @pytest.mark.asyncio
@@ -411,7 +403,7 @@ async def fake_model_inference(
     assert result.trials[0].metadata["model_id"] == "target-model"
     assert result.trials[0].output is not None
     assert result.trials[0].output.output_text == "Generated model answer"
-    assert _score(result.summary, "constant_metric.score").mean == 0.75
+    assert result.summary.score("constant_metric.score").mean == 0.75
 
 
 @pytest.mark.asyncio
@@ -524,9 +516,9 @@ def test_summary_reports_coverage_and_merges_views_into_scores() -> None:
 
     summary = AgentEvalSummary.from_scores(scores, tasks=[task])
 
-    assert _score(summary, "constant_metric.score").mean == 1.0
-    assert _score(summary, "other_metric.quality").mean == 0.0
-    assert _score(summary, "view.outcome_correctness").mean == 0.5
+    assert summary.score("constant_metric.score").mean == 1.0
+    assert summary.score("other_metric.quality").mean == 0.0
+    assert summary.score("view.outcome_correctness").mean == 0.5
     assert summary.metric_coverage["constant_metric"]["score"].total == 1
     assert summary.metric_coverage["constant_metric"]["score"].scored == 1
 

packages/nemo_evaluator_sdk/tests/agent_eval/test_gym_aggregate_scores.py

@@ -223,7 +223,7 @@ def test_median_matches_p50_wherever_both_are_reported() -> None:
     ]
 
     summary = AgentEvalSummary.from_scores(scores)
-    aggregate = next(s for s in summary.scores.scores if s.name == "m.score")
+    aggregate = summary.score("m.score")
 
     assert isinstance(aggregate, AggregateRangeScore)
     assert aggregate.percentiles is not None

packages/nemo_evaluator_sdk/tests/agent_eval/test_hermes_example.py

@@ -76,7 +76,7 @@ def handle(request: httpx.Request) -> httpx.Response:
     assert trial.output is not None
     assert trial.output.output_text == "streaming works"
 
-    aggregate = next(score for score in result.summary.scores.scores if score.name == "keyword_match.score")
+    aggregate = result.summary.score("keyword_match.score")
     assert aggregate.mean == 1.0
 
     assert trial.evidence is not None

packages/nemo_evaluator_sdk/tests/agent_eval/test_pass_at_k.py

@@ -136,9 +136,7 @@ def test_population_and_sample_stats_are_both_reported() -> None:
     values = [1.0, 1.0, 0.0, 0.0, 1.0, 1.0, 1.0, 1.0, 0.0, 0.0]  # n=10, mean=0.6, sum_sq_dev=2.4
     scores = [_score(f"t{1 + index % 2}", f"a{index}", "reward", "reward", value) for index, value in enumerate(values)]
 
-    aggregate = next(
-        s for s in AgentEvalSummary.from_scores(scores, tasks=tasks).scores.scores if s.name == "reward.reward"
-    )
+    aggregate = AgentEvalSummary.from_scores(scores, tasks=tasks).score("reward.reward")
 
     assert aggregate.count == 10
     assert aggregate.mean == pytest.approx(0.6)
@@ -153,9 +151,7 @@ def test_sample_stats_undefined_for_a_single_value() -> None:
     tasks = [_task("t1", _ScoreMetric("reward"))]
     scores = [_score("t1", "a0", "reward", "reward", 1.0)]
 
-    aggregate = next(
-        s for s in AgentEvalSummary.from_scores(scores, tasks=tasks).scores.scores if s.name == "reward.reward"
-    )
+    aggregate = AgentEvalSummary.from_scores(scores, tasks=tasks).score("reward.reward")
 
     assert aggregate.std_dev == 0.0  # population is well-defined for one value
     assert aggregate.sample_std_dev is None

packages/nemo_evaluator_sdk/tests/agent_eval/test_summary_accessors.py

@@ -0,0 +1,146 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+"""By-name aggregate lookup on AgentEvalSummary and the AggregatedMetricResult it delegates to."""
+
+from __future__ import annotations
+
+import pytest
+from nemo_evaluator_sdk.agent_eval.results import AgentEvalSummary
+from nemo_evaluator_sdk.values.results import AggregatedMetricResult, AggregateRangeScore
+
+
+def _aggregates(*names: str) -> AggregatedMetricResult:
+    return AggregatedMetricResult(
+        scores=[AggregateRangeScore(name=name, count=2, nan_count=0, mean=0.5) for name in names]
+    )
+
+
+def _summary(*names: str) -> AgentEvalSummary:
+    return AgentEvalSummary(scores=_aggregates(*names))
+
+
+def test_score_returns_the_aggregate_with_that_name() -> None:
+    summary = _summary("gym_reward.reward", "gym_reward.reward.pass@2")
+
+    assert summary.score("gym_reward.reward.pass@2").mean == 0.5
+
+
+def test_score_finds_a_name_that_is_not_the_first_in_the_list() -> None:
+    # Guards the scan itself: returning self.scores[0] regardless of name would satisfy a
+    # single-aggregate test but is plainly wrong.
+    summary = _summary("a.first", "b.second", "c.third")
+
+    assert summary.score("c.third").name == "c.third"
+
+
+def test_score_suggests_the_intended_name_when_the_lookup_looks_like_a_typo() -> None:
+    summary = _summary("gym_reward.reward", "view.solved")
+
+    with pytest.raises(KeyError, match="did you mean") as excinfo:
+        summary.score("gym_reward.rewrad")
+
+    # The suggestion is the whole point: a transposed name should be fixable from the message alone,
+    # without going back to the aggregation code to find out what was produced.
+    assert "gym_reward.reward" in str(excinfo.value)
+
+
+@pytest.mark.parametrize(
+    ("extra_names", "expected"),
+    [
+        # A suggestion that misses shouldn't be a dead end: the count tells the caller there is more
+        # to look at, rather than implying the offered names are the whole set.
+        (20, "(20 other aggregates in this result)"),
+        (1, "(1 other aggregate in this result)"),
+        (0, None),
+    ],
+)
+def test_score_reports_how_many_other_names_exist_alongside_a_suggestion(
+    extra_names: int, expected: str | None
+) -> None:
+    aggregates = _aggregates("reward.reward", *(f"metric_{index:02d}.zzz" for index in range(extra_names)))
+
+    with pytest.raises(KeyError) as excinfo:
+        aggregates.score("reward.rewrad")
+
+    message = str(excinfo.value)
+    assert "'reward.reward'" in message
+    if expected is None:
+        assert "in this result" not in message
+    else:
+        assert expected in message
+
+
+def test_miss_message_names_a_repeated_aggregate_only_once() -> None:
+    # A duplicate name is one name, not two near-misses: suggesting it twice, listing it twice, or
+    # counting it twice in the "other aggregates" tally all misrepresent what the result holds.
+    duplicated = _aggregates("reward.reward", "reward.reward", "view.solved")
+
+    with pytest.raises(KeyError) as excinfo:
+        duplicated.score("reward.rewrad")
+
+    message = str(excinfo.value)
+    assert message.count("'reward.reward'") == 1
+    assert "(1 other aggregate in this result)" in message
+
+
+def test_miss_message_does_not_repeat_a_duplicate_in_the_fallback_listing() -> None:
+    duplicated = _aggregates("alpha.one", "alpha.one", "beta.two")
+
+    with pytest.raises(KeyError) as excinfo:
+        duplicated.score("zzzzzz")
+
+    message = str(excinfo.value)
+    assert message.count("'alpha.one'") == 1
+
+
+def test_score_lists_available_names_when_nothing_is_close() -> None:
+    summary = _summary("gym_reward.reward", "view.solved")
+
+    with pytest.raises(KeyError) as excinfo:
+        summary.score("totally_unrelated")
+
+    message = str(excinfo.value)
+    assert "did you mean" not in message
+    assert "gym_reward.reward" in message
+    assert "view.solved" in message
+
+
+def test_score_truncates_a_long_name_list_rather_than_dumping_all_of_them() -> None:
+    # A real run carries several metrics times pass@k values; an untruncated dump buries the answer.
+    aggregates = _aggregates(*(f"metric_{index:02d}.zzz" for index in range(25)))
+
+    with pytest.raises(KeyError) as excinfo:
+        aggregates.score("qqq")
+
+    message = str(excinfo.value)
+    assert "(15 more)" in message
+    assert "metric_00.zzz" in message
+    assert "metric_24.zzz" not in message
+
+
+def test_score_says_so_when_the_result_has_no_aggregates_at_all() -> None:
+    # Distinct from a typo: nothing was produced, so no name would have worked.
+    with pytest.raises(KeyError, match="no aggregates at all"):
+        _summary().score("anything")
+
+
+def test_scores_by_name_supports_membership_and_get_for_optional_aggregates() -> None:
+    summary = _summary("gym_reward.reward")
+
+    assert "gym_reward.reward" in summary.scores_by_name
+    assert summary.scores_by_name.get("never_ran") is None
+
+
+def test_scores_by_name_keeps_the_first_of_a_repeated_name() -> None:
+    # Names are expected unique, but runner-contributed extras are appended as-is. First-wins matches
+    # the `next(...)` scans this replaced, so a collision behaves as it did before.
+    duplicated = AggregatedMetricResult(
+        scores=[
+            AggregateRangeScore(name="m.score", count=2, nan_count=0, mean=0.25),
+            AggregateRangeScore(name="m.score", count=2, nan_count=0, mean=0.75),
+        ]
+    )
+
+    assert duplicated.score("m.score").mean == 0.25
+    assert duplicated.scores_by_name["m.score"].mean == 0.25