chore(report): always return flattened report

AGENTS.md

@@ -202,7 +202,7 @@ All evaluators subclass `strands_evals.evaluators.Evaluator[InputT, OutputT]`:
 
 - `Case[InputT, OutputT]`: one test scenario (`input`, `expected_output`, optional `trajectory`, `metadata`)
 - `Experiment[InputT, OutputT]`: collection of Cases plus evaluators
-- Entry point: `experiment.run_evaluations(task_function)` returns a list of reports
+- Entry point: `experiment.run_evaluations(task_function)` returns a single `EvaluationReport`. With multiple evaluators, results are flattened across (case, evaluator) pairs and each row is tagged via `cases[i]["evaluator"]`.
 
 ### Session / Trace Types
 

README.md

@@ -93,8 +93,8 @@ def get_response(case: Case) -> str:
     return str(agent(case.input))
 
 # Run evaluations
-reports = experiment.run_evaluations(get_response)
-reports[0].run_display()
+report = experiment.run_evaluations(get_response)
+report.run_display()
 ```
 
 ## Installation
@@ -194,8 +194,8 @@ evaluators = [HelpfulnessEvaluator()]
 experiment = Experiment[str, str](cases=test_cases, evaluators=evaluators)
 
 # Run evaluations
-reports = experiment.run_evaluations(user_task_function)
-reports[0].run_display()
+report = experiment.run_evaluations(user_task_function)
+report.run_display()
 ```
 
 ### Multi-turn Conversation Simulation
@@ -255,7 +255,7 @@ evaluators = [
 ]
 
 experiment = Experiment(cases=test_cases, evaluators=evaluators)
-reports = experiment.run_evaluations(task_function)
+report = experiment.run_evaluations(task_function)
 ```
 
 **Key Benefits:**
@@ -310,7 +310,7 @@ def task_function(case: Case) -> dict:
 
 cases = [Case(name="heat_control", input="Turn on the heat to 72 degrees")]
 experiment = Experiment(cases=cases, evaluators=[GoalSuccessRateEvaluator()])
-reports = experiment.run_evaluations(task_function)
+report = experiment.run_evaluations(task_function)
 ```
 
 **Key Benefits:**
@@ -369,10 +369,10 @@ experiment = Experiment(
     ),
 )
 
-reports = experiment.run_evaluations(task_function)
+report = experiment.run_evaluations(task_function)
 
 # Display results with recommendations
-reports[0].display(include_recommendations=True)
+report.display(include_recommendations=True)
 ```
 
 You can also use the detectors standalone on any `Session` object (`strands_evals.types.trace.Session`):
@@ -498,8 +498,8 @@ def get_response(case: Case) -> str:
         {"text": case.input.instruction}
     ]))
 
-reports = experiment.run_evaluations(get_response)
-reports[0].run_display()
+report = experiment.run_evaluations(get_response)
+report.run_display()
 ```
 
 ## Available Evaluators
@@ -586,14 +586,13 @@ metrics = {
     "user_satisfaction": "Subjective helpfulness ratings"
 }
 
-# Generate analysis reports
-reports = experiment.run_evaluations(task_function)
-reports[0].run_display()  # Interactive display with metrics breakdown
+# Generate analysis report
+report = experiment.run_evaluations(task_function)
+report.run_display()  # Interactive display with metrics breakdown
 
-# Flatten multiple evaluator reports into a single combined view
-from strands_evals.types.evaluation_report import EvaluationReport
-combined = EvaluationReport.flatten(reports)
-combined.display(include_recommendations=True)
+# Multi-evaluator runs return a single flattened report; each row is tagged with its evaluator
+# via cases[i]["evaluator"], so you can filter or group without an extra flatten step.
+report.display(include_recommendations=True)
 ```
 
 ## Best Practices

SKILL.md

@@ -34,19 +34,13 @@ case = Case[str, str](
     metadata={"category": "knowledge"},
 )
 experiment = Experiment[str, str](cases=[case], evaluators=[...])
-reports = experiment.run_evaluations(task_function)
-reports[0].run_display()
+report = experiment.run_evaluations(task_function)
+report.run_display()
 ```
 
 `task_function(case: Case)` returns either a string output or, for trace-based evaluators, a dict like `{"output": ..., "trajectory": Session}`.
 
-Multiple reports can be flattened:
-
-```python
-from strands_evals.types.evaluation_report import EvaluationReport
-combined = EvaluationReport.flatten(reports)
-combined.display(include_recommendations=True)
-```
+`run_evaluations()` always returns a single `EvaluationReport`. With one evaluator, the report is keyed to that evaluator. With multiple, results are flattened into one report and each row is tagged via `report.cases[i]["evaluator"]`.
 
 Persist experiments:
 
@@ -249,7 +243,7 @@ rca = analyze_root_cause(session)
 Display recommendations on the report:
 
 ```python
-reports[0].display(include_recommendations=True)
+report.display(include_recommendations=True)
 ```
 
 ## ExperimentGenerator (auto test-case generation)

src/strands_evals/chaos/experiment.py

@@ -56,7 +56,7 @@ def my_task(case):
             evaluators=[my_evaluator],
         )
 
-        reports = experiment.run_evaluations(task=my_task)
+        report = experiment.run_evaluations(task=my_task)
     """
 
     def __init__(
@@ -125,7 +125,7 @@ def run_evaluations(
         self,
         task: Callable[[ChaosCase], Any],
         **kwargs,
-    ) -> list[EvaluationReport]:
+    ) -> EvaluationReport:
         """Run evaluations across all ChaosCase objects.
 
         Delegates to run_evaluations_async with max_workers=1, mirroring the
@@ -138,7 +138,7 @@ def run_evaluations(
             **kwargs: Additional kwargs passed to the base Experiment.run_evaluations_async.
 
         Returns:
-            List of EvaluationReport objects.
+            A single flattened EvaluationReport.
 
         Raises:
             ValueError: If an async task is passed (use run_evaluations_async instead).
@@ -157,7 +157,7 @@ async def run_evaluations_async(
         task: Callable[[ChaosCase], Any],
         max_workers: int = 10,
         **kwargs,
-    ) -> list[EvaluationReport]:
+    ) -> EvaluationReport:
         """Run evaluations asynchronously across all ChaosCase objects.
 
         Wraps the user's task to set the ContextVar before each case execution.
@@ -169,15 +169,15 @@ async def run_evaluations_async(
             **kwargs: Additional kwargs passed to the base Experiment.run_evaluations_async.
 
         Returns:
-            List of EvaluationReport objects.
+            A single flattened EvaluationReport.
         """
         wrapped = self._wrap_task(task)
-        reports = await self._experiment.run_evaluations_async(wrapped, max_workers=max_workers, **kwargs)
+        report = await self._experiment.run_evaluations_async(wrapped, max_workers=max_workers, **kwargs)
 
         logger.info(
-            "cases=<%d>, reports=<%d> | chaos experiment complete",
+            "cases=<%d>, scores=<%d> | chaos experiment complete",
             len(self._cases),
-            len(reports),
+            len(report.scores),
         )
 
-        return reports
+        return report

src/strands_evals/experiment.py

@@ -544,7 +544,7 @@ def run_evaluations(
         self,
         task: Callable[[Case[InputT, OutputT]], OutputT | dict[str, Any]],
         evaluation_data_store: EvaluationDataStore | None = None,
-    ) -> list[EvaluationReport]:
+    ) -> EvaluationReport:
         """
         Run the evaluations for all of the test cases with all evaluators.
 
@@ -557,8 +557,8 @@ def run_evaluations(
                 results are loaded instead of running the task, and new results are saved after task execution.
 
         Return:
-            A list of EvaluationReport objects, one for each evaluator, containing the overall score,
-            individual case results, and basic feedback for each test case.
+            A single EvaluationReport containing every (case, evaluator) result. Each case row is
+            tagged with its evaluator via the `evaluator` field on `cases`.
         """
         if asyncio.iscoroutinefunction(task):
             raise ValueError("Async task is not supported. Please use run_evaluations_async instead.")
@@ -570,7 +570,7 @@ async def run_evaluations_async(
         task: Callable,
         max_workers: int = 10,
         evaluation_data_store: EvaluationDataStore | None = None,
-    ) -> list[EvaluationReport]:
+    ) -> EvaluationReport:
         """
         Run evaluations asynchronously using a queue for parallel processing.
 
@@ -583,7 +583,8 @@ async def run_evaluations_async(
                 results are loaded instead of running the task, and new results are saved after task execution.
 
         Returns:
-            List of EvaluationReport objects, one for each evaluator, containing evaluation results
+            A single EvaluationReport flattened across every evaluator. Each row in `cases` carries
+            an `evaluator` key naming which evaluator produced it.
         """
         if evaluation_data_store is not None:
             self._validate_case_names()
@@ -625,7 +626,7 @@ async def run_evaluations_async(
             recommendation = result.get("recommendation")
             for eval_result in result["evaluator_results"]:
                 eval_name = eval_result["evaluator_name"]
-                evaluator_data[eval_name]["cases"].append(case_data)
+                evaluator_data[eval_name]["cases"].append({**case_data, "evaluator": eval_name})
                 evaluator_data[eval_name]["scores"].append(eval_result["score"])
                 evaluator_data[eval_name]["test_passes"].append(eval_result["test_pass"])
                 evaluator_data[eval_name]["reasons"].append(eval_result["reason"])
@@ -639,7 +640,6 @@ async def run_evaluations_async(
             data = evaluator_data[eval_name]
             scores = data["scores"]
             report = EvaluationReport(
-                evaluator_name=eval_name,
                 overall_score=sum(scores) / len(scores) if scores else 0,
                 scores=scores,
                 test_passes=data["test_passes"],
@@ -651,7 +651,11 @@ async def run_evaluations_async(
             )
             reports.append(report)
 
-        return reports
+        # Each case row already carries its evaluator tag (see worker aggregation above), so
+        # single-evaluator runs return as-is and multi-evaluator runs simply concatenate.
+        if len(reports) == 1:
+            return reports[0]
+        return EvaluationReport.flatten(reports)
 
     def to_dict(self) -> dict:
         """

src/strands_evals/experimental/redteam/experiment.py

@@ -75,10 +75,10 @@ async def run_evaluations_async(  # type: ignore[override]
     ) -> RedTeamReport:
         # max_workers=1: parallel runs would interleave on the shared target Agent.
         task = task or self._default_task()
-        reports = await super().run_evaluations_async(
+        report = await super().run_evaluations_async(
             task, max_workers=max_workers, evaluation_data_store=evaluation_data_store
         )
-        return RedTeamReport.from_evaluation_reports(reports)
+        return RedTeamReport.from_evaluation_report(report)
 
     def _default_task(self) -> Callable[[Case[InputT, OutputT]], Any]:
         if self._target is None:

src/strands_evals/experimental/redteam/report.py

@@ -50,40 +50,30 @@ class RedTeamReport(EvaluationReport):
     """Case-centric report for red team evaluation.
 
     Note:
-        ``trajectory`` holds raw tool I/O — sanitize before sharing if
+        `trajectory` holds raw tool I/O — sanitize before sharing if
         target tools return sensitive data.
     """
 
     @classmethod
-    def from_evaluation_reports(cls, reports: list[EvaluationReport]) -> RedTeamReport:
-        """Merge per-evaluator reports into a single case-centric report."""
-        scores: list[float] = []
-        cases: list[dict] = []
-        passes: list[bool] = []
-        reasons: list[str] = []
-        detailed: list = []
-
-        for report in reports:
-            evaluator = report.evaluator_name or "evaluator"
-            n = len(report.cases)
-            if not (len(report.scores) == n and len(report.test_passes) == n and len(report.reasons) == n):
-                raise ValueError(f"EvaluationReport {evaluator!r}: cases/scores/passes/reasons length mismatch")
-            # detailed_results is optional; pad with [] when shorter than cases.
-            for i, case_data in enumerate(report.cases):
-                cases.append({**case_data, "evaluator": evaluator})
-                scores.append(report.scores[i])
-                passes.append(report.test_passes[i])
-                reasons.append(report.reasons[i])
-                detailed.append(report.detailed_results[i] if i < len(report.detailed_results) else [])
+    def from_evaluation_report(cls, report: EvaluationReport) -> RedTeamReport:
+        """Wrap a flattened evaluation report as a case-centric red team report.
+
+        The base `Experiment.run_evaluations_async` already tags each case row with its
+        `evaluator` (regardless of evaluator count). We reuse that shape directly.
+        """
+        n = len(report.cases)
+        if not (len(report.scores) == n and len(report.test_passes) == n and len(report.reasons) == n):
+            raise ValueError("EvaluationReport: cases/scores/passes/reasons length mismatch")
+
+        cases = [{**case_data, "evaluator": case_data.get("evaluator", "evaluator")} for case_data in report.cases]
 
         return cls(
-            evaluator_name="RedTeam",
-            overall_score=sum(scores) / len(scores) if scores else 0.0,
-            scores=scores,
+            overall_score=report.overall_score,
+            scores=list(report.scores),
             cases=cases,
-            test_passes=passes,
-            reasons=reasons,
-            detailed_results=detailed,
+            test_passes=list(report.test_passes),
+            reasons=list(report.reasons),
+            detailed_results=[report.detailed_results[i] if i < len(report.detailed_results) else [] for i in range(n)],
         )
 
     def attack_results(self) -> list[AttackResult]:

src/strands_evals/types/evaluation_report.py

@@ -12,15 +12,14 @@ class EvaluationReport(BaseModel):
     A report of the evaluation of a task.
 
     Attributes:
-        evaluator_name: The name of the evaluator that produced this report.
         overall_score: The overall score of the task.
         scores: A list of the score for each test case in order.
-        cases: A list of records for each test case.
+        cases: A list of records for each test case. Each record carries an `evaluator` key naming
+            the evaluator that produced that row.
         test_passes: A list of booleans indicating whether the test pass or fail.
         reasons: A list of reason for each test case.
     """
 
-    evaluator_name: str = ""
     overall_score: float
     scores: list[float]
     cases: list[dict]
@@ -32,16 +31,20 @@ class EvaluationReport(BaseModel):
 
     @classmethod
     def flatten(cls, reports: list["EvaluationReport"]) -> "EvaluationReport":
-        """Flatten multiple evaluation reports into a single report."""
+        """Concatenate multiple evaluation reports into one.
+
+        The base `Experiment` already returns a flattened report; this helper exists for callers
+        that built reports separately (e.g., across multiple experiments) and want to merge them.
+        Each row's `evaluator` tag is preserved as-is.
+        """
         if not reports:
             return cls(overall_score=0.0, scores=[], cases=[], test_passes=[])
 
         scores, cases, passes, reasons, detailed, diags, recs = [], [], [], [], [], [], []
 
         for report in reports:
-            evaluator = report.evaluator_name or "Unknown"
             for i, case in enumerate(report.cases):
-                cases.append({**case, "evaluator": evaluator})
+                cases.append(dict(case))
                 scores.append(report.scores[i] if i < len(report.scores) else 0.0)
                 passes.append(report.test_passes[i] if i < len(report.test_passes) else False)
                 reasons.append(report.reasons[i] if i < len(report.reasons) else "")
@@ -50,7 +53,6 @@ def flatten(cls, reports: list["EvaluationReport"]) -> "EvaluationReport":
                 recs.append(report.recommendations[i] if i < len(report.recommendations) else None)
 
         return cls(
-            evaluator_name="Combined",
             overall_score=sum(scores) / len(scores) if scores else 0.0,
             scores=scores,
             cases=cases,

tests/strands_evals/chaos/test_experiment.py

@@ -124,10 +124,8 @@ def task(case: ChaosCase):
 
         chaos_cases = ChaosCase.expand(cases, effect_maps, include_no_effect_baseline=True)
         experiment = ChaosExperiment(cases=chaos_cases, evaluators=[evaluator])
-        reports = experiment.run_evaluations(task=task)
+        report = experiment.run_evaluations(task=task)
 
-        assert len(reports) >= 1
-        report = reports[0]
         # 2 cases × 3 conditions = 6 scores
         assert len(report.scores) == 6
 
@@ -154,8 +152,8 @@ async def async_task(case: ChaosCase):
             assert active is case
             return "async_output"
 
-        reports = await experiment.run_evaluations_async(task=async_task, max_workers=2)
-        assert len(reports) >= 1
+        report = await experiment.run_evaluations_async(task=async_task, max_workers=2)
+        assert len(report.scores) >= 1
 
     @pytest.mark.asyncio
     async def test_run_evaluations_async_context_var_reset(self, cases, effect_maps, evaluator):