feat: add reporters module for eval harness (RHAIENG-4149)

agents/autogen/templates/mcp_agent/tests/behavioral/test_cost_latency.py

@@ -15,14 +15,15 @@
 
 
 async def test_latency_single_tool(
-    run_eval: Any, autogen_mcp_thresholds: dict[str, Any]
+    run_eval: Any, autogen_mcp_thresholds: dict[str, Any], score_collector: Any
 ) -> None:
     """Response latency for a single-tool call must stay within the p95 threshold."""
     max_latency = autogen_mcp_thresholds["max_latency_p95"]
     result = await run_eval("Use the add tool to compute 55555 + 44444")
     assert result.success, f"Agent request failed: {result.error}"
 
     score = score_latency(result, max_latency)
+    score_collector.record("Use the add tool to compute 55555 + 44444", score)
     assert score.passed, (
         f"Latency exceeded threshold: {result.latency_seconds:.2f}s > "
         f"{max_latency}s (details: {score.details})"

agents/autogen/templates/mcp_agent/tests/behavioral/test_reliability.py

@@ -24,7 +24,7 @@
 
 
 async def test_pass_at_k_single_tool(
-    run_eval: Any, autogen_mcp_thresholds: dict[str, Any]
+    run_eval: Any, autogen_mcp_thresholds: dict[str, Any], score_collector: Any
 ) -> None:
     """Tool selection should succeed in >= threshold% of k runs.
 
@@ -49,6 +49,7 @@ async def test_pass_at_k_single_tool(
 
         if result.tool_calls:
             score = score_tool_selection(result, expected_tools)
+            score_collector.record(query, score)
             if score.passed:
                 passed_count += 1
         else:
@@ -67,7 +68,7 @@ async def test_pass_at_k_single_tool(
 
 
 async def test_pass_at_k_response_quality(
-    run_eval: Any, autogen_mcp_thresholds: dict[str, Any]
+    run_eval: Any, autogen_mcp_thresholds: dict[str, Any], score_collector: Any
 ) -> None:
     """Response coherence should pass in >= threshold% of k runs.
 
@@ -86,6 +87,7 @@ async def test_pass_at_k_response_quality(
             failures += 1
             continue
         score = score_plan_coherence(result)
+        score_collector.record(query, score)
         if score.passed:
             passed_count += 1
 

agents/autogen/templates/mcp_agent/tests/behavioral/test_response_quality.py

@@ -13,13 +13,16 @@
 pytestmark = pytest.mark.autogen_mcp
 
 
-async def test_plan_coherence(run_eval: Any) -> None:
+async def test_plan_coherence(run_eval: Any, score_collector: Any) -> None:
     """Response should have structure and substance (not a bare one-liner)."""
     result = await run_eval(
         "Use the add tool to compute 847392 + 293847 and explain the result"
     )
     assert result.success, f"Agent request failed: {result.error}"
     score = score_plan_coherence(result)
+    score_collector.record(
+        "Use the add tool to compute 847392 + 293847 and explain the result", score
+    )
     assert score.passed, (
         f"Plan coherence check failed (score={score.value:.2f}): {score.details}"
     )

agents/autogen/templates/mcp_agent/tests/behavioral/test_tool_usage.py

@@ -36,7 +36,9 @@ def _tool_queries() -> list[dict[str, Any]]:
     _tool_queries(),
     ids=lambda q: q["query"][:60],
 )
-async def test_single_tool_selection(run_eval: Any, golden: dict[str, Any]) -> None:
+async def test_single_tool_selection(
+    run_eval: Any, golden: dict[str, Any], score_collector: Any
+) -> None:
     """Correct tool should be selected for computation queries.
 
     Primary check: response contains the expected numeric result.
@@ -62,6 +64,7 @@ async def test_single_tool_selection(run_eval: Any, golden: dict[str, Any]) -> N
 
     if result.tool_calls:
         score = score_tool_selection(result, golden["expected_tools"])
+        score_collector.record(golden["query"], score)
         assert score.passed, (
             f"Tool selection failed: expected {golden['expected_tools']}, "
             f"got {score.details}"
@@ -93,7 +96,9 @@ async def test_adversarial_no_system_leakage(run_eval: Any) -> None:
     )
 
 
-async def test_no_hallucinated_tools(run_eval: Any, known_tools: list[str]) -> None:
+async def test_no_hallucinated_tools(
+    run_eval: Any, known_tools: list[str], score_collector: Any
+) -> None:
     """Agent must only call tools that exist in its schema.
 
     When tool_calls are not exposed, this test passes trivially.
@@ -105,12 +110,13 @@ async def test_no_hallucinated_tools(run_eval: Any, known_tools: list[str]) -> N
         pytest.skip("tool_calls not exposed in response — cannot verify")
 
     score = score_hallucinated_tools(result, known_tools)
+    score_collector.record("Use the add tool to compute 50 + 25", score)
     assert score.passed, (
         f"Hallucinated tools detected: {score.details.get('hallucinated')}"
     )
 
 
-async def test_tool_call_has_valid_args(run_eval: Any) -> None:
+async def test_tool_call_has_valid_args(run_eval: Any, score_collector: Any) -> None:
     """All tool call arguments must be valid JSON with required fields.
 
     When tool_calls are not exposed, this test is skipped.
@@ -122,6 +128,7 @@ async def test_tool_call_has_valid_args(run_eval: Any) -> None:
         pytest.skip("tool_calls not exposed in response — cannot verify")
 
     score = score_tool_call_validity(result)
+    score_collector.record("Use the add tool to compute 123 + 456", score)
     assert score.passed, f"Invalid tool call arguments: {score.details.get('invalid')}"
 
 

agents/crewai/templates/websearch_agent/tests/behavioral/test_cost_latency.py

@@ -15,14 +15,15 @@
 
 
 async def test_latency_under_threshold(
-    run_eval: Any, crewai_websearch_thresholds: dict[str, Any]
+    run_eval: Any, crewai_websearch_thresholds: dict[str, Any], score_collector: Any
 ) -> None:
     """Response latency must stay within the p95 threshold."""
     max_latency = crewai_websearch_thresholds["max_latency_p95"]
     result = await run_eval("What is the best platform for hosting AI workloads?")
     assert result.success, f"Agent request failed: {result.error}"
 
     score = score_latency(result, max_latency)
+    score_collector.record("What is the best platform for hosting AI workloads?", score)
     assert score.passed, (
         f"Latency exceeded threshold: {result.latency_seconds:.2f}s > "
         f"{max_latency}s (details: {score.details})"

agents/crewai/templates/websearch_agent/tests/behavioral/test_reliability.py

@@ -29,7 +29,7 @@
 
 
 async def test_pass_at_k_tool_usage(
-    run_eval: Any, crewai_websearch_thresholds: dict[str, Any]
+    run_eval: Any, crewai_websearch_thresholds: dict[str, Any], score_collector: Any
 ) -> None:
     """Tool selection should succeed in >= threshold% of k runs.
 
@@ -56,6 +56,7 @@ async def test_pass_at_k_tool_usage(
 
         if result.tool_calls:
             score = score_tool_selection(result, expected_tools)
+            score_collector.record(query, score)
             if score.passed:
                 passed_count += 1
         else:
@@ -80,7 +81,7 @@ async def test_pass_at_k_tool_usage(
 
 
 async def test_pass_at_k_response_quality(
-    run_eval: Any, crewai_websearch_thresholds: dict[str, Any]
+    run_eval: Any, crewai_websearch_thresholds: dict[str, Any], score_collector: Any
 ) -> None:
     """Response coherence should pass in >= threshold% of k runs.
 
@@ -99,6 +100,7 @@ async def test_pass_at_k_response_quality(
             failures += 1
             continue
         score = score_plan_coherence(result)
+        score_collector.record(query, score)
         if score.passed:
             passed_count += 1
 

agents/crewai/templates/websearch_agent/tests/behavioral/test_response_quality.py

@@ -19,13 +19,16 @@ def _queries_with_expected_elements() -> list[dict[str, Any]]:
     return [q for q in load_golden() if q.get("expected_elements")]
 
 
-async def test_plan_coherence(run_eval: Any) -> None:
+async def test_plan_coherence(run_eval: Any, score_collector: Any) -> None:
     """Response should have structure and substance (not a bare one-liner)."""
     result = await run_eval(
         "Explain how to deploy a machine learning model on OpenShift"
     )
     assert result.success, f"Agent request failed: {result.error}"
     score = score_plan_coherence(result)
+    score_collector.record(
+        "Explain how to deploy a machine learning model on OpenShift", score
+    )
     assert score.passed, (
         f"Plan coherence check failed (score={score.value:.2f}): {score.details}"
     )
@@ -36,7 +39,9 @@ async def test_plan_coherence(run_eval: Any) -> None:
     _queries_with_expected_elements(),
     ids=lambda q: q["query"][:60],
 )
-async def test_response_completeness(run_eval: Any, golden: dict[str, Any]) -> None:
+async def test_response_completeness(
+    run_eval: Any, golden: dict[str, Any], score_collector: Any
+) -> None:
     """Response should contain all expected elements from the golden dataset."""
     result = await run_eval(
         golden["query"],
@@ -45,6 +50,7 @@ async def test_response_completeness(run_eval: Any, golden: dict[str, Any]) -> N
     assert result.success, f"Agent request failed: {result.error}"
 
     score = score_completeness(result, golden["expected_elements"])
+    score_collector.record(golden["query"], score)
     assert score.passed, (
         f"Completeness check failed: missing {score.details.get('missing')} "
         f"(found {score.details.get('found')}). "

agents/crewai/templates/websearch_agent/tests/behavioral/test_tool_usage.py

@@ -37,7 +37,9 @@ def _factual_queries() -> list[dict[str, Any]]:
     _factual_queries(),
     ids=lambda q: q["query"][:60],
 )
-async def test_tool_selection_accuracy(run_eval: Any, golden: dict[str, Any]) -> None:
+async def test_tool_selection_accuracy(
+    run_eval: Any, golden: dict[str, Any], score_collector: Any
+) -> None:
     """Correct tool should be selected for information-seeking queries.
 
     Primary check: tool_calls extracted from MLflow traces via enrichment.
@@ -61,6 +63,7 @@ async def test_tool_selection_accuracy(run_eval: Any, golden: dict[str, Any]) ->
 
     if result.tool_calls:
         score = score_tool_selection(result, golden["expected_tools"])
+        score_collector.record(golden["query"], score)
         assert score.passed, (
             f"Tool selection failed: expected {golden['expected_tools']}, "
             f"got {score.details}"
@@ -75,7 +78,9 @@ async def test_tool_selection_accuracy(run_eval: Any, golden: dict[str, Any]) ->
         )
 
 
-async def test_no_hallucinated_tools(run_eval: Any, known_tools: list[str]) -> None:
+async def test_no_hallucinated_tools(
+    run_eval: Any, known_tools: list[str], score_collector: Any
+) -> None:
     """Agent must only call tools that exist in its schema.
 
     Requires MLflow trace enrichment to populate tool_calls.
@@ -87,12 +92,13 @@ async def test_no_hallucinated_tools(run_eval: Any, known_tools: list[str]) -> N
         pytest.skip("tool_calls not exposed — enable MLflow trace enrichment")
 
     score = score_hallucinated_tools(result, known_tools)
+    score_collector.record("Tell me about Kubernetes operators", score)
     assert score.passed, (
         f"Hallucinated tools detected: {score.details.get('hallucinated')}"
     )
 
 
-async def test_tool_call_has_valid_args(run_eval: Any) -> None:
+async def test_tool_call_has_valid_args(run_eval: Any, score_collector: Any) -> None:
     """All tool call arguments must be valid JSON with required fields.
 
     Requires MLflow trace enrichment to populate tool_calls.
@@ -104,6 +110,7 @@ async def test_tool_call_has_valid_args(run_eval: Any) -> None:
         pytest.skip("tool_calls not exposed — enable MLflow trace enrichment")
 
     score = score_tool_call_validity(result)
+    score_collector.record("What is OpenShift AI?", score)
     assert score.passed, f"Invalid tool call arguments: {score.details.get('invalid')}"
 
 

agents/google/templates/adk/tests/behavioral/test_cost_latency.py

@@ -15,14 +15,15 @@
 
 
 async def test_latency_under_threshold(
-    run_eval: Any, google_adk_thresholds: dict[str, Any]
+    run_eval: Any, google_adk_thresholds: dict[str, Any], score_collector: Any
 ) -> None:
     """Response latency must stay within the p95 threshold."""
     max_latency = google_adk_thresholds["max_latency_p95"]
     result = await run_eval("What is Red Hat OpenShift AI?")
     assert result.success, f"Agent request failed: {result.error}"
 
     score = score_latency(result, max_latency)
+    score_collector.record("What is Red Hat OpenShift AI?", score)
     assert score.passed, (
         f"Latency exceeded threshold: {result.latency_seconds:.2f}s > "
         f"{max_latency}s (details: {score.details})"

agents/google/templates/adk/tests/behavioral/test_reliability.py

@@ -25,7 +25,7 @@
 
 
 async def test_pass_at_k_tool_usage(
-    run_eval: Any, google_adk_thresholds: dict[str, Any]
+    run_eval: Any, google_adk_thresholds: dict[str, Any], score_collector: Any
 ) -> None:
     """Tool selection should succeed in >= threshold% of k runs.
 
@@ -50,6 +50,7 @@ async def test_pass_at_k_tool_usage(
 
         if result.tool_calls:
             score = score_tool_selection(result, expected_tools)
+            score_collector.record(query, score)
             if score.passed:
                 passed_count += 1
         else:
@@ -66,7 +67,7 @@ async def test_pass_at_k_tool_usage(
 
 
 async def test_pass_at_k_response_quality(
-    run_eval: Any, google_adk_thresholds: dict[str, Any]
+    run_eval: Any, google_adk_thresholds: dict[str, Any], score_collector: Any
 ) -> None:
     """Response coherence should pass in >= threshold% of k runs.
 
@@ -85,6 +86,7 @@ async def test_pass_at_k_response_quality(
             failures += 1
             continue
         score = score_plan_coherence(result)
+        score_collector.record(query, score)
         if score.passed:
             passed_count += 1
 

agents/google/templates/adk/tests/behavioral/test_response_quality.py

@@ -13,13 +13,16 @@
 pytestmark = pytest.mark.google_adk
 
 
-async def test_plan_coherence(run_eval: Any) -> None:
+async def test_plan_coherence(run_eval: Any, score_collector: Any) -> None:
     """Response should have structure and substance (not a bare one-liner)."""
     result = await run_eval(
         "Explain how to deploy a machine learning model on OpenShift"
     )
     assert result.success, f"Agent request failed: {result.error}"
     score = score_plan_coherence(result)
+    score_collector.record(
+        "Explain how to deploy a machine learning model on OpenShift", score
+    )
     assert score.passed, (
         f"Plan coherence check failed (score={score.value:.2f}): {score.details}"
     )

agents/google/templates/adk/tests/behavioral/test_tool_usage.py

@@ -39,7 +39,9 @@ def _factual_queries() -> list[dict[str, Any]]:
     _factual_queries(),
     ids=lambda q: q["query"][:60],
 )
-async def test_tool_selection_accuracy(run_eval: Any, golden: dict[str, Any]) -> None:
+async def test_tool_selection_accuracy(
+    run_eval: Any, golden: dict[str, Any], score_collector: Any
+) -> None:
     """Correct tool should be selected for information-seeking queries.
 
     Primary check: response contains content from the search tool's output.
@@ -63,6 +65,7 @@ async def test_tool_selection_accuracy(run_eval: Any, golden: dict[str, Any]) ->
 
     if result.tool_calls:
         score = score_tool_selection(result, golden["expected_tools"])
+        score_collector.record(golden["query"], score)
         assert score.passed, (
             f"Tool selection failed: expected {golden['expected_tools']}, "
             f"got {score.details}"
@@ -76,7 +79,9 @@ async def test_tool_selection_accuracy(run_eval: Any, golden: dict[str, Any]) ->
         )
 
 
-async def test_no_hallucinated_tools(run_eval: Any, known_tools: list[str]) -> None:
+async def test_no_hallucinated_tools(
+    run_eval: Any, known_tools: list[str], score_collector: Any
+) -> None:
     """Agent must only call tools that exist in its schema.
 
     When tool_calls are not exposed, this test passes trivially (no calls
@@ -89,12 +94,13 @@ async def test_no_hallucinated_tools(run_eval: Any, known_tools: list[str]) -> N
         pytest.skip("tool_calls not exposed in response — cannot verify")
 
     score = score_hallucinated_tools(result, known_tools)
+    score_collector.record("Tell me about Kubernetes operators", score)
     assert score.passed, (
         f"Hallucinated tools detected: {score.details.get('hallucinated')}"
     )
 
 
-async def test_tool_call_has_valid_args(run_eval: Any) -> None:
+async def test_tool_call_has_valid_args(run_eval: Any, score_collector: Any) -> None:
     """All tool call arguments must be valid JSON with required fields.
 
     When tool_calls are not exposed, this test is skipped.
@@ -106,6 +112,7 @@ async def test_tool_call_has_valid_args(run_eval: Any) -> None:
         pytest.skip("tool_calls not exposed in response — cannot verify")
 
     score = score_tool_call_validity(result)
+    score_collector.record("What is OpenShift AI?", score)
     assert score.passed, f"Invalid tool call arguments: {score.details.get('invalid')}"