diff --git a/openapi/ga/individual/platform.openapi.yaml b/openapi/ga/individual/platform.openapi.yaml index e2eedea801..1063e3647d 100644 --- a/openapi/ga/individual/platform.openapi.yaml +++ b/openapi/ga/individual/platform.openapi.yaml @@ -20974,6 +20974,20 @@ components: - prompt_template title: MetricOnlineJob description: A online metric job. + MetricOutput: + properties: + name: + type: string + title: Name + value: + title: Value + additionalProperties: false + type: object + required: + - name + - value + title: MetricOutput + description: One named value emitted by a metric. MetricRef: type: string pattern: ^[a-z0-9_-]+/[a-z0-9_-]+$ @@ -21031,24 +21045,6 @@ components: - dataset title: MetricRetrieverJob description: Evaluation with a retriever-based metric. - MetricScore: - properties: - name: - type: string - title: Name - value: - type: number - title: Value - stats: - allOf: - - $ref: '#/components/schemas/ScoreStats' - description: Computed score statistics for the score. - type: object - required: - - name - - value - title: MetricScore - description: A computed score for the metric MetricType: description: The predefined metric types. enum: @@ -26893,11 +26889,11 @@ components: metrics: additionalProperties: items: - $ref: '#/components/schemas/MetricScore' + $ref: '#/components/schemas/MetricOutput' type: array type: object title: Metrics - description: Metric-level row scores by metric key. + description: Metric-level row outputs by metric key. requests: items: additionalProperties: true @@ -27494,77 +27490,6 @@ components: type: object title: SafeSynthesizerTiming description: Wall-clock durations for each pipeline stage. - ScoreStats: - properties: - count: - title: Count - description: The number of values used for computing the score. - type: integer - sum: - anyOf: - - type: number - - type: string - title: Sum - description: The sum of all values used for computing the score. - sum_squared: - anyOf: - - type: number - - type: string - title: Sum Squared - description: The sum of the square of all values used for computing the - score. - min: - anyOf: - - type: number - - type: string - title: Min - description: The minimum of all values used for computing the score. - max: - anyOf: - - type: number - - type: string - title: Max - description: The maximum of all values used for computing the score. - mean: - anyOf: - - type: number - - type: string - title: Mean - description: The mean of all values used for computing the score. - variance: - anyOf: - - type: number - - type: string - title: Variance - description: 'The population variance, (note: not the sample variance).' - stddev: - anyOf: - - type: number - - type: string - title: Stddev - description: 'The population standard deviation, (note: not the sample standard - deviation).' - stderr: - anyOf: - - type: number - - type: string - title: Stderr - description: The standard error. - nan_count: - title: Nan Count - description: The number of values that are not a number (NaN) and are excluded - from the score stats calculations. - type: integer - rubric_distribution: - title: Rubric Distribution - description: The distribution of the rubric grading criteria for the score. - items: - $ref: '#/components/schemas/RubricScoreStat' - type: array - type: object - title: ScoreStats - description: Stats for a score. Fields that are NaN are serialized as the string - "NaN" in the API response. SecretRef: type: string pattern: ^[a-z0-9_-]+(/[a-z0-9_-]+)?$ diff --git a/openapi/ga/openapi.yaml b/openapi/ga/openapi.yaml index e2eedea801..1063e3647d 100644 --- a/openapi/ga/openapi.yaml +++ b/openapi/ga/openapi.yaml @@ -20974,6 +20974,20 @@ components: - prompt_template title: MetricOnlineJob description: A online metric job. + MetricOutput: + properties: + name: + type: string + title: Name + value: + title: Value + additionalProperties: false + type: object + required: + - name + - value + title: MetricOutput + description: One named value emitted by a metric. MetricRef: type: string pattern: ^[a-z0-9_-]+/[a-z0-9_-]+$ @@ -21031,24 +21045,6 @@ components: - dataset title: MetricRetrieverJob description: Evaluation with a retriever-based metric. - MetricScore: - properties: - name: - type: string - title: Name - value: - type: number - title: Value - stats: - allOf: - - $ref: '#/components/schemas/ScoreStats' - description: Computed score statistics for the score. - type: object - required: - - name - - value - title: MetricScore - description: A computed score for the metric MetricType: description: The predefined metric types. enum: @@ -26893,11 +26889,11 @@ components: metrics: additionalProperties: items: - $ref: '#/components/schemas/MetricScore' + $ref: '#/components/schemas/MetricOutput' type: array type: object title: Metrics - description: Metric-level row scores by metric key. + description: Metric-level row outputs by metric key. requests: items: additionalProperties: true @@ -27494,77 +27490,6 @@ components: type: object title: SafeSynthesizerTiming description: Wall-clock durations for each pipeline stage. - ScoreStats: - properties: - count: - title: Count - description: The number of values used for computing the score. - type: integer - sum: - anyOf: - - type: number - - type: string - title: Sum - description: The sum of all values used for computing the score. - sum_squared: - anyOf: - - type: number - - type: string - title: Sum Squared - description: The sum of the square of all values used for computing the - score. - min: - anyOf: - - type: number - - type: string - title: Min - description: The minimum of all values used for computing the score. - max: - anyOf: - - type: number - - type: string - title: Max - description: The maximum of all values used for computing the score. - mean: - anyOf: - - type: number - - type: string - title: Mean - description: The mean of all values used for computing the score. - variance: - anyOf: - - type: number - - type: string - title: Variance - description: 'The population variance, (note: not the sample variance).' - stddev: - anyOf: - - type: number - - type: string - title: Stddev - description: 'The population standard deviation, (note: not the sample standard - deviation).' - stderr: - anyOf: - - type: number - - type: string - title: Stderr - description: The standard error. - nan_count: - title: Nan Count - description: The number of values that are not a number (NaN) and are excluded - from the score stats calculations. - type: integer - rubric_distribution: - title: Rubric Distribution - description: The distribution of the rubric grading criteria for the score. - items: - $ref: '#/components/schemas/RubricScoreStat' - type: array - type: object - title: ScoreStats - description: Stats for a score. Fields that are NaN are serialized as the string - "NaN" in the API response. SecretRef: type: string pattern: ^[a-z0-9_-]+(/[a-z0-9_-]+)?$ diff --git a/openapi/openapi.yaml b/openapi/openapi.yaml index e2eedea801..1063e3647d 100644 --- a/openapi/openapi.yaml +++ b/openapi/openapi.yaml @@ -20974,6 +20974,20 @@ components: - prompt_template title: MetricOnlineJob description: A online metric job. + MetricOutput: + properties: + name: + type: string + title: Name + value: + title: Value + additionalProperties: false + type: object + required: + - name + - value + title: MetricOutput + description: One named value emitted by a metric. MetricRef: type: string pattern: ^[a-z0-9_-]+/[a-z0-9_-]+$ @@ -21031,24 +21045,6 @@ components: - dataset title: MetricRetrieverJob description: Evaluation with a retriever-based metric. - MetricScore: - properties: - name: - type: string - title: Name - value: - type: number - title: Value - stats: - allOf: - - $ref: '#/components/schemas/ScoreStats' - description: Computed score statistics for the score. - type: object - required: - - name - - value - title: MetricScore - description: A computed score for the metric MetricType: description: The predefined metric types. enum: @@ -26893,11 +26889,11 @@ components: metrics: additionalProperties: items: - $ref: '#/components/schemas/MetricScore' + $ref: '#/components/schemas/MetricOutput' type: array type: object title: Metrics - description: Metric-level row scores by metric key. + description: Metric-level row outputs by metric key. requests: items: additionalProperties: true @@ -27494,77 +27490,6 @@ components: type: object title: SafeSynthesizerTiming description: Wall-clock durations for each pipeline stage. - ScoreStats: - properties: - count: - title: Count - description: The number of values used for computing the score. - type: integer - sum: - anyOf: - - type: number - - type: string - title: Sum - description: The sum of all values used for computing the score. - sum_squared: - anyOf: - - type: number - - type: string - title: Sum Squared - description: The sum of the square of all values used for computing the - score. - min: - anyOf: - - type: number - - type: string - title: Min - description: The minimum of all values used for computing the score. - max: - anyOf: - - type: number - - type: string - title: Max - description: The maximum of all values used for computing the score. - mean: - anyOf: - - type: number - - type: string - title: Mean - description: The mean of all values used for computing the score. - variance: - anyOf: - - type: number - - type: string - title: Variance - description: 'The population variance, (note: not the sample variance).' - stddev: - anyOf: - - type: number - - type: string - title: Stddev - description: 'The population standard deviation, (note: not the sample standard - deviation).' - stderr: - anyOf: - - type: number - - type: string - title: Stderr - description: The standard error. - nan_count: - title: Nan Count - description: The number of values that are not a number (NaN) and are excluded - from the score stats calculations. - type: integer - rubric_distribution: - title: Rubric Distribution - description: The distribution of the rubric grading criteria for the score. - items: - $ref: '#/components/schemas/RubricScoreStat' - type: array - type: object - title: ScoreStats - description: Stats for a score. Fields that are NaN are serialized as the string - "NaN" in the API response. SecretRef: type: string pattern: ^[a-z0-9_-]+(/[a-z0-9_-]+)?$ diff --git a/packages/nemo_evaluator_sdk/examples/examples.py b/packages/nemo_evaluator_sdk/examples/examples.py index 5070a660cd..6c8a187c3f 100644 --- a/packages/nemo_evaluator_sdk/examples/examples.py +++ b/packages/nemo_evaluator_sdk/examples/examples.py @@ -21,12 +21,12 @@ from nemo_evaluator_sdk.execution.values import EvaluationError from nemo_evaluator_sdk.metrics.exact_match import ExactMatchMetric from nemo_evaluator_sdk.metrics.llm_judge import LLMJudgeMetric +from nemo_evaluator_sdk.metrics.protocol import MetricInput, MetricOutput, MetricOutputSpec from nemo_evaluator_sdk.metrics.string_check import StringCheckMetric from nemo_evaluator_sdk.values import ( InferenceParams, JSONScoreParser, MetricResult, - MetricScore, Model, RangeScore, RunConfig, @@ -315,32 +315,27 @@ def __init__(self, db_connection_string: str): """ self.db_connection_string = db_connection_string - async def compute_scores(self, item: dict, sample: dict) -> MetricResult: + def output_spec(self) -> list[MetricOutputSpec]: + """Return the outputs produced by the metric.""" + return [MetricOutputSpec.continuous_score(self.type)] + + async def compute_scores(self, input: MetricInput) -> MetricResult: """Convert the custom score into the SDK metric result shape. Args: - item: Original dataset row. - sample: Evaluated sample payload. + input: Original dataset row and evaluated candidate output. Returns: A metric result containing one exact-match score. """ - prediction = sample.get("output_text") - reference = item.get("actual") + prediction = input.candidate.output_text + reference = input.row.data.get("actual") if reference is None: - reference = item.get("reference") + reference = input.row.data.get("reference") if prediction is None or reference is None: - return MetricResult(scores=[MetricScore(name=self.type, value=0.0)]) + return MetricResult(outputs=[MetricOutput(name=self.type, value=0.0)]) score = 1.0 if prediction == reference else 0.0 - return MetricResult(scores=[MetricScore(name=self.type, value=score)]) - - def score_names(self) -> list[str]: - """Return the score names produced by the metric. - - Returns: - A one-element list containing the exact-match score name. - """ - return [self.type] + return MetricResult(outputs=[MetricOutput(name=self.type, value=score)]) class CustomFailingMetric: @@ -356,28 +351,24 @@ def __init__(self, message: str): """ self.message = message - async def compute_scores(self, item: dict, sample: dict) -> MetricResult: + def output_spec(self) -> list[MetricOutputSpec]: + """Return the outputs produced by the metric.""" + return [MetricOutputSpec.continuous_score(self.type)] + + async def compute_scores(self, input: MetricInput) -> MetricResult: """Convert the intentionally raised error into normal metric execution. Args: - item: Original dataset row. - sample: Evaluated sample payload. + input: Original dataset row and evaluated candidate output. Raises: RuntimeError: Always raised to exercise benchmark failure handling. """ + del input # This example metric is intentionally failing to demonstrate structured # benchmark error handling in local evaluator workflows. raise RuntimeError(self.message) - def score_names(self) -> list[str]: - """Return the score names produced by the metric. - - Returns: - A one-element list containing the intentionally failing score name. - """ - return [self.type] - # --- 2. Local evaluator workflows --- async def run_offline_local_exact_match_example() -> None: diff --git a/packages/nemo_evaluator_sdk/examples/plugin_examples.py b/packages/nemo_evaluator_sdk/examples/plugin_examples.py index be991f858b..0d8aea80e8 100644 --- a/packages/nemo_evaluator_sdk/examples/plugin_examples.py +++ b/packages/nemo_evaluator_sdk/examples/plugin_examples.py @@ -24,9 +24,9 @@ RunConfigOnlineModel, ) from nemo_evaluator_sdk.enums import MetricType -from nemo_evaluator_sdk.metrics.base import Metric from nemo_evaluator_sdk.metrics.exact_match import ExactMatchMetric from nemo_evaluator_sdk.metrics.llm_judge import LLMJudgeMetric +from nemo_evaluator_sdk.metrics.protocol import Metric from nemo_evaluator_sdk.values import ( InferenceParams, JSONScoreParser, diff --git a/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/__init__.py b/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/__init__.py index cacde568f2..9b9d07e44b 100644 --- a/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/__init__.py +++ b/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/__init__.py @@ -17,6 +17,10 @@ from nemo_evaluator_sdk.metrics.f1 import F1Metric from nemo_evaluator_sdk.metrics.llm_judge import LLMJudgeMetric from nemo_evaluator_sdk.metrics.number_check import NumberCheckMetric +from nemo_evaluator_sdk.metrics.protocol import ( + Metric, + validate_metric_result, +) from nemo_evaluator_sdk.metrics.remote import NemoAgentToolkitRemoteMetric, RemoteMetric from nemo_evaluator_sdk.metrics.rouge import ROUGEMetric from nemo_evaluator_sdk.metrics.string_check import StringCheckMetric @@ -31,10 +35,21 @@ ) from nemo_evaluator_sdk.values import ( Agent, + BooleanValue, + CandidateOutput, + ContinuousScore, + DatasetRow, DatasetRows, + DiscreteScore, EvaluationResult, InferenceParams, JSONScoreParser, + Label, + MetricDescriptor, + MetricInput, + MetricOutput, + MetricOutputSpec, + MetricResult, Model, RangeScore, ReasoningParams, @@ -69,7 +84,19 @@ "InferenceFn", "InferenceStructuredOutput", "JSONScoreParser", + "Metric", + "MetricDescriptor", + "MetricInput", + "MetricOutput", + "MetricOutputSpec", + "MetricResult", "LLMJudgeMetric", + "BooleanValue", + "CandidateOutput", + "ContinuousScore", + "DatasetRow", + "DiscreteScore", + "Label", "Model", "NemoAgentToolkitRemoteMetric", "NumberCheckMetric", @@ -88,5 +115,6 @@ "detect_structured_output_mode", "load_dataset", "load_dataset_as_dicts", + "validate_metric_result", "version", ] diff --git a/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/execution/backends/base.py b/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/execution/backends/base.py index f8664d21b1..5834c95517 100644 --- a/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/execution/backends/base.py +++ b/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/execution/backends/base.py @@ -9,7 +9,7 @@ from typing import Protocol from nemo_evaluator_sdk.execution.config import EvaluationRequest -from nemo_evaluator_sdk.metrics.base import Metric +from nemo_evaluator_sdk.metrics.protocol import Metric from nemo_evaluator_sdk.values.multi_metric_results import BenchmarkEvaluationResult from nemo_evaluator_sdk.values.results import EvaluationResult diff --git a/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/execution/backends/local/backend.py b/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/execution/backends/local/backend.py index ce80afe333..f5e49f2ec8 100644 --- a/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/execution/backends/local/backend.py +++ b/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/execution/backends/local/backend.py @@ -14,7 +14,7 @@ from nemo_evaluator_sdk.execution.config import EvaluationRequest from nemo_evaluator_sdk.execution.metric_execution import _merge_online_hooks, evaluate_metric from nemo_evaluator_sdk.execution.utils import prepare_metric_for_local_execution, unique_metric_keys -from nemo_evaluator_sdk.metrics.base import Metric +from nemo_evaluator_sdk.metrics.protocol import Metric from nemo_evaluator_sdk.metrics.utils import metric_type_name from nemo_evaluator_sdk.values.multi_metric_results import BenchmarkEvaluationResult, namespace_result from nemo_evaluator_sdk.values.results import EvaluationResult diff --git a/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/execution/benchmark_execution.py b/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/execution/benchmark_execution.py index c9c0b4e940..ef39d689eb 100644 --- a/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/execution/benchmark_execution.py +++ b/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/execution/benchmark_execution.py @@ -35,7 +35,7 @@ generate_online_sample, generate_online_sample_agent, ) -from nemo_evaluator_sdk.execution.scoring import nan_metric_result +from nemo_evaluator_sdk.execution.scoring import build_metric_input, corpus_output_spec, nan_metric_result from nemo_evaluator_sdk.execution.values import EvaluationError, EvaluationPhase from nemo_evaluator_sdk.inference import ( InferenceFn, @@ -45,8 +45,18 @@ new_inference_client, requests_log_var, ) -from nemo_evaluator_sdk.metrics.aggregation import add_corpus_scores, aggregate_metrics -from nemo_evaluator_sdk.metrics.base import CorpusMetric, Metric +from nemo_evaluator_sdk.metrics.aggregation import ( + add_corpus_scores, + aggregate_metrics, + rubric_definitions_from_metric, +) +from nemo_evaluator_sdk.metrics.protocol import ( + CorpusMetric, + Metric, + MetricOutputSpec, + MetricResult, + validate_metric_result, +) from nemo_evaluator_sdk.resilience.api import use_resilience_session from nemo_evaluator_sdk.resilience.errors import first_failure_cause, iter_leaf_causes from nemo_evaluator_sdk.values import ( @@ -54,8 +64,6 @@ AggregatedMetricResult, AggregateFieldName, EvaluationResult, - MetricResult, - MetricScore, Model, RowScore, RunConfig, @@ -105,21 +113,16 @@ class _MetricPipeline: metric_ref: str metric: Metric - score_names: list[str] + output_spec: list[MetricOutputSpec] queue: asyncio.Queue results: list[MetricResult | None] -def _normalize_metric_result(metric_result: MetricResult, expected_score_names: list[str]) -> MetricResult: - """Normalize score ordering and backfill missing expected scores with NaN.""" - actual_scores = {score.name: score for score in metric_result.scores} - normalized: list[MetricScore] = [] - for name in expected_score_names: - normalized.append(actual_scores.get(name, MetricScore(name=name, value=float("nan")))) - for score in metric_result.scores: - if score.name not in expected_score_names: - normalized.append(score) - return MetricResult(scores=normalized) +def _normalize_metric_result(metric_result: MetricResult, expected_outputs: list[MetricOutputSpec]) -> MetricResult: + """Validate output names and normalize output ordering to the declared spec.""" + validated = validate_metric_result(metric_result, expected_outputs) + actual_outputs = {output.name: output for output in validated.outputs} + return MetricResult(outputs=[actual_outputs[output.name] for output in expected_outputs]) def _benchmark_error_from_exception(exc: BaseException) -> EvaluationError | None: @@ -149,14 +152,14 @@ def _build_metric_pipelines( """ pipelines: list[_MetricPipeline] = [] for metric_ref, metric in metrics: - score_names = list(metric.score_names()) - if not score_names: - raise RuntimeError(f"Metric '{metric_ref}' does not declare any score names") + output_spec = list(metric.output_spec()) + if not output_spec: + raise RuntimeError(f"Metric '{metric_ref}' does not declare any outputs") pipelines.append( _MetricPipeline( metric_ref=metric_ref, metric=metric, - score_names=score_names, + output_spec=output_spec, queue=asyncio.Queue(maxsize=queue_capacity), results=[None] * item_count, ) @@ -206,7 +209,13 @@ async def _finalize_benchmark_metric_result( should only see successful rows so failed rows with empty samples do not skew corpus metrics. """ - aggregated = aggregate_metrics([result for result in results if result is not None]) + output_spec = metric.output_spec() + metric_results = [result for result in results if result is not None] + rubric_definitions = rubric_definitions_from_metric(metric) + if rubric_definitions: + aggregated = aggregate_metrics(metric_results, output_spec, rubric_definitions=rubric_definitions) + else: + aggregated = aggregate_metrics(metric_results, output_spec) if isinstance(metric, CorpusMetric): # Ignored sample-generation failures intentionally keep metric_errors # empty to match the previous service benchmark row artifacts, so @@ -218,11 +227,13 @@ async def _finalize_benchmark_metric_result( ] if corpus_rows: corpus_result = await metric.compute_corpus_scores( - items=[row_score.item for row_score in corpus_rows], - samples=[row_score.sample for row_score in corpus_rows], + inputs=[ + build_metric_input(row_score.item, row_score.sample, row_score.row_index) + for row_score in corpus_rows + ], ) if corpus_result is not None: - add_corpus_scores(aggregated, corpus_result) + add_corpus_scores(aggregated, corpus_result, corpus_output_spec(metric, output_spec)) return EvaluationResult(row_scores=row_scores, aggregate_scores=aggregated) @@ -392,8 +403,10 @@ async def _metric_worker( requests_log_var.set(requests_log) try: metric_result = _normalize_metric_result( - await pipeline.metric.compute_scores(dict(event.item), dict(event.sample)), - pipeline.score_names, + await pipeline.metric.compute_scores( + build_metric_input(dict(event.item), dict(event.sample), event.row_index) + ), + pipeline.output_spec, ) except Exception as e: if not tolerate_failure: @@ -408,7 +421,7 @@ async def _metric_worker( "Evaluation failed, marking as NaN", extra={"metric_ref": pipeline.metric_ref, "item_index": event.row_index, "error": error_message}, ) - metric_result = nan_metric_result(pipeline.score_names) + metric_result = nan_metric_result(pipeline.output_spec) # Record the swallowed metric exception on the row while keeping # the NaN score result. Example: if the "judge" metric raises # "bad output", the row gets metric_errors={"judge": "bad output"}. @@ -420,7 +433,7 @@ async def _metric_worker( pipeline.queue.task_done() pipeline.results[event.row_index] = metric_result - row_scores[event.row_index].metrics[pipeline.metric_ref] = metric_result.scores + row_scores[event.row_index].metrics[pipeline.metric_ref] = metric_result.outputs if progress is not None: progress.increment_work() diff --git a/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/execution/evaluator.py b/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/execution/evaluator.py index f4d5b6dc50..95258e2884 100644 --- a/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/execution/evaluator.py +++ b/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/execution/evaluator.py @@ -13,7 +13,7 @@ import nemo_evaluator_sdk.inference as inference from nemo_evaluator_sdk.execution.metric_execution import run_sync -from nemo_evaluator_sdk.metrics.base import Metric +from nemo_evaluator_sdk.metrics.protocol import Metric from nemo_evaluator_sdk.values.agents import Agent from nemo_evaluator_sdk.values.datasets import DatasetInput from nemo_evaluator_sdk.values.models import Model diff --git a/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/execution/metric_execution.py b/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/execution/metric_execution.py index 0e7d718932..4a188d8ac4 100644 --- a/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/execution/metric_execution.py +++ b/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/execution/metric_execution.py @@ -38,7 +38,7 @@ from nemo_evaluator_sdk.execution.utils import prepare_metric_for_local_execution from nemo_evaluator_sdk.execution.values import EvaluationError, EvaluationPhase from nemo_evaluator_sdk.inference import InferenceMetricBase -from nemo_evaluator_sdk.metrics.base import Metric +from nemo_evaluator_sdk.metrics.protocol import Metric, MetricResult from nemo_evaluator_sdk.metrics.utils import metric_type_name from nemo_evaluator_sdk.resilience.api import run_indexed_tasks, use_resilience_session from nemo_evaluator_sdk.resilience.errors import get_evaluation_error @@ -46,7 +46,6 @@ from nemo_evaluator_sdk.values import ( Agent, EvaluationResult, - MetricResult, Model, RowScore, RunConfig, @@ -214,7 +213,7 @@ def _merge_online_hooks( """Build deterministic hook lists for SDK local online generation. Online sample generation should only use run-level generation hooks. - Metric hooks belong to metric.compute_scores(item, sample) and must not + Metric hooks belong to metric.compute_scores(input) and must not affect the evaluated-model generation stage. """ @@ -658,7 +657,7 @@ def handle_generation_error( log.warning("Inference failed, marking as NaN", extra={"item_index": index, "error": error_message}) sample = {"output_text": None, "response": {}, "inference_error": error_message} - nan_result = nan_metric_result(self.metric.score_names()) + nan_result = nan_metric_result(self.metric.output_spec()) return ( index, @@ -667,7 +666,7 @@ def handle_generation_error( row_index=index, item=row, sample=sample, - metrics={}, + metrics={self.metric_key: nan_result.outputs}, requests=generation_requests, metric_errors={self.metric_key: error_message}, ), diff --git a/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/execution/pipeline.py b/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/execution/pipeline.py index 6854c43e68..b3c097a1c4 100644 --- a/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/execution/pipeline.py +++ b/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/execution/pipeline.py @@ -10,7 +10,8 @@ from types import MappingProxyType from typing import Any, Protocol -from nemo_evaluator_sdk.values.results import MetricResult, RowScore +from nemo_evaluator_sdk.metrics.protocol import MetricResult +from nemo_evaluator_sdk.values.results import RowScore @dataclass diff --git a/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/execution/scoring.py b/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/execution/scoring.py index 2661b1df85..eac992d742 100644 --- a/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/execution/scoring.py +++ b/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/execution/scoring.py @@ -9,22 +9,75 @@ from nemo_evaluator_sdk.execution.values import EvaluationError, EvaluationPhase from nemo_evaluator_sdk.inference import requests_log_var -from nemo_evaluator_sdk.metrics.aggregation import add_corpus_scores, aggregate_metrics -from nemo_evaluator_sdk.metrics.base import CorpusMetric, Metric +from nemo_evaluator_sdk.metrics.aggregation import ( + add_corpus_scores, + aggregate_metrics, + is_aggregateable_output_spec, + rubric_definitions_from_metric, +) +from nemo_evaluator_sdk.metrics.protocol import ( + CandidateOutput, + CorpusMetric, + DatasetRow, + Metric, + MetricInput, + MetricOutput, + MetricOutputSpec, + MetricResult, + validate_metric_result, +) from nemo_evaluator_sdk.metrics.utils import metric_type_name -from nemo_evaluator_sdk.values import EvaluationResult, MetricResult, MetricScore, RowScore +from nemo_evaluator_sdk.values import ( + EvaluationResult, + RowScore, +) logger = getLogger(__name__) -def nan_metric_result(score_names: str | Iterable[str]) -> MetricResult: - """Build the NaN score payload used for ignored scoring failures. +_CANDIDATE_SAMPLE_FIELDS = frozenset({"output_text", "response", "trajectory"}) + + +def build_metric_input(row: dict[str, Any], sample: dict[str, Any], index: int | None = None) -> MetricInput: + """Build the metric protocol input from dataset row and generated sample payloads.""" + output_text = sample.get("output_text") + metadata = { + key: value + for key, value in sample.items() + if key not in _CANDIDATE_SAMPLE_FIELDS or (key == "output_text" and not isinstance(output_text, str)) + } + return MetricInput( + row=DatasetRow(row_index=index, data=row), + candidate=CandidateOutput( + output_text=output_text if isinstance(output_text, str) else None, + response=sample.get("response"), + trajectory=sample.get("trajectory"), + metadata=metadata, + ), + ) + - Accepts either a single score name (single-metric pipelines) or an - iterable of score names (multi-score/benchmark pipelines). +def nan_metric_result(outputs: Iterable[MetricOutputSpec]) -> MetricResult: + """Build the NaN output payload used for ignored scoring failures. + + Only aggregateable outputs receive NaN placeholders; non-score outputs are + not synthesized for failed rows. """ - names: Iterable[str] = (score_names,) if isinstance(score_names, str) else score_names - return MetricResult(scores=[MetricScore(name=name, value=float("nan")) for name in names]) + return MetricResult( + outputs=[ + MetricOutput(name=output.name, value=float("nan")) + for output in outputs + if is_aggregateable_output_spec(output) + ] + ) + + +def corpus_output_spec(metric: Metric, fallback: list[MetricOutputSpec] | None = None) -> list[MetricOutputSpec]: + """Return corpus-level output specs when a metric declares them.""" + corpus_spec = getattr(metric, "corpus_output_spec", None) + if callable(corpus_spec): + return list(corpus_spec()) + return list(fallback if fallback is not None else metric.output_spec()) CompletedRowEvaluation = tuple[int, MetricResult | None, RowScore] @@ -32,7 +85,7 @@ def nan_metric_result(score_names: str | Iterable[str]) -> MetricResult: def empty_evaluation_result() -> EvaluationResult: """Return the canonical empty evaluation result payload.""" - return EvaluationResult(row_scores=[], aggregate_scores=aggregate_metrics([])) + return EvaluationResult(row_scores=[], aggregate_scores=aggregate_metrics([], [])) async def finalize_evaluation_result( @@ -65,15 +118,22 @@ async def finalize_evaluation_result( # aggregation and corpus inputs honor ``skip_errored``. row_scores = [row_score for _, _, row_score in eval_results] - aggregated_result = aggregate_metrics(metric_results) + output_spec = metric.output_spec() + rubric_definitions = rubric_definitions_from_metric(metric) + if rubric_definitions: + aggregated_result = aggregate_metrics(metric_results, output_spec, rubric_definitions=rubric_definitions) + else: + aggregated_result = aggregate_metrics(metric_results, output_spec) if valid_eval_results and isinstance(metric, CorpusMetric): corpus_metric_result = await metric.compute_corpus_scores( - items=[row_score.item for _, row_score in valid_eval_results], - samples=[row_score.sample for _, row_score in valid_eval_results], + inputs=[ + build_metric_input(row_score.item, row_score.sample, row_score.row_index) + for _, row_score in valid_eval_results + ], ) if corpus_metric_result: - add_corpus_scores(aggregated_result, corpus_metric_result) + add_corpus_scores(aggregated_result, corpus_metric_result, corpus_output_spec(metric, output_spec)) return EvaluationResult( row_scores=row_scores, @@ -117,13 +177,16 @@ async def score_row( active_logger = logger or globals()["logger"] try: - result = await metric.compute_scores(row, sample) + output_spec = metric.output_spec() + result = validate_metric_result( + await metric.compute_scores(build_metric_input(row, sample, index)), output_spec + ) active_logger.debug( "Computed metric", extra={ "item_index": index, "metric_type": metric_type_name(metric), - "scores": [score.model_dump() for score in result.scores], + "outputs": [output.model_dump() for output in result.outputs], }, ) return ( @@ -133,7 +196,7 @@ async def score_row( row_index=index, item=row, sample=sample, - metrics={metric_key: result.scores}, + metrics={metric_key: result.outputs}, requests=[*generation_requests, *metric_requests], ), ) @@ -146,7 +209,7 @@ async def score_row( metric_key=metric_key, ) from e active_logger.warning("Evaluation failed, marking as NaN", extra={"item_index": index, "error": str(e)}) - result = nan_metric_result(metric.score_names()) + result = nan_metric_result(metric.output_spec()) return ( index, result, @@ -154,7 +217,7 @@ async def score_row( row_index=index, item=row, sample=sample, - metrics={metric_key: result.scores}, + metrics={metric_key: result.outputs}, requests=[*generation_requests, *metric_requests], metric_errors={metric_key: str(e)}, ), diff --git a/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/execution/utils.py b/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/execution/utils.py index 52bb499370..0060a061c8 100644 --- a/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/execution/utils.py +++ b/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/execution/utils.py @@ -11,7 +11,7 @@ from typing import cast from nemo_evaluator_sdk.execution._protocols import JobParamsConfigurableMetric -from nemo_evaluator_sdk.metrics.base import Metric, MetricWithPreflight, MetricWithSecrets +from nemo_evaluator_sdk.metrics.protocol import Metric, MetricWithPreflight, MetricWithSecrets from nemo_evaluator_sdk.metrics.utils import metric_type_name from nemo_evaluator_sdk.values.params import RunConfig from pydantic import BaseModel diff --git a/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/metrics/aggregation.py b/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/metrics/aggregation.py index 350006de0f..a0261250dd 100644 --- a/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/metrics/aggregation.py +++ b/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/metrics/aggregation.py @@ -7,7 +7,17 @@ import math from collections import OrderedDict, defaultdict - +from collections.abc import Mapping, Sequence +from typing import Protocol, cast, runtime_checkable + +from nemo_evaluator_sdk.metrics.protocol import ( + BooleanValue, + ContinuousScore, + DiscreteScore, + MetricOutput, + MetricOutputSpec, + MetricResult, +) from nemo_evaluator_sdk.values.results import ( AggregatedMetricResult, AggregateRangeScore, @@ -15,15 +25,100 @@ AggregateScore, Histogram, HistogramBin, - MetricResult, MetricScore, Percentiles, RubricScoreStat, ScoreStats, ) +from nemo_evaluator_sdk.values.scores import RubricScore, Score + + +def is_aggregateable_output_spec(output_spec: MetricOutputSpec) -> bool: + """Return whether an output should contribute aggregate statistics.""" + return issubclass(output_spec.value_schema, (ContinuousScore, DiscreteScore, BooleanValue)) + + +@runtime_checkable +class MetricWithScores(Protocol): + """Metric config protocol for metrics that carry rubric score definitions.""" + + @property + def scores(self) -> Sequence[Score]: ... + + +def _coerce_aggregate_output(output: MetricOutput, output_spec: MetricOutputSpec) -> MetricScore | None: + """Convert one declared aggregateable metric output into MetricScore form.""" + if not is_aggregateable_output_spec(output_spec): + return None + if ( + issubclass(output_spec.value_schema, BooleanValue) + and isinstance(output.value, float) + and math.isnan(output.value) + ): + return MetricScore(name=output.name, value=output.value) + coerced = cast(ContinuousScore | DiscreteScore | BooleanValue, output_spec.coerce_output(output)) + value = coerced.root + if isinstance(value, bool): + value = 1.0 if value else 0.0 + return MetricScore(name=output.name, value=value) + + +def _attach_rubric_stats( + score: MetricScore, + output_by_name: Mapping[str, MetricOutput], + rubric_definitions: Mapping[str, Sequence[RubricScoreStat]], +) -> MetricScore: + """Attach per-row rubric bucket stats from companion label outputs.""" + rubric_definition = rubric_definitions.get(score.name) + if not rubric_definition: + return score + + label_output = output_by_name.get(f"{score.name}.label") + selected_label = label_output.value if label_output is not None else None + if isinstance(score.value, float) and math.isnan(score.value): + selected_label = None + + rubric_distribution = [ + RubricScoreStat( + label=rubric.label, + description=rubric.description, + value=rubric.value, + count=int(isinstance(selected_label, str) and selected_label == rubric.label), + ) + for rubric in rubric_definition + ] + return MetricScore( + name=score.name, + value=score.value, + stats=ScoreStats(rubric_distribution=rubric_distribution), + ) -def add_corpus_scores(aggregated_result: AggregatedMetricResult, corpus_result: MetricResult) -> None: +def _aggregateable_scores( + result: MetricResult, + output_specs: list[MetricOutputSpec], + rubric_definitions: Mapping[str, Sequence[RubricScoreStat]] | None = None, +) -> list[MetricScore]: + """Extract score-like outputs from a metric result using declared output specs.""" + specs_by_name = {output_spec.name: output_spec for output_spec in output_specs} + output_by_name = {output.name: output for output in result.outputs} + scores: list[MetricScore] = [] + for output in result.outputs: + output_spec = specs_by_name.get(output.name) + if output_spec is None: + continue + score = _coerce_aggregate_output(output, output_spec) + if score is not None: + score = _attach_rubric_stats(score, output_by_name, rubric_definitions or {}) + scores.append(score) + return scores + + +def add_corpus_scores( + aggregated_result: AggregatedMetricResult, + corpus_result: MetricResult, + output_specs: list[MetricOutputSpec], +) -> None: """Append corpus-level scores using aggregate-score schema fields. Args: @@ -33,7 +128,7 @@ def add_corpus_scores(aggregated_result: AggregatedMetricResult, corpus_result: Returns: ``None``. The ``aggregated_result`` object is updated in place. """ - for score in corpus_result.scores: + for score in _aggregateable_scores(corpus_result, output_specs): value = score.value # Corpus-level metrics contribute one already-aggregated value, so # expose them through the same aggregate schema with count=1. @@ -148,7 +243,11 @@ def _compute_histogram(values: list[float], num_bins: int = 10) -> Histogram: return Histogram(bins=bins) -def aggregate_metrics(items: list[MetricResult]) -> AggregatedMetricResult: +def aggregate_metrics( + items: list[MetricResult], + output_specs: list[MetricOutputSpec], + rubric_definitions: Mapping[str, Sequence[RubricScoreStat]] | None = None, +) -> AggregatedMetricResult: """Aggregate row-level metric results into range or rubric summaries. This function performs two logical passes: @@ -158,6 +257,11 @@ def aggregate_metrics(items: list[MetricResult]) -> AggregatedMetricResult: Args: items: Row-level metric results to aggregate. + output_specs: Declared outputs for the metric. Only continuous, + discrete, and boolean output values contribute to aggregate scores. + rubric_definitions: Optional rubric bucket definitions keyed by numeric + output name. This is aggregation metadata, not metric protocol + metadata, and is usually derived from LLM judge score config. Returns: Aggregate metric result with one aggregate score per score name. @@ -169,7 +273,7 @@ def aggregate_metrics(items: list[MetricResult]) -> AggregatedMetricResult: has_rubric: dict[str, bool] = {} for item in items: - for score in item.scores: + for score in _aggregateable_scores(item, output_specs, rubric_definitions): if score.name not in aggregated_results: # Keep one running accumulator per score name; distribution # details are materialized in a second pass once all values exist. @@ -317,3 +421,31 @@ def aggregate_metrics(items: list[MetricResult]) -> AggregatedMetricResult: ) return AggregatedMetricResult(scores=aggregated_scores) + + +def rubric_definitions_from_scores(scores: Sequence[Score]) -> dict[str, list[RubricScoreStat]]: + """Return declared rubric buckets keyed by score name.""" + definitions: dict[str, list[RubricScoreStat]] = {} + for score in scores: + if not isinstance(score, RubricScore): + continue + definitions[score.name] = [ + RubricScoreStat( + label=rubric.label, + description=rubric.description, + value=rubric.value, + count=0, + ) + for rubric in score.rubric + ] + return definitions + + +def rubric_definitions_from_metric(metric: object) -> dict[str, list[RubricScoreStat]]: + """Return rubric bucket definitions for metrics that carry score config.""" + if not isinstance(metric, MetricWithScores): + return {} + scores = metric.scores + if not isinstance(scores, Sequence) or isinstance(scores, (str, bytes)): + return {} + return rubric_definitions_from_scores(scores) diff --git a/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/metrics/base.py b/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/metrics/base.py deleted file mode 100644 index e90fbfe5cb..0000000000 --- a/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/metrics/base.py +++ /dev/null @@ -1,117 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""Optional metric capabilities and shared helpers for evaluator SDK runtime.""" - -# Migrated from: services/evaluator/src/nmp/evaluator/app/metrics/base.py - -import re -import string -from typing import Awaitable, Callable, Protocol, runtime_checkable - -from nemo_evaluator_sdk.values import SecretRef -from nemo_evaluator_sdk.values.results import MetricResult - -SecretResolver = Callable[[str], Awaitable[str | None]] - - -@runtime_checkable -class Metric(Protocol): - """Structural contract for SDK runtime metrics used by generic evaluator code. - - This protocol describes what execution and orchestration code may rely on - when working with a metric instance. In particular, ``type`` is treated as - a string identifier. Built-in metrics may expose existing ``MetricType`` - values for schema compatibility, but custom metrics should use plain - strings: - - - a built-in ``MetricType`` member - - a plain string such as ``"my-custom-metric"`` - - Generic consumers must therefore treat ``type`` as a string identifier and - must not depend on enum-only APIs such as ``.value``. Callers that need to - normalize supported runtime shapes should use ``metric_type_name(...)``. - """ - - @property - def type(self) -> str: - """Return the public metric key/type identifier. - - Examples: - Built-in runtime metrics may expose ``MetricType.BLEU``. - - Custom metrics may expose a plain string such as - ``"my-custom-metric"``. - """ - ... - - async def compute_scores(self, item: dict, sample: dict) -> MetricResult: - """Compute structured score output for one item/sample pair.""" - ... - - def score_names(self) -> list[str]: - """Return canonical score names emitted by this metric.""" - ... - - -@runtime_checkable -class CorpusMetric(Protocol): - """Protocol for metrics that also emit corpus-level scores.""" - - async def compute_corpus_scores(self, items: list[dict], samples: list[dict]) -> MetricResult | None: - """Compute corpus-level scores across all evaluated rows. - - Args: - items: Original dataset rows. - samples: Sample payloads paired to ``items``. - - Returns: - Optional corpus-level metric result. - """ - ... - - -@runtime_checkable -class MetricWithSecrets(Protocol): - """Protocol for metrics that require secrets (e.g., API keys).""" - - def secrets(self) -> dict[str, SecretRef]: - """ - Returns a dictionary of environment variables to the secret reference. - Used by the job flow to set up environment variables. - """ - ... - - async def resolve_secrets(self, secret_resolver: SecretResolver) -> None: - """ - Resolve secrets using the provided resolver function. - Called before the metric is used for evaluation. - """ - ... - - -@runtime_checkable -class MetricWithPreflight(Protocol): - """Protocol for metrics that need one-time setup before parallel evaluation starts.""" - - async def preflight(self) -> None: - """Run one-time preflight (e.g., capability detection) before processing rows.""" - ... - - -# TODO: migrate the rest of the protocols from services/evaluator/src/nmp/evaluator/app/metrics/base.py - - -def normalize_text(s: str) -> str: - """Normalize free-form text for token/equality-based metric comparisons.""" - if not s: - return "" - # lower case - s = s.lower() - # remove punctuation - s = "".join(ch for ch in s if ch not in set(string.punctuation)) - # remove articles - s = re.sub(r"\b(a|an|the)\b", " ", s) - # collapse whitespace - s = " ".join(s.split()) - return s diff --git a/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/metrics/bleu.py b/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/metrics/bleu.py index b14195a095..8c08460791 100644 --- a/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/metrics/bleu.py +++ b/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/metrics/bleu.py @@ -4,14 +4,15 @@ """BLEU metric runtime implementation.""" import sacrebleu +from nemo_evaluator_sdk.metrics.protocol import MetricInput, MetricOutput, MetricOutputSpec, MetricResult from nemo_evaluator_sdk.metrics.template_rendering import ( + TemplateSample, build_template_context, render_default_output_text_candidate_or_raise, render_template_or_raise, template_metric_repr, ) from nemo_evaluator_sdk.values.metrics import BLEU -from nemo_evaluator_sdk.values.results import MetricResult, MetricScore __all__ = ["BLEUMetric"] @@ -24,11 +25,15 @@ class BLEUMetric(BLEU): ``sample.output_text`` when the evaluator generates model outputs online. """ - def score_names(self) -> list[str]: - """Return score keys emitted by this metric.""" - return ["sentence"] + def output_spec(self) -> list[MetricOutputSpec]: + """Return row-level outputs emitted by this metric.""" + return [MetricOutputSpec.continuous_score("sentence")] - def _render_references(self, item: dict, sample: dict) -> list[str]: + def corpus_output_spec(self) -> list[MetricOutputSpec]: + """Return corpus-level outputs emitted by this metric.""" + return [MetricOutputSpec.continuous_score("corpus")] + + def _render_references(self, item: dict, sample: TemplateSample) -> list[str]: """Render all reference templates for one row.""" context = build_template_context(item, sample) metric_repr = template_metric_repr(self) @@ -47,7 +52,7 @@ def _render_references(self, item: dict, sample: dict) -> list[str]: references.append(rendered_reference) return references - def _render_candidate(self, item: dict, sample: dict) -> str: + def _render_candidate(self, item: dict, sample: TemplateSample) -> str: """Render the candidate text for one row.""" if self.candidate: context = build_template_context(item, sample) @@ -69,18 +74,21 @@ def _render_candidate(self, item: dict, sample: dict) -> str: raise TypeError("The candidate must be a string.") return prediction - async def compute_scores(self, item: dict, sample: dict) -> MetricResult: + async def compute_scores(self, input: MetricInput) -> MetricResult: """Compute the scores for the metric.""" + item = input.row.data + sample = input.candidate references = self._render_references(item, sample) candidate = self._render_candidate(item, sample) score = sacrebleu.sentence_bleu(candidate, references).score.real - return MetricResult(scores=[MetricScore(name="sentence", value=score)]) + return MetricResult(outputs=[MetricOutput(name="sentence", value=score)]) - async def compute_corpus_scores(self, items: list[dict], samples: list[dict]) -> MetricResult | None: + async def compute_corpus_scores(self, inputs: list[MetricInput]) -> MetricResult | None: """Compute the corpus-level BLEU metric.""" - references_raw = [self._render_references(item, sample) for item, sample in zip(items, samples)] + item_sample_pairs = [(input.row.data, input.candidate) for input in inputs] + references_raw = [self._render_references(item, sample) for item, sample in item_sample_pairs] # NOTE: because of the bug in sacrebleu, we need to flatten the references references = [[reference_set[0] for reference_set in references_raw]] - candidates = [self._render_candidate(item, sample) for item, sample in zip(items, samples)] + candidates = [self._render_candidate(item, sample) for item, sample in item_sample_pairs] score = sacrebleu.corpus_bleu(candidates, references) - return MetricResult(scores=[MetricScore(name="corpus", value=score.score.real)]) + return MetricResult(outputs=[MetricOutput(name="corpus", value=score.score.real)]) diff --git a/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/metrics/exact_match.py b/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/metrics/exact_match.py index e0f1c5e8eb..758bfc1186 100644 --- a/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/metrics/exact_match.py +++ b/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/metrics/exact_match.py @@ -3,10 +3,10 @@ """Exact-match metric runtime implementation.""" -from nemo_evaluator_sdk.metrics.base import normalize_text +from nemo_evaluator_sdk.metrics.protocol import MetricInput, MetricOutput, MetricOutputSpec, MetricResult from nemo_evaluator_sdk.metrics.template_rendering import render_reference_and_candidate, template_metric_repr +from nemo_evaluator_sdk.metrics.utils import normalize_text from nemo_evaluator_sdk.values.metrics import ExactMatch -from nemo_evaluator_sdk.values.results import MetricResult, MetricScore __all__ = ["ExactMatchMetric"] @@ -18,15 +18,11 @@ class ExactMatchMetric(ExactMatch): model outputs through ``sample.output_text`` for online execution. """ - def score_names(self) -> list[str]: - """Return score keys emitted by this metric. + def output_spec(self) -> list[MetricOutputSpec]: + """Return outputs emitted by this metric.""" + return [MetricOutputSpec.continuous_score(self.type.value)] - Returns: - Single-item list containing the exact-match metric type name. - """ - return [self.type.value] - - async def compute_scores(self, item: dict, sample: dict) -> MetricResult: + async def compute_scores(self, input: MetricInput) -> MetricResult: """Compute structured metric output for one item/sample pair. The algorithm renders reference and candidate text from templates, then @@ -34,8 +30,7 @@ async def compute_scores(self, item: dict, sample: dict) -> MetricResult: before equality comparison. Args: - item: Original dataset row. - sample: Generated-sample payload used for candidate extraction. + input: Original dataset row paired with candidate output. Returns: ``MetricResult`` with one exact-match score entry. @@ -50,8 +45,8 @@ async def compute_scores(self, item: dict, sample: dict) -> MetricResult: metric_name=self.__class__.__name__, reference_template=self.reference, candidate_template=self.candidate, - item=item, - sample=sample, + item=input.row.data, + sample=input.candidate, ) score = int(normalize_text(prediction) == normalize_text(ground_truth)) - return MetricResult(scores=[MetricScore(name=self.type.value, value=score)]) + return MetricResult(outputs=[MetricOutput(name=self.type.value, value=score)]) diff --git a/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/metrics/f1.py b/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/metrics/f1.py index 7c761e7811..a099d5c7c1 100644 --- a/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/metrics/f1.py +++ b/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/metrics/f1.py @@ -5,10 +5,10 @@ import collections -from nemo_evaluator_sdk.metrics.base import normalize_text +from nemo_evaluator_sdk.metrics.protocol import MetricInput, MetricOutput, MetricOutputSpec, MetricResult from nemo_evaluator_sdk.metrics.template_rendering import render_reference_and_candidate, template_metric_repr +from nemo_evaluator_sdk.metrics.utils import normalize_text from nemo_evaluator_sdk.values.metrics import F1 -from nemo_evaluator_sdk.values.results import MetricResult, MetricScore __all__ = ["F1Metric"] @@ -16,19 +16,19 @@ class F1Metric(F1): """F1 metric for token-overlap similarity scoring.""" - def score_names(self) -> list[str]: - """Return score keys emitted by this metric.""" - return [self.type.value] + def output_spec(self) -> list[MetricOutputSpec]: + """Return outputs emitted by this metric.""" + return [MetricOutputSpec.continuous_score(self.type.value)] - async def compute_scores(self, item: dict, sample: dict) -> MetricResult: + async def compute_scores(self, input: MetricInput) -> MetricResult: """Compute the scores for the metric.""" ground_truth, prediction = render_reference_and_candidate( metric_repr=template_metric_repr(self), metric_name=self.__class__.__name__, reference_template=self.reference, candidate_template=self.candidate, - item=item, - sample=sample, + item=input.row.data, + sample=input.candidate, ) prediction_tokens = normalize_text(prediction).split() @@ -37,14 +37,14 @@ async def compute_scores(self, item: dict, sample: dict) -> MetricResult: # If either token list is empty, the F1 is 1.0 if they agree, 0.0 otherwise. if len(ground_truth_tokens) == 0 or len(prediction_tokens) == 0: score = float(ground_truth_tokens == prediction_tokens) - return MetricResult(scores=[MetricScore(name=self.type.value, value=score)]) + return MetricResult(outputs=[MetricOutput(name=self.type.value, value=score)]) common = collections.Counter(prediction_tokens) & collections.Counter(ground_truth_tokens) num_same = sum(common.values()) if num_same == 0: - return MetricResult(scores=[MetricScore(name=self.type.value, value=0.0)]) + return MetricResult(outputs=[MetricOutput(name=self.type.value, value=0.0)]) precision = 1.0 * num_same / len(prediction_tokens) recall = 1.0 * num_same / len(ground_truth_tokens) score = (2 * precision * recall) / (precision + recall) - return MetricResult(scores=[MetricScore(name=self.type.value, value=score)]) + return MetricResult(outputs=[MetricOutput(name=self.type.value, value=score)]) diff --git a/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/metrics/llm_judge.py b/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/metrics/llm_judge.py index 3a922e42fc..34312ae93b 100644 --- a/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/metrics/llm_judge.py +++ b/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/metrics/llm_judge.py @@ -13,7 +13,17 @@ from nemo_evaluator_sdk.inference import InferenceFn, InferenceHookParams from nemo_evaluator_sdk.inference import new_hooks as _new_inference_hooks from nemo_evaluator_sdk.metrics.hooks import HooksBase -from nemo_evaluator_sdk.metrics.template_rendering import build_template_context +from nemo_evaluator_sdk.metrics.protocol import ( + MetricInput, + MetricOutput, + MetricOutputSpec, + MetricResult, +) +from nemo_evaluator_sdk.metrics.template_rendering import ( + TemplateSample, + build_template_context, + sample_template_payload, +) from nemo_evaluator_sdk.structured_output import InferenceStructuredOutput, detect_structured_output_mode from nemo_evaluator_sdk.templates import render_request from nemo_evaluator_sdk.values.common import SecretRef, SupportedJobTypes @@ -26,7 +36,7 @@ ) from nemo_evaluator_sdk.values.models import Model from nemo_evaluator_sdk.values.params import InferenceParams, ReasoningParams -from nemo_evaluator_sdk.values.results import MetricResult, MetricScore +from nemo_evaluator_sdk.values.results import MetricScore from nemo_evaluator_sdk.values.scores import ( JSONScoreParser, RangeScore, @@ -107,9 +117,21 @@ def set_inference_fn(self, inference_fn: InferenceFn) -> None: """Set the inference function to use for LLM calls.""" self._inference_fn = inference_fn - def score_names(self) -> list[str]: - """Return score keys emitted by this metric.""" - return list(self._parsers.keys()) + def output_spec(self) -> list[MetricOutputSpec]: + """Return outputs emitted by this metric.""" + specs: list[MetricOutputSpec] = [] + for score in self.scores: + if isinstance(score, RubricScore): + specs.append(MetricOutputSpec.continuous_score(score.name, description=score.description)) + specs.append( + MetricOutputSpec.label( + f"{score.name}.label", + description=f"Selected rubric label for {score.name}", + ) + ) + else: + specs.append(MetricOutputSpec.continuous_score(score.name, description=score.description)) + return specs def _handle_none_output_error(self, response: dict) -> ValueError: error_message = "LLM judge returned no usable textual content for score parsing" @@ -141,7 +163,12 @@ def _handle_invalid_output(self, error: Exception, fallback: MetricResult, messa raise error def _nan_result(self) -> MetricResult: - return MetricResult(scores=[MetricScore(name=name, value=float("nan")) for name in self._parsers]) + outputs: list[MetricOutput] = [] + for score in self.scores: + outputs.append(MetricOutput(name=score.name, value=float("nan"))) + if isinstance(score, RubricScore): + outputs.append(MetricOutput(name=f"{score.name}.label", value="")) + return MetricResult(outputs=outputs) async def resolve_secrets(self, secret_resolver: Callable[[str], Awaitable[str | None]]) -> None: """Resolve API key secret if configured and reinitialize AsyncOpenAI client. Must be called before using the metric.""" @@ -228,8 +255,9 @@ def _initialize_score_parsers(self) -> None: self._parsers[score.name] = parser self._score_dumps[score.name] = score.model_dump(mode="json", exclude={"parser"}) - def _render_request(self, item: dict, sample: dict) -> dict: - overlapping_keys = set(item.keys()) & set(sample.keys()) + def _render_request(self, item: dict, sample: TemplateSample) -> dict: + sample_payload = sample_template_payload(sample) + overlapping_keys = set(item.keys()) & set(sample_payload.keys()) if overlapping_keys: _logger.warning( "Dataset columns %s overlap with model response keys. " @@ -261,8 +289,10 @@ def _retry_with_max_completion_tokens(self, request: dict) -> dict: del request["max_tokens"] return request - async def compute_scores(self, item: dict, sample: dict) -> MetricResult: + async def compute_scores(self, input: MetricInput) -> MetricResult: """Compute structured score output for one item/sample pair.""" + item = input.row.data + sample = input.candidate request = self._render_request(item, sample) try: @@ -290,14 +320,27 @@ async def compute_scores(self, item: dict, sample: dict) -> MetricResult: "LLM judge returned invalid output, marking as NaN", ) - result = MetricResult(scores=[]) + result = MetricResult(outputs=[]) for score_name, parser in self._parsers.items(): score = parser.parse(output_text) _logger.debug("Parsed score %s: %s", score_name, score.value) - result.scores.append(score) + result.outputs.append(MetricOutput(name=score.name, value=score.value)) + label = _selected_rubric_label(score) + if label is not None: + result.outputs.append(MetricOutput(name=f"{score.name}.label", value=label)) return result +def _selected_rubric_label(score: MetricScore) -> str | None: + """Return the selected rubric label recorded by the parser, if any.""" + if not score.stats or not score.stats.rubric_distribution: + return None + for rubric_stat in score.stats.rubric_distribution: + if rubric_stat.count: + return rubric_stat.label + return "" + + def new_hooks(params: _LLMJudgeHookParams | None): """Initialize preprocess and postprocess hooks for the LLM judge.""" model_format = params.model.format if params else ModelFormat.NVIDIA_NIM diff --git a/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/metrics/number_check.py b/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/metrics/number_check.py index 639d7a5f09..58c532fcba 100644 --- a/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/metrics/number_check.py +++ b/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/metrics/number_check.py @@ -6,13 +6,14 @@ import math import re +from nemo_evaluator_sdk.metrics.protocol import MetricInput, MetricOutput, MetricOutputSpec, MetricResult from nemo_evaluator_sdk.metrics.template_rendering import ( + TemplateSample, build_template_context, render_template_or_raise, template_metric_repr, ) from nemo_evaluator_sdk.values.metrics import NumberCheck, NumberCheckOperation -from nemo_evaluator_sdk.values.results import MetricResult, MetricScore __all__ = ["NumberCheckMetric", "NumberCheckOperation"] @@ -35,12 +36,14 @@ def _parse_number_answer(answer: str) -> int | float: class NumberCheckMetric(NumberCheck): """Numeric-comparison metric with template-driven operands.""" - def score_names(self) -> list[str]: - """Return score keys emitted by this metric.""" - return [self.type.value] + def output_spec(self) -> list[MetricOutputSpec]: + """Return outputs emitted by this metric.""" + return [MetricOutputSpec.continuous_score(self.type.value)] - async def compute_scores(self, item: dict, sample: dict) -> MetricResult: + async def compute_scores(self, input: MetricInput) -> MetricResult: """Compute the scores for the metric.""" + item = input.row.data + sample: TemplateSample = input.candidate context = build_template_context(item, sample) metric_repr = template_metric_repr(self) left_value = render_template_or_raise( @@ -65,9 +68,9 @@ async def compute_scores(self, item: dict, sample: dict) -> MetricResult: # Preserve the legacy behavior: if either side fails to parse as a # number, return NaN instead of raising. if math.isnan(left_number): - return MetricResult(scores=[MetricScore(name=self.type.value, value=left_number)]) + return MetricResult(outputs=[MetricOutput(name=self.type.value, value=left_number)]) if math.isnan(right_number): - return MetricResult(scores=[MetricScore(name=self.type.value, value=right_number)]) + return MetricResult(outputs=[MetricOutput(name=self.type.value, value=right_number)]) # Perform the requested numeric comparison on the parsed operands. if self.operation in ["equals", "=="]: @@ -89,4 +92,4 @@ async def compute_scores(self, item: dict, sample: dict) -> MetricResult: else: raise ValueError(f"Unsupported operation: {self.operation}") - return MetricResult(scores=[MetricScore(name=self.type.value, value=1.0 if score else 0.0)]) + return MetricResult(outputs=[MetricOutput(name=self.type.value, value=1.0 if score else 0.0)]) diff --git a/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/metrics/protocol.py b/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/metrics/protocol.py new file mode 100644 index 0000000000..f93f9a34bd --- /dev/null +++ b/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/metrics/protocol.py @@ -0,0 +1,241 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Runtime protocol for implementing Evaluator metrics.""" + +from __future__ import annotations + +import math +from collections.abc import Awaitable, Callable +from typing import Any, Protocol, runtime_checkable + +from nemo_evaluator_sdk.values.common import SecretRef +from pydantic import BaseModel, ConfigDict, Field, RootModel, field_serializer, field_validator + +SecretResolver = Callable[[str], Awaitable[str | None]] + + +class DatasetRow(BaseModel): + """Original dataset row plus optional stable row identity.""" + + model_config = ConfigDict(extra="forbid") + + row_index: int | None = None + data: dict[str, Any] + + +class CandidateOutput(BaseModel): + """Candidate or prediction output being scored for one dataset row.""" + + model_config = ConfigDict(extra="forbid") + + output_text: str | None = None + response: Any | None = None + trajectory: Any | None = None + metadata: dict[str, Any] = Field(default_factory=dict) + + def as_sample(self) -> dict[str, Any]: + """Return a sample-shaped payload for template rendering helpers.""" + sample = dict(self.metadata) + if self.output_text is not None: + sample["output_text"] = self.output_text + if self.response is not None: + sample["response"] = self.response + if self.trajectory is not None: + sample["trajectory"] = self.trajectory + return sample + + +class MetricInput(BaseModel): + """Complete per-row scoring input passed to a metric.""" + + model_config = ConfigDict(extra="forbid") + + row: DatasetRow + candidate: CandidateOutput + + +class ContinuousScore(RootModel[float]): + """Continuous numeric metric value.""" + + +class DiscreteScore(RootModel[int]): + """Discrete numeric metric value.""" + + +class Label(RootModel[str]): + """String label metric value.""" + + +class BooleanValue(RootModel[bool]): + """Boolean metric value.""" + + +class MetricOutputSpec(BaseModel): + """Schema for one named value emitted by a metric.""" + + model_config = ConfigDict(extra="forbid", arbitrary_types_allowed=True) + + name: str + description: str | None = None + value_schema: type[BaseModel] + + @field_validator("name") + @classmethod + def _name_must_not_be_empty(cls, value: str) -> str: + if not value: + raise ValueError("metric output name must not be empty") + return value + + @staticmethod + def continuous_score(name: str, description: str | None = None) -> "MetricOutputSpec": + return MetricOutputSpec(name=name, description=description, value_schema=ContinuousScore) + + @staticmethod + def discrete_score(name: str, description: str | None = None) -> "MetricOutputSpec": + return MetricOutputSpec(name=name, description=description, value_schema=DiscreteScore) + + @staticmethod + def label(name: str, description: str | None = None) -> "MetricOutputSpec": + return MetricOutputSpec(name=name, description=description, value_schema=Label) + + @staticmethod + def boolean(name: str, description: str | None = None) -> "MetricOutputSpec": + return MetricOutputSpec(name=name, description=description, value_schema=BooleanValue) + + @staticmethod + def model(name: str, value_schema: type[BaseModel], description: str | None = None) -> "MetricOutputSpec": + return MetricOutputSpec(name=name, description=description, value_schema=value_schema) + + def coerce_value(self, value: Any) -> BaseModel: + """Validate and coerce a raw output value to this spec's declared schema.""" + return self.value_schema.model_validate(value) + + def coerce_output(self, output: "MetricOutput") -> BaseModel: + """Validate and coerce a named metric output against this spec.""" + if output.name != self.name: + raise ValueError(f"Expected metric output {self.name!r}, got {output.name!r}") + return self.coerce_value(output.value) + + def value_json_schema(self) -> dict[str, Any]: + return self.value_schema.model_json_schema() + + +class MetricDescriptor(BaseModel): + """Metadata describing a metric implementation and its declared outputs.""" + + model_config = ConfigDict(extra="forbid") + + type: str + outputs: list[MetricOutputSpec] = Field(min_length=1) + + @field_validator("type") + @classmethod + def _type_must_not_be_empty(cls, value: str) -> str: + if not value: + raise ValueError("metric type must not be empty") + return value + + @field_validator("outputs") + @classmethod + def _output_names_must_be_unique(cls, value: list[MetricOutputSpec]) -> list[MetricOutputSpec]: + names = [output.name for output in value] + duplicates = sorted({name for name in names if names.count(name) > 1}) + if duplicates: + raise ValueError(f"duplicate metric output names: {duplicates}") + return value + + +class MetricOutput(BaseModel): + """One named value emitted by a metric.""" + + model_config = ConfigDict(extra="forbid") + + name: str + value: Any + + @field_serializer("value") + def serialize_nan(self, value: Any) -> Any: + if isinstance(value, float) and math.isnan(value): + return "NaN" + return value + + +class MetricResult(BaseModel): + """Structured row-level metric result.""" + + model_config = ConfigDict(extra="forbid") + + outputs: list[MetricOutput] + + +@runtime_checkable +class Metric(Protocol): + """Shared row-scoring primitive for SDK runtime metrics.""" + + @property + def type(self) -> str: + """Return the public metric key/type identifier.""" + ... + + def output_spec(self) -> list[MetricOutputSpec]: + """Return declared row-level outputs emitted by this metric.""" + ... + + async def compute_scores(self, input: MetricInput) -> MetricResult: + """Compute structured output for one row/candidate pair.""" + ... + + +@runtime_checkable +class CorpusMetric(Protocol): + """Protocol for metrics that also emit corpus-level scores.""" + + async def compute_corpus_scores(self, inputs: list[MetricInput]) -> MetricResult | None: + """Compute corpus-level scores across all evaluated rows.""" + ... + + +@runtime_checkable +class MetricWithSecrets(Protocol): + """Protocol for metrics that require secrets.""" + + def secrets(self) -> dict[str, SecretRef]: + """Return environment variables mapped to secret references.""" + ... + + async def resolve_secrets(self, secret_resolver: SecretResolver) -> None: + """Resolve secrets before the metric is used for evaluation.""" + ... + + +@runtime_checkable +class MetricWithPreflight(Protocol): + """Protocol for metrics that need one-time setup before parallel evaluation starts.""" + + async def preflight(self) -> None: + """Run one-time preflight before processing rows.""" + ... + + +def validate_metric_result(result: MetricResult, outputs: list[MetricOutputSpec]) -> MetricResult: + """Validate a metric result against its declared outputs.""" + returned_names = [output.name for output in result.outputs] + duplicates = sorted({name for name in returned_names if returned_names.count(name) > 1}) + if duplicates: + raise ValueError(f"Duplicate metric output names: {duplicates}") + + outputs_by_name = {output.name: output for output in outputs} + declared_names = [output.name for output in outputs] + declared = set(declared_names) + returned = set(returned_names) + missing = [name for name in declared_names if name not in returned] + undeclared = [name for name in returned_names if name not in declared] + + if missing: + raise ValueError(f"Missing declared metric outputs: {missing}") + if undeclared: + raise ValueError(f"Undeclared metric outputs: {undeclared}") + for output in result.outputs: + outputs_by_name[output.name].coerce_output(output) + return result diff --git a/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/metrics/ragas/base.py b/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/metrics/ragas/base.py index fb4a7ed9c6..525391d5c0 100644 --- a/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/metrics/ragas/base.py +++ b/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/metrics/ragas/base.py @@ -16,6 +16,7 @@ import nemo_evaluator_sdk.constants as constants from nemo_evaluator_sdk.enums import MetricType from nemo_evaluator_sdk.inference import get_logger, requests_log_var +from nemo_evaluator_sdk.metrics.protocol import MetricInput, MetricOutput, MetricOutputSpec, MetricResult # Lazy imports for RAGAS - these are getter functions that defer the expensive # RAGAS/langchain imports (~20-30s) until first use, improving startup time. @@ -29,8 +30,6 @@ from nemo_evaluator_sdk.templates import render_request from nemo_evaluator_sdk.values import ( MetricBase, - MetricResult, - MetricScore, Model, SecretRef, ) @@ -108,9 +107,10 @@ class BaseRAGASMetric(MetricBase): _secrets: dict[str, SecretRef] = PrivateAttr(default_factory=dict) _log: logging.Logger = logging.getLogger(__name__) - def score_names(self) -> list[str]: + def output_spec(self) -> list[MetricOutputSpec]: + """Return outputs emitted by this metric.""" if isinstance(self.type, MetricType): - return [self.type.value] + return [MetricOutputSpec.continuous_score(self.type.value)] return [] def __init__(self, logger: logging.Logger | None = None, **data): @@ -220,7 +220,7 @@ def _nan_scores_for_metrics(self, metrics: list) -> dict[str, float]: metric_names.append(metric_name) if not metric_names: - metric_names = self.score_names() + metric_names = [output.name for output in self.output_spec()] return {metric_name: float("nan") for metric_name in metric_names} @@ -363,11 +363,15 @@ def _create_evaluation_dataset(self, item: dict, sample: dict) -> EvaluationData if response: payload["response"] = response - return EvaluationDatasetCls.from_list([payload]) + return cast(Any, EvaluationDatasetCls).from_list([payload]) - async def compute_scores(self, item: dict, sample: dict) -> MetricResult: + async def compute_scores(self, input: MetricInput) -> MetricResult: """Compute the scores for the metric.""" - return await _run_function_in_plain_loop(self.compute_scores_async, item, sample) + return await _run_function_in_plain_loop( + self.compute_scores_async, + input.row.data, + input.candidate.as_sample(), + ) async def compute_scores_async(self, item: dict, sample: dict) -> MetricResult: """Compute the scores for the metric asynchronously.""" @@ -376,7 +380,9 @@ async def compute_scores_async(self, item: dict, sample: dict) -> MetricResult: llm_judge = self._get_llm_judge(client) scores = self._metric(data, llm_judge) return MetricResult( - scores=[MetricScore(name=metric_name, value=score_value) for metric_name, score_value in scores.items()] + outputs=[ + MetricOutput(name=metric_name, value=score_value) for metric_name, score_value in scores.items() + ] ) def _metric(self, data: EvaluationDataset, llm_judge: LangchainLLMWrapper | None) -> dict[str, float]: diff --git a/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/metrics/remote.py b/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/metrics/remote.py index 78bb48ca9f..25cc00adbd 100644 --- a/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/metrics/remote.py +++ b/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/metrics/remote.py @@ -7,14 +7,16 @@ import os from abc import ABC, abstractmethod from collections.abc import Awaitable, Callable -from typing import Any +from typing import Any, cast import httpx from httpx import Timeout from jsonpath_ng import parse as jsonpath_parse from jsonpath_ng.exceptions import JsonPathParserError from nemo_evaluator_sdk.inference import requests_log_var +from nemo_evaluator_sdk.metrics.protocol import MetricInput, MetricOutput, MetricOutputSpec, MetricResult from nemo_evaluator_sdk.metrics.template_rendering import ( + TemplateSample, build_template_context, render_template_or_raise, template_metric_repr, @@ -23,7 +25,6 @@ from nemo_evaluator_sdk.resilience.classifier import endpoint_identity from nemo_evaluator_sdk.values.common import SecretRef from nemo_evaluator_sdk.values.metrics import NemoAgentToolkitRemote, Remote, _RemoteBase -from nemo_evaluator_sdk.values.results import MetricResult, MetricScore from nemo_evaluator_sdk.values.scores import RemoteScore from pydantic import Field, SecretStr, field_validator @@ -124,10 +125,10 @@ def secrets(self) -> dict[str, SecretRef]: def _select_metric_score(self, metric_result: MetricResult) -> float: """Select the default score value for one-row metric results.""" - return metric_result.scores[0].value + return float(metric_result.outputs[0].value) @abstractmethod - async def compute_scores(self, item: dict[str, Any], sample: dict[str, Any]) -> MetricResult: + async def compute_scores(self, input: MetricInput) -> MetricResult: """Compute structured score output for one item/sample pair.""" ... @@ -135,9 +136,9 @@ async def compute_scores(self, item: dict[str, Any], sample: dict[str, Any]) -> class RemoteMetric(Remote, _RemoteMetricBase): """A metric that computes scores via a remote endpoint.""" - def score_names(self) -> list[str]: - """Return score keys emitted by this metric.""" - return [score.name for score in self.scores] + def output_spec(self) -> list[MetricOutputSpec]: + """Return outputs emitted by this metric.""" + return [MetricOutputSpec.continuous_score(score.name) for score in self.scores] @field_validator("scores") @classmethod @@ -154,24 +155,28 @@ def _validate_scores(cls, scores: list[RemoteScore]) -> list[RemoteScore]: def _select_metric_score(self, metric_result: MetricResult) -> float: """Select the score value used for single-score consumers.""" if self.metric_threshold_score: - score_names = [score.name for score in metric_result.scores] - if self.metric_threshold_score not in score_names: + output_names = [output.name for output in metric_result.outputs] + if self.metric_threshold_score not in output_names: raise ValueError( f"Score name '{self.metric_threshold_score}' not found in remote metric response. " - f"Available scores: {score_names}" + f"Available scores: {output_names}" ) - return next(score for score in metric_result.scores if score.name == self.metric_threshold_score).value + return float( + next(output for output in metric_result.outputs if output.name == self.metric_threshold_score).value + ) - if len(metric_result.scores) == 1: - return metric_result.scores[0].value + if len(metric_result.outputs) == 1: + return float(metric_result.outputs[0].value) raise ValueError( - f"Remote metric returned multiple scores {[score.name for score in metric_result.scores]}. " + f"Remote metric returned multiple scores {[output.name for output in metric_result.outputs]}. " "Please set metric_threshold_score to specify which score to use." ) - async def compute_scores(self, item: dict[str, Any], sample: dict[str, Any]) -> MetricResult: + async def compute_scores(self, input: MetricInput) -> MetricResult: """Compute structured score output via the remote endpoint.""" + item = input.row.data + sample: TemplateSample = input.candidate context = build_template_context(item, sample) rendered_args = render_template_or_raise( template_name="body", @@ -181,12 +186,12 @@ async def compute_scores(self, item: dict[str, Any], sample: dict[str, Any]) -> sample=sample, metric_repr=template_metric_repr(self), ) - payload = rendered_args if isinstance(rendered_args, dict) else {"args": rendered_args} + payload = cast(dict[str, Any], rendered_args) if isinstance(rendered_args, dict) else {"args": rendered_args} result_data = await self._post_payload(payload) try: _logger.debug("Remote metric result received for payload: %r", payload) - scores: list[MetricScore] = [] + outputs: list[MetricOutput] = [] for score_config in self.scores: jsonpath_expr = jsonpath_parse(score_config.parser.json_path) matches = jsonpath_expr.find(result_data) @@ -196,12 +201,12 @@ async def compute_scores(self, item: dict[str, Any], sample: dict[str, Any]) -> score_config.name, score_config.parser.json_path, ) - scores.append(MetricScore(name=score_config.name, value=float("nan"))) + outputs.append(MetricOutput(name=score_config.name, value=float("nan"))) else: score_value = matches[0].value - scores.append(MetricScore(name=score_config.name, value=float(score_value))) + outputs.append(MetricOutput(name=score_config.name, value=float(score_value))) - return MetricResult(scores=scores) + return MetricResult(outputs=outputs) except Exception: _logger.exception("Error validating remote metric response") raise @@ -212,12 +217,14 @@ class NemoAgentToolkitRemoteMetric(NemoAgentToolkitRemote, _RemoteMetricBase): _RESULT_SCORE_JSONPATH = jsonpath_parse("$.result.score") - def score_names(self) -> list[str]: - """Return score keys emitted by this metric.""" - return [self.evaluator_name] + def output_spec(self) -> list[MetricOutputSpec]: + """Return outputs emitted by this metric.""" + return [MetricOutputSpec.continuous_score(self.evaluator_name)] - async def compute_scores(self, item: dict[str, Any], sample: dict[str, Any]) -> MetricResult: + async def compute_scores(self, input: MetricInput) -> MetricResult: """Compute structured score output via the NeMo Agent Toolkit evaluator endpoint.""" + item = input.row.data + sample: TemplateSample = input.candidate context = build_template_context(item, sample) rendered_item = render_template_or_raise( template_name="body.item", @@ -240,4 +247,4 @@ async def compute_scores(self, item: dict[str, Any], sample: dict[str, Any]) -> else: score = float(matches[0].value) - return MetricResult(scores=[MetricScore(name=self.evaluator_name, value=score)]) + return MetricResult(outputs=[MetricOutput(name=self.evaluator_name, value=score)]) diff --git a/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/metrics/rouge.py b/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/metrics/rouge.py index 735e3a887d..62f69ed822 100644 --- a/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/metrics/rouge.py +++ b/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/metrics/rouge.py @@ -6,9 +6,13 @@ from functools import cached_property from typing import ClassVar, Literal -from nemo_evaluator_sdk.metrics.template_rendering import render_reference_and_candidate, template_metric_repr +from nemo_evaluator_sdk.metrics.protocol import MetricInput, MetricOutput, MetricOutputSpec, MetricResult +from nemo_evaluator_sdk.metrics.template_rendering import ( + TemplateSample, + render_reference_and_candidate, + template_metric_repr, +) from nemo_evaluator_sdk.values.metrics import ROUGE -from nemo_evaluator_sdk.values.results import MetricResult, MetricScore __all__ = ["ROUGEMetric", "RougeScoreName"] @@ -40,11 +44,11 @@ def _scorer(self): return rouge_scorer.RougeScorer(["rouge1", "rouge2", "rouge3", "rougeL"], use_stemmer=True) - def score_names(self) -> list[str]: - """Return score keys emitted by this metric.""" - return list(self.scores_mapping.keys()) + def output_spec(self) -> list[MetricOutputSpec]: + """Return outputs emitted by this metric.""" + return [MetricOutputSpec.continuous_score(score_name) for score_name in self.scores_mapping] - def _metric(self, item: dict, sample: dict) -> dict: + def _metric(self, item: dict, sample: TemplateSample) -> dict: """Compute raw ROUGE scores for one item/sample pair.""" ground_truth, prediction = render_reference_and_candidate( metric_repr=template_metric_repr(self), @@ -56,12 +60,12 @@ def _metric(self, item: dict, sample: dict) -> dict: ) return self._scorer.score(ground_truth, prediction) - async def compute_scores(self, item: dict, sample: dict) -> MetricResult: + async def compute_scores(self, input: MetricInput) -> MetricResult: """Compute structured score output for one item/sample pair.""" - scores = self._metric(item, sample) + scores = self._metric(input.row.data, input.candidate) return MetricResult( - scores=[ - MetricScore(name=score_name, value=scores[score_key].fmeasure) + outputs=[ + MetricOutput(name=score_name, value=scores[score_key].fmeasure) for score_name, score_key in self.scores_mapping.items() ] ) diff --git a/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/metrics/string_check.py b/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/metrics/string_check.py index 96fe61a169..17053390a1 100644 --- a/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/metrics/string_check.py +++ b/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/metrics/string_check.py @@ -3,13 +3,14 @@ """String-check metric runtime implementation.""" +from nemo_evaluator_sdk.metrics.protocol import MetricInput, MetricOutput, MetricOutputSpec, MetricResult from nemo_evaluator_sdk.metrics.template_rendering import ( + TemplateSample, build_template_context, render_template_or_raise, template_metric_repr, ) from nemo_evaluator_sdk.values.metrics import StringCheck, StringCheckOperation -from nemo_evaluator_sdk.values.results import MetricResult, MetricScore __all__ = ["StringCheckMetric", "StringCheckOperation"] @@ -17,12 +18,14 @@ class StringCheckMetric(StringCheck): """String-comparison metric with operator-based checks.""" - def score_names(self) -> list[str]: - """Return score keys emitted by this metric.""" - return [self.type.value] + def output_spec(self) -> list[MetricOutputSpec]: + """Return outputs emitted by this metric.""" + return [MetricOutputSpec.continuous_score(self.type.value)] - async def compute_scores(self, item: dict, sample: dict) -> MetricResult: + async def compute_scores(self, input: MetricInput) -> MetricResult: """Compute the scores for the metric.""" + item = input.row.data + sample: TemplateSample = input.candidate context = build_template_context(item, sample) metric_repr = template_metric_repr(self) left_value = render_template_or_raise( @@ -63,4 +66,4 @@ async def compute_scores(self, item: dict, sample: dict) -> MetricResult: else: raise ValueError(f"Unsupported operation: {self.operation}") - return MetricResult(scores=[MetricScore(name=self.type.value, value=1.0 if score else 0.0)]) + return MetricResult(outputs=[MetricOutput(name=self.type.value, value=1.0 if score else 0.0)]) diff --git a/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/metrics/template_rendering.py b/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/metrics/template_rendering.py index a955686530..127d6f8f4c 100644 --- a/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/metrics/template_rendering.py +++ b/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/metrics/template_rendering.py @@ -8,17 +8,27 @@ from typing import Any from jinja2 import UndefinedError +from nemo_evaluator_sdk.metrics.protocol import CandidateOutput from nemo_evaluator_sdk.templates import render_template from pydantic import BaseModel TemplateValue = str | dict[Any, Any] | list[Any] +TemplateSample = dict[str, Any] | CandidateOutput _DICT_ATTRIBUTE_ERROR_RE = re.compile(r"^'dict object' has no attribute '(?P[^']+)'$") _UNDEFINED_NAME_ERROR_RE = re.compile(r"^'(?P[^']+)' is undefined$") -def build_template_context(item: dict[str, Any], sample: dict[str, Any]) -> dict[str, Any]: +def sample_template_payload(sample: TemplateSample) -> dict[str, Any]: + """Return a sample-shaped dictionary for template rendering helpers.""" + if isinstance(sample, CandidateOutput): + return sample.as_sample() + return sample + + +def build_template_context(item: dict[str, Any], sample: TemplateSample) -> dict[str, Any]: """Build the template context shared by item and sample rendering.""" - return {**item, **sample, "item": item, "sample": sample} + sample_payload = sample_template_payload(sample) + return {**item, **sample_payload, "item": item, "sample": sample_payload} def template_metric_repr(metric: BaseModel | object) -> str: @@ -56,12 +66,13 @@ def render_template_or_raise( template: TemplateValue, context: dict[str, Any], item: dict[str, Any], - sample: dict[str, Any], + sample: TemplateSample, metric_repr: str, item_keys_label: str = "item", sample_keys_label: str = "sample", ) -> object: """Render one template and raise a specific validation error on missing keys.""" + sample_payload = sample_template_payload(sample) try: return render_template(template, context) except UndefinedError as exc: @@ -69,7 +80,7 @@ def render_template_or_raise( base_message = ( f"{metric_repr} could not render its '{template_name}' template for this row.\n" f"Available {item_keys_label} keys={sorted(item.keys())}. \n" - f"Available {sample_keys_label} keys={sorted(sample.keys())}.\n" + f"Available {sample_keys_label} keys={sorted(sample_payload.keys())}.\n" ) if missing_key is not None: detail = f"Dataset item has missing_key='{missing_key}' but the '{template_name}' template references it.\n" @@ -80,9 +91,9 @@ def render_template_or_raise( ) from exc -def render_default_output_text_candidate_or_raise(*, sample: dict[str, Any], metric_name: str) -> object: +def render_default_output_text_candidate_or_raise(*, sample: TemplateSample, metric_name: str) -> object: """Return the default output-text candidate or raise a clear guidance error.""" - prediction = sample.get("output_text") + prediction = sample_template_payload(sample).get("output_text") if prediction is None: raise ValueError( f"{metric_name} has missing `candidate` field.\n" @@ -99,7 +110,7 @@ def render_reference_and_candidate( reference_template: str, candidate_template: str | None, item: dict[str, Any], - sample: dict[str, Any], + sample: TemplateSample, ) -> tuple[str, str]: """Render reference and candidate templates, returning validated strings.""" context = build_template_context(item, sample) diff --git a/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/metrics/tool_calling.py b/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/metrics/tool_calling.py index 8ab7f5f81e..c54e346ba1 100644 --- a/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/metrics/tool_calling.py +++ b/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/metrics/tool_calling.py @@ -8,13 +8,15 @@ from collections.abc import Mapping from typing import ClassVar, cast +from nemo_evaluator_sdk.metrics.protocol import MetricInput, MetricOutput, MetricOutputSpec, MetricResult from nemo_evaluator_sdk.metrics.template_rendering import ( + TemplateSample, build_template_context, render_template_or_raise, + sample_template_payload, template_metric_repr, ) from nemo_evaluator_sdk.values.metrics import ToolCalling -from nemo_evaluator_sdk.values.results import MetricResult, MetricScore __all__ = ["ToolCallingMetric"] @@ -41,12 +43,13 @@ class ToolCallingMetric(ToolCalling): _score_names: ClassVar[list[str]] = ["function_name_accuracy", "function_name_and_args_accuracy"] - def score_names(self) -> list[str]: - """Return score keys emitted by this metric.""" - return list(self._score_names) + def output_spec(self) -> list[MetricOutputSpec]: + """Return outputs emitted by this metric.""" + return [MetricOutputSpec.continuous_score(score_name) for score_name in self._score_names] - def _metric(self, item: dict, sample: dict) -> dict[str, float]: + def _metric(self, item: dict, sample: TemplateSample) -> dict[str, float]: """Compute raw tool-calling scores for one item/sample pair.""" + sample_payload = sample_template_payload(sample) context = build_template_context(item, sample) ground_truth = render_template_or_raise( template_name="reference", @@ -90,7 +93,7 @@ def _metric(self, item: dict, sample: dict) -> dict[str, float]: ) from e # Parse tool calls: check sample (online) first, then item (offline). - response_data = sample.get("response") or item.get("response") + response_data = sample_payload.get("response") or item.get("response") if not response_data: raise ValueError("No response found in sample or item - tool-calling metric requires model response data") if not isinstance(response_data, dict): @@ -107,7 +110,7 @@ def _metric(self, item: dict, sample: dict) -> dict[str, float]: message = first_choice["message"] if not message.get("tool_calls"): - _logger.info("No tool calls found in %s", sample) + _logger.info("No tool calls found in %s", sample_payload) message["tool_calls"] = [] tool_calls = message["tool_calls"] @@ -160,7 +163,11 @@ def _metric(self, item: dict, sample: dict) -> dict[str, float]: "function_name_and_args_accuracy": fn_name_and_args_accuracy_score, } - async def compute_scores(self, item: dict, sample: dict) -> MetricResult: + async def compute_scores(self, input: MetricInput) -> MetricResult: """Compute the scores for the metric.""" + item = input.row.data + sample = input.candidate scores = self._metric(item, sample) - return MetricResult(scores=[MetricScore(name=score_name, value=score) for score_name, score in scores.items()]) + return MetricResult( + outputs=[MetricOutput(name=score_name, value=score) for score_name, score in scores.items()] + ) diff --git a/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/metrics/utils.py b/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/metrics/utils.py index 3970929467..fbe8ca0b37 100644 --- a/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/metrics/utils.py +++ b/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/metrics/utils.py @@ -1,10 +1,23 @@ # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -"""Shared helpers for working with runtime metric identifiers.""" +"""Shared helpers for runtime metrics.""" + +import re +import string from nemo_evaluator_sdk.enums import MetricType -from nemo_evaluator_sdk.metrics.base import Metric +from nemo_evaluator_sdk.metrics.protocol import Metric + + +def normalize_text(s: str) -> str: + """Normalize free-form text for token/equality-based metric comparisons.""" + if not s: + return "" + s = s.lower() + s = "".join(ch for ch in s if ch not in set(string.punctuation)) + s = re.sub(r"\b(a|an|the)\b", " ", s) + return " ".join(s.split()) def metric_type_name(metric: Metric) -> str: diff --git a/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/values/__init__.py b/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/values/__init__.py index de8f4e5805..f6d6c68699 100644 --- a/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/values/__init__.py +++ b/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/values/__init__.py @@ -3,6 +3,19 @@ """Public value types for evaluator SDK runtime.""" +from nemo_evaluator_sdk.metrics.protocol import ( + BooleanValue, + CandidateOutput, + ContinuousScore, + DatasetRow, + DiscreteScore, + Label, + MetricDescriptor, + MetricInput, + MetricOutput, + MetricOutputSpec, + MetricResult, +) from nemo_evaluator_sdk.values.agents import Agent from nemo_evaluator_sdk.values.common import SecretRef, SupportedJobTypes from nemo_evaluator_sdk.values.dataset_schemas import ( @@ -53,7 +66,6 @@ EvaluationResult, Histogram, HistogramBin, - MetricResult, MetricScore, Percentiles, RowScore, @@ -81,8 +93,13 @@ "AggregateRubricScore", "AggregateScore", "AggregateScoreBase", + "BooleanValue", + "CandidateOutput", + "ContinuousScore", + "DatasetRow", "DatasetRows", "DefaultAggregateFieldName", + "DiscreteScore", "RunConfig", "RunConfigOnline", "RunConfigOnlineModel", @@ -91,6 +108,11 @@ "HistogramBin", "InferenceParams", "JSONScoreParser", + "Label", + "MetricDescriptor", + "MetricInput", + "MetricOutput", + "MetricOutputSpec", "MetricResult", "MetricScore", "Model", diff --git a/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/values/multi_metric_results.py b/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/values/multi_metric_results.py index 637a5fd63f..b330d724e8 100644 --- a/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/values/multi_metric_results.py +++ b/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/values/multi_metric_results.py @@ -10,11 +10,11 @@ from pydantic import BaseModel +from nemo_evaluator_sdk.metrics.protocol import MetricOutput from nemo_evaluator_sdk.values.results import ( AggregatedMetricResult, AggregateFieldName, EvaluationResult, - MetricScore, ResultView, RowScore, flatten_dict, @@ -49,8 +49,8 @@ def _filter_aggregate_fields( return AggregatedMetricResult(scores=filtered_scores) -def _extract_metric_scores(row_score: RowScore, expected_key: str) -> list[MetricScore]: - """Resolve the score list for one metric from a row result. +def _extract_metric_outputs(row_score: RowScore, expected_key: str) -> list[MetricOutput]: + """Resolve the output list for one metric from a row result. This exists because local and remote execution paths may return row scores either already keyed by the final metric key or as a single unnamed metric @@ -61,7 +61,7 @@ def _extract_metric_scores(row_score: RowScore, expected_key: str) -> list[Metri expected_key: Metric key the caller expects to find on the row. Returns: - The score list for the requested metric key, or an empty list when the + The output list for the requested metric key, or an empty list when the row has no metric output because evaluation failed. Raises: @@ -74,7 +74,7 @@ def _extract_metric_scores(row_score: RowScore, expected_key: str) -> list[Metri return [] if len(row_score.metrics) == 1: return next(iter(row_score.metrics.values())) - raise ValueError(f"Unable to resolve row metric scores for key {expected_key!r}") + raise ValueError(f"Unable to resolve row metric outputs for key {expected_key!r}") def _extract_metric_error(row_score: RowScore, expected_key: str) -> str | None: @@ -123,7 +123,7 @@ def namespace_result( row_index=row_score.row_index, item=row_score.item, sample=row_score.sample, - metrics={metric_key: _extract_metric_scores(row_score, metric_key)}, + metrics={metric_key: _extract_metric_outputs(row_score, metric_key)}, requests=row_score.requests, metric_errors={metric_key: error} if (error := _extract_metric_error(row_score, metric_key)) else None, ) @@ -176,12 +176,12 @@ def collapse_results( combined_rows: list[RowScore] = [] for index in range(row_count): first_row = results_by_key[ordered_keys[0]].row_scores[index] - metrics: dict[str, list[MetricScore]] = {} + metrics: dict[str, list[MetricOutput]] = {} requests: list[dict[str, Any]] = [] metric_errors: dict[str, str] = {} for metric_key in ordered_keys: row_score = results_by_key[metric_key].row_scores[index] - metrics[metric_key] = _extract_metric_scores(row_score, metric_key) + metrics[metric_key] = _extract_metric_outputs(row_score, metric_key) requests.extend(row_score.requests) if row_score.metric_errors: metric_errors.update(row_score.metric_errors) @@ -252,8 +252,8 @@ def to_records(self, view: ResultView = "rows") -> list[dict[str, Any]]: if error_text := row_error_text(row_score): record["error"] = error_text for metric_key, metric_scores in row_score.metrics.items(): - for score in metric_scores: - record[f"score.{metric_key}.{score.name}"] = serialize_value(score.value) + for output in metric_scores: + record[f"output.{metric_key}.{output.name}"] = serialize_value(output.value) records.append(record) return records diff --git a/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/values/results.py b/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/values/results.py index c46945a0e9..34d221006e 100644 --- a/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/values/results.py +++ b/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/values/results.py @@ -15,6 +15,8 @@ import pyarrow as pa from pydantic import BaseModel, ConfigDict, Field, field_serializer, field_validator, model_serializer +from nemo_evaluator_sdk.metrics.protocol import MetricOutput, MetricResult + ResultView = Literal["rows", "aggregate"] AggregateFieldName = Literal[ # Base statistics @@ -240,12 +242,6 @@ def serialize_nan(self, v): return v -class MetricResult(BaseModel): - """Evaluation results for the metric""" - - scores: list[MetricScore] - - class Percentiles(BaseModel): """Percentile distribution of scores.""" @@ -364,7 +360,7 @@ class RowScore(BaseModel): row_index: int | None = Field(default=None, description="Stable row position used for result alignment.", ge=0) item: dict[str, Any] = Field(description="Input item metadata for the evaluated row.") sample: dict[str, Any] = Field(description="Sample output payload for the evaluated row.") - metrics: dict[str, list[MetricScore]] = Field(description="Metric-level row scores by metric key.") + metrics: dict[str, list[MetricOutput]] = Field(description="Metric-level row outputs by metric key.") requests: list[dict[str, Any]] = Field(description="Request details captured during evaluation.") metric_errors: dict[str, str] | None = Field( default=None, @@ -593,8 +589,8 @@ def to_records(self, view: ResultView = "rows") -> list[dict[str, Any]]: if error_text := row_error_text(row_score): record["error"] = error_text for metric_scores in row_score.metrics.values(): - for score in metric_scores: - record[f"score.{score.name}"] = serialize_value(score.value) + for output in metric_scores: + record[f"output.{output.name}"] = serialize_value(output.value) records.append(record) return records @@ -658,8 +654,8 @@ def format_summary(self, max_rows: int = 10, *, max_error_rows: int | None = Non for index, row_score in enumerate(self.row_scores[:max_rows]): record = summary_row_base_record(row_score, index) for metric_scores in row_score.metrics.values(): - for score in metric_scores: - record[f"score.{score.name}"] = serialize_value(score.value) + for output in metric_scores: + record[f"output.{output.name}"] = serialize_value(output.value) preview_records.append(record) parts = [ summary_header("EvaluationResult", self.row_scores, len(self.aggregate_scores.scores)), diff --git a/packages/nemo_evaluator_sdk/tests/conftest.py b/packages/nemo_evaluator_sdk/tests/conftest.py index 8fab93662b..e0e19f3684 100644 --- a/packages/nemo_evaluator_sdk/tests/conftest.py +++ b/packages/nemo_evaluator_sdk/tests/conftest.py @@ -1,11 +1,14 @@ # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 +import sys from pathlib import Path import pytest _TESTS_DIR = Path(__file__).resolve().parent +if str(_TESTS_DIR) not in sys.path: + sys.path.insert(0, str(_TESTS_DIR)) _CATEGORY_MARKERS = { "unit": "Unit tests for the SDK package.", "e2e": "End-to-end tests.", diff --git a/packages/nemo_evaluator_sdk/tests/execution/backends/local/_stubs.py b/packages/nemo_evaluator_sdk/tests/execution/backends/local/_stubs.py index 5b93dd41ec..56819500cf 100644 --- a/packages/nemo_evaluator_sdk/tests/execution/backends/local/_stubs.py +++ b/packages/nemo_evaluator_sdk/tests/execution/backends/local/_stubs.py @@ -8,8 +8,8 @@ from collections.abc import Awaitable, Callable from nemo_evaluator_sdk.inference import PostprocessResponse, PreprocessRequest +from nemo_evaluator_sdk.metrics.protocol import MetricInput, MetricOutput, MetricOutputSpec, MetricResult from nemo_evaluator_sdk.values import RunConfig -from nemo_evaluator_sdk.values.results import MetricResult, MetricScore from pydantic import BaseModel @@ -39,14 +39,14 @@ def type(self) -> str: """Return the metric key used by the backend when namespacing results.""" return "duplicate" - def score_names(self) -> list[str]: - """Return the score names exposed by this metric.""" - return ["score"] + def output_spec(self) -> list[MetricOutputSpec]: + """Return the outputs exposed by this metric.""" + return [MetricOutputSpec.continuous_score("score")] - async def compute_scores(self, item: dict, sample: dict) -> MetricResult: + async def compute_scores(self, input: MetricInput) -> MetricResult: """Return one structured score result for protocol conformance in tests.""" - del item, sample - return MetricResult(scores=[MetricScore(name="score", value=1.0)]) + del input + return MetricResult(outputs=[MetricOutput(name="score", value=1.0)]) class PreparedBenchmarkMetric(BaseModel): @@ -57,9 +57,9 @@ class PreparedBenchmarkMetric(BaseModel): secrets_resolved: bool = False preflight_ran: bool = False - def score_names(self) -> list[str]: - """Return the score names exposed by this metric.""" - return ["score"] + def output_spec(self) -> list[MetricOutputSpec]: + """Return the outputs exposed by this metric.""" + return [MetricOutputSpec.continuous_score("score")] def secrets(self) -> dict[str, object]: """Return no concrete secret refs while satisfying the secrets protocol.""" @@ -78,7 +78,7 @@ async def preflight(self) -> None: """Record that local metric preparation ran preflight.""" self.preflight_ran = True - async def compute_scores(self, item: dict, sample: dict) -> MetricResult: + async def compute_scores(self, input: MetricInput) -> MetricResult: """Return one structured score result for protocol conformance in tests.""" - del item, sample - return MetricResult(scores=[MetricScore(name="score", value=1.0)]) + del input + return MetricResult(outputs=[MetricOutput(name="score", value=1.0)]) diff --git a/packages/nemo_evaluator_sdk/tests/execution/test_benchmark_execution.py b/packages/nemo_evaluator_sdk/tests/execution/test_benchmark_execution.py index 1f5a1ffafe..6327b77deb 100644 --- a/packages/nemo_evaluator_sdk/tests/execution/test_benchmark_execution.py +++ b/packages/nemo_evaluator_sdk/tests/execution/test_benchmark_execution.py @@ -25,10 +25,11 @@ evaluate_benchmark, ) from nemo_evaluator_sdk.execution.values import EvaluationError, EvaluationPhase +from nemo_evaluator_sdk.metrics.protocol import MetricInput, MetricOutput, MetricOutputSpec, MetricResult from nemo_evaluator_sdk.values import Agent from nemo_evaluator_sdk.values.models import Model from nemo_evaluator_sdk.values.params import RunConfig, RunConfigOnlineModel -from nemo_evaluator_sdk.values.results import MetricResult, MetricScore, RowScore +from nemo_evaluator_sdk.values.results import RowScore from pytest_mock import MockerFixture @@ -50,19 +51,21 @@ def type(self) -> str: """Return the public metric type identifier.""" return self._name - def score_names(self) -> list[str]: - """Return the score names exposed by this metric.""" - return ["score"] + def output_spec(self) -> list[MetricOutputSpec]: + """Return the outputs exposed by this metric.""" + return [MetricOutputSpec.continuous_score("score")] def metric(self, item: dict, sample: dict, trace=None) -> float: """Return a raw score for protocol conformance.""" del trace return float(self._score_fn(item, sample)) - async def compute_scores(self, item: dict, sample: dict) -> MetricResult: + async def compute_scores(self, input: MetricInput) -> MetricResult: """Record the call and produce a single-score metric result.""" + item = input.row.data + sample = input.candidate.as_sample() self.calls.append((dict(item), dict(sample))) - return MetricResult(scores=[MetricScore(name="score", value=float(self._score_fn(item, sample)))]) + return MetricResult(outputs=[MetricOutput(name="score", value=float(self._score_fn(item, sample)))]) class _RaisingMetric: @@ -78,26 +81,26 @@ def type(self) -> str: """Return the public metric type identifier.""" return self._name - def score_names(self) -> list[str]: - """Return the score names exposed by this metric.""" - return ["score"] + def output_spec(self) -> list[MetricOutputSpec]: + """Return the outputs exposed by this metric.""" + return [MetricOutputSpec.continuous_score("score")] def metric(self, item: dict, sample: dict, trace=None) -> float: """Raise the configured exception for protocol conformance.""" del item, sample, trace raise self._exc - async def compute_scores(self, item: dict, sample: dict) -> MetricResult: + async def compute_scores(self, input: MetricInput) -> MetricResult: """Raise the configured exception to exercise failure handling.""" - del item, sample + del input raise self._exc -class _NoScoreNamesMetric(_ScriptedMetric): - """Metric stub that deliberately declares no score names.""" +class _NoOutputsMetric(_ScriptedMetric): + """Metric stub that deliberately declares no outputs.""" - def score_names(self) -> list[str]: - """Return no score names to exercise pipeline validation.""" + def output_spec(self) -> list[MetricOutputSpec]: + """Return no outputs to exercise pipeline validation.""" return [] @@ -115,12 +118,16 @@ class _CorpusMetric(_ScriptedMetric): def __init__(self, name: str, score_fn): """Configure row scoring and call tracking for corpus scoring.""" super().__init__(name, score_fn) - self.corpus_calls: list[tuple[list[dict], list[dict]]] = [] + self.corpus_calls: list[list[MetricInput]] = [] - async def compute_corpus_scores(self, items: list[dict], samples: list[dict]) -> MetricResult: + def corpus_output_spec(self) -> list[MetricOutputSpec]: + """Return corpus-level outputs exposed by this metric.""" + return [MetricOutputSpec.continuous_score("corpus")] + + async def compute_corpus_scores(self, inputs: list[MetricInput]) -> MetricResult: """Record corpus inputs and return one corpus score.""" - self.corpus_calls.append((items, samples)) - return MetricResult(scores=[MetricScore(name="corpus", value=42.0)]) + self.corpus_calls.append(inputs) + return MetricResult(outputs=[MetricOutput(name="corpus", value=42.0)]) def _make_model() -> Model: @@ -633,15 +640,15 @@ def type(self) -> str: """Return the public metric type identifier.""" return self._name - def score_names(self) -> list[str]: - """Return the score names exposed by this metric.""" - return ["score"] + def output_spec(self) -> list[MetricOutputSpec]: + """Return the outputs exposed by this metric.""" + return [MetricOutputSpec.continuous_score("score")] - async def compute_scores(self, item: dict, sample: dict) -> MetricResult: + async def compute_scores(self, input: MetricInput) -> MetricResult: """Record a metric-specific request and return the configured score.""" - del item, sample + del input inference.requests_log_var.get().append({"phase": "metric", "metric": self._name}) - return MetricResult(scores=[MetricScore(name="score", value=self._value)]) + return MetricResult(outputs=[MetricOutput(name="score", value=self._value)]) mocker.patch( "nemo_evaluator_sdk.execution.benchmark_execution.generate_online_sample", @@ -737,7 +744,10 @@ async def test_corpus_scores_are_namespaced_at_aggregate_level(self) -> None: params=RunConfig(parallelism=1), ) - assert metric.corpus_calls == [([{"prompt": "a"}, {"prompt": "b"}], [{}, {}])] + assert [[input.row.data for input in call] for call in metric.corpus_calls] == [ + [{"prompt": "a"}, {"prompt": "b"}] + ] + assert [[input.candidate.as_sample() for input in call] for call in metric.corpus_calls] == [[{}, {}]] assert [score.name for score in result.per_metric["custom.ref"].aggregate_scores.scores] == [ "custom.ref.score", "custom.ref.corpus", @@ -776,7 +786,10 @@ def _score(item: dict, sample: dict) -> float: prompt_template="{{prompt}}", ) - assert metric.corpus_calls == [([{"prompt": "ok"}], [{"output_text": "ok", "response": {}}])] + assert [[input.row.data for input in call] for call in metric.corpus_calls] == [[{"prompt": "ok"}]] + assert [[input.candidate.as_sample() for input in call] for call in metric.corpus_calls] == [ + [{"output_text": "ok", "response": {}}] + ] aggregate_scores = result.per_metric["custom.ref"].aggregate_scores.scores assert [(score.name, score.count, score.nan_count) for score in aggregate_scores] == [ ("custom.ref.score", 1, 1), @@ -823,7 +836,10 @@ def _score(_item: dict, sample: dict) -> float: "inference_error": "generation boom", } assert failed_row.metric_errors is None - assert metric.corpus_calls == [([{"prompt": "ok"}], [{"output_text": "ok", "response": {}}])] + assert [[input.row.data for input in call] for call in metric.corpus_calls] == [[{"prompt": "ok"}]] + assert [[input.candidate.as_sample() for input in call] for call in metric.corpus_calls] == [ + [{"output_text": "ok", "response": {}}] + ] aggregate_scores = result.per_metric["custom.ref"].aggregate_scores.scores assert [(score.name, score.count, score.nan_count) for score in aggregate_scores] == [ ("custom.ref.score", 1, 1), @@ -872,23 +888,27 @@ def test_protocol_increment_work_body_is_callable(self) -> None: reporter: ProgressReporter = _NoOpProgressReporter() assert reporter.increment_work() is None - def test_normalize_metric_result_preserves_unexpected_scores_after_expected_scores(self) -> None: + def test_normalize_metric_result_orders_declared_outputs(self) -> None: result = _normalize_metric_result( - MetricResult(scores=[MetricScore(name="extra", value=3.0)]), - ["score"], + MetricResult( + outputs=[ + MetricOutput(name="second", value=2.0), + MetricOutput(name="first", value=1.0), + ] + ), + [MetricOutputSpec.continuous_score("first"), MetricOutputSpec.continuous_score("second")], ) - assert [score.name for score in result.scores] == ["score", "extra"] - assert math.isnan(result.scores[0].value) - assert result.scores[1].value == 3.0 + assert [output.name for output in result.outputs] == ["first", "second"] + assert [output.value for output in result.outputs] == [1.0, 2.0] def test_benchmark_error_from_exception_returns_none_without_typed_leaf(self) -> None: assert _benchmark_error_from_exception(ValueError("plain failure")) is None - def test_build_metric_pipelines_rejects_metric_without_score_names(self) -> None: - metric = _NoScoreNamesMetric("empty", lambda item, sample: 1.0) + def test_build_metric_pipelines_rejects_metric_without_outputs(self) -> None: + metric = _NoOutputsMetric("empty", lambda item, sample: 1.0) - with pytest.raises(RuntimeError, match="does not declare any score names"): + with pytest.raises(RuntimeError, match="does not declare any outputs"): _build_metric_pipelines([("empty", metric)], item_count=1, queue_capacity=1) @@ -902,7 +922,7 @@ async def test_raises_for_unexpected_queue_item(self) -> None: pipeline = _MetricPipeline( metric_ref="a", metric=_ScriptedMetric("a", lambda item, sample: 1.0), - score_names=["score"], + output_spec=[MetricOutputSpec.continuous_score("score")], queue=queue, results=[None], ) @@ -930,7 +950,7 @@ async def test_put_pipeline_sentinels_uses_put_nowait_when_cancelling(self, mock pipeline = _MetricPipeline( metric_ref="a", metric=_ScriptedMetric("a", lambda item, sample: 1.0), - score_names=["score"], + output_spec=[MetricOutputSpec.continuous_score("score")], queue=queue, results=[None], ) diff --git a/packages/nemo_evaluator_sdk/tests/execution/test_evaluator.py b/packages/nemo_evaluator_sdk/tests/execution/test_evaluator.py index b217f86330..1e091fe88c 100644 --- a/packages/nemo_evaluator_sdk/tests/execution/test_evaluator.py +++ b/packages/nemo_evaluator_sdk/tests/execution/test_evaluator.py @@ -16,14 +16,12 @@ RunConfigOnlineModel, ) from nemo_evaluator_sdk.execution.evaluator import Evaluator -from nemo_evaluator_sdk.metrics.base import Metric from nemo_evaluator_sdk.metrics.exact_match import ExactMatchMetric +from nemo_evaluator_sdk.metrics.protocol import Metric, MetricInput, MetricOutput, MetricOutputSpec, MetricResult from nemo_evaluator_sdk.values.multi_metric_results import BenchmarkEvaluationResult from nemo_evaluator_sdk.values.results import ( AggregatedMetricResult, EvaluationResult, - MetricResult, - MetricScore, ) from pydantic import ValidationError from pytest_mock import MockerFixture @@ -34,13 +32,12 @@ class _CustomMetric: def type(self) -> str: return MetricType.STRING_CHECK.value - async def compute_scores(self, item: dict, sample: dict) -> MetricResult: - del sample - score = 1.0 if item["expected"] == item["model_output"] else 0.0 - return MetricResult(scores=[MetricScore(name="string-check", value=score)]) + async def compute_scores(self, input: MetricInput) -> MetricResult: + score = 1.0 if input.row.data["expected"] == input.row.data["model_output"] else 0.0 + return MetricResult(outputs=[MetricOutput(name="string-check", value=score)]) - def score_names(self) -> list[str]: - return ["string-check"] + def output_spec(self) -> list[MetricOutputSpec]: + return [MetricOutputSpec.continuous_score("string-check")] _DATASET = [ diff --git a/packages/nemo_evaluator_sdk/tests/execution/test_metric_execution.py b/packages/nemo_evaluator_sdk/tests/execution/test_metric_execution.py index 6403fec4d3..6fc1396652 100644 --- a/packages/nemo_evaluator_sdk/tests/execution/test_metric_execution.py +++ b/packages/nemo_evaluator_sdk/tests/execution/test_metric_execution.py @@ -32,7 +32,7 @@ run_sync, ) from nemo_evaluator_sdk.execution.pipeline import PipelineRuntime -from nemo_evaluator_sdk.execution.scoring import empty_evaluation_result, finalize_evaluation_result +from nemo_evaluator_sdk.execution.scoring import build_metric_input, empty_evaluation_result, finalize_evaluation_result from nemo_evaluator_sdk.execution.utils import ( _candidate_env_names, _copy_metric, @@ -40,9 +40,9 @@ prepare_metric_for_local_execution, ) from nemo_evaluator_sdk.execution.values import EvaluationError, EvaluationPhase -from nemo_evaluator_sdk.metrics.base import Metric from nemo_evaluator_sdk.metrics.hooks import HooksBase from nemo_evaluator_sdk.metrics.llm_judge import LLMJudgeMetric as RuntimeLLMJudgeMetric +from nemo_evaluator_sdk.metrics.protocol import Metric, MetricInput, MetricOutput, MetricOutputSpec, MetricResult from nemo_evaluator_sdk.metrics.utils import metric_type_name from nemo_evaluator_sdk.structured_output import StructuredOutputMode from nemo_evaluator_sdk.values.agents import Agent @@ -58,8 +58,6 @@ from nemo_evaluator_sdk.values.results import ( AggregatedMetricResult, EvaluationResult, - MetricResult, - MetricScore, RowScore, ) from nemo_evaluator_sdk.values.scores import JSONScoreParser, RangeScore @@ -105,7 +103,11 @@ def test_format_exception_summary_falls_back_to_exception_type() -> None: def _make_metric_result(*scores: tuple[str, float]) -> MetricResult: - return MetricResult(scores=[MetricScore(name=n, value=v) for n, v in scores]) + return MetricResult(outputs=[MetricOutput(name=n, value=v) for n, v in scores]) + + +def _score_spec(*names: str) -> list[MetricOutputSpec]: + return [MetricOutputSpec.continuous_score(name) for name in names] def _make_mock_metric(mocker: MockerFixture, results: list[MetricResult] | None = None) -> Mock: @@ -123,6 +125,8 @@ def _make_mock_metric(mocker: MockerFixture, results: list[MetricResult] | None default_result = _make_metric_result(("accuracy", 1.0)) metric.compute_scores = mocker.AsyncMock(return_value=default_result) metric.compute_corpus_scores = mocker.AsyncMock(return_value=None) + metric.output_spec = mocker.Mock(return_value=_score_spec("accuracy", "score", "corpus_score")) + metric.corpus_output_spec = mocker.Mock(return_value=_score_spec("corpus_score")) return metric @@ -163,8 +167,8 @@ def __deepcopy__(self, memo: dict[int, object]) -> "_TestMetric": copied.compute_corpus_scores = self.compute_corpus_scores return copied - def score_names(self) -> list[str]: - return [TEST_METRIC_KEY] + def output_spec(self) -> list[MetricOutputSpec]: + return [MetricOutputSpec.continuous_score("score")] def _make_test_metric() -> Metric: @@ -176,9 +180,9 @@ class _DistinctScoreNameMetric(_TestMetric): type: ClassVar[str] = "metric-a" - def score_names(self) -> list[str]: - """Return a score name that differs from the metric type.""" - return ["score"] + def output_spec(self) -> list[MetricOutputSpec]: + """Return an output name that differs from the metric type.""" + return [MetricOutputSpec.continuous_score("score")] def _make_distinct_score_name_metric() -> Metric: @@ -219,12 +223,12 @@ class _HookedMetric(HooksBase): type: ClassVar[MetricType] = MetricType.STRING_CHECK _result: MetricResult = PrivateAttr(default_factory=lambda: _make_metric_result(("score", 1.0))) - async def compute_scores(self, item: dict, sample: dict) -> MetricResult: - del item, sample + async def compute_scores(self, input: MetricInput) -> MetricResult: + del input return self._result - def score_names(self) -> list[str]: - return ["score"] + def output_spec(self) -> list[MetricOutputSpec]: + return [MetricOutputSpec.continuous_score("score")] class _PreparedMetric(BaseModel): @@ -233,12 +237,12 @@ class _PreparedMetric(BaseModel): _events: list[tuple[str, str | None]] = PrivateAttr(default_factory=list) _resolved_secret: str | None = PrivateAttr(default=None) - async def compute_scores(self, item: dict, sample: dict) -> MetricResult: - del item, sample + async def compute_scores(self, input: MetricInput) -> MetricResult: + del input return _make_metric_result(("score", 1.0)) - def score_names(self) -> list[str]: - return ["score"] + def output_spec(self) -> list[MetricOutputSpec]: + return [MetricOutputSpec.continuous_score("score")] def secrets(self) -> dict[str, SecretRef]: return {"NVIDIA_BUILD_API_KEY": SecretRef(root="nvidia-build-api-key")} @@ -664,7 +668,7 @@ async def test_preserves_input_order_and_filters_none_from_aggregate(self, mocke row_index=0, item={"idx": 0}, sample={"value": "a"}, - metrics={"mock": success_result.scores}, + metrics={"mock": success_result.outputs}, requests=[], metric_errors=None, ), @@ -686,7 +690,7 @@ async def test_preserves_input_order_and_filters_none_from_aggregate(self, mocke result = await finalize_evaluation_result(metric, completed) assert [row_score.row_index for row_score in result.row_scores] == [0, 1] - mock_agg.assert_called_once_with([success_result]) + mock_agg.assert_called_once_with([success_result], metric.output_spec()) assert result.aggregate_scores is aggregate_result @pytest.mark.asyncio @@ -711,7 +715,7 @@ async def test_applies_corpus_scores(self, mocker: MockerFixture): row_index=0, item={"idx": 0}, sample={"value": "a"}, - metrics={"mock": row_result.scores}, + metrics={"mock": row_result.outputs}, requests=[], metric_errors=None, ), @@ -720,8 +724,11 @@ async def test_applies_corpus_scores(self, mocker: MockerFixture): await finalize_evaluation_result(metric, completed) - metric.compute_corpus_scores.assert_awaited_once_with(items=[{"idx": 0}], samples=[{"value": "a"}]) - mock_add.assert_called_once_with(aggregate_result, corpus_result) + metric.compute_corpus_scores.assert_awaited_once() + assert metric.compute_corpus_scores.await_args.kwargs["inputs"] == [ + build_metric_input({"idx": 0}, {"value": "a"}, 0) + ] + mock_add.assert_called_once_with(aggregate_result, corpus_result, metric.corpus_output_spec()) @pytest.mark.asyncio async def test_skip_errored_excludes_nan_placeholder_from_aggregate(self, mocker: MockerFixture): @@ -762,7 +769,7 @@ async def test_skip_errored_excludes_nan_placeholder_from_aggregate(self, mocker result = await finalize_evaluation_result(metric, completed, skip_errored=True) - mock_agg.assert_called_once_with([success_result]) + mock_agg.assert_called_once_with([success_result], metric.output_spec()) assert [rs.row_index for rs in result.row_scores] == [0, 1] @pytest.mark.asyncio @@ -793,7 +800,7 @@ async def test_skip_errored_excludes_errored_rows_from_corpus_inputs(self, mocke row_index=0, item={"idx": 0}, sample={"value": "a"}, - metrics={"mock": success_result.scores}, + metrics={"mock": success_result.outputs}, requests=[], metric_errors=None, ), @@ -805,7 +812,7 @@ async def test_skip_errored_excludes_errored_rows_from_corpus_inputs(self, mocke row_index=1, item={"idx": 1}, sample={"value": "b"}, - metrics={"mock": nan_result.scores}, + metrics={"mock": nan_result.outputs}, requests=[], metric_errors={"mock": "ignored failure"}, ), @@ -815,7 +822,10 @@ async def test_skip_errored_excludes_errored_rows_from_corpus_inputs(self, mocke result = await finalize_evaluation_result(metric, completed, skip_errored=True) # Only row 0 feeds the corpus metric; row 1 is the errored/skipped row. - metric.compute_corpus_scores.assert_awaited_once_with(items=[{"idx": 0}], samples=[{"value": "a"}]) + metric.compute_corpus_scores.assert_awaited_once() + assert metric.compute_corpus_scores.await_args.kwargs["inputs"] == [ + build_metric_input({"idx": 0}, {"value": "a"}, 0) + ] # Both rows still surface in ``row_scores`` for reporting. assert [rs.row_index for rs in result.row_scores] == [0, 1] @@ -837,7 +847,7 @@ async def test_skip_errored_bypasses_corpus_scoring_when_all_rows_are_skipped(se row_index=0, item={"idx": 0}, sample={"value": "a"}, - metrics={"mock": nan_result.scores}, + metrics={"mock": nan_result.outputs}, requests=[], metric_errors={"mock": "ignored failure"}, ), @@ -846,7 +856,7 @@ async def test_skip_errored_bypasses_corpus_scoring_when_all_rows_are_skipped(se result = await finalize_evaluation_result(metric, completed, skip_errored=True) - mock_agg.assert_called_once_with([]) + mock_agg.assert_called_once_with([], metric.output_spec()) metric.compute_corpus_scores.assert_not_awaited() mock_add.assert_not_called() assert [rs.row_index for rs in result.row_scores] == [0] @@ -1192,8 +1202,9 @@ async def test_preserves_call_assertions_after_copy(self): prepared = await prepare_metric_for_local_execution(metric, RunConfig()) assert prepared is not metric - await prepared.compute_scores({"prompt": "hello"}, {"output_text": "world"}) - impl.compute_scores.assert_awaited_once_with({"prompt": "hello"}, {"output_text": "world"}) + metric_input = build_metric_input({"prompt": "hello"}, {"output_text": "world"}) + await prepared.compute_scores(metric_input) + impl.compute_scores.assert_awaited_once_with(metric_input) @pytest.mark.asyncio async def test_resolves_env_secret_before_preflight(self, monkeypatch: pytest.MonkeyPatch): @@ -1256,9 +1267,12 @@ async def _fake_inference( inference.requests_log_var.get([]).append(generation_request) return {"choices": [{"message": {"content": "world"}}]} - async def _compute_scores(item: dict[str, str], sample: dict[str, str]) -> MetricResult: - assert item == {"prompt": "hello"} - assert sample == {"output_text": "world", "response": {"choices": [{"message": {"content": "world"}}]}} + async def _compute_scores(input: MetricInput) -> MetricResult: + assert input.row.data == {"prompt": "hello"} + assert input.candidate.as_sample() == { + "output_text": "world", + "response": {"choices": [{"message": {"content": "world"}}]}, + } inference.requests_log_var.get([]).append(metric_request) return _make_metric_result(("score", 1.0)) @@ -1287,8 +1301,8 @@ async def _compute_scores(item: dict[str, str], sample: dict[str, str]) -> Metri _, debug_kwargs = execution_logger.debug.call_args assert debug_kwargs["extra"]["item_index"] == 0 assert debug_kwargs["extra"]["metric_type"] == MetricType.STRING_CHECK.value - assert debug_kwargs["extra"]["scores"][0]["name"] == "score" - assert debug_kwargs["extra"]["scores"][0]["value"] == 1.0 + assert debug_kwargs["extra"]["outputs"][0]["name"] == "score" + assert debug_kwargs["extra"]["outputs"][0]["value"] == 1.0 @pytest.mark.asyncio async def test_metric_failure_preserves_generation_and_metric_requests(self, mocker: MockerFixture): @@ -1306,8 +1320,8 @@ async def _fake_inference( inference.requests_log_var.get([]).append(generation_request) return {"choices": [{"message": {"content": "world"}}]} - async def _compute_scores(item: dict[str, str], sample: dict[str, str]) -> MetricResult: - del item, sample + async def _compute_scores(input: MetricInput) -> MetricResult: + del input inference.requests_log_var.get([]).append(metric_request) raise ValueError("bad score") @@ -1372,7 +1386,8 @@ async def _fake_inference( } assert row_score.error == f"{TEST_METRIC_KEY}: {GENERATION_FAILURE_MESSAGE}" assert row_score.metric_errors == {TEST_METRIC_KEY: GENERATION_FAILURE_MESSAGE} - assert row_score.metrics == {} + assert row_score.metrics[TEST_METRIC_KEY][0].name == "score" + assert math.isnan(row_score.metrics[TEST_METRIC_KEY][0].value) assert result.aggregate_scores.scores[0].count == 0 assert result.aggregate_scores.scores[0].nan_count == 1 execution_logger.warning.assert_any_call( @@ -1403,7 +1418,8 @@ async def test_template_generation_failure_maps_to_nan_with_ignore_request_failu ) row_score = result.row_scores[0] - assert row_score.metrics == {} + assert row_score.metrics[TEST_METRIC_KEY][0].name == "score" + assert math.isnan(row_score.metrics[TEST_METRIC_KEY][0].value) assert row_score.metric_errors is not None error_message = row_score.metric_errors[TEST_METRIC_KEY] assert "'dict object' has no attribute 'missing'" in error_message @@ -1478,7 +1494,8 @@ async def _fake_inference( ) row_score = result.row_scores[0] - assert row_score.metrics == {} + assert row_score.metrics["metric-a"][0].name == "score" + assert math.isnan(row_score.metrics["metric-a"][0].value) assert row_score.metric_errors == {"metric-a": GENERATION_FAILURE_MESSAGE} assert result.aggregate_scores.scores[0].name == "score" assert result.aggregate_scores.scores[0].count == 0 @@ -1532,9 +1549,9 @@ async def _fake_inference( ) -> dict: return {"choices": [{"message": {"content": ""}}]} - async def _compute_scores(item: dict[str, str], sample: dict[str, str]) -> MetricResult: - del item - assert sample == {"response": {"choices": [{"message": {"content": ""}}]}} + async def _compute_scores(input: MetricInput) -> MetricResult: + assert input.row.data == {"prompt": "hello"} + assert input.candidate.as_sample() == {"response": {"choices": [{"message": {"content": ""}}]}} return _make_metric_result(("score", 1.0)) metric.compute_scores = mocker.AsyncMock(side_effect=_compute_scores) diff --git a/packages/nemo_evaluator_sdk/tests/execution/test_metric_output_plumbing.py b/packages/nemo_evaluator_sdk/tests/execution/test_metric_output_plumbing.py new file mode 100644 index 0000000000..f24ecee73e --- /dev/null +++ b/packages/nemo_evaluator_sdk/tests/execution/test_metric_output_plumbing.py @@ -0,0 +1,117 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Focused tests for MetricInput/MetricOutput execution plumbing.""" + +from __future__ import annotations + +import math + +import pytest +from nemo_evaluator_sdk.execution.benchmark_execution import evaluate_benchmark +from nemo_evaluator_sdk.execution.scoring import finalize_evaluation_result, score_row +from nemo_evaluator_sdk.metrics.protocol import MetricInput, MetricOutput, MetricOutputSpec, MetricResult +from nemo_evaluator_sdk.values import ( + RunConfig, +) + + +class _OutputMetric: + @property + def type(self) -> str: + return "test-output-metric" + + def output_spec(self) -> list[MetricOutputSpec]: + return [ + MetricOutputSpec.continuous_score("score"), + MetricOutputSpec.boolean("passed"), + MetricOutputSpec.label("reason"), + ] + + async def compute_scores(self, input: MetricInput) -> MetricResult: + assert input.row.row_index is not None + assert input.row.data["expected"] == "yes" + assert input.candidate.output_text in {"yes", "no", None} + passed = input.candidate.output_text == input.row.data["expected"] + return MetricResult( + outputs=[ + MetricOutput(name="score", value=1.0 if passed else 0.0), + MetricOutput(name="passed", value=passed), + MetricOutput(name="reason", value="matched" if passed else "mismatched"), + ] + ) + + +class _RaisingMetric(_OutputMetric): + async def compute_scores(self, input: MetricInput) -> MetricResult: + del input + raise RuntimeError("metric failed") + + +@pytest.mark.asyncio +async def test_score_row_stores_outputs_and_aggregates_score_like_specs() -> None: + metric = _OutputMetric() + completed = [ + await score_row( + metric=metric, + row={"expected": "yes"}, + sample={"output_text": "yes", "response": {"id": "ok"}}, + index=0, + metric_key="metric", + fail_fast=True, + generation_requests=[], + ) + ] + + result = await finalize_evaluation_result(metric, completed) + + row = result.row_scores[0] + assert [output.name for output in row.metrics["metric"]] == ["score", "passed", "reason"] + assert result.to_records()[0]["output.score"] == 1.0 + assert result.to_records()[0]["output.reason"] == "matched" + assert [score.name for score in result.aggregate_scores.scores] == ["score", "passed"] + assert [score.mean for score in result.aggregate_scores.scores] == [1.0, 1.0] + + +@pytest.mark.asyncio +async def test_lenient_score_row_creates_nan_outputs_for_aggregateable_specs() -> None: + metric = _RaisingMetric() + + _, metric_result, row = await score_row( + metric=metric, + row={"expected": "yes"}, + sample={"output_text": "no"}, + index=0, + metric_key="metric", + fail_fast=False, + generation_requests=[], + ) + + assert metric_result is not None + assert [output.name for output in metric_result.outputs] == ["score", "passed"] + assert all(math.isnan(output.value) for output in metric_result.outputs) + assert [output.name for output in row.metrics["metric"]] == ["score", "passed"] + assert row.metric_errors == {"metric": "metric failed"} + + result = await finalize_evaluation_result(metric, [(0, metric_result, row)]) + + assert [(score.name, score.count, score.nan_count) for score in result.aggregate_scores.scores] == [ + ("score", 0, 1), + ("passed", 0, 1), + ] + + +@pytest.mark.asyncio +async def test_benchmark_namespaces_output_rows_and_aggregates() -> None: + rows = [{"expected": "yes"}, {"expected": "yes"}] + + result = await evaluate_benchmark( + metrics=[("custom", _OutputMetric())], + rows=rows, + target=None, + params=RunConfig(parallelism=1), + ) + + assert result.row_scores[0].metrics["custom"][0] == MetricOutput(name="score", value=0.0) + assert result.to_records()[0]["output.custom.reason"] == "mismatched" + assert [score.name for score in result.aggregate_scores.scores] == ["custom.score", "custom.passed"] diff --git a/packages/nemo_evaluator_sdk/tests/metrics/helpers.py b/packages/nemo_evaluator_sdk/tests/metrics/helpers.py new file mode 100644 index 0000000000..1c5d340c3b --- /dev/null +++ b/packages/nemo_evaluator_sdk/tests/metrics/helpers.py @@ -0,0 +1,42 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Test helpers for the MetricInput/MetricOutput protocol.""" + +from typing import Any + +from nemo_evaluator_sdk.execution.scoring import build_metric_input +from nemo_evaluator_sdk.metrics.protocol import CorpusMetric, Metric +from nemo_evaluator_sdk.values import MetricInput, MetricResult + + +def metric_input(item: dict[str, Any], sample: dict[str, Any] | None = None) -> MetricInput: + """Build a MetricInput for concise metric unit tests.""" + return build_metric_input(item, sample or {}) + + +async def compute_scores( + metric: Metric, + item: dict[str, Any] | None = None, + sample: dict[str, Any] | None = None, +) -> MetricResult: + """Invoke the new metric protocol using test row/sample dictionaries.""" + return await metric.compute_scores(metric_input(item or {}, sample or {})) + + +async def compute_corpus_scores( + metric: CorpusMetric, + items: list[dict[str, Any]] | None = None, + samples: list[dict[str, Any]] | None = None, +) -> MetricResult | None: + """Invoke the new corpus metric protocol using test row/sample dictionaries.""" + rows = items or [] + sample_rows = samples or [{} for _ in rows] + return await metric.compute_corpus_scores( + [build_metric_input(item, sample) for item, sample in zip(rows, sample_rows)] + ) + + +def output_names(metric: Metric) -> list[str]: + """Return declared metric output names.""" + return [output.name for output in metric.output_spec()] diff --git a/packages/nemo_evaluator_sdk/tests/metrics/ragas/test_ragas.py b/packages/nemo_evaluator_sdk/tests/metrics/ragas/test_ragas.py index 33db632518..ab23e512ae 100644 --- a/packages/nemo_evaluator_sdk/tests/metrics/ragas/test_ragas.py +++ b/packages/nemo_evaluator_sdk/tests/metrics/ragas/test_ragas.py @@ -10,6 +10,7 @@ import httpx import pytest +from metrics.helpers import compute_scores, output_names from nemo_evaluator_sdk.constants import PLACEHOLDER_INFERENCE_API_KEY from nemo_evaluator_sdk.enums import MetricType from nemo_evaluator_sdk.metrics.ragas import ( @@ -67,11 +68,11 @@ def _is_chat_handler_factory(value: object) -> TypeGuard[_ChatHandlerFactory]: def _get_score_value(result: MetricResult, score_name: str) -> float: - """Helper to get score value from v2 MetricResult (list of MetricScore).""" - for score in result.scores: - if score.name == score_name: - return score.value - raise KeyError(f"Score '{score_name}' not found in result") + """Helper to get output value from MetricResult.""" + for output in result.outputs: + if output.name == score_name: + return output.value + raise KeyError(f"Output '{score_name}' not found in result") @pytest.mark.asyncio @@ -104,7 +105,7 @@ async def test_metric_types(metric_class, expected_type, params): def test_score_names_defaults_to_metric_type_value(): metric = TopicAdherenceMetric(metric_mode="f1", judge_model=MOCK_JUDGE_MODEL) - assert metric.score_names() == [metric.type.value] + assert output_names(metric) == [metric.type.value] def test_llm_backed_ragas_metric_exposes_ignore_request_failure(): @@ -128,7 +129,7 @@ async def test_topic_adherence_metric(): mock_evaluate = MagicMock() mock_evaluate.return_value.scores = [{"topic_relevance": 0.85}] with patch("nemo_evaluator_sdk.metrics.ragas.base.get_evaluate_function", return_value=mock_evaluate): - result = await metric.compute_scores(MOCK_ITEM, MOCK_SAMPLE) + result = await compute_scores(metric, MOCK_ITEM, MOCK_SAMPLE) assert isinstance(result, MetricResult) assert _get_score_value(result, "topic_relevance") == 0.85 @@ -141,7 +142,7 @@ async def test_tool_call_accuracy_metric(): mock_evaluate = MagicMock() mock_evaluate.return_value.scores = [{"tool_call_accuracy": 0.9}] with patch("nemo_evaluator_sdk.metrics.ragas.base.get_evaluate_function", return_value=mock_evaluate): - result = await metric.compute_scores(MOCK_ITEM, MOCK_SAMPLE) + result = await compute_scores(metric, MOCK_ITEM, MOCK_SAMPLE) assert isinstance(result, MetricResult) assert _get_score_value(result, "tool_call_accuracy") == 0.9 @@ -157,7 +158,7 @@ async def test_agent_goal_accuracy_metric(): mock_evaluate = MagicMock() mock_evaluate.return_value.scores = [{"agent_goal_accuracy": 0.95}] with patch("nemo_evaluator_sdk.metrics.ragas.base.get_evaluate_function", return_value=mock_evaluate): - result = await metric_with_ref.compute_scores(MOCK_ITEM, MOCK_SAMPLE) + result = await compute_scores(metric_with_ref, MOCK_ITEM, MOCK_SAMPLE) assert _get_score_value(result, "agent_goal_accuracy") == 0.95 # Test without reference @@ -168,7 +169,7 @@ async def test_agent_goal_accuracy_metric(): mock_evaluate2 = MagicMock() mock_evaluate2.return_value.scores = [{"agent_goal_accuracy": 0.85}] with patch("nemo_evaluator_sdk.metrics.ragas.base.get_evaluate_function", return_value=mock_evaluate2): - result = await metric_without_ref.compute_scores(MOCK_ITEM, MOCK_SAMPLE) + result = await compute_scores(metric_without_ref, MOCK_ITEM, MOCK_SAMPLE) assert _get_score_value(result, "agent_goal_accuracy") == 0.85 @@ -508,7 +509,7 @@ async def test_embeddings_client_validation(): mock_evaluate = MagicMock() mock_evaluate.return_value.scores = [{"answer_relevancy": 0.9}] with patch("nemo_evaluator_sdk.metrics.ragas.base.get_evaluate_function", return_value=mock_evaluate): - result = await metric.compute_scores(MOCK_ITEM, MOCK_SAMPLE) + result = await compute_scores(metric, MOCK_ITEM, MOCK_SAMPLE) assert isinstance(result, MetricResult) @@ -600,15 +601,15 @@ async def test_metric_result_format(): mock_evaluate = MagicMock() mock_evaluate.return_value.scores = [{"topic_relevance": 0.85, "another_score": 0.9}] with patch("nemo_evaluator_sdk.metrics.ragas.base.get_evaluate_function", return_value=mock_evaluate): - result = await metric.compute_scores(MOCK_ITEM, MOCK_SAMPLE) + result = await compute_scores(metric, MOCK_ITEM, MOCK_SAMPLE) - # v2 format: MetricResult.scores is a list of MetricScore + # MetricResult.outputs is a list of MetricOutput. assert isinstance(result, MetricResult) - assert isinstance(result.scores, list) - assert len(result.scores) == 2 + assert isinstance(result.outputs, list) + assert len(result.outputs) == 2 - # Check that scores are MetricScore objects with name and value - score_dict = {s.name: s.value for s in result.scores} + # Check that outputs have name and value. + score_dict = {s.name: s.value for s in result.outputs} assert score_dict["topic_relevance"] == 0.85 assert score_dict["another_score"] == 0.9 diff --git a/packages/nemo_evaluator_sdk/tests/metrics/test_bleu.py b/packages/nemo_evaluator_sdk/tests/metrics/test_bleu.py index 8368aeab9e..09fc846e65 100644 --- a/packages/nemo_evaluator_sdk/tests/metrics/test_bleu.py +++ b/packages/nemo_evaluator_sdk/tests/metrics/test_bleu.py @@ -2,6 +2,7 @@ # SPDX-License-Identifier: Apache-2.0 import pytest +from metrics.helpers import compute_corpus_scores, compute_scores, output_names from nemo_evaluator_sdk.execution.evaluator import Evaluator from nemo_evaluator_sdk.metrics.bleu import BLEUMetric, MetricResult @@ -11,7 +12,9 @@ class TestBLEUMetric: async def test_metric_default_candidate(self): metric = BLEUMetric(references=["{{item.reference}}"]) score = ( - (await metric.compute_scores({"reference": "the cat sat"}, {"output_text": "the cat sat"})).scores[0].value + (await compute_scores(metric, {"reference": "the cat sat"}, {"output_text": "the cat sat"})) + .outputs[0] + .value ) assert score == pytest.approx(100.0) @@ -20,11 +23,11 @@ async def test_metric_with_custom_candidate(self): metric = BLEUMetric(references=["{{item.reference}}"], candidate="{{item.pred}}") score = ( ( - await metric.compute_scores( - {"reference": "the cat sat", "pred": "the cat sat"}, {"output_text": "ignored"} + await compute_scores( + metric, {"reference": "the cat sat", "pred": "the cat sat"}, {"output_text": "ignored"} ) ) - .scores[0] + .outputs[0] .value ) assert score == pytest.approx(100.0) @@ -32,10 +35,10 @@ async def test_metric_with_custom_candidate(self): @pytest.mark.asyncio async def test_metric_scores(self): metric = BLEUMetric(references=["{{item.reference}}"]) - assert (await metric.compute_scores({"reference": "the cat"}, {"output_text": "the cat"})).scores[ + assert (await compute_scores(metric, {"reference": "the cat"}, {"output_text": "the cat"})).outputs[ 0 ].value == pytest.approx(100.0) - assert (await metric.compute_scores({"reference": "the cat"}, {"output_text": "dog barked"})).scores[ + assert (await compute_scores(metric, {"reference": "the cat"}, {"output_text": "dog barked"})).outputs[ 0 ].value == pytest.approx(0.0) @@ -43,41 +46,43 @@ async def test_metric_scores(self): async def test_metric_validates_reference_and_candidate_types(self): metric = BLEUMetric(references=["{{item.reference}}"]) with pytest.raises(TypeError, match="The reference must be a string"): - await metric.compute_scores({"reference": 1}, {"output_text": "1"}) + await compute_scores(metric, {"reference": 1}, {"output_text": "1"}) bad_candidate = BLEUMetric(references=["{{item.reference}}"], candidate="{{item.pred}}") with pytest.raises(TypeError, match="The candidate must be a string"): - await bad_candidate.compute_scores({"reference": "x", "pred": 1}, {"output_text": "ignored"}) + await compute_scores(bad_candidate, {"reference": "x", "pred": 1}, {"output_text": "ignored"}) @pytest.mark.asyncio async def test_metric_default_candidate_non_string_raises_value_error(self): metric = BLEUMetric(references=["{{item.reference}}"]) with pytest.raises(TypeError, match="The candidate must be a string"): - await metric.compute_scores({"reference": "x"}, {"output_text": 123}) + await compute_scores(metric, {"reference": "x"}, {"output_text": 123}) @pytest.mark.asyncio async def test_metric_default_candidate_missing_output_text_raises_clear_error(self): metric = BLEUMetric(references=["{{item.reference}}"]) with pytest.raises(ValueError, match=r"candidate=\.\.\."): - await metric.compute_scores({"reference": "x"}, {}) + await compute_scores(metric, {"reference": "x"}, {}) @pytest.mark.asyncio async def test_compute_scores_and_corpus_scores(self): metric = BLEUMetric(references=["{{item.reference}}"]) - row_result = await metric.compute_scores({"reference": "the cat"}, {"output_text": "the cat"}) - assert row_result.scores[0].name == "sentence" + row_result = await compute_scores(metric, {"reference": "the cat"}, {"output_text": "the cat"}) + assert row_result.outputs[0].name == "sentence" - corpus_result = await metric.compute_corpus_scores( + corpus_result = await compute_corpus_scores( + metric, items=[{"reference": "the cat"}, {"reference": "a dog"}], samples=[{"output_text": "the cat"}, {"output_text": "a dog"}], ) assert corpus_result is not None - assert corpus_result.scores[0].name == "corpus" + assert corpus_result.outputs[0].name == "corpus" @pytest.mark.asyncio async def test_corpus_uses_candidate_template_and_validates_type(self): metric = BLEUMetric(references=["{{item.reference}}"], candidate="{{item.pred}}") - corpus_result = await metric.compute_corpus_scores( + corpus_result = await compute_corpus_scores( + metric, items=[{"reference": "the cat", "pred": "the cat"}], samples=[{"output_text": "ignored"}], ) @@ -85,7 +90,8 @@ async def test_corpus_uses_candidate_template_and_validates_type(self): bad_metric = BLEUMetric(references=["{{item.reference}}"], candidate="{{item.pred}}") with pytest.raises(TypeError, match="The candidate must be a string"): - await bad_metric.compute_corpus_scores( + await compute_corpus_scores( + bad_metric, items=[{"reference": "the cat", "pred": 1}], samples=[{"output_text": "ignored"}], ) @@ -94,13 +100,13 @@ async def test_corpus_uses_candidate_template_and_validates_type(self): async def test_raises_clear_error_for_missing_reference_field(self): metric = BLEUMetric(references=["{{item.reference}}"], candidate="{{sample.output_text}}") with pytest.raises(ValueError) as exc_info: - await metric.compute_scores(item={"prompt": "hello"}, sample={"output_text": "hi"}) + await compute_scores(metric, item={"prompt": "hello"}, sample={"output_text": "hi"}) assert "could not render its 'references[0]' template for this row" in str(exc_info.value) assert "missing_key='reference'" in str(exc_info.value) def test_score_names(self): metric = BLEUMetric(references=["{{item.reference}}"]) - assert metric.score_names() == ["sentence"] + assert output_names(metric) == ["sentence"] def test_run_sync_adds_corpus_score(self): metric = BLEUMetric(references=["{{item.reference}}"], candidate="{{item.pred}}") @@ -125,19 +131,19 @@ async def test_compute_scores_exact_match(self): item = {"reference": "The cat sat on the mat."} sample = {"output_text": "The cat sat on the mat."} - result = await metric.compute_scores(item, sample) + result = await compute_scores(metric, item, sample) assert isinstance(result, MetricResult) - assert len(result.scores) == 1 - assert result.scores[0].name == "sentence" + assert len(result.outputs) == 1 + assert result.outputs[0].name == "sentence" # Exact match should give a perfect or near-perfect BLEU score - assert result.scores[0].value > 90.0 + assert result.outputs[0].value > 90.0 @pytest.mark.asyncio async def test_score_names_match_compute_scores(self): metric = BLEUMetric(references=["{{item.reference}}"]) - result = await metric.compute_scores({"reference": "a"}, {"output_text": "a"}) - assert {score.name for score in result.scores} == set(metric.score_names()) + result = await compute_scores(metric, {"reference": "a"}, {"output_text": "a"}) + assert {score.name for score in result.outputs} == set(output_names(metric)) @pytest.mark.asyncio async def test_compute_scores_partial_match(self): @@ -149,13 +155,13 @@ async def test_compute_scores_partial_match(self): item = {"reference": "The cat sat on the mat."} sample = {"output_text": "The dog sat on the floor."} - result = await metric.compute_scores(item, sample) + result = await compute_scores(metric, item, sample) assert isinstance(result, MetricResult) - assert len(result.scores) == 1 - assert result.scores[0].name == "sentence" + assert len(result.outputs) == 1 + assert result.outputs[0].name == "sentence" # Partial match should give a lower score - assert 0.0 <= result.scores[0].value < 100.0 + assert 0.0 <= result.outputs[0].value < 100.0 @pytest.mark.asyncio async def test_compute_scores_no_match(self): @@ -167,13 +173,13 @@ async def test_compute_scores_no_match(self): item = {"reference": "The cat sat on the mat."} sample = {"output_text": "Completely different sentence with no overlap."} - result = await metric.compute_scores(item, sample) + result = await compute_scores(metric, item, sample) assert isinstance(result, MetricResult) - assert len(result.scores) == 1 - assert result.scores[0].name == "sentence" + assert len(result.outputs) == 1 + assert result.outputs[0].name == "sentence" # No match should give a very low score - assert result.scores[0].value >= 0.0 + assert result.outputs[0].value >= 0.0 @pytest.mark.asyncio async def test_compute_scores_with_custom_candidate(self): @@ -186,11 +192,11 @@ async def test_compute_scores_with_custom_candidate(self): item = {"reference": "The cat sat on the mat.", "custom_output": "The cat sat on the mat."} sample = {"output_text": "This should be ignored."} - result = await metric.compute_scores(item, sample) + result = await compute_scores(metric, item, sample) assert isinstance(result, MetricResult) - assert len(result.scores) == 1 - assert result.scores[0].value > 90.0 + assert len(result.outputs) == 1 + assert result.outputs[0].value > 90.0 @pytest.mark.asyncio async def test_compute_corpus_scores(self): @@ -208,13 +214,13 @@ async def test_compute_corpus_scores(self): {"output_text": "The dog ran in the park."}, ] - result = await metric.compute_corpus_scores(items, samples) + result = await compute_corpus_scores(metric, items, samples) assert isinstance(result, MetricResult) - assert len(result.scores) == 1 - assert result.scores[0].name == "corpus" + assert len(result.outputs) == 1 + assert result.outputs[0].name == "corpus" # Good matches should give high corpus score - assert result.scores[0].value > 90.0 + assert result.outputs[0].value > 90.0 @pytest.mark.asyncio async def test_compute_scores_multiple_references(self): @@ -226,9 +232,9 @@ async def test_compute_scores_multiple_references(self): item = {"reference1": "The cat sat on the mat.", "reference2": "A cat was sitting on the mat."} sample = {"output_text": "The cat sat on the mat."} - result = await metric.compute_scores(item, sample) + result = await compute_scores(metric, item, sample) assert isinstance(result, MetricResult) - assert len(result.scores) == 1 + assert len(result.outputs) == 1 # With multiple references, should still compute properly - assert result.scores[0].value >= 0.0 + assert result.outputs[0].value >= 0.0 diff --git a/packages/nemo_evaluator_sdk/tests/metrics/test_exact_match.py b/packages/nemo_evaluator_sdk/tests/metrics/test_exact_match.py index e204ef81e0..e4a9b4068d 100644 --- a/packages/nemo_evaluator_sdk/tests/metrics/test_exact_match.py +++ b/packages/nemo_evaluator_sdk/tests/metrics/test_exact_match.py @@ -2,18 +2,19 @@ # SPDX-License-Identifier: Apache-2.0 import pytest +from metrics.helpers import compute_scores, output_names from nemo_evaluator_sdk.metrics.exact_match import ExactMatchMetric, MetricResult class TestExactMatchMetric: def test_score_names(self): metric = ExactMatchMetric(reference="{{item.reference}}") - assert metric.score_names() == ["exact-match"] + assert output_names(metric) == ["exact-match"] @pytest.mark.asyncio async def test_metric_returns_score(self): metric = ExactMatchMetric(reference="{{item.reference}}") - assert (await metric.compute_scores({"reference": "The Cat"}, {"output_text": "the cat"})).scores[ + assert (await compute_scores(metric, {"reference": "The Cat"}, {"output_text": "the cat"})).outputs[ 0 ].value == 1.0 @@ -56,18 +57,18 @@ async def test_compute_scores_exact_match(self): item = {"reference": "The cat sat on the mat."} sample = {"output_text": "The cat sat on the mat."} - result = await metric.compute_scores(item, sample) + result = await compute_scores(metric, item, sample) assert isinstance(result, MetricResult) - assert len(result.scores) == 1 - assert result.scores[0].name == "exact-match" - assert result.scores[0].value == 1 + assert len(result.outputs) == 1 + assert result.outputs[0].name == "exact-match" + assert result.outputs[0].value == 1 @pytest.mark.asyncio async def test_score_names_match_compute_scores(self): metric = ExactMatchMetric(reference="{{item.reference}}") - result = await metric.compute_scores({"reference": "a"}, {"output_text": "a"}) - assert {score.name for score in result.scores} == set(metric.score_names()) + result = await compute_scores(metric, {"reference": "a"}, {"output_text": "a"}) + assert {score.name for score in result.outputs} == set(output_names(metric)) @pytest.mark.asyncio async def test_compute_scores_case_insensitive(self): @@ -79,11 +80,11 @@ async def test_compute_scores_case_insensitive(self): item = {"reference": "The Cat Sat On The Mat."} sample = {"output_text": "the cat sat on the mat."} - result = await metric.compute_scores(item, sample) + result = await compute_scores(metric, item, sample) assert isinstance(result, MetricResult) - assert len(result.scores) == 1 - assert result.scores[0].value == 1 + assert len(result.outputs) == 1 + assert result.outputs[0].value == 1 @pytest.mark.asyncio async def test_compute_scores_ignores_punctuation(self): @@ -95,11 +96,11 @@ async def test_compute_scores_ignores_punctuation(self): item = {"reference": "The cat sat on the mat."} sample = {"output_text": "The cat sat on the mat!"} - result = await metric.compute_scores(item, sample) + result = await compute_scores(metric, item, sample) assert isinstance(result, MetricResult) - assert len(result.scores) == 1 - assert result.scores[0].value == 1 + assert len(result.outputs) == 1 + assert result.outputs[0].value == 1 @pytest.mark.asyncio async def test_compute_scores_ignores_articles(self): @@ -111,11 +112,11 @@ async def test_compute_scores_ignores_articles(self): item = {"reference": "The cat sat on a mat."} sample = {"output_text": "cat sat on mat"} - result = await metric.compute_scores(item, sample) + result = await compute_scores(metric, item, sample) assert isinstance(result, MetricResult) - assert len(result.scores) == 1 - assert result.scores[0].value == 1 + assert len(result.outputs) == 1 + assert result.outputs[0].value == 1 @pytest.mark.asyncio async def test_compute_scores_no_match(self): @@ -127,11 +128,11 @@ async def test_compute_scores_no_match(self): item = {"reference": "The cat sat on the mat."} sample = {"output_text": "The dog ran in the park."} - result = await metric.compute_scores(item, sample) + result = await compute_scores(metric, item, sample) assert isinstance(result, MetricResult) - assert len(result.scores) == 1 - assert result.scores[0].value == 0 + assert len(result.outputs) == 1 + assert result.outputs[0].value == 0 @pytest.mark.asyncio async def test_compute_scores_with_custom_candidate(self): @@ -144,11 +145,11 @@ async def test_compute_scores_with_custom_candidate(self): item = {"reference": "The cat sat on the mat.", "custom_output": "The cat sat on the mat."} sample = {"output_text": "This should be ignored."} - result = await metric.compute_scores(item, sample) + result = await compute_scores(metric, item, sample) assert isinstance(result, MetricResult) - assert len(result.scores) == 1 - assert result.scores[0].value == 1 + assert len(result.outputs) == 1 + assert result.outputs[0].value == 1 @pytest.mark.asyncio async def test_compute_scores_whitespace_handling(self): @@ -160,11 +161,11 @@ async def test_compute_scores_whitespace_handling(self): item = {"reference": "The cat sat on the mat."} sample = {"output_text": "The cat sat on the mat."} - result = await metric.compute_scores(item, sample) + result = await compute_scores(metric, item, sample) assert isinstance(result, MetricResult) - assert len(result.scores) == 1 - assert result.scores[0].value == 1 + assert len(result.outputs) == 1 + assert result.outputs[0].value == 1 @pytest.mark.asyncio async def test_compute_scores_empty_strings(self): @@ -176,8 +177,8 @@ async def test_compute_scores_empty_strings(self): item = {"reference": ""} sample = {"output_text": ""} - result = await metric.compute_scores(item, sample) + result = await compute_scores(metric, item, sample) assert isinstance(result, MetricResult) - assert len(result.scores) == 1 - assert result.scores[0].value == 1 + assert len(result.outputs) == 1 + assert result.outputs[0].value == 1 diff --git a/packages/nemo_evaluator_sdk/tests/metrics/test_f1.py b/packages/nemo_evaluator_sdk/tests/metrics/test_f1.py index ced55fc2d2..cd3c2245a6 100644 --- a/packages/nemo_evaluator_sdk/tests/metrics/test_f1.py +++ b/packages/nemo_evaluator_sdk/tests/metrics/test_f1.py @@ -2,6 +2,7 @@ # SPDX-License-Identifier: Apache-2.0 import pytest +from metrics.helpers import compute_scores, output_names from nemo_evaluator_sdk.execution.evaluator import Evaluator from nemo_evaluator_sdk.metrics.f1 import F1Metric, MetricResult @@ -19,41 +20,41 @@ class TestF1Metric: @pytest.mark.asyncio async def test_metric_default_candidate(self, item, sample, expected): metric = F1Metric(reference="{{item.reference}}") - assert (await metric.compute_scores(item, sample)).scores[0].value == expected + assert (await compute_scores(metric, item, sample)).outputs[0].value == expected @pytest.mark.asyncio async def test_metric_with_custom_candidate(self): metric = F1Metric(reference="{{item.reference}}", candidate="{{item.pred}}") - assert (await metric.compute_scores({"reference": "a b c", "pred": "a b"}, {"output_text": "ignored"})).scores[ - 0 - ].value == pytest.approx(0.6666666666666666) + assert ( + await compute_scores(metric, {"reference": "a b c", "pred": "a b"}, {"output_text": "ignored"}) + ).outputs[0].value == pytest.approx(0.6666666666666666) @pytest.mark.asyncio async def test_metric_validates_rendered_types(self): metric = F1Metric(reference="{{item.reference}}") with pytest.raises(TypeError, match="The reference must be a string"): - await metric.compute_scores({"reference": 1}, {"output_text": "1"}) + await compute_scores(metric, {"reference": 1}, {"output_text": "1"}) with pytest.raises(TypeError, match="The candidate must be a string"): - await metric.compute_scores({"reference": "1"}, {"output_text": 1}) + await compute_scores(metric, {"reference": "1"}, {"output_text": 1}) @pytest.mark.asyncio async def test_compute_scores(self): metric = F1Metric(reference="{{item.reference}}") - result = await metric.compute_scores({"reference": "cat"}, {"output_text": "cat"}) - assert result.scores[0].name == "f1" - assert result.scores[0].value == 1.0 + result = await compute_scores(metric, {"reference": "cat"}, {"output_text": "cat"}) + assert result.outputs[0].name == "f1" + assert result.outputs[0].value == 1.0 @pytest.mark.asyncio async def test_raises_clear_error_for_missing_reference_field(self): metric = F1Metric(reference="{{item.reference}}", candidate="{{sample.output_text}}") with pytest.raises(ValueError) as exc_info: - await metric.compute_scores(item={"prompt": "hello"}, sample={"output_text": "hi"}) + await compute_scores(metric, item={"prompt": "hello"}, sample={"output_text": "hi"}) assert "could not render its 'reference' template for this row" in str(exc_info.value) assert "missing_key='reference'" in str(exc_info.value) def test_score_names(self): metric = F1Metric(reference="{{item.reference}}") - assert metric.score_names() == ["f1"] + assert output_names(metric) == ["f1"] def test_run_sync(self): metric = F1Metric(reference="{{item.reference}}", candidate="{{item.prediction}}") @@ -92,18 +93,18 @@ async def test_compute_scores_exact_match(self): item = {"reference": "The cat sat on the mat"} sample = {"output_text": "The cat sat on the mat"} - result = await metric.compute_scores(item, sample) + result = await compute_scores(metric, item, sample) assert isinstance(result, MetricResult) - assert len(result.scores) == 1 - assert result.scores[0].name == "f1" - assert result.scores[0].value == 1.0 + assert len(result.outputs) == 1 + assert result.outputs[0].name == "f1" + assert result.outputs[0].value == 1.0 @pytest.mark.asyncio async def test_score_names_match_compute_scores(self): metric = F1Metric(reference="{{item.reference}}") - result = await metric.compute_scores({"reference": "a"}, {"output_text": "a"}) - assert {score.name for score in result.scores} == set(metric.score_names()) + result = await compute_scores(metric, {"reference": "a"}, {"output_text": "a"}) + assert {score.name for score in result.outputs} == set(output_names(metric)) @pytest.mark.asyncio async def test_compute_scores_partial_overlap(self): @@ -115,13 +116,13 @@ async def test_compute_scores_partial_overlap(self): item = {"reference": "The cat sat on the mat"} sample = {"output_text": "The cat slept on the floor"} - result = await metric.compute_scores(item, sample) + result = await compute_scores(metric, item, sample) assert isinstance(result, MetricResult) - assert len(result.scores) == 1 - assert result.scores[0].name == "f1" + assert len(result.outputs) == 1 + assert result.outputs[0].name == "f1" # Partial overlap should give score between 0 and 1 - assert 0.0 < result.scores[0].value < 1.0 + assert 0.0 < result.outputs[0].value < 1.0 @pytest.mark.asyncio async def test_compute_scores_no_overlap(self): @@ -133,11 +134,11 @@ async def test_compute_scores_no_overlap(self): item = {"reference": "hello world"} sample = {"output_text": "goodbye universe"} - result = await metric.compute_scores(item, sample) + result = await compute_scores(metric, item, sample) assert isinstance(result, MetricResult) - assert len(result.scores) == 1 - assert result.scores[0].value == 0.0 + assert len(result.outputs) == 1 + assert result.outputs[0].value == 0.0 @pytest.mark.asyncio async def test_compute_scores_both_empty(self): @@ -149,12 +150,12 @@ async def test_compute_scores_both_empty(self): item = {"reference": ""} sample = {"output_text": ""} - result = await metric.compute_scores(item, sample) + result = await compute_scores(metric, item, sample) assert isinstance(result, MetricResult) - assert len(result.scores) == 1 + assert len(result.outputs) == 1 # Empty strings that agree should return 1 - assert result.scores[0].value == 1 + assert result.outputs[0].value == 1 @pytest.mark.asyncio async def test_compute_scores_one_empty(self): @@ -166,12 +167,12 @@ async def test_compute_scores_one_empty(self): item = {"reference": ""} sample = {"output_text": "some text"} - result = await metric.compute_scores(item, sample) + result = await compute_scores(metric, item, sample) assert isinstance(result, MetricResult) - assert len(result.scores) == 1 + assert len(result.outputs) == 1 # One empty, one not should return 0 - assert result.scores[0].value == 0 + assert result.outputs[0].value == 0 @pytest.mark.asyncio async def test_compute_scores_case_insensitive(self): @@ -183,11 +184,11 @@ async def test_compute_scores_case_insensitive(self): item = {"reference": "The Cat Sat"} sample = {"output_text": "the cat sat"} - result = await metric.compute_scores(item, sample) + result = await compute_scores(metric, item, sample) assert isinstance(result, MetricResult) - assert len(result.scores) == 1 - assert result.scores[0].value == 1.0 + assert len(result.outputs) == 1 + assert result.outputs[0].value == 1.0 @pytest.mark.asyncio async def test_compute_scores_ignores_punctuation(self): @@ -199,11 +200,11 @@ async def test_compute_scores_ignores_punctuation(self): item = {"reference": "Hello, world!"} sample = {"output_text": "Hello world"} - result = await metric.compute_scores(item, sample) + result = await compute_scores(metric, item, sample) assert isinstance(result, MetricResult) - assert len(result.scores) == 1 - assert result.scores[0].value == 1.0 + assert len(result.outputs) == 1 + assert result.outputs[0].value == 1.0 @pytest.mark.asyncio async def test_compute_scores_ignores_articles(self): @@ -215,11 +216,11 @@ async def test_compute_scores_ignores_articles(self): item = {"reference": "The cat sat on a mat"} sample = {"output_text": "cat sat on mat"} - result = await metric.compute_scores(item, sample) + result = await compute_scores(metric, item, sample) assert isinstance(result, MetricResult) - assert len(result.scores) == 1 - assert result.scores[0].value == 1.0 + assert len(result.outputs) == 1 + assert result.outputs[0].value == 1.0 @pytest.mark.asyncio async def test_compute_scores_duplicate_tokens(self): @@ -231,9 +232,9 @@ async def test_compute_scores_duplicate_tokens(self): item = {"reference": "cat cat dog"} sample = {"output_text": "cat dog dog"} - result = await metric.compute_scores(item, sample) + result = await compute_scores(metric, item, sample) assert isinstance(result, MetricResult) - assert len(result.scores) == 1 + assert len(result.outputs) == 1 # Should handle duplicate tokens with Counter intersection - assert 0.0 < result.scores[0].value < 1.0 + assert 0.0 < result.outputs[0].value < 1.0 diff --git a/packages/nemo_evaluator_sdk/tests/metrics/test_llm_judge.py b/packages/nemo_evaluator_sdk/tests/metrics/test_llm_judge.py index 266e8aa614..c4583b287e 100644 --- a/packages/nemo_evaluator_sdk/tests/metrics/test_llm_judge.py +++ b/packages/nemo_evaluator_sdk/tests/metrics/test_llm_judge.py @@ -8,6 +8,7 @@ import pytest from jsonschema.exceptions import SchemaError +from metrics.helpers import compute_scores, output_names from nemo_evaluator_sdk.enums import ModelFormat from nemo_evaluator_sdk.inference import ( AddInferenceParameter, @@ -24,11 +25,11 @@ default_judge_prompt_template_completions, generate_structured_output, ) +from nemo_evaluator_sdk.metrics.protocol import MetricOutput, MetricResult from nemo_evaluator_sdk.structured_output import InferenceStructuredOutput, StructuredOutputMode from nemo_evaluator_sdk.values.common import SecretRef, SupportedJobTypes from nemo_evaluator_sdk.values.models import Model from nemo_evaluator_sdk.values.params import InferenceParams -from nemo_evaluator_sdk.values.results import MetricResult, MetricScore from nemo_evaluator_sdk.values.scores import ( JSONScoreParser, RangeScore, @@ -478,7 +479,23 @@ def test_initializes_json_parser_with_auto_generated_structured_output(self): def test_score_names_match_initialized_parsers(self): metric = LLMJudgeMetric(model=_make_model(), scores=[_make_metric_score()]) - assert metric.score_names() == ["helpfulness"] + assert output_names(metric) == ["helpfulness"] + + def test_rubric_output_spec_includes_numeric_score_and_label(self): + metric = LLMJudgeMetric(model=_make_model(), scores=[_new_rubric_score(None)]) + + assert output_names(metric) == ["length", "length.label"] + + @pytest.mark.asyncio + async def test_rubric_compute_scores_emits_numeric_score_and_label(self, mocker: MockerFixture): + metric = LLMJudgeMetric(model=_make_model(), scores=[_new_rubric_score(None)]) + metric.set_inference_fn( + mocker.AsyncMock(return_value={"choices": [{"message": {"content": '{"length":"short"}'}}]}) + ) + + result = await compute_scores(metric, {"prompt": "hello"}, {"output_text": "world"}) + + assert [(output.name, output.value) for output in result.outputs] == [("length", 0), ("length.label", "short")] def test_unique_scores_allows_empty_scores_for_constructed_model(self): metric = LLMJudgeMetric.model_construct(model=_make_model(), scores=[], _fields_set={"model", "scores"}) @@ -643,12 +660,12 @@ def test_handle_none_output_error_mentions_reasoning_only_responses(self): def test_handle_invalid_output_returns_fallback_when_ignore_request_failure_enabled(self): metric = LLMJudgeMetric(model=_make_model(), scores=[_make_metric_score()], ignore_request_failure=True) - fallback = MetricResult(scores=[MetricScore(name="helpfulness", value=float("nan"))]) + fallback = MetricResult(outputs=[MetricOutput(name="helpfulness", value=float("nan"))]) assert metric._handle_invalid_output(ValueError("boom"), fallback, "ignored") is fallback def test_handle_invalid_output_raises_when_ignore_request_failure_disabled(self): metric = LLMJudgeMetric(model=_make_model(), scores=[_make_metric_score()]) - fallback = MetricResult(scores=[MetricScore(name="helpfulness", value=float("nan"))]) + fallback = MetricResult(outputs=[MetricOutput(name="helpfulness", value=float("nan"))]) with pytest.raises(ValueError, match="boom"): metric._handle_invalid_output(ValueError("boom"), fallback, "ignored") @@ -709,7 +726,7 @@ async def inference_fn(*args, **kwargs): metric.set_inference_fn(inference_fn) - assert (await metric.compute_scores({"prompt": "hello"}, {"output_text": "world"})).scores[0].value == 3 + assert (await compute_scores(metric, {"prompt": "hello"}, {"output_text": "world"})).outputs[0].value == 3 assert len(captured_requests) == 2 first_request = captured_requests[0] second_request = captured_requests[1] @@ -726,7 +743,7 @@ async def test_metric_returns_score_when_trace_is_disabled(self, mocker: MockerF mocker.AsyncMock(return_value={"choices": [{"message": {"content": '{"helpfulness": 3}'}}]}) ) - assert (await metric.compute_scores({"prompt": "hello"}, {"output_text": "world"})).scores[0].value == 3 + assert (await compute_scores(metric, {"prompt": "hello"}, {"output_text": "world"})).outputs[0].value == 3 @pytest.mark.asyncio async def test_metric_invalid_output_returns_nan_when_ignore_enabled(self, mocker: MockerFixture): @@ -739,10 +756,10 @@ async def test_metric_invalid_output_returns_nan_when_ignore_enabled(self, mocke mocker.AsyncMock(return_value={"choices": [{"message": {"content": None, "reasoning_content": None}}]}) ) - result = await metric.compute_scores({"prompt": "hello"}, {"output_text": "world"}) - assert len(result.scores) == 1 - assert result.scores[0].name == "helpfulness" - assert math.isnan(result.scores[0].value) + result = await compute_scores(metric, {"prompt": "hello"}, {"output_text": "world"}) + assert len(result.outputs) == 1 + assert result.outputs[0].name == "helpfulness" + assert math.isnan(result.outputs[0].value) @pytest.mark.asyncio async def test_compute_scores_returns_nan_when_inference_failure_is_ignored(self, mocker: MockerFixture): @@ -754,11 +771,11 @@ async def test_compute_scores_returns_nan_when_inference_failure_is_ignored(self error = ClientInferenceError(mocker.Mock(status_code=500, response=mocker.Mock(text="boom"))) metric.set_inference_fn(mocker.AsyncMock(side_effect=error)) - result = await metric.compute_scores({"prompt": "hello"}, {"output_text": "world"}) + result = await compute_scores(metric, {"prompt": "hello"}, {"output_text": "world"}) - assert len(result.scores) == 1 - assert result.scores[0].name == "helpfulness" - assert math.isnan(result.scores[0].value) + assert len(result.outputs) == 1 + assert result.outputs[0].name == "helpfulness" + assert math.isnan(result.outputs[0].value) @pytest.mark.asyncio async def test_compute_scores_retries_with_max_completion_tokens(self, mocker: MockerFixture): @@ -777,9 +794,9 @@ async def inference_fn(*args, **kwargs): metric.set_inference_fn(inference_fn) - result = await metric.compute_scores({"prompt": "hello"}, {"output_text": "world"}) + result = await compute_scores(metric, {"prompt": "hello"}, {"output_text": "world"}) - assert result.scores[0].value == 4 + assert result.outputs[0].value == 4 assert "max_tokens" in captured_requests[0] assert "max_completion_tokens" in captured_requests[1] @@ -791,7 +808,7 @@ async def test_compute_scores_raises_invalid_output_when_ignore_failure_disabled ) with pytest.raises(ValueError, match="LLM judge returned no usable textual content"): - await metric.compute_scores({"prompt": "hello"}, {"output_text": "world"}) + await compute_scores(metric, {"prompt": "hello"}, {"output_text": "world"}) @pytest.mark.asyncio async def test_preflight_selects_structured_output_mode_for_nim(self, mocker: MockerFixture): @@ -864,9 +881,9 @@ async def inference_fn( metric.set_inference_fn(inference_fn) - result = await metric.compute_scores({"prompt": "hello"}, {"output_text": "world"}) + result = await compute_scores(metric, {"prompt": "hello"}, {"output_text": "world"}) - assert result.scores[0].value == 4 + assert result.outputs[0].value == 4 assert captured["default_headers"] == {"special-header": "evaluator"} def test_render_request_default_prompt_includes_rubric_details(self): @@ -900,12 +917,13 @@ async def test_ignore_request_failure_empty_output_returns_nan(self, mocker: Moc metric = LLMJudgeMetric.model_validate(llm_judge_param_dict({"ignore_request_failure": True})) metric.set_inference_fn(mocker.AsyncMock(return_value=_empty_judge_response())) - result = await metric.compute_scores( - {"question": "What is AI?"}, {"output_text": "AI is artificial intelligence"} + result = await compute_scores( + metric, {"question": "What is AI?"}, {"output_text": "AI is artificial intelligence"} ) - assert len(result.scores) > 0 - for score in result.scores: - assert math.isnan(score.value) + assert len(result.outputs) == 2 + assert result.outputs[0].name == "quality" + assert math.isnan(result.outputs[0].value) + assert result.outputs[1] == MetricOutput(name="quality.label", value="") @pytest.mark.asyncio async def test_empty_output_raises(self, mocker: MockerFixture): @@ -913,7 +931,7 @@ async def test_empty_output_raises(self, mocker: MockerFixture): metric.set_inference_fn(mocker.AsyncMock(return_value=_empty_judge_response())) with pytest.raises(ValueError, match="LLM judge returned no usable textual content for score parsing"): - await metric.compute_scores({"question": "What is AI?"}, {"output_text": "AI is artificial intelligence"}) + await compute_scores(metric, {"question": "What is AI?"}, {"output_text": "AI is artificial intelligence"}) @pytest.mark.asyncio async def test_none_output_raises(self, mocker: MockerFixture): @@ -924,7 +942,7 @@ async def test_none_output_raises(self, mocker: MockerFixture): ValueError, match="inference\\.extra_body\\.nvext\\.max_thinking_tokens", ): - await metric.compute_scores({"question": "What is AI?"}, {"output_text": "AI is artificial intelligence"}) + await compute_scores(metric, {"question": "What is AI?"}, {"output_text": "AI is artificial intelligence"}) class TestGenerateStructuredOutput: @@ -1268,7 +1286,7 @@ async def fake_inference_fn(model, request, max_retries, **kwargs): item = {"question": "What is AI?"} sample = {"output_text": "AI is artificial intelligence"} - await metric.compute_scores(item, sample) + await compute_scores(metric, item, sample) # Verify max_retries was passed as 3, not None assert captured_max_retries == 3, f"Expected max_retries=3, got {captured_max_retries}" @@ -1298,7 +1316,7 @@ async def fake_inference_fn(model, request, max_retries, **kwargs): item = {"question": "What is AI?"} sample = {"output_text": "AI is artificial intelligence"} - await metric.compute_scores(item, sample) + await compute_scores(metric, item, sample) # Verify max_retries was passed as 3, not None assert captured_max_retries == 3, f"Expected max_retries=3, got {captured_max_retries}" diff --git a/packages/nemo_evaluator_sdk/tests/metrics/test_metric_contract.py b/packages/nemo_evaluator_sdk/tests/metrics/test_metric_contract.py new file mode 100644 index 0000000000..1cbd03af29 --- /dev/null +++ b/packages/nemo_evaluator_sdk/tests/metrics/test_metric_contract.py @@ -0,0 +1,170 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Tests for the shared MetricInput -> MetricResult contract.""" + +from __future__ import annotations + +from typing import cast + +import pytest +from nemo_evaluator_sdk.metrics.protocol import ( + CandidateOutput, + ContinuousScore, + DatasetRow, + Label, + Metric, + MetricDescriptor, + MetricInput, + MetricOutput, + MetricOutputSpec, + MetricResult, + validate_metric_result, +) +from pydantic import BaseModel, ValidationError + + +class JudgeDetails(BaseModel): + label: str + rationale: str + confidence: float + + +def test_metric_input_groups_row_and_candidate() -> None: + metric_input = MetricInput( + row=DatasetRow(row_index=7, data={"answer": "Paris", "category": "geography"}), + candidate=CandidateOutput(output_text="Paris", metadata={"model": "mock"}), + ) + + assert metric_input.row.row_index == 7 + assert metric_input.candidate.output_text == "Paris" + assert metric_input.row.data["answer"] == "Paris" + assert metric_input.candidate.as_sample() == {"model": "mock", "output_text": "Paris"} + + +def test_metric_output_spec_convenience_constructors_and_json_schema() -> None: + score = MetricOutputSpec.continuous_score("reward", "Reward score") + label = MetricOutputSpec.label("judge_label") + details = MetricOutputSpec.model("judge_details", JudgeDetails) + + assert score.name == "reward" + assert score.description == "Reward score" + assert score.value_schema is ContinuousScore + assert score.value_json_schema()["type"] == "number" + assert label.value_schema is Label + assert details.value_schema is JudgeDetails + schema_properties = cast(dict[str, object], details.value_json_schema()["properties"]) + confidence_schema = cast(dict[str, object], schema_properties["confidence"]) + assert confidence_schema["type"] == "number" + + +def test_metric_output_spec_coerces_values_to_declared_schema() -> None: + reward = MetricOutputSpec.continuous_score("reward") + details = MetricOutputSpec.model("judge_details", JudgeDetails) + + coerced_reward = reward.coerce_output(MetricOutput(name="reward", value=1)) + coerced_details = details.coerce_value({"label": "pass", "rationale": "all checks passed", "confidence": 0.9}) + + assert isinstance(coerced_reward, ContinuousScore) + assert coerced_reward.root == 1.0 + assert isinstance(coerced_details, JudgeDetails) + assert coerced_details.label == "pass" + + with pytest.raises(ValueError, match="Expected metric output"): + reward.coerce_output(MetricOutput(name="other", value=1)) + + +def test_metric_descriptor_rejects_duplicate_outputs() -> None: + with pytest.raises(ValueError, match="duplicate metric output names"): + MetricDescriptor( + type="tests.duplicate", + outputs=[ + MetricOutputSpec.continuous_score("reward"), + MetricOutputSpec.boolean("reward"), + ], + ) + + +def test_validate_metric_result_accepts_declared_outputs() -> None: + outputs = [ + MetricOutputSpec.continuous_score("reward"), + MetricOutputSpec.boolean("correct"), + MetricOutputSpec.label("label"), + ] + result = MetricResult( + outputs=[ + MetricOutput(name="reward", value=True), + MetricOutput(name="correct", value=True), + MetricOutput(name="label", value="yes"), + ] + ) + + validated = validate_metric_result(result, outputs) + + assert validated is result + + +def test_validate_metric_result_rejects_duplicate_output_names() -> None: + outputs = [MetricOutputSpec.continuous_score("reward")] + result = MetricResult( + outputs=[ + MetricOutput(name="reward", value=1.0), + MetricOutput(name="reward", value=0.0), + ] + ) + + with pytest.raises(ValueError, match="Duplicate metric output"): + validate_metric_result(result, outputs) + + +def test_validate_metric_result_rejects_missing_or_undeclared_outputs() -> None: + outputs = [MetricOutputSpec.continuous_score("reward"), MetricOutputSpec.continuous_score("format")] + + with pytest.raises(ValueError, match="Missing declared metric outputs"): + validate_metric_result(MetricResult(outputs=[MetricOutput(name="reward", value=1.0)]), outputs) + + with pytest.raises(ValueError, match="Undeclared metric outputs"): + validate_metric_result( + MetricResult( + outputs=[ + MetricOutput(name="reward", value=1.0), + MetricOutput(name="format", value=1.0), + MetricOutput(name="extra", value=1.0), + ] + ), + outputs, + ) + + +def test_validate_metric_result_rejects_value_that_does_not_match_schema() -> None: + outputs = [MetricOutputSpec.model("judge_details", JudgeDetails)] + result = MetricResult(outputs=[MetricOutput(name="judge_details", value={"label": "pass"})]) + + with pytest.raises(ValidationError): + validate_metric_result(result, outputs) + + +@pytest.mark.asyncio +async def test_minimal_metric_conforms_to_protocol() -> None: + class MinimalMetric: + type = "tests.minimal" + + def output_spec(self) -> list[MetricOutputSpec]: + return [MetricOutputSpec.continuous_score("reward")] + + async def compute_scores(self, input: MetricInput) -> MetricResult: + assert input.candidate.output_text == "candidate" + return MetricResult(outputs=[MetricOutput(name="reward", value=1.0)]) + + metric = MinimalMetric() + + assert isinstance(metric, Metric) + result = await metric.compute_scores( + MetricInput(row=DatasetRow(data={}), candidate=CandidateOutput(output_text="candidate")) + ) + assert result.outputs == [MetricOutput(name="reward", value=1.0)] + + +def test_metric_result_rejects_legacy_scores_field() -> None: + with pytest.raises(ValidationError): + MetricResult.model_validate({"scores": [{"name": "reward", "value": 1.0}]}) diff --git a/packages/nemo_evaluator_sdk/tests/metrics/test_number_check.py b/packages/nemo_evaluator_sdk/tests/metrics/test_number_check.py index a803cff8c7..1d0818fc7d 100644 --- a/packages/nemo_evaluator_sdk/tests/metrics/test_number_check.py +++ b/packages/nemo_evaluator_sdk/tests/metrics/test_number_check.py @@ -5,6 +5,7 @@ from typing import Literal import pytest +from metrics.helpers import compute_scores, output_names from nemo_evaluator_sdk.execution.evaluator import Evaluator from nemo_evaluator_sdk.metrics.number_check import NumberCheckMetric, _parse_number_answer from pydantic_core import ValidationError @@ -107,8 +108,8 @@ async def test_score_names_match_compute_scores(self): left_template="{{item.reference_answer}}", right_template="{{output_text}}", ) - result = await metric.compute_scores({"reference_answer": "1"}, {"output_text": "1"}) - assert {score.name for score in result.scores} == set(metric.score_names()) + result = await compute_scores(metric, {"reference_answer": "1"}, {"output_text": "1"}) + assert {score.name for score in result.outputs} == set(output_names(metric)) def test_absolute_difference_requires_epsilon(self): with pytest.raises(ValueError, match="epsilon value is required with operation absolute difference"): @@ -145,7 +146,7 @@ async def test_operations(self, operation, left, right, expected): left_template="{{item.left}}", right_template="{{item.right}}", ) - assert (await metric.compute_scores({"left": left, "right": right}, {})).scores[0].value == ( + assert (await compute_scores(metric, {"left": left, "right": right}, {})).outputs[0].value == ( 1.0 if expected else 0.0 ) @@ -157,8 +158,8 @@ async def test_absolute_difference(self): right_template="{{item.right}}", epsilon=0.25, ) - assert (await metric.compute_scores({"left": "3.0", "right": "3.1"}, {})).scores[0].value == 1.0 - assert (await metric.compute_scores({"left": "3.0", "right": "3.4"}, {})).scores[0].value == 0.0 + assert (await compute_scores(metric, {"left": "3.0", "right": "3.1"}, {})).outputs[0].value == 1.0 + assert (await compute_scores(metric, {"left": "3.0", "right": "3.4"}, {})).outputs[0].value == 0.0 @pytest.mark.asyncio async def test_absolute_difference_raises_when_epsilon_removed_after_init(self): @@ -171,33 +172,33 @@ async def test_absolute_difference_raises_when_epsilon_removed_after_init(self): metric.epsilon = None with pytest.raises(ValueError, match="epsilon value is required"): - await metric.compute_scores({"left": "3.0", "right": "3.1"}, {}) + await compute_scores(metric, {"left": "3.0", "right": "3.1"}, {}) @pytest.mark.asyncio async def test_nan_short_circuit(self): metric = NumberCheckMetric(operation="equals", left_template="{{item.left}}", right_template="{{item.right}}") - assert math.isnan((await metric.compute_scores({"left": "abc", "right": "1"}, {})).scores[0].value) - assert math.isnan((await metric.compute_scores({"left": "1", "right": "abc"}, {})).scores[0].value) + assert math.isnan((await compute_scores(metric, {"left": "abc", "right": "1"}, {})).outputs[0].value) + assert math.isnan((await compute_scores(metric, {"left": "1", "right": "abc"}, {})).outputs[0].value) @pytest.mark.asyncio async def test_unsupported_operation_raises(self): metric = NumberCheckMetric(operation="equals", left_template="{{item.left}}", right_template="{{item.right}}") metric.operation = "unsupported" # ty: ignore[invalid-assignment] with pytest.raises(ValueError, match="Unsupported operation"): - await metric.compute_scores({"left": "1", "right": "1"}, {}) + await compute_scores(metric, {"left": "1", "right": "1"}, {}) @pytest.mark.asyncio async def test_compute_scores_and_score_names(self): metric = NumberCheckMetric(operation="equals", left_template="{{item.left}}", right_template="{{item.right}}") - result = await metric.compute_scores({"left": "1", "right": "1"}, {}) - assert result.scores[0].name == "number-check" - assert metric.score_names() == ["number-check"] + result = await compute_scores(metric, {"left": "1", "right": "1"}, {}) + assert result.outputs[0].name == "number-check" + assert output_names(metric) == ["number-check"] @pytest.mark.asyncio async def test_raises_clear_error_for_missing_template_field(self): metric = NumberCheckMetric(operation="equals", left_template="{{item.left}}", right_template="{{item.right}}") with pytest.raises(ValueError) as exc_info: - await metric.compute_scores({"left": "1"}, {}) + await compute_scores(metric, {"left": "1"}, {}) assert "could not render its 'right_template' template for this row" in str(exc_info.value) assert "missing_key='right'" in str(exc_info.value) @@ -297,9 +298,9 @@ async def test_equals(item: dict, sample: dict, expected_score: float, desc: str right_template="{{output_text | trim}}", ) - metric_result = await metric.compute_scores(item, sample) - assert len(metric_result.scores) == 1 - score = metric_result.scores[0] + metric_result = await compute_scores(metric, item, sample) + assert len(metric_result.outputs) == 1 + score = metric_result.outputs[0] assert score.name == "number-check" if math.isnan(expected_score): assert math.isnan(score.value) @@ -364,9 +365,9 @@ async def test_not_equals(item: dict, sample: dict, expected_score: float, desc: right_template="{{output_text | trim}}", ) - metric_result = await metric.compute_scores(item, sample) - assert len(metric_result.scores) == 1 - score = metric_result.scores[0] + metric_result = await compute_scores(metric, item, sample) + assert len(metric_result.outputs) == 1 + score = metric_result.outputs[0] assert score.name == "number-check" if math.isnan(expected_score): assert math.isnan(score.value) @@ -436,9 +437,9 @@ async def test_gte(item: dict, sample: dict, expected_score: float, desc: str): right_template="{{output_text | trim}}", ) - metric_result = await metric.compute_scores(item, sample) - assert len(metric_result.scores) == 1 - score = metric_result.scores[0] + metric_result = await compute_scores(metric, item, sample) + assert len(metric_result.outputs) == 1 + score = metric_result.outputs[0] assert score.name == "number-check" if math.isnan(expected_score): assert math.isnan(score.value) @@ -508,9 +509,9 @@ async def test_gt(item: dict, sample: dict, expected_score: float, desc: str): right_template="{{output_text | trim}}", ) - metric_result = await metric.compute_scores(item, sample) - assert len(metric_result.scores) == 1 - score = metric_result.scores[0] + metric_result = await compute_scores(metric, item, sample) + assert len(metric_result.outputs) == 1 + score = metric_result.outputs[0] assert score.name == "number-check" if math.isnan(expected_score): assert math.isnan(score.value) @@ -575,9 +576,9 @@ async def test_lte(item: dict, sample: dict, expected_score: float, desc: str): right_template="{{output_text | trim}}", ) - metric_result = await metric.compute_scores(item, sample) - assert len(metric_result.scores) == 1 - score = metric_result.scores[0] + metric_result = await compute_scores(metric, item, sample) + assert len(metric_result.outputs) == 1 + score = metric_result.outputs[0] assert score.name == "number-check" if math.isnan(expected_score): assert math.isnan(score.value) @@ -647,9 +648,9 @@ async def test_lt(item: dict, sample: dict, expected_score: float, desc: str): right_template="{{output_text | trim}}", ) - metric_result = await metric.compute_scores(item, sample) - assert len(metric_result.scores) == 1 - score = metric_result.scores[0] + metric_result = await compute_scores(metric, item, sample) + assert len(metric_result.outputs) == 1 + score = metric_result.outputs[0] assert score.name == "number-check" if math.isnan(expected_score): assert math.isnan(score.value) @@ -731,9 +732,9 @@ async def test_abs_diff(item: dict, sample: dict, expected_score: float, desc: s epsilon=2, ) - metric_result = await metric.compute_scores(item, sample) - assert len(metric_result.scores) == 1 - score = metric_result.scores[0] + metric_result = await compute_scores(metric, item, sample) + assert len(metric_result.outputs) == 1 + score = metric_result.outputs[0] assert score.name == "number-check" if math.isnan(expected_score): assert math.isnan(score.value) diff --git a/packages/nemo_evaluator_sdk/tests/metrics/test_remote.py b/packages/nemo_evaluator_sdk/tests/metrics/test_remote.py index 8a66e7f078..a390fb7f7e 100644 --- a/packages/nemo_evaluator_sdk/tests/metrics/test_remote.py +++ b/packages/nemo_evaluator_sdk/tests/metrics/test_remote.py @@ -5,10 +5,11 @@ from unittest.mock import AsyncMock import pytest +from metrics.helpers import compute_scores, output_names from nemo_evaluator_sdk.inference import requests_log_var +from nemo_evaluator_sdk.metrics.protocol import MetricOutput, MetricResult from nemo_evaluator_sdk.metrics.remote import NemoAgentToolkitRemoteMetric, RemoteMetric, _post_to_remote_endpoint from nemo_evaluator_sdk.values.common import SecretRef -from nemo_evaluator_sdk.values.results import MetricResult, MetricScore from nemo_evaluator_sdk.values.scores import JSONScoreParser, RemoteScore from pytest_mock import MockerFixture @@ -71,7 +72,7 @@ def test_score_names_match_declared_scores(self): body={"input": "{{item.prompt}}"}, scores=[RemoteScore(name="quality", parser=JSONScoreParser(json_path="$.result.quality"))], ) - assert metric.score_names() == ["quality"] + assert output_names(metric) == ["quality"] @pytest.mark.asyncio async def test_compute_scores(self, mocker: MockerFixture): @@ -86,11 +87,11 @@ async def test_compute_scores(self, mocker: MockerFixture): scores=[RemoteScore(name="quality", parser=JSONScoreParser(json_path="$.result.quality"))], ) - result = await metric.compute_scores({"prompt": "hello"}, {}) + result = await compute_scores(metric, {"prompt": "hello"}, {}) mock_post.assert_awaited_once() - assert result.scores[0].name == "quality" - assert result.scores[0].value == 0.75 + assert result.outputs[0].name == "quality" + assert result.outputs[0].value == 0.75 @pytest.mark.asyncio async def test_missing_template_key_raises_clear_error(self): @@ -101,7 +102,7 @@ async def test_missing_template_key_raises_clear_error(self): ) with pytest.raises(ValueError) as exc_info: - await metric.compute_scores({}, {}) + await compute_scores(metric, {}, {}) assert "could not render its 'body' template for this row" in str(exc_info.value) @@ -118,8 +119,8 @@ async def test_missing_extracted_score_returns_nan(self, mocker: MockerFixture): scores=[RemoteScore(name="quality", parser=JSONScoreParser(json_path="$.result.quality"))], ) - result = await metric.compute_scores({"prompt": "hello"}, {}) - assert math.isnan(result.scores[0].value) + result = await compute_scores(metric, {"prompt": "hello"}, {}) + assert math.isnan(result.outputs[0].value) @pytest.mark.asyncio async def test_compute_scores_passes_rendered_body_dict_to_remote_endpoint(self, mocker: MockerFixture): @@ -134,7 +135,7 @@ async def test_compute_scores_passes_rendered_body_dict_to_remote_endpoint(self, scores=[RemoteScore(name="quality", parser=JSONScoreParser(json_path="$.result.quality"))], ) - await metric.compute_scores({"prompt": "hello"}, {}) + await compute_scores(metric, {"prompt": "hello"}, {}) assert mock_post.await_args.kwargs["payload"] == {"input": "hello"} @@ -153,7 +154,7 @@ async def test_compute_scores_logs_and_reraises_invalid_score_values(self, mocke ) with pytest.raises(TypeError): - await metric.compute_scores({"prompt": "hello"}, {}) + await compute_scores(metric, {"prompt": "hello"}, {}) log_exception.assert_called_once() @@ -172,16 +173,16 @@ async def test_metric_returns_multiple_scores_when_configured(self, mocker: Mock "compute_scores", new=AsyncMock( return_value=MetricResult( - scores=[ - MetricScore(name="quality", value=0.7), - MetricScore(name="helpfulness", value=0.8), + outputs=[ + MetricOutput(name="quality", value=0.7), + MetricOutput(name="helpfulness", value=0.8), ] ) ), ) - result = await metric.compute_scores({"prompt": "hello"}, {}) - assert [score.name for score in result.scores] == ["quality", "helpfulness"] - assert [score.value for score in result.scores] == [0.7, 0.8] + result = await compute_scores(metric, {"prompt": "hello"}, {}) + assert [score.name for score in result.outputs] == ["quality", "helpfulness"] + assert [score.value for score in result.outputs] == [0.7, 0.8] @pytest.mark.asyncio async def test_metric_uses_single_score_when_only_one_score_is_returned(self, mocker: MockerFixture): @@ -193,9 +194,9 @@ async def test_metric_uses_single_score_when_only_one_score_is_returned(self, mo mocker.patch.object( RemoteMetric, "compute_scores", - new=AsyncMock(return_value=MetricResult(scores=[MetricScore(name="quality", value=0.7)])), + new=AsyncMock(return_value=MetricResult(outputs=[MetricOutput(name="quality", value=0.7)])), ) - assert (await metric.compute_scores({"prompt": "hello"}, {})).scores[0].value == 0.7 + assert (await compute_scores(metric, {"prompt": "hello"}, {})).outputs[0].value == 0.7 @pytest.mark.asyncio async def test_metric_runs_inside_existing_event_loop(self, mocker: MockerFixture): @@ -207,10 +208,10 @@ async def test_metric_runs_inside_existing_event_loop(self, mocker: MockerFixtur mocker.patch.object( RemoteMetric, "compute_scores", - new=AsyncMock(return_value=MetricResult(scores=[MetricScore(name="quality", value=0.7)])), + new=AsyncMock(return_value=MetricResult(outputs=[MetricOutput(name="quality", value=0.7)])), ) - assert (await metric.compute_scores({"prompt": "hello"}, {})).scores[0].value == 0.7 + assert (await compute_scores(metric, {"prompt": "hello"}, {})).outputs[0].value == 0.7 @pytest.mark.asyncio async def test_resolve_secrets(self, mocker: MockerFixture): @@ -237,7 +238,7 @@ async def fake_resolver(_: str) -> str: return "secret-value" await metric.resolve_secrets(fake_resolver) - await metric.compute_scores({"prompt": "hello"}, {}) + await compute_scores(metric, {"prompt": "hello"}, {}) assert captured_api_keys == ["secret-value"] @@ -296,7 +297,7 @@ async def fake_post(*, api_key: str | None = None, **kwargs): ) assert metric._get_api_key() == "env-secret" - await metric.compute_scores({"prompt": "hello"}, {}) + await compute_scores(metric, {"prompt": "hello"}, {}) assert captured_api_keys == ["env-secret"] @@ -315,7 +316,7 @@ async def test_compute_scores_appends_request_log(self, mocker: MockerFixture): token = requests_log_var.set([]) try: - await metric.compute_scores({"prompt": "hello"}, {}) + await compute_scores(metric, {"prompt": "hello"}, {}) assert requests_log_var.get() == [ {"request": {"input": "hello"}, "response": {"result": {"quality": 0.75}}} ] @@ -336,12 +337,12 @@ async def test_nemo_agent_toolkit_remote_metric(self, mocker: MockerFixture): evaluator_name="tool-accuracy", ) - result = await metric.compute_scores({"prompt": "hello"}, {}) + result = await compute_scores(metric, {"prompt": "hello"}, {}) payload = mock_post.await_args.kwargs["payload"] assert payload["evaluator_name"] == "tool-accuracy" assert payload["item"] == {"prompt": "hello"} - assert result.scores[0].name == "tool-accuracy" + assert result.outputs[0].name == "tool-accuracy" @pytest.mark.asyncio async def test_nemo_agent_toolkit_missing_score_returns_nan(self, mocker: MockerFixture): @@ -355,16 +356,16 @@ async def test_nemo_agent_toolkit_missing_score_returns_nan(self, mocker: Mocker evaluator_name="tool-accuracy", ) - result = await metric.compute_scores({"prompt": "hello"}, {}) + result = await compute_scores(metric, {"prompt": "hello"}, {}) - assert math.isnan(result.scores[0].value) + assert math.isnan(result.outputs[0].value) def test_nemo_agent_toolkit_score_names(self): metric = NemoAgentToolkitRemoteMetric( url="https://remote.example.test", evaluator_name="tool-accuracy", ) - assert metric.score_names() == ["tool-accuracy"] + assert output_names(metric) == ["tool-accuracy"] @pytest.mark.asyncio async def test_nemo_agent_toolkit_resolve_secrets(self, mocker: MockerFixture): @@ -390,7 +391,7 @@ async def fake_resolver(_: str) -> str: return "secret-value" await metric.resolve_secrets(fake_resolver) - await metric.compute_scores({"prompt": "hello"}, {}) + await compute_scores(metric, {"prompt": "hello"}, {}) assert metric._get_api_key() == "secret-value" assert captured_api_keys == ["secret-value"] @@ -418,7 +419,7 @@ async def fake_post(*, api_key: str | None = None, **kwargs): api_key_secret=SecretRef(root="remote-api-key"), ) - await metric.compute_scores({"prompt": "hello"}, {}) + await compute_scores(metric, {"prompt": "hello"}, {}) assert metric._get_api_key() == "env-secret" assert captured_api_keys == ["env-secret"] @@ -437,7 +438,7 @@ async def test_nemo_agent_toolkit_compute_scores_appends_request_log(self, mocke token = requests_log_var.set([]) try: - await metric.compute_scores({"prompt": "hello"}, {}) + await compute_scores(metric, {"prompt": "hello"}, {}) assert requests_log_var.get() == [ { "request": {"evaluator_name": "tool-accuracy", "item": {"prompt": "hello"}}, @@ -459,4 +460,4 @@ async def test_nemo_agent_toolkit_metric_returns_score(self, mocker: MockerFixtu return_value={"result": {"score": 0.9}}, ) - assert (await metric.compute_scores({"prompt": "hello"}, {})).scores[0].value == 0.9 + assert (await compute_scores(metric, {"prompt": "hello"}, {})).outputs[0].value == 0.9 diff --git a/packages/nemo_evaluator_sdk/tests/metrics/test_rouge.py b/packages/nemo_evaluator_sdk/tests/metrics/test_rouge.py index c5d0db2865..ac7dba25ab 100644 --- a/packages/nemo_evaluator_sdk/tests/metrics/test_rouge.py +++ b/packages/nemo_evaluator_sdk/tests/metrics/test_rouge.py @@ -5,6 +5,7 @@ import types import pytest +from metrics.helpers import compute_scores, output_names from nemo_evaluator_sdk.execution.evaluator import Evaluator from nemo_evaluator_sdk.metrics.rouge import MetricResult, ROUGEMetric @@ -46,7 +47,7 @@ def __init__(self, keys: list[str], *, use_stemmer: bool): def test_score_names(self): metric = ROUGEMetric(reference="{{item.reference}}") - assert metric.score_names() == ["rouge_1_score", "rouge_2_score", "rouge_3_score", "rouge_L_score"] + assert output_names(metric) == ["rouge_1_score", "rouge_2_score", "rouge_3_score", "rouge_L_score"] @pytest.mark.asyncio async def test_compute_scores_default_candidate(self): @@ -59,8 +60,8 @@ async def test_compute_scores_default_candidate(self): "rougeL": _FakeRougeScore(1.0), } ) - result = await metric.compute_scores({"reference": "the cat sat"}, {"output_text": "the cat sat"}) - assert {score.name for score in result.scores} == { + result = await compute_scores(metric, {"reference": "the cat sat"}, {"output_text": "the cat sat"}) + assert {score.name for score in result.outputs} == { "rouge_1_score", "rouge_2_score", "rouge_3_score", @@ -78,8 +79,8 @@ async def test_compute_scores_custom_candidate(self): "rougeL": _FakeRougeScore(1.0), } ) - result = await metric.compute_scores({"reference": "the cat sat", "pred": "the cat sat"}, {"output_text": "x"}) - assert all(score.value == pytest.approx(1.0) for score in result.scores) + result = await compute_scores(metric, {"reference": "the cat sat", "pred": "the cat sat"}, {"output_text": "x"}) + assert all(score.value == pytest.approx(1.0) for score in result.outputs) @pytest.mark.asyncio async def test_metric_scores(self): @@ -92,7 +93,7 @@ async def test_metric_scores(self): "rougeL": _FakeRougeScore(0.0), } ) - assert (await metric.compute_scores({"reference": "the cat"}, {"output_text": "the cat"})).scores[ + assert (await compute_scores(metric, {"reference": "the cat"}, {"output_text": "the cat"})).outputs[ 0 ].value == 1.0 metric.__dict__["_scorer"] = _FakeRougeScorer( @@ -103,9 +104,11 @@ async def test_metric_scores(self): "rougeL": _FakeRougeScore(0.0), } ) - assert (await metric.compute_scores({"reference": "the cat"}, {"output_text": "a dog"})).scores[0].value == 0.0 + assert (await compute_scores(metric, {"reference": "the cat"}, {"output_text": "a dog"})).outputs[ + 0 + ].value == 0.0 assert isinstance( - (await metric.compute_scores({"reference": "the cat"}, {"output_text": "the cat"})).scores[0].value, float + (await compute_scores(metric, {"reference": "the cat"}, {"output_text": "the cat"})).outputs[0].value, float ) def test_metric_validates_rendered_types(self): @@ -131,7 +134,8 @@ async def test_raises_clear_error_for_missing_reference_field(self): metric = ROUGEMetric(reference="{{item.reference}}", candidate="{{sample.output_text}}") with pytest.raises(ValueError) as exc_info: - await metric.compute_scores( + await compute_scores( + metric, item={"prompt": "Summarize this"}, sample={"output_text": "summary"}, ) @@ -144,7 +148,8 @@ async def test_raises_clear_error_for_default_candidate_missing_output_text(self metric = ROUGEMetric(reference="{{item.reference}}") with pytest.raises(ValueError) as exc_info: - await metric.compute_scores( + await compute_scores( + metric, item={"reference": "Paris"}, sample={"some_other_field": "Paris"}, ) @@ -208,22 +213,22 @@ async def test_compute_scores_exact_match(self): item = {"reference": "The cat sat on the mat."} sample = {"output_text": "The cat sat on the mat."} - result = await metric.compute_scores(item, sample) + result = await compute_scores(metric, item, sample) assert isinstance(result, MetricResult) # Should return all 4 ROUGE scores - assert len(result.scores) == 4 - score_names = {s.name for s in result.scores} + assert len(result.outputs) == 4 + score_names = {s.name for s in result.outputs} assert score_names == {"rouge_1_score", "rouge_2_score", "rouge_3_score", "rouge_L_score"} # All scores should be high for exact match - for score in result.scores: + for score in result.outputs: assert score.value >= 0.9 @pytest.mark.asyncio async def test_score_names_match_compute_scores(self): metric = ROUGEMetric(reference="{{item.reference}}") - result = await metric.compute_scores({"reference": "a"}, {"output_text": "a"}) - assert {score.name for score in result.scores} == set(metric.score_names()) + result = await compute_scores(metric, {"reference": "a"}, {"output_text": "a"}) + assert {score.name for score in result.outputs} == set(output_names(metric)) @pytest.mark.asyncio async def test_compute_scores_partial_match(self): @@ -235,12 +240,12 @@ async def test_compute_scores_partial_match(self): item = {"reference": "The cat sat on the mat."} sample = {"output_text": "The dog sat on the floor."} - result = await metric.compute_scores(item, sample) + result = await compute_scores(metric, item, sample) assert isinstance(result, MetricResult) - assert len(result.scores) == 4 + assert len(result.outputs) == 4 # All scores should be between 0 and 1 - for score in result.scores: + for score in result.outputs: assert 0.0 <= score.value <= 1.0 @pytest.mark.asyncio @@ -253,12 +258,12 @@ async def test_compute_scores_no_match(self): item = {"reference": "hello world"} sample = {"output_text": "goodbye universe"} - result = await metric.compute_scores(item, sample) + result = await compute_scores(metric, item, sample) assert isinstance(result, MetricResult) - assert len(result.scores) == 4 + assert len(result.outputs) == 4 # All scores should be very low or 0 for no match - for score in result.scores: + for score in result.outputs: assert score.value < 0.1 @pytest.mark.asyncio @@ -272,12 +277,12 @@ async def test_compute_scores_with_custom_candidate(self): item = {"reference": "The cat sat on the mat.", "custom_output": "The cat sat on the mat."} sample = {"output_text": "This should be ignored."} - result = await metric.compute_scores(item, sample) + result = await compute_scores(metric, item, sample) assert isinstance(result, MetricResult) - assert len(result.scores) == 4 + assert len(result.outputs) == 4 # All scores should be high for exact match - for score in result.scores: + for score in result.outputs: assert score.value >= 0.9 @pytest.mark.asyncio @@ -291,10 +296,10 @@ async def test_compute_scores_rouge_1(self): item = {"reference": "cat dog"} sample = {"output_text": "cat bird"} - result = await metric.compute_scores(item, sample) + result = await compute_scores(metric, item, sample) # ROUGE-1 should detect unigram overlap (cat) - rouge_1_score = next(s for s in result.scores if s.name == "rouge_1_score") + rouge_1_score = next(s for s in result.outputs if s.name == "rouge_1_score") assert rouge_1_score.value > 0.0 @pytest.mark.asyncio @@ -308,10 +313,10 @@ async def test_compute_scores_rouge_2(self): item = {"reference": "the cat sat"} sample = {"output_text": "the cat slept"} - result = await metric.compute_scores(item, sample) + result = await compute_scores(metric, item, sample) # ROUGE-2 should detect bigram overlap (the cat) - rouge_2_score = next(s for s in result.scores if s.name == "rouge_2_score") + rouge_2_score = next(s for s in result.outputs if s.name == "rouge_2_score") assert rouge_2_score.value > 0.0 @pytest.mark.asyncio @@ -325,8 +330,8 @@ async def test_compute_scores_rouge_l(self): item = {"reference": "the quick brown fox"} sample = {"output_text": "the brown fox"} - result = await metric.compute_scores(item, sample) + result = await compute_scores(metric, item, sample) # ROUGE-L should detect LCS - rouge_l_score = next(s for s in result.scores if s.name == "rouge_L_score") + rouge_l_score = next(s for s in result.outputs if s.name == "rouge_L_score") assert rouge_l_score.value > 0.0 diff --git a/packages/nemo_evaluator_sdk/tests/metrics/test_string_check.py b/packages/nemo_evaluator_sdk/tests/metrics/test_string_check.py index 7122b73f04..7d760b73fb 100644 --- a/packages/nemo_evaluator_sdk/tests/metrics/test_string_check.py +++ b/packages/nemo_evaluator_sdk/tests/metrics/test_string_check.py @@ -2,6 +2,7 @@ # SPDX-License-Identifier: Apache-2.0 import pytest +from metrics.helpers import compute_scores, output_names from nemo_evaluator_sdk.execution.evaluator import Evaluator from nemo_evaluator_sdk.metrics.string_check import MetricResult, StringCheckMetric, StringCheckOperation @@ -27,7 +28,7 @@ async def test_operations(self, operation, left, right, expected): left_template="{{item.left}}", right_template="{{item.right}}", ) - assert (await metric.compute_scores({"left": left, "right": right}, {})).scores[0].value == ( + assert (await compute_scores(metric, {"left": left, "right": right}, {})).outputs[0].value == ( 1.0 if expected else 0.0 ) @@ -36,28 +37,28 @@ async def test_unsupported_operation_raises(self): metric = StringCheckMetric(operation="equals", left_template="{{item.left}}", right_template="{{item.right}}") metric.operation = "unknown" # ty: ignore[invalid-assignment] with pytest.raises(ValueError, match="Unsupported operation"): - await metric.compute_scores({"left": "a", "right": "a"}, {}) + await compute_scores(metric, {"left": "a", "right": "a"}, {}) @pytest.mark.asyncio async def test_validates_rendered_types(self): metric = StringCheckMetric(operation="equals", left_template="{{item.left}}", right_template="{{item.right}}") with pytest.raises(TypeError, match="The left value must be a string"): - await metric.compute_scores({"left": 1, "right": "1"}, {}) + await compute_scores(metric, {"left": 1, "right": "1"}, {}) with pytest.raises(TypeError, match="The right value must be a string"): - await metric.compute_scores({"left": "1", "right": 1}, {}) + await compute_scores(metric, {"left": "1", "right": 1}, {}) @pytest.mark.asyncio async def test_compute_scores_and_score_names(self): metric = StringCheckMetric(operation="equals", left_template="{{item.left}}", right_template="{{item.right}}") - result = await metric.compute_scores({"left": "x", "right": "x"}, {}) - assert result.scores[0].name == "string-check" - assert metric.score_names() == ["string-check"] + result = await compute_scores(metric, {"left": "x", "right": "x"}, {}) + assert result.outputs[0].name == "string-check" + assert output_names(metric) == ["string-check"] @pytest.mark.asyncio async def test_raises_clear_error_for_missing_template_field(self): metric = StringCheckMetric(operation="equals", left_template="{{item.left}}", right_template="{{item.right}}") with pytest.raises(ValueError) as exc_info: - await metric.compute_scores({"left": "x"}, {}) + await compute_scores(metric, {"left": "x"}, {}) assert "could not render its 'right_template' template for this row" in str(exc_info.value) assert "missing_key='right'" in str(exc_info.value) @@ -95,12 +96,12 @@ async def test_compute_scores_equals_true(self): item = {"expected": "hello"} sample = {"output_text": "hello"} - result = await metric.compute_scores(item, sample) + result = await compute_scores(metric, item, sample) assert isinstance(result, MetricResult) - assert len(result.scores) == 1 - assert result.scores[0].name == "string-check" - assert result.scores[0].value == 1.0 + assert len(result.outputs) == 1 + assert result.outputs[0].name == "string-check" + assert result.outputs[0].value == 1.0 @pytest.mark.asyncio async def test_score_names_match_compute_scores(self): @@ -109,8 +110,8 @@ async def test_score_names_match_compute_scores(self): left_template="{{item.expected}}", right_template="{{sample.output_text}}", ) - result = await metric.compute_scores({"expected": "a"}, {"output_text": "a"}) - assert {score.name for score in result.scores} == set(metric.score_names()) + result = await compute_scores(metric, {"expected": "a"}, {"output_text": "a"}) + assert {score.name for score in result.outputs} == set(output_names(metric)) @pytest.mark.asyncio async def test_compute_scores_equals_false(self): @@ -124,11 +125,11 @@ async def test_compute_scores_equals_false(self): item = {"expected": "hello"} sample = {"output_text": "goodbye"} - result = await metric.compute_scores(item, sample) + result = await compute_scores(metric, item, sample) assert isinstance(result, MetricResult) - assert len(result.scores) == 1 - assert result.scores[0].value == 0.0 + assert len(result.outputs) == 1 + assert result.outputs[0].value == 0.0 @pytest.mark.asyncio async def test_compute_scores_double_equals(self): @@ -142,9 +143,9 @@ async def test_compute_scores_double_equals(self): item = {"expected": "hello"} sample = {"output_text": "hello"} - result = await metric.compute_scores(item, sample) + result = await compute_scores(metric, item, sample) - assert result.scores[0].value == 1.0 + assert result.outputs[0].value == 1.0 @pytest.mark.asyncio async def test_compute_scores_not_equals(self): @@ -158,9 +159,9 @@ async def test_compute_scores_not_equals(self): item = {"expected": "hello"} sample = {"output_text": "goodbye"} - result = await metric.compute_scores(item, sample) + result = await compute_scores(metric, item, sample) - assert result.scores[0].value == 1.0 + assert result.outputs[0].value == 1.0 @pytest.mark.asyncio async def test_compute_scores_not_equals_variants(self): @@ -175,8 +176,8 @@ async def test_compute_scores_not_equals_variants(self): item = {"expected": "hello"} sample = {"output_text": "goodbye"} - result = await metric.compute_scores(item, sample) - assert result.scores[0].value == 1.0 + result = await compute_scores(metric, item, sample) + assert result.outputs[0].value == 1.0 @pytest.mark.asyncio async def test_compute_scores_contains_true(self): @@ -190,9 +191,9 @@ async def test_compute_scores_contains_true(self): item = {"haystack": "hello world", "needle": "world"} sample = {} - result = await metric.compute_scores(item, sample) + result = await compute_scores(metric, item, sample) - assert result.scores[0].value == 1.0 + assert result.outputs[0].value == 1.0 @pytest.mark.asyncio async def test_compute_scores_contains_false(self): @@ -206,9 +207,9 @@ async def test_compute_scores_contains_false(self): item = {"haystack": "hello world", "needle": "goodbye"} sample = {} - result = await metric.compute_scores(item, sample) + result = await compute_scores(metric, item, sample) - assert result.scores[0].value == 0.0 + assert result.outputs[0].value == 0.0 @pytest.mark.asyncio async def test_compute_scores_not_contains(self): @@ -222,9 +223,9 @@ async def test_compute_scores_not_contains(self): item = {"haystack": "hello world", "needle": "goodbye"} sample = {} - result = await metric.compute_scores(item, sample) + result = await compute_scores(metric, item, sample) - assert result.scores[0].value == 1.0 + assert result.outputs[0].value == 1.0 @pytest.mark.asyncio async def test_compute_scores_startswith_true(self): @@ -238,9 +239,9 @@ async def test_compute_scores_startswith_true(self): item = {"text": "hello world", "prefix": "hello"} sample = {} - result = await metric.compute_scores(item, sample) + result = await compute_scores(metric, item, sample) - assert result.scores[0].value == 1.0 + assert result.outputs[0].value == 1.0 @pytest.mark.asyncio async def test_compute_scores_startswith_false(self): @@ -254,9 +255,9 @@ async def test_compute_scores_startswith_false(self): item = {"text": "hello world", "prefix": "world"} sample = {} - result = await metric.compute_scores(item, sample) + result = await compute_scores(metric, item, sample) - assert result.scores[0].value == 0.0 + assert result.outputs[0].value == 0.0 @pytest.mark.asyncio async def test_compute_scores_endswith_true(self): @@ -270,9 +271,9 @@ async def test_compute_scores_endswith_true(self): item = {"text": "hello world", "suffix": "world"} sample = {} - result = await metric.compute_scores(item, sample) + result = await compute_scores(metric, item, sample) - assert result.scores[0].value == 1.0 + assert result.outputs[0].value == 1.0 @pytest.mark.asyncio async def test_compute_scores_endswith_false(self): @@ -286,9 +287,9 @@ async def test_compute_scores_endswith_false(self): item = {"text": "hello world", "suffix": "hello"} sample = {} - result = await metric.compute_scores(item, sample) + result = await compute_scores(metric, item, sample) - assert result.scores[0].value == 0.0 + assert result.outputs[0].value == 0.0 @pytest.mark.asyncio async def test_compute_scores_case_sensitive(self): @@ -302,10 +303,10 @@ async def test_compute_scores_case_sensitive(self): item = {"expected": "Hello"} sample = {"output_text": "hello"} - result = await metric.compute_scores(item, sample) + result = await compute_scores(metric, item, sample) # Should be case sensitive - assert result.scores[0].value == 0.0 + assert result.outputs[0].value == 0.0 @pytest.mark.asyncio async def test_compute_scores_with_jinja_filters(self): @@ -319,9 +320,9 @@ async def test_compute_scores_with_jinja_filters(self): item = {"expected": " hello "} sample = {"output_text": "hello"} - result = await metric.compute_scores(item, sample) + result = await compute_scores(metric, item, sample) - assert result.scores[0].value == 1.0 + assert result.outputs[0].value == 1.0 @pytest.mark.asyncio async def test_metric_method_unsupported_operation(self): @@ -338,4 +339,4 @@ async def test_metric_method_unsupported_operation(self): sample = {"output_text": "hello"} with pytest.raises(ValueError, match="Unsupported operation"): - await metric.compute_scores(item, sample) + await compute_scores(metric, item, sample) diff --git a/packages/nemo_evaluator_sdk/tests/metrics/test_tool_calling.py b/packages/nemo_evaluator_sdk/tests/metrics/test_tool_calling.py index b420c11b27..33fec0b309 100644 --- a/packages/nemo_evaluator_sdk/tests/metrics/test_tool_calling.py +++ b/packages/nemo_evaluator_sdk/tests/metrics/test_tool_calling.py @@ -4,6 +4,7 @@ import math import pytest +from metrics.helpers import compute_scores, output_names from nemo_evaluator_sdk.execution.evaluator import Evaluator from nemo_evaluator_sdk.metrics.tool_calling import MetricResult, ToolCallingMetric @@ -15,14 +16,14 @@ def _response(tool_calls): class TestToolCallingMetric: def test_score_names(self): metric = ToolCallingMetric(reference="{{item.reference}}") - assert metric.score_names() == ["function_name_accuracy", "function_name_and_args_accuracy"] + assert output_names(metric) == ["function_name_accuracy", "function_name_and_args_accuracy"] @pytest.mark.asyncio async def test_metric(self): metric = ToolCallingMetric(reference="{{item.reference}}") item = {"reference": [{"function": {"name": "sum", "arguments": {"x": 1}}}]} sample = {"response": _response([{"function": {"name": "sum", "arguments": '{"x": 1}'}}])} - assert (await metric.compute_scores(item, sample)).scores[0].value == 1.0 + assert (await compute_scores(metric, item, sample)).outputs[0].value == 1.0 def test_metric_uses_item_response_when_sample_missing(self): metric = ToolCallingMetric(reference="{{item.reference}}") @@ -139,15 +140,16 @@ async def test_compute_scores_score_names(self): metric = ToolCallingMetric(reference="{{item.reference}}") item = {"reference": [{"function": {"name": "sum", "arguments": {"x": 1}}}]} sample = {"response": _response([{"function": {"name": "sum", "arguments": '{"x": 1}'}}])} - result = await metric.compute_scores(item, sample) - assert {score.name for score in result.scores} == {"function_name_accuracy", "function_name_and_args_accuracy"} + result = await compute_scores(metric, item, sample) + assert {score.name for score in result.outputs} == {"function_name_accuracy", "function_name_and_args_accuracy"} @pytest.mark.asyncio async def test_raises_clear_error_for_missing_reference_field(self): metric = ToolCallingMetric(reference="{{item.reference}}") with pytest.raises(ValueError) as exc_info: - await metric.compute_scores( + await compute_scores( + metric, item={"prompt": "call sum"}, sample={"response": _response([])}, ) @@ -204,14 +206,14 @@ async def test_compute_scores_exact_match(self): } } - result = await metric.compute_scores(item, sample) + result = await compute_scores(metric, item, sample) assert isinstance(result, MetricResult) - assert len(result.scores) == 2 - score_names = {s.name for s in result.scores} + assert len(result.outputs) == 2 + score_names = {s.name for s in result.outputs} assert score_names == {"function_name_accuracy", "function_name_and_args_accuracy"} # Both should be 1.0 for exact match - for score in result.scores: + for score in result.outputs: assert score.value == 1.0 @pytest.mark.asyncio @@ -235,8 +237,8 @@ async def test_score_names_match_compute_scores(self): ] } } - result = await metric.compute_scores(item, sample) - assert {score.name for score in result.scores} == set(metric.score_names()) + result = await compute_scores(metric, item, sample) + assert {score.name for score in result.outputs} == set(output_names(metric)) @pytest.mark.asyncio async def test_compute_scores_reference_without_tojson_preserves_list(self): @@ -273,10 +275,10 @@ async def test_compute_scores_reference_without_tojson_preserves_list(self): } } - result = await metric.compute_scores(item, sample) + result = await compute_scores(metric, item, sample) assert isinstance(result, MetricResult) - for score in result.scores: + for score in result.outputs: assert score.value == 1.0 @pytest.mark.asyncio @@ -315,11 +317,11 @@ async def test_compute_scores_function_name_match_only(self): } } - result = await metric.compute_scores(item, sample) + result = await compute_scores(metric, item, sample) assert isinstance(result, MetricResult) - function_name_accuracy = next(s for s in result.scores if s.name == "function_name_accuracy") - function_name_and_args_accuracy = next(s for s in result.scores if s.name == "function_name_and_args_accuracy") + function_name_accuracy = next(s for s in result.outputs if s.name == "function_name_accuracy") + function_name_and_args_accuracy = next(s for s in result.outputs if s.name == "function_name_and_args_accuracy") # Function names match assert function_name_accuracy.value == 1.0 @@ -362,11 +364,11 @@ async def test_compute_scores_no_match(self): } } - result = await metric.compute_scores(item, sample) + result = await compute_scores(metric, item, sample) assert isinstance(result, MetricResult) # Both should be 0.0 when function names don't match - for score in result.scores: + for score in result.outputs: assert score.value == 0.0 @pytest.mark.asyncio @@ -388,11 +390,11 @@ async def test_compute_scores_no_tool_calls(self): } sample = {"response": {"choices": [{"message": {}}]}} - result = await metric.compute_scores(item, sample) + result = await compute_scores(metric, item, sample) assert isinstance(result, MetricResult) # Both should be 0.0 when no tool calls present - for score in result.scores: + for score in result.outputs: assert score.value == 0.0 @pytest.mark.asyncio @@ -443,11 +445,11 @@ async def test_compute_scores_multiple_tool_calls_order_insensitive(self): } } - result = await metric.compute_scores(item, sample) + result = await compute_scores(metric, item, sample) assert isinstance(result, MetricResult) # Both should be 1.0 even though order is different - for score in result.scores: + for score in result.outputs: assert score.value == 1.0 @pytest.mark.asyncio @@ -486,11 +488,11 @@ async def test_compute_scores_invalid_json_arguments_returns_nan(self): } } - result = await metric.compute_scores(item, sample) + result = await compute_scores(metric, item, sample) assert isinstance(result, MetricResult) - function_name_accuracy = next(s for s in result.scores if s.name == "function_name_accuracy") - function_name_and_args_accuracy = next(s for s in result.scores if s.name == "function_name_and_args_accuracy") + function_name_accuracy = next(s for s in result.outputs if s.name == "function_name_accuracy") + function_name_and_args_accuracy = next(s for s in result.outputs if s.name == "function_name_and_args_accuracy") # Function names still match assert function_name_accuracy.value == 1.0 @@ -533,10 +535,10 @@ async def test_compute_scores_case_sensitive(self): } } - result = await metric.compute_scores(item, sample) + result = await compute_scores(metric, item, sample) # Case sensitivity means this should not match - for score in result.scores: + for score in result.outputs: assert score.value == 0.0 @pytest.mark.asyncio @@ -575,7 +577,7 @@ async def test_compute_scores_value(self): } } - score = (await metric.compute_scores(item, sample)).scores[0].value + score = (await compute_scores(metric, item, sample)).outputs[0].value assert score == 1.0 @pytest.mark.asyncio @@ -619,11 +621,11 @@ async def test_offline_evaluation_response_in_item(self): # Empty sample - no live inference sample: dict = {} - result = await metric.compute_scores(item, sample) + result = await compute_scores(metric, item, sample) assert isinstance(result, MetricResult) - assert len(result.scores) == 2 - for score in result.scores: + assert len(result.outputs) == 2 + for score in result.outputs: assert score.value == 1.0 @pytest.mark.asyncio @@ -639,4 +641,4 @@ async def test_offline_evaluation_no_response_raises_error(self): sample: dict = {} with pytest.raises(ValueError, match="No response found"): - await metric.compute_scores(item, sample) + await compute_scores(metric, item, sample) diff --git a/packages/nemo_evaluator_sdk/tests/test_api.py b/packages/nemo_evaluator_sdk/tests/test_api.py index 29c50327ac..53d8f82a9f 100644 --- a/packages/nemo_evaluator_sdk/tests/test_api.py +++ b/packages/nemo_evaluator_sdk/tests/test_api.py @@ -7,14 +7,15 @@ import pytest from jinja2 import UndefinedError from nemo_evaluator_sdk import Evaluator +from nemo_evaluator_sdk.execution.scoring import build_metric_input from nemo_evaluator_sdk.metrics.exact_match import ExactMatchMetric +from nemo_evaluator_sdk.metrics.protocol import MetricInput, MetricOutput from nemo_evaluator_sdk.values.results import ( AggregatedMetricResult, AggregateRangeScore, AggregateRubricScore, EvaluationResult, Histogram, - MetricScore, Percentiles, RowScore, RubricScoreStat, @@ -55,6 +56,10 @@ def _make_range_score(name: str, *, count: int, nan_count: int = 0) -> Aggregate ) +def _metric_input(item: dict[str, str], sample: dict[str, str]) -> MetricInput: + return build_metric_input(item, sample, 0) + + class TestExactMatchMetric: def test_top_level_exports_include_evaluators(self): assert Evaluator is not None @@ -130,17 +135,11 @@ async def test_evaluate_runs_inside_active_event_loop(self): @pytest.mark.asyncio async def test_default_candidate_uses_output_text(self): metric = ExactMatchMetric(reference="{{item.expected}}") - match = await metric.compute_scores( - item={"expected": "blue"}, - sample={"output_text": "Blue"}, - ) - mismatch = await metric.compute_scores( - item={"expected": "Jupiter"}, - sample={"output_text": "Saturn"}, - ) + match = await metric.compute_scores(_metric_input({"expected": "blue"}, {"output_text": "Blue"})) + mismatch = await metric.compute_scores(_metric_input({"expected": "Jupiter"}, {"output_text": "Saturn"})) - assert match.scores[0].value == 1.0 - assert mismatch.scores[0].value == 0.0 + assert match.outputs[0].value == 1.0 + assert mismatch.outputs[0].value == 0.0 class TestComputeScores: @@ -150,8 +149,7 @@ async def test_raises_clear_error_for_missing_reference_field(self): with pytest.raises(ValueError) as exc_info: await metric.compute_scores( - item={"prompt": "What is the capital of France?"}, - sample={"output_text": "Paris"}, + _metric_input({"prompt": "What is the capital of France?"}, {"output_text": "Paris"}) ) assert str(exc_info.value) == ( @@ -168,10 +166,7 @@ async def test_raises_clear_error_for_missing_candidate_field(self): metric = ExactMatchMetric(reference="{{item.reference}}", candidate="{{sample.prediction}}") with pytest.raises(ValueError) as exc_info: - await metric.compute_scores( - item={"reference": "Paris"}, - sample={"output_text": "Paris"}, - ) + await metric.compute_scores(_metric_input({"reference": "Paris"}, {"output_text": "Paris"})) assert str(exc_info.value) == ( "ExactMatchMetric(reference='{{item.reference}}', candidate='{{sample.prediction}}') " @@ -188,8 +183,7 @@ async def test_raises_clear_error_for_missing_bare_reference_name(self): with pytest.raises(ValueError) as exc_info: await metric.compute_scores( - item={"prompt": "What is the capital of France?"}, - sample={"output_text": "Paris"}, + _metric_input({"prompt": "What is the capital of France?"}, {"output_text": "Paris"}) ) assert str(exc_info.value) == ( @@ -206,10 +200,7 @@ async def test_raises_clear_error_for_default_candidate_missing_output_text(self metric = ExactMatchMetric(reference="{{item.reference}}") with pytest.raises(ValueError) as exc_info: - await metric.compute_scores( - item={"reference": "Paris"}, - sample={"some_other_field": "Paris"}, - ) + await metric.compute_scores(_metric_input({"reference": "Paris"}, {"some_other_field": "Paris"})) assert str(exc_info.value) == ( "ExactMatchMetric has missing `candidate` field.\n" @@ -227,8 +218,7 @@ async def test_includes_original_jinja_error_when_missing_key_cannot_be_inferred with pytest.raises(ValueError, match=r"jinja_error='custom undefined message'"): await metric.compute_scores( - item={"prompt": "What is the capital of France?"}, - sample={"output_text": "Paris"}, + _metric_input({"prompt": "What is the capital of France?"}, {"output_text": "Paris"}) ) @@ -247,8 +237,8 @@ def result(self): def test_to_records_rows(self, result): records = result.to_records(view="rows") assert len(records) == 2 - assert records[0]["score.exact-match"] == 1.0 - assert records[1]["score.exact-match"] == 0.0 + assert records[0]["output.exact-match"] == 1.0 + assert records[1]["output.exact-match"] == 0.0 assert records[0]["item.expected"] == "blue" assert records[1]["item.expected"] == "Jupiter" assert records[0]["item.model_output"] == "Blue" @@ -264,13 +254,13 @@ def test_to_records_aggregate(self, result): def test_to_table_returns_pyarrow_table(self, result): table = result.to_table(view="rows") assert table.num_rows == 2 - assert "score.exact-match" in table.column_names + assert "output.exact-match" in table.column_names def test_to_pandas_returns_dataframe_when_available(self, result): pd = pytest.importorskip("pandas") dataframe = result.to_pandas(view="rows") assert isinstance(dataframe, pd.DataFrame) - assert "score.exact-match" in dataframe.columns + assert "output.exact-match" in dataframe.columns def test_format_summary_and_str(self, result): formatted = result.format_summary(max_rows=1) @@ -314,7 +304,7 @@ def test_to_records_rows_marks_scored_rows_with_errors_as_error(self): row_index=3, item={"prompt": "hello"}, sample={"output_text": "world"}, - metrics={"judge": [MetricScore(name="judge", value=0.5)]}, + metrics={"judge": [MetricOutput(name="judge", value=0.5)]}, requests=[], metric_errors={"judge": "bad but scored"}, ) @@ -331,7 +321,7 @@ def test_to_records_rows_marks_scored_rows_with_errors_as_error(self): "item.prompt": "hello", "sample.output_text": "world", "error": "judge: bad but scored", - "score.judge": 0.5, + "output.judge": 0.5, } ] @@ -393,7 +383,7 @@ def test_format_summary_handles_scored_and_unscored_error_rows(self): row_index=3, item={"prompt": "first"}, sample={"response": {"id": "hidden"}, "output_text": "text"}, - metrics={"judge": [MetricScore(name="judge", value=0.5)]}, + metrics={"judge": [MetricOutput(name="judge", value=0.5)]}, requests=[], metric_errors={"judge": "bad but scored"}, ), diff --git a/packages/nemo_evaluator_sdk/tests/values/test_multi_metric_results.py b/packages/nemo_evaluator_sdk/tests/values/test_multi_metric_results.py index ac17ad7441..6d620d0452 100644 --- a/packages/nemo_evaluator_sdk/tests/values/test_multi_metric_results.py +++ b/packages/nemo_evaluator_sdk/tests/values/test_multi_metric_results.py @@ -11,7 +11,7 @@ AggregatedMetricResult, AggregateRangeScore, EvaluationResult, - MetricScore, + MetricOutput, Percentiles, RowScore, ) @@ -59,7 +59,7 @@ def test_prefixes_metric_names_and_aggregate_scores(self): row_index=7, item={"prompt": "q"}, sample={"output_text": "a"}, - metrics={"score": [MetricScore(name="score", value=1.0)]}, + metrics={"score": [MetricOutput(name="score", value=1.0)]}, requests=[{"id": "req-1"}], metric_errors=None, ) @@ -76,7 +76,7 @@ def test_prefixes_metric_names_and_aggregate_scores(self): row_index=7, item={"prompt": "q"}, sample={"output_text": "a"}, - metrics={"exact-match": [MetricScore(name="score", value=1.0)]}, + metrics={"exact-match": [MetricOutput(name="score", value=1.0)]}, requests=[{"id": "req-1"}], metric_errors=None, ) @@ -91,7 +91,7 @@ def test_filters_aggregate_fields(self): row_index=0, item={}, sample={}, - metrics={"score": [MetricScore(name="score", value=1.0)]}, + metrics={"score": [MetricOutput(name="score", value=1.0)]}, requests=[], metric_errors=None, ) @@ -131,7 +131,7 @@ def test_uses_single_metric_fallback_and_preserves_metric_error(self): row_index=1, item={"id": "row-1"}, sample={}, - metrics={"raw": [MetricScore(name="score", value=0.0)]}, + metrics={"raw": [MetricOutput(name="score", value=0.0)]}, requests=[], metric_errors={"raw": "boom"}, ) @@ -147,7 +147,7 @@ def test_uses_single_metric_fallback_and_preserves_metric_error(self): row_index=1, item={"id": "row-1"}, sample={}, - metrics={"metric-a": [MetricScore(name="score", value=0.0)]}, + metrics={"metric-a": [MetricOutput(name="score", value=0.0)]}, requests=[], metric_errors={"metric-a": "boom"}, ) @@ -191,7 +191,7 @@ def test_merges_rows_aggregate_scores_and_errors(self): row_index=10, item={"id": "row-1"}, sample={"output_text": "first"}, - metrics={"metric-a": [MetricScore(name="score", value=1.0)]}, + metrics={"metric-a": [MetricOutput(name="score", value=1.0)]}, requests=[{"request_id": "a-1"}], metric_errors=None, ), @@ -213,7 +213,7 @@ def test_merges_rows_aggregate_scores_and_errors(self): row_index=10, item={"id": "row-1"}, sample={"output_text": "first"}, - metrics={"metric-b": [MetricScore(name="score", value=0.5)]}, + metrics={"metric-b": [MetricOutput(name="score", value=0.5)]}, requests=[{"request_id": "b-1"}], metric_errors=None, ), @@ -239,8 +239,8 @@ def test_merges_rows_aggregate_scores_and_errors(self): item={"id": "row-1"}, sample={"output_text": "first"}, metrics={ - "metric-a": [MetricScore(name="score", value=1.0)], - "metric-b": [MetricScore(name="score", value=0.5)], + "metric-a": [MetricOutput(name="score", value=1.0)], + "metric-b": [MetricOutput(name="score", value=0.5)], }, requests=[{"request_id": "a-1"}, {"request_id": "b-1"}], metric_errors=None, @@ -270,7 +270,7 @@ def test_to_records_marks_rows_with_metric_errors_as_error(self): row_index=0, item={"id": "row-1"}, sample={"output_text": "ok"}, - metrics={"metric-a": [MetricScore(name="score", value=1.0)]}, + metrics={"metric-a": [MetricOutput(name="score", value=1.0)]}, requests=[], metric_errors=None, ), @@ -292,7 +292,7 @@ def test_to_records_marks_rows_with_metric_errors_as_error(self): row_index=0, item={"id": "row-1"}, sample={"output_text": "ok"}, - metrics={"metric-b": [MetricScore(name="score", value=0.5)]}, + metrics={"metric-b": [MetricOutput(name="score", value=0.5)]}, requests=[], metric_errors=None, ), @@ -300,7 +300,7 @@ def test_to_records_marks_rows_with_metric_errors_as_error(self): row_index=1, item={"id": "row-2"}, sample={"output_text": "bad"}, - metrics={"metric-b": [MetricScore(name="score", value=0.0)]}, + metrics={"metric-b": [MetricOutput(name="score", value=0.0)]}, requests=[], metric_errors=None, ), @@ -337,7 +337,7 @@ def test_format_summary_includes_multi_metric_row_errors(self): row_index=0, item={"id": "row-1"}, sample={"output_text": "bad"}, - metrics={"metric-b": [MetricScore(name="score", value=0.0)]}, + metrics={"metric-b": [MetricOutput(name="score", value=0.0)]}, requests=[], metric_errors=None, ) @@ -376,7 +376,7 @@ def test_imports_from_multi_metric_results_module(self): row_index=0, item={}, sample={}, - metrics={"score": [MetricScore(name="score", value=1.0)]}, + metrics={"score": [MetricOutput(name="score", value=1.0)]}, requests=[], metric_errors=None, ) @@ -401,7 +401,7 @@ def test_applies_aggregate_field_filter_to_combined_result(self): row_index=0, item={}, sample={}, - metrics={"metric-a": [MetricScore(name="score", value=1.0)]}, + metrics={"metric-a": [MetricOutput(name="score", value=1.0)]}, requests=[], metric_errors=None, ) @@ -427,7 +427,7 @@ def test_applies_aggregate_field_filter_to_combined_result(self): row_index=0, item={}, sample={}, - metrics={"metric-a": [MetricScore(name="score", value=1.0)]}, + metrics={"metric-a": [MetricOutput(name="score", value=1.0)]}, requests=[], metric_errors=None, ) @@ -451,7 +451,7 @@ def test_applies_aggregate_field_filter_to_combined_result(self): row_index=0, item={}, sample={}, - metrics={"metric-a": [MetricScore(name="score", value=1.0)]}, + metrics={"metric-a": [MetricOutput(name="score", value=1.0)]}, requests=[], metric_errors=None, ) @@ -465,7 +465,7 @@ def test_applies_aggregate_field_filter_to_combined_result(self): row_index=1, item={}, sample={}, - metrics={"metric-b": [MetricScore(name="score", value=0.0)]}, + metrics={"metric-b": [MetricOutput(name="score", value=0.0)]}, requests=[], metric_errors=None, ) diff --git a/packages/nmp_platform/tests/test_main.py b/packages/nmp_platform/tests/test_main.py index 5bcee2167b..bfbbb1e93d 100644 --- a/packages/nmp_platform/tests/test_main.py +++ b/packages/nmp_platform/tests/test_main.py @@ -67,6 +67,7 @@ def test_service_startup_exits_with_code_1_and_guidance(self, tmp_path): timeout=30, ) + stderr = result.stderr.decode() assert result.returncode == 2 - assert "invalid choice" in result.stderr.decode() - assert "choose from 'task'" in result.stderr.decode() + assert "invalid choice" in stderr + assert "choose from 'task'" in stderr or "choose from task" in stderr diff --git a/plugins/nemo-evaluator/src/nemo_evaluator/sdk/_executor.py b/plugins/nemo-evaluator/src/nemo_evaluator/sdk/_executor.py index a738472557..86bb9f38e1 100644 --- a/plugins/nemo-evaluator/src/nemo_evaluator/sdk/_executor.py +++ b/plugins/nemo-evaluator/src/nemo_evaluator/sdk/_executor.py @@ -22,7 +22,7 @@ from nemo_evaluator.sdk.utils import filter_benchmark_result, filter_evaluation_result from nemo_evaluator_sdk.datasets.loader import prepare_dataset_rows from nemo_evaluator_sdk.execution.config import EvaluationRequest, normalize_params -from nemo_evaluator_sdk.metrics.base import Metric +from nemo_evaluator_sdk.metrics.protocol import Metric from nemo_evaluator_sdk.values import ( Agent, Model, diff --git a/plugins/nemo-evaluator/src/nemo_evaluator/sdk/resources.py b/plugins/nemo-evaluator/src/nemo_evaluator/sdk/resources.py index ffbddffc79..1339136d81 100644 --- a/plugins/nemo-evaluator/src/nemo_evaluator/sdk/resources.py +++ b/plugins/nemo-evaluator/src/nemo_evaluator/sdk/resources.py @@ -24,7 +24,7 @@ RunConfigOnline, RunConfigOnlineModel, ) -from nemo_evaluator_sdk.metrics.base import Metric +from nemo_evaluator_sdk.metrics.protocol import Metric from nemo_evaluator_sdk.values import ( Agent, AggregateFieldName, diff --git a/plugins/nemo-evaluator/src/nemo_evaluator/sdk/standalone_sdk/backend.py b/plugins/nemo-evaluator/src/nemo_evaluator/sdk/standalone_sdk/backend.py index 29b15ca054..43af13a789 100644 --- a/plugins/nemo-evaluator/src/nemo_evaluator/sdk/standalone_sdk/backend.py +++ b/plugins/nemo-evaluator/src/nemo_evaluator/sdk/standalone_sdk/backend.py @@ -11,7 +11,7 @@ from nemo_evaluator.sdk.resources import AsyncEvaluator, Evaluator from nemo_evaluator.sdk.types import ExecutionMode from nemo_evaluator_sdk.execution.config import EvaluationRequest -from nemo_evaluator_sdk.metrics.base import Metric +from nemo_evaluator_sdk.metrics.protocol import Metric from nemo_evaluator_sdk.values.multi_metric_results import BenchmarkEvaluationResult from nemo_evaluator_sdk.values.results import EvaluationResult diff --git a/plugins/nemo-evaluator/tests/test_sdk_job_resources.py b/plugins/nemo-evaluator/tests/test_sdk_job_resources.py index bac86f1c9e..a52bc5b8a4 100644 --- a/plugins/nemo-evaluator/tests/test_sdk_job_resources.py +++ b/plugins/nemo-evaluator/tests/test_sdk_job_resources.py @@ -33,7 +33,7 @@ AggregatedMetricResult, AggregateRangeScore, EvaluationResult, - MetricScore, + MetricOutput, RowScore, ) from nemo_platform_plugin.jobs.schemas import PlatformJobStatus, PlatformJobStatusResponse @@ -140,7 +140,7 @@ def _evaluation_result_parts() -> tuple[AggregatedMetricResult, list[RowScore]]: row_index=0, item={"expected": "a", "output": "a"}, sample={}, - metrics={"serializable": [MetricScore(name="score", value=1.0)]}, + metrics={"serializable": [MetricOutput(name="score", value=1.0)]}, requests=[], ) ] diff --git a/sdk/python/nemo-platform/.nmpcontext/openapi.yaml b/sdk/python/nemo-platform/.nmpcontext/openapi.yaml index e2eedea801..1063e3647d 100644 --- a/sdk/python/nemo-platform/.nmpcontext/openapi.yaml +++ b/sdk/python/nemo-platform/.nmpcontext/openapi.yaml @@ -20974,6 +20974,20 @@ components: - prompt_template title: MetricOnlineJob description: A online metric job. + MetricOutput: + properties: + name: + type: string + title: Name + value: + title: Value + additionalProperties: false + type: object + required: + - name + - value + title: MetricOutput + description: One named value emitted by a metric. MetricRef: type: string pattern: ^[a-z0-9_-]+/[a-z0-9_-]+$ @@ -21031,24 +21045,6 @@ components: - dataset title: MetricRetrieverJob description: Evaluation with a retriever-based metric. - MetricScore: - properties: - name: - type: string - title: Name - value: - type: number - title: Value - stats: - allOf: - - $ref: '#/components/schemas/ScoreStats' - description: Computed score statistics for the score. - type: object - required: - - name - - value - title: MetricScore - description: A computed score for the metric MetricType: description: The predefined metric types. enum: @@ -26893,11 +26889,11 @@ components: metrics: additionalProperties: items: - $ref: '#/components/schemas/MetricScore' + $ref: '#/components/schemas/MetricOutput' type: array type: object title: Metrics - description: Metric-level row scores by metric key. + description: Metric-level row outputs by metric key. requests: items: additionalProperties: true @@ -27494,77 +27490,6 @@ components: type: object title: SafeSynthesizerTiming description: Wall-clock durations for each pipeline stage. - ScoreStats: - properties: - count: - title: Count - description: The number of values used for computing the score. - type: integer - sum: - anyOf: - - type: number - - type: string - title: Sum - description: The sum of all values used for computing the score. - sum_squared: - anyOf: - - type: number - - type: string - title: Sum Squared - description: The sum of the square of all values used for computing the - score. - min: - anyOf: - - type: number - - type: string - title: Min - description: The minimum of all values used for computing the score. - max: - anyOf: - - type: number - - type: string - title: Max - description: The maximum of all values used for computing the score. - mean: - anyOf: - - type: number - - type: string - title: Mean - description: The mean of all values used for computing the score. - variance: - anyOf: - - type: number - - type: string - title: Variance - description: 'The population variance, (note: not the sample variance).' - stddev: - anyOf: - - type: number - - type: string - title: Stddev - description: 'The population standard deviation, (note: not the sample standard - deviation).' - stderr: - anyOf: - - type: number - - type: string - title: Stderr - description: The standard error. - nan_count: - title: Nan Count - description: The number of values that are not a number (NaN) and are excluded - from the score stats calculations. - type: integer - rubric_distribution: - title: Rubric Distribution - description: The distribution of the rubric grading criteria for the score. - items: - $ref: '#/components/schemas/RubricScoreStat' - type: array - type: object - title: ScoreStats - description: Stats for a score. Fields that are NaN are serialized as the string - "NaN" in the API response. SecretRef: type: string pattern: ^[a-z0-9_-]+(/[a-z0-9_-]+)?$ diff --git a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/__init__.py b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/__init__.py index 514eb8a7ad..6d33101875 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/__init__.py +++ b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/__init__.py @@ -17,6 +17,10 @@ from nemo_platform.beta.evaluator.metrics.f1 import F1Metric from nemo_platform.beta.evaluator.metrics.llm_judge import LLMJudgeMetric from nemo_platform.beta.evaluator.metrics.number_check import NumberCheckMetric +from nemo_platform.beta.evaluator.metrics.protocol import ( + Metric, + validate_metric_result, +) from nemo_platform.beta.evaluator.metrics.remote import NemoAgentToolkitRemoteMetric, RemoteMetric from nemo_platform.beta.evaluator.metrics.rouge import ROUGEMetric from nemo_platform.beta.evaluator.metrics.string_check import StringCheckMetric @@ -31,10 +35,21 @@ ) from nemo_platform.beta.evaluator.values import ( Agent, + BooleanValue, + CandidateOutput, + ContinuousScore, + DatasetRow, DatasetRows, + DiscreteScore, EvaluationResult, InferenceParams, JSONScoreParser, + Label, + MetricDescriptor, + MetricInput, + MetricOutput, + MetricOutputSpec, + MetricResult, Model, RangeScore, ReasoningParams, @@ -69,7 +84,19 @@ "InferenceFn", "InferenceStructuredOutput", "JSONScoreParser", + "Metric", + "MetricDescriptor", + "MetricInput", + "MetricOutput", + "MetricOutputSpec", + "MetricResult", "LLMJudgeMetric", + "BooleanValue", + "CandidateOutput", + "ContinuousScore", + "DatasetRow", + "DiscreteScore", + "Label", "Model", "NemoAgentToolkitRemoteMetric", "NumberCheckMetric", @@ -88,5 +115,6 @@ "detect_structured_output_mode", "load_dataset", "load_dataset_as_dicts", + "validate_metric_result", "version", ] diff --git a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/execution/backends/base.py b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/execution/backends/base.py index c50a6d87ce..0239a64e02 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/execution/backends/base.py +++ b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/execution/backends/base.py @@ -9,7 +9,7 @@ from typing import Protocol from nemo_platform.beta.evaluator.execution.config import EvaluationRequest -from nemo_platform.beta.evaluator.metrics.base import Metric +from nemo_platform.beta.evaluator.metrics.protocol import Metric from nemo_platform.beta.evaluator.values.multi_metric_results import BenchmarkEvaluationResult from nemo_platform.beta.evaluator.values.results import EvaluationResult diff --git a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/execution/backends/local/backend.py b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/execution/backends/local/backend.py index dd881ae011..c32d4f809a 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/execution/backends/local/backend.py +++ b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/execution/backends/local/backend.py @@ -14,7 +14,7 @@ from nemo_platform.beta.evaluator.execution.config import EvaluationRequest from nemo_platform.beta.evaluator.execution.metric_execution import _merge_online_hooks, evaluate_metric from nemo_platform.beta.evaluator.execution.utils import prepare_metric_for_local_execution, unique_metric_keys -from nemo_platform.beta.evaluator.metrics.base import Metric +from nemo_platform.beta.evaluator.metrics.protocol import Metric from nemo_platform.beta.evaluator.metrics.utils import metric_type_name from nemo_platform.beta.evaluator.values.multi_metric_results import BenchmarkEvaluationResult, namespace_result from nemo_platform.beta.evaluator.values.results import EvaluationResult diff --git a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/execution/benchmark_execution.py b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/execution/benchmark_execution.py index f3f4efe45a..35c19284c1 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/execution/benchmark_execution.py +++ b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/execution/benchmark_execution.py @@ -35,7 +35,7 @@ generate_online_sample, generate_online_sample_agent, ) -from nemo_platform.beta.evaluator.execution.scoring import nan_metric_result +from nemo_platform.beta.evaluator.execution.scoring import build_metric_input, corpus_output_spec, nan_metric_result from nemo_platform.beta.evaluator.execution.values import EvaluationError, EvaluationPhase from nemo_platform.beta.evaluator.inference import ( InferenceFn, @@ -45,8 +45,18 @@ new_inference_client, requests_log_var, ) -from nemo_platform.beta.evaluator.metrics.aggregation import add_corpus_scores, aggregate_metrics -from nemo_platform.beta.evaluator.metrics.base import CorpusMetric, Metric +from nemo_platform.beta.evaluator.metrics.aggregation import ( + add_corpus_scores, + aggregate_metrics, + rubric_definitions_from_metric, +) +from nemo_platform.beta.evaluator.metrics.protocol import ( + CorpusMetric, + Metric, + MetricOutputSpec, + MetricResult, + validate_metric_result, +) from nemo_platform.beta.evaluator.resilience.api import use_resilience_session from nemo_platform.beta.evaluator.resilience.errors import first_failure_cause, iter_leaf_causes from nemo_platform.beta.evaluator.values import ( @@ -54,8 +64,6 @@ AggregatedMetricResult, AggregateFieldName, EvaluationResult, - MetricResult, - MetricScore, Model, RowScore, RunConfig, @@ -105,21 +113,16 @@ class _MetricPipeline: metric_ref: str metric: Metric - score_names: list[str] + output_spec: list[MetricOutputSpec] queue: asyncio.Queue results: list[MetricResult | None] -def _normalize_metric_result(metric_result: MetricResult, expected_score_names: list[str]) -> MetricResult: - """Normalize score ordering and backfill missing expected scores with NaN.""" - actual_scores = {score.name: score for score in metric_result.scores} - normalized: list[MetricScore] = [] - for name in expected_score_names: - normalized.append(actual_scores.get(name, MetricScore(name=name, value=float("nan")))) - for score in metric_result.scores: - if score.name not in expected_score_names: - normalized.append(score) - return MetricResult(scores=normalized) +def _normalize_metric_result(metric_result: MetricResult, expected_outputs: list[MetricOutputSpec]) -> MetricResult: + """Validate output names and normalize output ordering to the declared spec.""" + validated = validate_metric_result(metric_result, expected_outputs) + actual_outputs = {output.name: output for output in validated.outputs} + return MetricResult(outputs=[actual_outputs[output.name] for output in expected_outputs]) def _benchmark_error_from_exception(exc: BaseException) -> EvaluationError | None: @@ -149,14 +152,14 @@ def _build_metric_pipelines( """ pipelines: list[_MetricPipeline] = [] for metric_ref, metric in metrics: - score_names = list(metric.score_names()) - if not score_names: - raise RuntimeError(f"Metric '{metric_ref}' does not declare any score names") + output_spec = list(metric.output_spec()) + if not output_spec: + raise RuntimeError(f"Metric '{metric_ref}' does not declare any outputs") pipelines.append( _MetricPipeline( metric_ref=metric_ref, metric=metric, - score_names=score_names, + output_spec=output_spec, queue=asyncio.Queue(maxsize=queue_capacity), results=[None] * item_count, ) @@ -206,7 +209,13 @@ async def _finalize_benchmark_metric_result( should only see successful rows so failed rows with empty samples do not skew corpus metrics. """ - aggregated = aggregate_metrics([result for result in results if result is not None]) + output_spec = metric.output_spec() + metric_results = [result for result in results if result is not None] + rubric_definitions = rubric_definitions_from_metric(metric) + if rubric_definitions: + aggregated = aggregate_metrics(metric_results, output_spec, rubric_definitions=rubric_definitions) + else: + aggregated = aggregate_metrics(metric_results, output_spec) if isinstance(metric, CorpusMetric): # Ignored sample-generation failures intentionally keep metric_errors # empty to match the previous service benchmark row artifacts, so @@ -218,11 +227,13 @@ async def _finalize_benchmark_metric_result( ] if corpus_rows: corpus_result = await metric.compute_corpus_scores( - items=[row_score.item for row_score in corpus_rows], - samples=[row_score.sample for row_score in corpus_rows], + inputs=[ + build_metric_input(row_score.item, row_score.sample, row_score.row_index) + for row_score in corpus_rows + ], ) if corpus_result is not None: - add_corpus_scores(aggregated, corpus_result) + add_corpus_scores(aggregated, corpus_result, corpus_output_spec(metric, output_spec)) return EvaluationResult(row_scores=row_scores, aggregate_scores=aggregated) @@ -392,8 +403,10 @@ async def _metric_worker( requests_log_var.set(requests_log) try: metric_result = _normalize_metric_result( - await pipeline.metric.compute_scores(dict(event.item), dict(event.sample)), - pipeline.score_names, + await pipeline.metric.compute_scores( + build_metric_input(dict(event.item), dict(event.sample), event.row_index) + ), + pipeline.output_spec, ) except Exception as e: if not tolerate_failure: @@ -408,7 +421,7 @@ async def _metric_worker( "Evaluation failed, marking as NaN", extra={"metric_ref": pipeline.metric_ref, "item_index": event.row_index, "error": error_message}, ) - metric_result = nan_metric_result(pipeline.score_names) + metric_result = nan_metric_result(pipeline.output_spec) # Record the swallowed metric exception on the row while keeping # the NaN score result. Example: if the "judge" metric raises # "bad output", the row gets metric_errors={"judge": "bad output"}. @@ -420,7 +433,7 @@ async def _metric_worker( pipeline.queue.task_done() pipeline.results[event.row_index] = metric_result - row_scores[event.row_index].metrics[pipeline.metric_ref] = metric_result.scores + row_scores[event.row_index].metrics[pipeline.metric_ref] = metric_result.outputs if progress is not None: progress.increment_work() diff --git a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/execution/evaluator.py b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/execution/evaluator.py index 0a0682b419..3d04878475 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/execution/evaluator.py +++ b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/execution/evaluator.py @@ -13,7 +13,7 @@ import nemo_platform.beta.evaluator.inference as inference from nemo_platform.beta.evaluator.execution.metric_execution import run_sync -from nemo_platform.beta.evaluator.metrics.base import Metric +from nemo_platform.beta.evaluator.metrics.protocol import Metric from nemo_platform.beta.evaluator.values.agents import Agent from nemo_platform.beta.evaluator.values.datasets import DatasetInput from nemo_platform.beta.evaluator.values.models import Model diff --git a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/execution/metric_execution.py b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/execution/metric_execution.py index 75883652b0..7febeacc64 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/execution/metric_execution.py +++ b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/execution/metric_execution.py @@ -38,7 +38,7 @@ from nemo_platform.beta.evaluator.execution.utils import prepare_metric_for_local_execution from nemo_platform.beta.evaluator.execution.values import EvaluationError, EvaluationPhase from nemo_platform.beta.evaluator.inference import InferenceMetricBase -from nemo_platform.beta.evaluator.metrics.base import Metric +from nemo_platform.beta.evaluator.metrics.protocol import Metric, MetricResult from nemo_platform.beta.evaluator.metrics.utils import metric_type_name from nemo_platform.beta.evaluator.resilience.api import run_indexed_tasks, use_resilience_session from nemo_platform.beta.evaluator.resilience.errors import get_evaluation_error @@ -46,7 +46,6 @@ from nemo_platform.beta.evaluator.values import ( Agent, EvaluationResult, - MetricResult, Model, RowScore, RunConfig, @@ -214,7 +213,7 @@ def _merge_online_hooks( """Build deterministic hook lists for SDK local online generation. Online sample generation should only use run-level generation hooks. - Metric hooks belong to metric.compute_scores(item, sample) and must not + Metric hooks belong to metric.compute_scores(input) and must not affect the evaluated-model generation stage. """ @@ -658,7 +657,7 @@ def handle_generation_error( log.warning("Inference failed, marking as NaN", extra={"item_index": index, "error": error_message}) sample = {"output_text": None, "response": {}, "inference_error": error_message} - nan_result = nan_metric_result(self.metric.score_names()) + nan_result = nan_metric_result(self.metric.output_spec()) return ( index, @@ -667,7 +666,7 @@ def handle_generation_error( row_index=index, item=row, sample=sample, - metrics={}, + metrics={self.metric_key: nan_result.outputs}, requests=generation_requests, metric_errors={self.metric_key: error_message}, ), diff --git a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/execution/pipeline.py b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/execution/pipeline.py index 72452dbfcc..e44a45020a 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/execution/pipeline.py +++ b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/execution/pipeline.py @@ -10,7 +10,8 @@ from types import MappingProxyType from typing import Any, Protocol -from nemo_platform.beta.evaluator.values.results import MetricResult, RowScore +from nemo_platform.beta.evaluator.metrics.protocol import MetricResult +from nemo_platform.beta.evaluator.values.results import RowScore @dataclass diff --git a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/execution/scoring.py b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/execution/scoring.py index 0777b92790..41b7b1a0d1 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/execution/scoring.py +++ b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/execution/scoring.py @@ -9,22 +9,75 @@ from nemo_platform.beta.evaluator.execution.values import EvaluationError, EvaluationPhase from nemo_platform.beta.evaluator.inference import requests_log_var -from nemo_platform.beta.evaluator.metrics.aggregation import add_corpus_scores, aggregate_metrics -from nemo_platform.beta.evaluator.metrics.base import CorpusMetric, Metric +from nemo_platform.beta.evaluator.metrics.aggregation import ( + add_corpus_scores, + aggregate_metrics, + is_aggregateable_output_spec, + rubric_definitions_from_metric, +) +from nemo_platform.beta.evaluator.metrics.protocol import ( + CandidateOutput, + CorpusMetric, + DatasetRow, + Metric, + MetricInput, + MetricOutput, + MetricOutputSpec, + MetricResult, + validate_metric_result, +) from nemo_platform.beta.evaluator.metrics.utils import metric_type_name -from nemo_platform.beta.evaluator.values import EvaluationResult, MetricResult, MetricScore, RowScore +from nemo_platform.beta.evaluator.values import ( + EvaluationResult, + RowScore, +) logger = getLogger(__name__) -def nan_metric_result(score_names: str | Iterable[str]) -> MetricResult: - """Build the NaN score payload used for ignored scoring failures. +_CANDIDATE_SAMPLE_FIELDS = frozenset({"output_text", "response", "trajectory"}) + + +def build_metric_input(row: dict[str, Any], sample: dict[str, Any], index: int | None = None) -> MetricInput: + """Build the metric protocol input from dataset row and generated sample payloads.""" + output_text = sample.get("output_text") + metadata = { + key: value + for key, value in sample.items() + if key not in _CANDIDATE_SAMPLE_FIELDS or (key == "output_text" and not isinstance(output_text, str)) + } + return MetricInput( + row=DatasetRow(row_index=index, data=row), + candidate=CandidateOutput( + output_text=output_text if isinstance(output_text, str) else None, + response=sample.get("response"), + trajectory=sample.get("trajectory"), + metadata=metadata, + ), + ) + - Accepts either a single score name (single-metric pipelines) or an - iterable of score names (multi-score/benchmark pipelines). +def nan_metric_result(outputs: Iterable[MetricOutputSpec]) -> MetricResult: + """Build the NaN output payload used for ignored scoring failures. + + Only aggregateable outputs receive NaN placeholders; non-score outputs are + not synthesized for failed rows. """ - names: Iterable[str] = (score_names,) if isinstance(score_names, str) else score_names - return MetricResult(scores=[MetricScore(name=name, value=float("nan")) for name in names]) + return MetricResult( + outputs=[ + MetricOutput(name=output.name, value=float("nan")) + for output in outputs + if is_aggregateable_output_spec(output) + ] + ) + + +def corpus_output_spec(metric: Metric, fallback: list[MetricOutputSpec] | None = None) -> list[MetricOutputSpec]: + """Return corpus-level output specs when a metric declares them.""" + corpus_spec = getattr(metric, "corpus_output_spec", None) + if callable(corpus_spec): + return list(corpus_spec()) + return list(fallback if fallback is not None else metric.output_spec()) CompletedRowEvaluation = tuple[int, MetricResult | None, RowScore] @@ -32,7 +85,7 @@ def nan_metric_result(score_names: str | Iterable[str]) -> MetricResult: def empty_evaluation_result() -> EvaluationResult: """Return the canonical empty evaluation result payload.""" - return EvaluationResult(row_scores=[], aggregate_scores=aggregate_metrics([])) + return EvaluationResult(row_scores=[], aggregate_scores=aggregate_metrics([], [])) async def finalize_evaluation_result( @@ -65,15 +118,22 @@ async def finalize_evaluation_result( # aggregation and corpus inputs honor ``skip_errored``. row_scores = [row_score for _, _, row_score in eval_results] - aggregated_result = aggregate_metrics(metric_results) + output_spec = metric.output_spec() + rubric_definitions = rubric_definitions_from_metric(metric) + if rubric_definitions: + aggregated_result = aggregate_metrics(metric_results, output_spec, rubric_definitions=rubric_definitions) + else: + aggregated_result = aggregate_metrics(metric_results, output_spec) if valid_eval_results and isinstance(metric, CorpusMetric): corpus_metric_result = await metric.compute_corpus_scores( - items=[row_score.item for _, row_score in valid_eval_results], - samples=[row_score.sample for _, row_score in valid_eval_results], + inputs=[ + build_metric_input(row_score.item, row_score.sample, row_score.row_index) + for _, row_score in valid_eval_results + ], ) if corpus_metric_result: - add_corpus_scores(aggregated_result, corpus_metric_result) + add_corpus_scores(aggregated_result, corpus_metric_result, corpus_output_spec(metric, output_spec)) return EvaluationResult( row_scores=row_scores, @@ -117,13 +177,16 @@ async def score_row( active_logger = logger or globals()["logger"] try: - result = await metric.compute_scores(row, sample) + output_spec = metric.output_spec() + result = validate_metric_result( + await metric.compute_scores(build_metric_input(row, sample, index)), output_spec + ) active_logger.debug( "Computed metric", extra={ "item_index": index, "metric_type": metric_type_name(metric), - "scores": [score.model_dump() for score in result.scores], + "outputs": [output.model_dump() for output in result.outputs], }, ) return ( @@ -133,7 +196,7 @@ async def score_row( row_index=index, item=row, sample=sample, - metrics={metric_key: result.scores}, + metrics={metric_key: result.outputs}, requests=[*generation_requests, *metric_requests], ), ) @@ -146,7 +209,7 @@ async def score_row( metric_key=metric_key, ) from e active_logger.warning("Evaluation failed, marking as NaN", extra={"item_index": index, "error": str(e)}) - result = nan_metric_result(metric.score_names()) + result = nan_metric_result(metric.output_spec()) return ( index, result, @@ -154,7 +217,7 @@ async def score_row( row_index=index, item=row, sample=sample, - metrics={metric_key: result.scores}, + metrics={metric_key: result.outputs}, requests=[*generation_requests, *metric_requests], metric_errors={metric_key: str(e)}, ), diff --git a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/execution/utils.py b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/execution/utils.py index 5621c192bd..bf7337388f 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/execution/utils.py +++ b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/execution/utils.py @@ -11,7 +11,7 @@ from typing import cast from nemo_platform.beta.evaluator.execution._protocols import JobParamsConfigurableMetric -from nemo_platform.beta.evaluator.metrics.base import Metric, MetricWithPreflight, MetricWithSecrets +from nemo_platform.beta.evaluator.metrics.protocol import Metric, MetricWithPreflight, MetricWithSecrets from nemo_platform.beta.evaluator.metrics.utils import metric_type_name from nemo_platform.beta.evaluator.values.params import RunConfig from pydantic import BaseModel diff --git a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/metrics/aggregation.py b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/metrics/aggregation.py index 517ca5d4db..90523d7793 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/metrics/aggregation.py +++ b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/metrics/aggregation.py @@ -7,7 +7,17 @@ import math from collections import OrderedDict, defaultdict - +from collections.abc import Mapping, Sequence +from typing import Protocol, cast, runtime_checkable + +from nemo_platform.beta.evaluator.metrics.protocol import ( + BooleanValue, + ContinuousScore, + DiscreteScore, + MetricOutput, + MetricOutputSpec, + MetricResult, +) from nemo_platform.beta.evaluator.values.results import ( AggregatedMetricResult, AggregateRangeScore, @@ -15,15 +25,100 @@ AggregateScore, Histogram, HistogramBin, - MetricResult, MetricScore, Percentiles, RubricScoreStat, ScoreStats, ) +from nemo_platform.beta.evaluator.values.scores import RubricScore, Score + + +def is_aggregateable_output_spec(output_spec: MetricOutputSpec) -> bool: + """Return whether an output should contribute aggregate statistics.""" + return issubclass(output_spec.value_schema, (ContinuousScore, DiscreteScore, BooleanValue)) + + +@runtime_checkable +class MetricWithScores(Protocol): + """Metric config protocol for metrics that carry rubric score definitions.""" + + @property + def scores(self) -> Sequence[Score]: ... + + +def _coerce_aggregate_output(output: MetricOutput, output_spec: MetricOutputSpec) -> MetricScore | None: + """Convert one declared aggregateable metric output into MetricScore form.""" + if not is_aggregateable_output_spec(output_spec): + return None + if ( + issubclass(output_spec.value_schema, BooleanValue) + and isinstance(output.value, float) + and math.isnan(output.value) + ): + return MetricScore(name=output.name, value=output.value) + coerced = cast(ContinuousScore | DiscreteScore | BooleanValue, output_spec.coerce_output(output)) + value = coerced.root + if isinstance(value, bool): + value = 1.0 if value else 0.0 + return MetricScore(name=output.name, value=value) + + +def _attach_rubric_stats( + score: MetricScore, + output_by_name: Mapping[str, MetricOutput], + rubric_definitions: Mapping[str, Sequence[RubricScoreStat]], +) -> MetricScore: + """Attach per-row rubric bucket stats from companion label outputs.""" + rubric_definition = rubric_definitions.get(score.name) + if not rubric_definition: + return score + + label_output = output_by_name.get(f"{score.name}.label") + selected_label = label_output.value if label_output is not None else None + if isinstance(score.value, float) and math.isnan(score.value): + selected_label = None + + rubric_distribution = [ + RubricScoreStat( + label=rubric.label, + description=rubric.description, + value=rubric.value, + count=int(isinstance(selected_label, str) and selected_label == rubric.label), + ) + for rubric in rubric_definition + ] + return MetricScore( + name=score.name, + value=score.value, + stats=ScoreStats(rubric_distribution=rubric_distribution), + ) -def add_corpus_scores(aggregated_result: AggregatedMetricResult, corpus_result: MetricResult) -> None: +def _aggregateable_scores( + result: MetricResult, + output_specs: list[MetricOutputSpec], + rubric_definitions: Mapping[str, Sequence[RubricScoreStat]] | None = None, +) -> list[MetricScore]: + """Extract score-like outputs from a metric result using declared output specs.""" + specs_by_name = {output_spec.name: output_spec for output_spec in output_specs} + output_by_name = {output.name: output for output in result.outputs} + scores: list[MetricScore] = [] + for output in result.outputs: + output_spec = specs_by_name.get(output.name) + if output_spec is None: + continue + score = _coerce_aggregate_output(output, output_spec) + if score is not None: + score = _attach_rubric_stats(score, output_by_name, rubric_definitions or {}) + scores.append(score) + return scores + + +def add_corpus_scores( + aggregated_result: AggregatedMetricResult, + corpus_result: MetricResult, + output_specs: list[MetricOutputSpec], +) -> None: """Append corpus-level scores using aggregate-score schema fields. Args: @@ -33,7 +128,7 @@ def add_corpus_scores(aggregated_result: AggregatedMetricResult, corpus_result: Returns: ``None``. The ``aggregated_result`` object is updated in place. """ - for score in corpus_result.scores: + for score in _aggregateable_scores(corpus_result, output_specs): value = score.value # Corpus-level metrics contribute one already-aggregated value, so # expose them through the same aggregate schema with count=1. @@ -148,7 +243,11 @@ def _compute_histogram(values: list[float], num_bins: int = 10) -> Histogram: return Histogram(bins=bins) -def aggregate_metrics(items: list[MetricResult]) -> AggregatedMetricResult: +def aggregate_metrics( + items: list[MetricResult], + output_specs: list[MetricOutputSpec], + rubric_definitions: Mapping[str, Sequence[RubricScoreStat]] | None = None, +) -> AggregatedMetricResult: """Aggregate row-level metric results into range or rubric summaries. This function performs two logical passes: @@ -158,6 +257,11 @@ def aggregate_metrics(items: list[MetricResult]) -> AggregatedMetricResult: Args: items: Row-level metric results to aggregate. + output_specs: Declared outputs for the metric. Only continuous, + discrete, and boolean output values contribute to aggregate scores. + rubric_definitions: Optional rubric bucket definitions keyed by numeric + output name. This is aggregation metadata, not metric protocol + metadata, and is usually derived from LLM judge score config. Returns: Aggregate metric result with one aggregate score per score name. @@ -169,7 +273,7 @@ def aggregate_metrics(items: list[MetricResult]) -> AggregatedMetricResult: has_rubric: dict[str, bool] = {} for item in items: - for score in item.scores: + for score in _aggregateable_scores(item, output_specs, rubric_definitions): if score.name not in aggregated_results: # Keep one running accumulator per score name; distribution # details are materialized in a second pass once all values exist. @@ -317,3 +421,31 @@ def aggregate_metrics(items: list[MetricResult]) -> AggregatedMetricResult: ) return AggregatedMetricResult(scores=aggregated_scores) + + +def rubric_definitions_from_scores(scores: Sequence[Score]) -> dict[str, list[RubricScoreStat]]: + """Return declared rubric buckets keyed by score name.""" + definitions: dict[str, list[RubricScoreStat]] = {} + for score in scores: + if not isinstance(score, RubricScore): + continue + definitions[score.name] = [ + RubricScoreStat( + label=rubric.label, + description=rubric.description, + value=rubric.value, + count=0, + ) + for rubric in score.rubric + ] + return definitions + + +def rubric_definitions_from_metric(metric: object) -> dict[str, list[RubricScoreStat]]: + """Return rubric bucket definitions for metrics that carry score config.""" + if not isinstance(metric, MetricWithScores): + return {} + scores = metric.scores + if not isinstance(scores, Sequence) or isinstance(scores, (str, bytes)): + return {} + return rubric_definitions_from_scores(scores) diff --git a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/metrics/base.py b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/metrics/base.py deleted file mode 100644 index ef8d3cc4ba..0000000000 --- a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/metrics/base.py +++ /dev/null @@ -1,117 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""Optional metric capabilities and shared helpers for evaluator SDK runtime.""" - -# Migrated from: services/evaluator/src/nmp/evaluator/app/metrics/base.py - -import re -import string -from typing import Awaitable, Callable, Protocol, runtime_checkable - -from nemo_platform.beta.evaluator.values import SecretRef -from nemo_platform.beta.evaluator.values.results import MetricResult - -SecretResolver = Callable[[str], Awaitable[str | None]] - - -@runtime_checkable -class Metric(Protocol): - """Structural contract for SDK runtime metrics used by generic evaluator code. - - This protocol describes what execution and orchestration code may rely on - when working with a metric instance. In particular, ``type`` is treated as - a string identifier. Built-in metrics may expose existing ``MetricType`` - values for schema compatibility, but custom metrics should use plain - strings: - - - a built-in ``MetricType`` member - - a plain string such as ``"my-custom-metric"`` - - Generic consumers must therefore treat ``type`` as a string identifier and - must not depend on enum-only APIs such as ``.value``. Callers that need to - normalize supported runtime shapes should use ``metric_type_name(...)``. - """ - - @property - def type(self) -> str: - """Return the public metric key/type identifier. - - Examples: - Built-in runtime metrics may expose ``MetricType.BLEU``. - - Custom metrics may expose a plain string such as - ``"my-custom-metric"``. - """ - ... - - async def compute_scores(self, item: dict, sample: dict) -> MetricResult: - """Compute structured score output for one item/sample pair.""" - ... - - def score_names(self) -> list[str]: - """Return canonical score names emitted by this metric.""" - ... - - -@runtime_checkable -class CorpusMetric(Protocol): - """Protocol for metrics that also emit corpus-level scores.""" - - async def compute_corpus_scores(self, items: list[dict], samples: list[dict]) -> MetricResult | None: - """Compute corpus-level scores across all evaluated rows. - - Args: - items: Original dataset rows. - samples: Sample payloads paired to ``items``. - - Returns: - Optional corpus-level metric result. - """ - ... - - -@runtime_checkable -class MetricWithSecrets(Protocol): - """Protocol for metrics that require secrets (e.g., API keys).""" - - def secrets(self) -> dict[str, SecretRef]: - """ - Returns a dictionary of environment variables to the secret reference. - Used by the job flow to set up environment variables. - """ - ... - - async def resolve_secrets(self, secret_resolver: SecretResolver) -> None: - """ - Resolve secrets using the provided resolver function. - Called before the metric is used for evaluation. - """ - ... - - -@runtime_checkable -class MetricWithPreflight(Protocol): - """Protocol for metrics that need one-time setup before parallel evaluation starts.""" - - async def preflight(self) -> None: - """Run one-time preflight (e.g., capability detection) before processing rows.""" - ... - - -# TODO: migrate the rest of the protocols from services/evaluator/src/nmp/evaluator/app/metrics/base.py - - -def normalize_text(s: str) -> str: - """Normalize free-form text for token/equality-based metric comparisons.""" - if not s: - return "" - # lower case - s = s.lower() - # remove punctuation - s = "".join(ch for ch in s if ch not in set(string.punctuation)) - # remove articles - s = re.sub(r"\b(a|an|the)\b", " ", s) - # collapse whitespace - s = " ".join(s.split()) - return s diff --git a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/metrics/bleu.py b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/metrics/bleu.py index 0156080e68..f9b756f48f 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/metrics/bleu.py +++ b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/metrics/bleu.py @@ -4,14 +4,15 @@ """BLEU metric runtime implementation.""" import sacrebleu +from nemo_platform.beta.evaluator.metrics.protocol import MetricInput, MetricOutput, MetricOutputSpec, MetricResult from nemo_platform.beta.evaluator.metrics.template_rendering import ( + TemplateSample, build_template_context, render_default_output_text_candidate_or_raise, render_template_or_raise, template_metric_repr, ) from nemo_platform.beta.evaluator.values.metrics import BLEU -from nemo_platform.beta.evaluator.values.results import MetricResult, MetricScore __all__ = ["BLEUMetric"] @@ -24,11 +25,15 @@ class BLEUMetric(BLEU): ``sample.output_text`` when the evaluator generates model outputs online. """ - def score_names(self) -> list[str]: - """Return score keys emitted by this metric.""" - return ["sentence"] + def output_spec(self) -> list[MetricOutputSpec]: + """Return row-level outputs emitted by this metric.""" + return [MetricOutputSpec.continuous_score("sentence")] - def _render_references(self, item: dict, sample: dict) -> list[str]: + def corpus_output_spec(self) -> list[MetricOutputSpec]: + """Return corpus-level outputs emitted by this metric.""" + return [MetricOutputSpec.continuous_score("corpus")] + + def _render_references(self, item: dict, sample: TemplateSample) -> list[str]: """Render all reference templates for one row.""" context = build_template_context(item, sample) metric_repr = template_metric_repr(self) @@ -47,7 +52,7 @@ def _render_references(self, item: dict, sample: dict) -> list[str]: references.append(rendered_reference) return references - def _render_candidate(self, item: dict, sample: dict) -> str: + def _render_candidate(self, item: dict, sample: TemplateSample) -> str: """Render the candidate text for one row.""" if self.candidate: context = build_template_context(item, sample) @@ -69,18 +74,21 @@ def _render_candidate(self, item: dict, sample: dict) -> str: raise TypeError("The candidate must be a string.") return prediction - async def compute_scores(self, item: dict, sample: dict) -> MetricResult: + async def compute_scores(self, input: MetricInput) -> MetricResult: """Compute the scores for the metric.""" + item = input.row.data + sample = input.candidate references = self._render_references(item, sample) candidate = self._render_candidate(item, sample) score = sacrebleu.sentence_bleu(candidate, references).score.real - return MetricResult(scores=[MetricScore(name="sentence", value=score)]) + return MetricResult(outputs=[MetricOutput(name="sentence", value=score)]) - async def compute_corpus_scores(self, items: list[dict], samples: list[dict]) -> MetricResult | None: + async def compute_corpus_scores(self, inputs: list[MetricInput]) -> MetricResult | None: """Compute the corpus-level BLEU metric.""" - references_raw = [self._render_references(item, sample) for item, sample in zip(items, samples)] + item_sample_pairs = [(input.row.data, input.candidate) for input in inputs] + references_raw = [self._render_references(item, sample) for item, sample in item_sample_pairs] # NOTE: because of the bug in sacrebleu, we need to flatten the references references = [[reference_set[0] for reference_set in references_raw]] - candidates = [self._render_candidate(item, sample) for item, sample in zip(items, samples)] + candidates = [self._render_candidate(item, sample) for item, sample in item_sample_pairs] score = sacrebleu.corpus_bleu(candidates, references) - return MetricResult(scores=[MetricScore(name="corpus", value=score.score.real)]) + return MetricResult(outputs=[MetricOutput(name="corpus", value=score.score.real)]) diff --git a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/metrics/exact_match.py b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/metrics/exact_match.py index 480d619d8e..1c65fc0b2b 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/metrics/exact_match.py +++ b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/metrics/exact_match.py @@ -3,10 +3,10 @@ """Exact-match metric runtime implementation.""" -from nemo_platform.beta.evaluator.metrics.base import normalize_text +from nemo_platform.beta.evaluator.metrics.protocol import MetricInput, MetricOutput, MetricOutputSpec, MetricResult from nemo_platform.beta.evaluator.metrics.template_rendering import render_reference_and_candidate, template_metric_repr +from nemo_platform.beta.evaluator.metrics.utils import normalize_text from nemo_platform.beta.evaluator.values.metrics import ExactMatch -from nemo_platform.beta.evaluator.values.results import MetricResult, MetricScore __all__ = ["ExactMatchMetric"] @@ -18,15 +18,11 @@ class ExactMatchMetric(ExactMatch): model outputs through ``sample.output_text`` for online execution. """ - def score_names(self) -> list[str]: - """Return score keys emitted by this metric. + def output_spec(self) -> list[MetricOutputSpec]: + """Return outputs emitted by this metric.""" + return [MetricOutputSpec.continuous_score(self.type.value)] - Returns: - Single-item list containing the exact-match metric type name. - """ - return [self.type.value] - - async def compute_scores(self, item: dict, sample: dict) -> MetricResult: + async def compute_scores(self, input: MetricInput) -> MetricResult: """Compute structured metric output for one item/sample pair. The algorithm renders reference and candidate text from templates, then @@ -34,8 +30,7 @@ async def compute_scores(self, item: dict, sample: dict) -> MetricResult: before equality comparison. Args: - item: Original dataset row. - sample: Generated-sample payload used for candidate extraction. + input: Original dataset row paired with candidate output. Returns: ``MetricResult`` with one exact-match score entry. @@ -50,8 +45,8 @@ async def compute_scores(self, item: dict, sample: dict) -> MetricResult: metric_name=self.__class__.__name__, reference_template=self.reference, candidate_template=self.candidate, - item=item, - sample=sample, + item=input.row.data, + sample=input.candidate, ) score = int(normalize_text(prediction) == normalize_text(ground_truth)) - return MetricResult(scores=[MetricScore(name=self.type.value, value=score)]) + return MetricResult(outputs=[MetricOutput(name=self.type.value, value=score)]) diff --git a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/metrics/f1.py b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/metrics/f1.py index 3a35927db7..2a2d7f5933 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/metrics/f1.py +++ b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/metrics/f1.py @@ -5,10 +5,10 @@ import collections -from nemo_platform.beta.evaluator.metrics.base import normalize_text +from nemo_platform.beta.evaluator.metrics.protocol import MetricInput, MetricOutput, MetricOutputSpec, MetricResult from nemo_platform.beta.evaluator.metrics.template_rendering import render_reference_and_candidate, template_metric_repr +from nemo_platform.beta.evaluator.metrics.utils import normalize_text from nemo_platform.beta.evaluator.values.metrics import F1 -from nemo_platform.beta.evaluator.values.results import MetricResult, MetricScore __all__ = ["F1Metric"] @@ -16,19 +16,19 @@ class F1Metric(F1): """F1 metric for token-overlap similarity scoring.""" - def score_names(self) -> list[str]: - """Return score keys emitted by this metric.""" - return [self.type.value] + def output_spec(self) -> list[MetricOutputSpec]: + """Return outputs emitted by this metric.""" + return [MetricOutputSpec.continuous_score(self.type.value)] - async def compute_scores(self, item: dict, sample: dict) -> MetricResult: + async def compute_scores(self, input: MetricInput) -> MetricResult: """Compute the scores for the metric.""" ground_truth, prediction = render_reference_and_candidate( metric_repr=template_metric_repr(self), metric_name=self.__class__.__name__, reference_template=self.reference, candidate_template=self.candidate, - item=item, - sample=sample, + item=input.row.data, + sample=input.candidate, ) prediction_tokens = normalize_text(prediction).split() @@ -37,14 +37,14 @@ async def compute_scores(self, item: dict, sample: dict) -> MetricResult: # If either token list is empty, the F1 is 1.0 if they agree, 0.0 otherwise. if len(ground_truth_tokens) == 0 or len(prediction_tokens) == 0: score = float(ground_truth_tokens == prediction_tokens) - return MetricResult(scores=[MetricScore(name=self.type.value, value=score)]) + return MetricResult(outputs=[MetricOutput(name=self.type.value, value=score)]) common = collections.Counter(prediction_tokens) & collections.Counter(ground_truth_tokens) num_same = sum(common.values()) if num_same == 0: - return MetricResult(scores=[MetricScore(name=self.type.value, value=0.0)]) + return MetricResult(outputs=[MetricOutput(name=self.type.value, value=0.0)]) precision = 1.0 * num_same / len(prediction_tokens) recall = 1.0 * num_same / len(ground_truth_tokens) score = (2 * precision * recall) / (precision + recall) - return MetricResult(scores=[MetricScore(name=self.type.value, value=score)]) + return MetricResult(outputs=[MetricOutput(name=self.type.value, value=score)]) diff --git a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/metrics/llm_judge.py b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/metrics/llm_judge.py index 9a73335e4a..9ca7c25538 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/metrics/llm_judge.py +++ b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/metrics/llm_judge.py @@ -13,7 +13,17 @@ from nemo_platform.beta.evaluator.inference import InferenceFn, InferenceHookParams from nemo_platform.beta.evaluator.inference import new_hooks as _new_inference_hooks from nemo_platform.beta.evaluator.metrics.hooks import HooksBase -from nemo_platform.beta.evaluator.metrics.template_rendering import build_template_context +from nemo_platform.beta.evaluator.metrics.protocol import ( + MetricInput, + MetricOutput, + MetricOutputSpec, + MetricResult, +) +from nemo_platform.beta.evaluator.metrics.template_rendering import ( + TemplateSample, + build_template_context, + sample_template_payload, +) from nemo_platform.beta.evaluator.structured_output import InferenceStructuredOutput, detect_structured_output_mode from nemo_platform.beta.evaluator.templates import render_request from nemo_platform.beta.evaluator.values.common import SecretRef, SupportedJobTypes @@ -26,7 +36,7 @@ ) from nemo_platform.beta.evaluator.values.models import Model from nemo_platform.beta.evaluator.values.params import InferenceParams, ReasoningParams -from nemo_platform.beta.evaluator.values.results import MetricResult, MetricScore +from nemo_platform.beta.evaluator.values.results import MetricScore from nemo_platform.beta.evaluator.values.scores import ( JSONScoreParser, RangeScore, @@ -107,9 +117,21 @@ def set_inference_fn(self, inference_fn: InferenceFn) -> None: """Set the inference function to use for LLM calls.""" self._inference_fn = inference_fn - def score_names(self) -> list[str]: - """Return score keys emitted by this metric.""" - return list(self._parsers.keys()) + def output_spec(self) -> list[MetricOutputSpec]: + """Return outputs emitted by this metric.""" + specs: list[MetricOutputSpec] = [] + for score in self.scores: + if isinstance(score, RubricScore): + specs.append(MetricOutputSpec.continuous_score(score.name, description=score.description)) + specs.append( + MetricOutputSpec.label( + f"{score.name}.label", + description=f"Selected rubric label for {score.name}", + ) + ) + else: + specs.append(MetricOutputSpec.continuous_score(score.name, description=score.description)) + return specs def _handle_none_output_error(self, response: dict) -> ValueError: error_message = "LLM judge returned no usable textual content for score parsing" @@ -141,7 +163,12 @@ def _handle_invalid_output(self, error: Exception, fallback: MetricResult, messa raise error def _nan_result(self) -> MetricResult: - return MetricResult(scores=[MetricScore(name=name, value=float("nan")) for name in self._parsers]) + outputs: list[MetricOutput] = [] + for score in self.scores: + outputs.append(MetricOutput(name=score.name, value=float("nan"))) + if isinstance(score, RubricScore): + outputs.append(MetricOutput(name=f"{score.name}.label", value="")) + return MetricResult(outputs=outputs) async def resolve_secrets(self, secret_resolver: Callable[[str], Awaitable[str | None]]) -> None: """Resolve API key secret if configured and reinitialize AsyncOpenAI client. Must be called before using the metric.""" @@ -228,8 +255,9 @@ def _initialize_score_parsers(self) -> None: self._parsers[score.name] = parser self._score_dumps[score.name] = score.model_dump(mode="json", exclude={"parser"}) - def _render_request(self, item: dict, sample: dict) -> dict: - overlapping_keys = set(item.keys()) & set(sample.keys()) + def _render_request(self, item: dict, sample: TemplateSample) -> dict: + sample_payload = sample_template_payload(sample) + overlapping_keys = set(item.keys()) & set(sample_payload.keys()) if overlapping_keys: _logger.warning( "Dataset columns %s overlap with model response keys. " @@ -261,8 +289,10 @@ def _retry_with_max_completion_tokens(self, request: dict) -> dict: del request["max_tokens"] return request - async def compute_scores(self, item: dict, sample: dict) -> MetricResult: + async def compute_scores(self, input: MetricInput) -> MetricResult: """Compute structured score output for one item/sample pair.""" + item = input.row.data + sample = input.candidate request = self._render_request(item, sample) try: @@ -290,14 +320,27 @@ async def compute_scores(self, item: dict, sample: dict) -> MetricResult: "LLM judge returned invalid output, marking as NaN", ) - result = MetricResult(scores=[]) + result = MetricResult(outputs=[]) for score_name, parser in self._parsers.items(): score = parser.parse(output_text) _logger.debug("Parsed score %s: %s", score_name, score.value) - result.scores.append(score) + result.outputs.append(MetricOutput(name=score.name, value=score.value)) + label = _selected_rubric_label(score) + if label is not None: + result.outputs.append(MetricOutput(name=f"{score.name}.label", value=label)) return result +def _selected_rubric_label(score: MetricScore) -> str | None: + """Return the selected rubric label recorded by the parser, if any.""" + if not score.stats or not score.stats.rubric_distribution: + return None + for rubric_stat in score.stats.rubric_distribution: + if rubric_stat.count: + return rubric_stat.label + return "" + + def new_hooks(params: _LLMJudgeHookParams | None): """Initialize preprocess and postprocess hooks for the LLM judge.""" model_format = params.model.format if params else ModelFormat.NVIDIA_NIM diff --git a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/metrics/number_check.py b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/metrics/number_check.py index 786d31e0ef..b4fffe4c6a 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/metrics/number_check.py +++ b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/metrics/number_check.py @@ -6,13 +6,14 @@ import math import re +from nemo_platform.beta.evaluator.metrics.protocol import MetricInput, MetricOutput, MetricOutputSpec, MetricResult from nemo_platform.beta.evaluator.metrics.template_rendering import ( + TemplateSample, build_template_context, render_template_or_raise, template_metric_repr, ) from nemo_platform.beta.evaluator.values.metrics import NumberCheck, NumberCheckOperation -from nemo_platform.beta.evaluator.values.results import MetricResult, MetricScore __all__ = ["NumberCheckMetric", "NumberCheckOperation"] @@ -35,12 +36,14 @@ def _parse_number_answer(answer: str) -> int | float: class NumberCheckMetric(NumberCheck): """Numeric-comparison metric with template-driven operands.""" - def score_names(self) -> list[str]: - """Return score keys emitted by this metric.""" - return [self.type.value] + def output_spec(self) -> list[MetricOutputSpec]: + """Return outputs emitted by this metric.""" + return [MetricOutputSpec.continuous_score(self.type.value)] - async def compute_scores(self, item: dict, sample: dict) -> MetricResult: + async def compute_scores(self, input: MetricInput) -> MetricResult: """Compute the scores for the metric.""" + item = input.row.data + sample: TemplateSample = input.candidate context = build_template_context(item, sample) metric_repr = template_metric_repr(self) left_value = render_template_or_raise( @@ -65,9 +68,9 @@ async def compute_scores(self, item: dict, sample: dict) -> MetricResult: # Preserve the legacy behavior: if either side fails to parse as a # number, return NaN instead of raising. if math.isnan(left_number): - return MetricResult(scores=[MetricScore(name=self.type.value, value=left_number)]) + return MetricResult(outputs=[MetricOutput(name=self.type.value, value=left_number)]) if math.isnan(right_number): - return MetricResult(scores=[MetricScore(name=self.type.value, value=right_number)]) + return MetricResult(outputs=[MetricOutput(name=self.type.value, value=right_number)]) # Perform the requested numeric comparison on the parsed operands. if self.operation in ["equals", "=="]: @@ -89,4 +92,4 @@ async def compute_scores(self, item: dict, sample: dict) -> MetricResult: else: raise ValueError(f"Unsupported operation: {self.operation}") - return MetricResult(scores=[MetricScore(name=self.type.value, value=1.0 if score else 0.0)]) + return MetricResult(outputs=[MetricOutput(name=self.type.value, value=1.0 if score else 0.0)]) diff --git a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/metrics/protocol.py b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/metrics/protocol.py new file mode 100644 index 0000000000..21b00d74f5 --- /dev/null +++ b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/metrics/protocol.py @@ -0,0 +1,241 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Runtime protocol for implementing Evaluator metrics.""" + +from __future__ import annotations + +import math +from collections.abc import Awaitable, Callable +from typing import Any, Protocol, runtime_checkable + +from nemo_platform.beta.evaluator.values.common import SecretRef +from pydantic import BaseModel, ConfigDict, Field, RootModel, field_serializer, field_validator + +SecretResolver = Callable[[str], Awaitable[str | None]] + + +class DatasetRow(BaseModel): + """Original dataset row plus optional stable row identity.""" + + model_config = ConfigDict(extra="forbid") + + row_index: int | None = None + data: dict[str, Any] + + +class CandidateOutput(BaseModel): + """Candidate or prediction output being scored for one dataset row.""" + + model_config = ConfigDict(extra="forbid") + + output_text: str | None = None + response: Any | None = None + trajectory: Any | None = None + metadata: dict[str, Any] = Field(default_factory=dict) + + def as_sample(self) -> dict[str, Any]: + """Return a sample-shaped payload for template rendering helpers.""" + sample = dict(self.metadata) + if self.output_text is not None: + sample["output_text"] = self.output_text + if self.response is not None: + sample["response"] = self.response + if self.trajectory is not None: + sample["trajectory"] = self.trajectory + return sample + + +class MetricInput(BaseModel): + """Complete per-row scoring input passed to a metric.""" + + model_config = ConfigDict(extra="forbid") + + row: DatasetRow + candidate: CandidateOutput + + +class ContinuousScore(RootModel[float]): + """Continuous numeric metric value.""" + + +class DiscreteScore(RootModel[int]): + """Discrete numeric metric value.""" + + +class Label(RootModel[str]): + """String label metric value.""" + + +class BooleanValue(RootModel[bool]): + """Boolean metric value.""" + + +class MetricOutputSpec(BaseModel): + """Schema for one named value emitted by a metric.""" + + model_config = ConfigDict(extra="forbid", arbitrary_types_allowed=True) + + name: str + description: str | None = None + value_schema: type[BaseModel] + + @field_validator("name") + @classmethod + def _name_must_not_be_empty(cls, value: str) -> str: + if not value: + raise ValueError("metric output name must not be empty") + return value + + @staticmethod + def continuous_score(name: str, description: str | None = None) -> "MetricOutputSpec": + return MetricOutputSpec(name=name, description=description, value_schema=ContinuousScore) + + @staticmethod + def discrete_score(name: str, description: str | None = None) -> "MetricOutputSpec": + return MetricOutputSpec(name=name, description=description, value_schema=DiscreteScore) + + @staticmethod + def label(name: str, description: str | None = None) -> "MetricOutputSpec": + return MetricOutputSpec(name=name, description=description, value_schema=Label) + + @staticmethod + def boolean(name: str, description: str | None = None) -> "MetricOutputSpec": + return MetricOutputSpec(name=name, description=description, value_schema=BooleanValue) + + @staticmethod + def model(name: str, value_schema: type[BaseModel], description: str | None = None) -> "MetricOutputSpec": + return MetricOutputSpec(name=name, description=description, value_schema=value_schema) + + def coerce_value(self, value: Any) -> BaseModel: + """Validate and coerce a raw output value to this spec's declared schema.""" + return self.value_schema.model_validate(value) + + def coerce_output(self, output: "MetricOutput") -> BaseModel: + """Validate and coerce a named metric output against this spec.""" + if output.name != self.name: + raise ValueError(f"Expected metric output {self.name!r}, got {output.name!r}") + return self.coerce_value(output.value) + + def value_json_schema(self) -> dict[str, Any]: + return self.value_schema.model_json_schema() + + +class MetricDescriptor(BaseModel): + """Metadata describing a metric implementation and its declared outputs.""" + + model_config = ConfigDict(extra="forbid") + + type: str + outputs: list[MetricOutputSpec] = Field(min_length=1) + + @field_validator("type") + @classmethod + def _type_must_not_be_empty(cls, value: str) -> str: + if not value: + raise ValueError("metric type must not be empty") + return value + + @field_validator("outputs") + @classmethod + def _output_names_must_be_unique(cls, value: list[MetricOutputSpec]) -> list[MetricOutputSpec]: + names = [output.name for output in value] + duplicates = sorted({name for name in names if names.count(name) > 1}) + if duplicates: + raise ValueError(f"duplicate metric output names: {duplicates}") + return value + + +class MetricOutput(BaseModel): + """One named value emitted by a metric.""" + + model_config = ConfigDict(extra="forbid") + + name: str + value: Any + + @field_serializer("value") + def serialize_nan(self, value: Any) -> Any: + if isinstance(value, float) and math.isnan(value): + return "NaN" + return value + + +class MetricResult(BaseModel): + """Structured row-level metric result.""" + + model_config = ConfigDict(extra="forbid") + + outputs: list[MetricOutput] + + +@runtime_checkable +class Metric(Protocol): + """Shared row-scoring primitive for SDK runtime metrics.""" + + @property + def type(self) -> str: + """Return the public metric key/type identifier.""" + ... + + def output_spec(self) -> list[MetricOutputSpec]: + """Return declared row-level outputs emitted by this metric.""" + ... + + async def compute_scores(self, input: MetricInput) -> MetricResult: + """Compute structured output for one row/candidate pair.""" + ... + + +@runtime_checkable +class CorpusMetric(Protocol): + """Protocol for metrics that also emit corpus-level scores.""" + + async def compute_corpus_scores(self, inputs: list[MetricInput]) -> MetricResult | None: + """Compute corpus-level scores across all evaluated rows.""" + ... + + +@runtime_checkable +class MetricWithSecrets(Protocol): + """Protocol for metrics that require secrets.""" + + def secrets(self) -> dict[str, SecretRef]: + """Return environment variables mapped to secret references.""" + ... + + async def resolve_secrets(self, secret_resolver: SecretResolver) -> None: + """Resolve secrets before the metric is used for evaluation.""" + ... + + +@runtime_checkable +class MetricWithPreflight(Protocol): + """Protocol for metrics that need one-time setup before parallel evaluation starts.""" + + async def preflight(self) -> None: + """Run one-time preflight before processing rows.""" + ... + + +def validate_metric_result(result: MetricResult, outputs: list[MetricOutputSpec]) -> MetricResult: + """Validate a metric result against its declared outputs.""" + returned_names = [output.name for output in result.outputs] + duplicates = sorted({name for name in returned_names if returned_names.count(name) > 1}) + if duplicates: + raise ValueError(f"Duplicate metric output names: {duplicates}") + + outputs_by_name = {output.name: output for output in outputs} + declared_names = [output.name for output in outputs] + declared = set(declared_names) + returned = set(returned_names) + missing = [name for name in declared_names if name not in returned] + undeclared = [name for name in returned_names if name not in declared] + + if missing: + raise ValueError(f"Missing declared metric outputs: {missing}") + if undeclared: + raise ValueError(f"Undeclared metric outputs: {undeclared}") + for output in result.outputs: + outputs_by_name[output.name].coerce_output(output) + return result diff --git a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/metrics/ragas/base.py b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/metrics/ragas/base.py index a4cbaf06d0..6345cb00f4 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/metrics/ragas/base.py +++ b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/metrics/ragas/base.py @@ -16,6 +16,7 @@ import nemo_platform.beta.evaluator.constants as constants from nemo_platform.beta.evaluator.enums import MetricType from nemo_platform.beta.evaluator.inference import get_logger, requests_log_var +from nemo_platform.beta.evaluator.metrics.protocol import MetricInput, MetricOutput, MetricOutputSpec, MetricResult # Lazy imports for RAGAS - these are getter functions that defer the expensive # RAGAS/langchain imports (~20-30s) until first use, improving startup time. @@ -29,8 +30,6 @@ from nemo_platform.beta.evaluator.templates import render_request from nemo_platform.beta.evaluator.values import ( MetricBase, - MetricResult, - MetricScore, Model, SecretRef, ) @@ -108,9 +107,10 @@ class BaseRAGASMetric(MetricBase): _secrets: dict[str, SecretRef] = PrivateAttr(default_factory=dict) _log: logging.Logger = logging.getLogger(__name__) - def score_names(self) -> list[str]: + def output_spec(self) -> list[MetricOutputSpec]: + """Return outputs emitted by this metric.""" if isinstance(self.type, MetricType): - return [self.type.value] + return [MetricOutputSpec.continuous_score(self.type.value)] return [] def __init__(self, logger: logging.Logger | None = None, **data): @@ -220,7 +220,7 @@ def _nan_scores_for_metrics(self, metrics: list) -> dict[str, float]: metric_names.append(metric_name) if not metric_names: - metric_names = self.score_names() + metric_names = [output.name for output in self.output_spec()] return {metric_name: float("nan") for metric_name in metric_names} @@ -363,11 +363,15 @@ def _create_evaluation_dataset(self, item: dict, sample: dict) -> EvaluationData if response: payload["response"] = response - return EvaluationDatasetCls.from_list([payload]) + return cast(Any, EvaluationDatasetCls).from_list([payload]) - async def compute_scores(self, item: dict, sample: dict) -> MetricResult: + async def compute_scores(self, input: MetricInput) -> MetricResult: """Compute the scores for the metric.""" - return await _run_function_in_plain_loop(self.compute_scores_async, item, sample) + return await _run_function_in_plain_loop( + self.compute_scores_async, + input.row.data, + input.candidate.as_sample(), + ) async def compute_scores_async(self, item: dict, sample: dict) -> MetricResult: """Compute the scores for the metric asynchronously.""" @@ -376,7 +380,9 @@ async def compute_scores_async(self, item: dict, sample: dict) -> MetricResult: llm_judge = self._get_llm_judge(client) scores = self._metric(data, llm_judge) return MetricResult( - scores=[MetricScore(name=metric_name, value=score_value) for metric_name, score_value in scores.items()] + outputs=[ + MetricOutput(name=metric_name, value=score_value) for metric_name, score_value in scores.items() + ] ) def _metric(self, data: EvaluationDataset, llm_judge: LangchainLLMWrapper | None) -> dict[str, float]: diff --git a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/metrics/remote.py b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/metrics/remote.py index 4a2e55aca1..4a056b310b 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/metrics/remote.py +++ b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/metrics/remote.py @@ -7,14 +7,16 @@ import os from abc import ABC, abstractmethod from collections.abc import Awaitable, Callable -from typing import Any +from typing import Any, cast import httpx from httpx import Timeout from jsonpath_ng import parse as jsonpath_parse from jsonpath_ng.exceptions import JsonPathParserError from nemo_platform.beta.evaluator.inference import requests_log_var +from nemo_platform.beta.evaluator.metrics.protocol import MetricInput, MetricOutput, MetricOutputSpec, MetricResult from nemo_platform.beta.evaluator.metrics.template_rendering import ( + TemplateSample, build_template_context, render_template_or_raise, template_metric_repr, @@ -23,7 +25,6 @@ from nemo_platform.beta.evaluator.resilience.classifier import endpoint_identity from nemo_platform.beta.evaluator.values.common import SecretRef from nemo_platform.beta.evaluator.values.metrics import NemoAgentToolkitRemote, Remote, _RemoteBase -from nemo_platform.beta.evaluator.values.results import MetricResult, MetricScore from nemo_platform.beta.evaluator.values.scores import RemoteScore from pydantic import Field, SecretStr, field_validator @@ -124,10 +125,10 @@ def secrets(self) -> dict[str, SecretRef]: def _select_metric_score(self, metric_result: MetricResult) -> float: """Select the default score value for one-row metric results.""" - return metric_result.scores[0].value + return float(metric_result.outputs[0].value) @abstractmethod - async def compute_scores(self, item: dict[str, Any], sample: dict[str, Any]) -> MetricResult: + async def compute_scores(self, input: MetricInput) -> MetricResult: """Compute structured score output for one item/sample pair.""" ... @@ -135,9 +136,9 @@ async def compute_scores(self, item: dict[str, Any], sample: dict[str, Any]) -> class RemoteMetric(Remote, _RemoteMetricBase): """A metric that computes scores via a remote endpoint.""" - def score_names(self) -> list[str]: - """Return score keys emitted by this metric.""" - return [score.name for score in self.scores] + def output_spec(self) -> list[MetricOutputSpec]: + """Return outputs emitted by this metric.""" + return [MetricOutputSpec.continuous_score(score.name) for score in self.scores] @field_validator("scores") @classmethod @@ -154,24 +155,28 @@ def _validate_scores(cls, scores: list[RemoteScore]) -> list[RemoteScore]: def _select_metric_score(self, metric_result: MetricResult) -> float: """Select the score value used for single-score consumers.""" if self.metric_threshold_score: - score_names = [score.name for score in metric_result.scores] - if self.metric_threshold_score not in score_names: + output_names = [output.name for output in metric_result.outputs] + if self.metric_threshold_score not in output_names: raise ValueError( f"Score name '{self.metric_threshold_score}' not found in remote metric response. " - f"Available scores: {score_names}" + f"Available scores: {output_names}" ) - return next(score for score in metric_result.scores if score.name == self.metric_threshold_score).value + return float( + next(output for output in metric_result.outputs if output.name == self.metric_threshold_score).value + ) - if len(metric_result.scores) == 1: - return metric_result.scores[0].value + if len(metric_result.outputs) == 1: + return float(metric_result.outputs[0].value) raise ValueError( - f"Remote metric returned multiple scores {[score.name for score in metric_result.scores]}. " + f"Remote metric returned multiple scores {[output.name for output in metric_result.outputs]}. " "Please set metric_threshold_score to specify which score to use." ) - async def compute_scores(self, item: dict[str, Any], sample: dict[str, Any]) -> MetricResult: + async def compute_scores(self, input: MetricInput) -> MetricResult: """Compute structured score output via the remote endpoint.""" + item = input.row.data + sample: TemplateSample = input.candidate context = build_template_context(item, sample) rendered_args = render_template_or_raise( template_name="body", @@ -181,12 +186,12 @@ async def compute_scores(self, item: dict[str, Any], sample: dict[str, Any]) -> sample=sample, metric_repr=template_metric_repr(self), ) - payload = rendered_args if isinstance(rendered_args, dict) else {"args": rendered_args} + payload = cast(dict[str, Any], rendered_args) if isinstance(rendered_args, dict) else {"args": rendered_args} result_data = await self._post_payload(payload) try: _logger.debug("Remote metric result received for payload: %r", payload) - scores: list[MetricScore] = [] + outputs: list[MetricOutput] = [] for score_config in self.scores: jsonpath_expr = jsonpath_parse(score_config.parser.json_path) matches = jsonpath_expr.find(result_data) @@ -196,12 +201,12 @@ async def compute_scores(self, item: dict[str, Any], sample: dict[str, Any]) -> score_config.name, score_config.parser.json_path, ) - scores.append(MetricScore(name=score_config.name, value=float("nan"))) + outputs.append(MetricOutput(name=score_config.name, value=float("nan"))) else: score_value = matches[0].value - scores.append(MetricScore(name=score_config.name, value=float(score_value))) + outputs.append(MetricOutput(name=score_config.name, value=float(score_value))) - return MetricResult(scores=scores) + return MetricResult(outputs=outputs) except Exception: _logger.exception("Error validating remote metric response") raise @@ -212,12 +217,14 @@ class NemoAgentToolkitRemoteMetric(NemoAgentToolkitRemote, _RemoteMetricBase): _RESULT_SCORE_JSONPATH = jsonpath_parse("$.result.score") - def score_names(self) -> list[str]: - """Return score keys emitted by this metric.""" - return [self.evaluator_name] + def output_spec(self) -> list[MetricOutputSpec]: + """Return outputs emitted by this metric.""" + return [MetricOutputSpec.continuous_score(self.evaluator_name)] - async def compute_scores(self, item: dict[str, Any], sample: dict[str, Any]) -> MetricResult: + async def compute_scores(self, input: MetricInput) -> MetricResult: """Compute structured score output via the NeMo Agent Toolkit evaluator endpoint.""" + item = input.row.data + sample: TemplateSample = input.candidate context = build_template_context(item, sample) rendered_item = render_template_or_raise( template_name="body.item", @@ -240,4 +247,4 @@ async def compute_scores(self, item: dict[str, Any], sample: dict[str, Any]) -> else: score = float(matches[0].value) - return MetricResult(scores=[MetricScore(name=self.evaluator_name, value=score)]) + return MetricResult(outputs=[MetricOutput(name=self.evaluator_name, value=score)]) diff --git a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/metrics/rouge.py b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/metrics/rouge.py index 7d3fbe667d..b6973d2274 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/metrics/rouge.py +++ b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/metrics/rouge.py @@ -6,9 +6,13 @@ from functools import cached_property from typing import ClassVar, Literal -from nemo_platform.beta.evaluator.metrics.template_rendering import render_reference_and_candidate, template_metric_repr +from nemo_platform.beta.evaluator.metrics.protocol import MetricInput, MetricOutput, MetricOutputSpec, MetricResult +from nemo_platform.beta.evaluator.metrics.template_rendering import ( + TemplateSample, + render_reference_and_candidate, + template_metric_repr, +) from nemo_platform.beta.evaluator.values.metrics import ROUGE -from nemo_platform.beta.evaluator.values.results import MetricResult, MetricScore __all__ = ["ROUGEMetric", "RougeScoreName"] @@ -40,11 +44,11 @@ def _scorer(self): return rouge_scorer.RougeScorer(["rouge1", "rouge2", "rouge3", "rougeL"], use_stemmer=True) - def score_names(self) -> list[str]: - """Return score keys emitted by this metric.""" - return list(self.scores_mapping.keys()) + def output_spec(self) -> list[MetricOutputSpec]: + """Return outputs emitted by this metric.""" + return [MetricOutputSpec.continuous_score(score_name) for score_name in self.scores_mapping] - def _metric(self, item: dict, sample: dict) -> dict: + def _metric(self, item: dict, sample: TemplateSample) -> dict: """Compute raw ROUGE scores for one item/sample pair.""" ground_truth, prediction = render_reference_and_candidate( metric_repr=template_metric_repr(self), @@ -56,12 +60,12 @@ def _metric(self, item: dict, sample: dict) -> dict: ) return self._scorer.score(ground_truth, prediction) - async def compute_scores(self, item: dict, sample: dict) -> MetricResult: + async def compute_scores(self, input: MetricInput) -> MetricResult: """Compute structured score output for one item/sample pair.""" - scores = self._metric(item, sample) + scores = self._metric(input.row.data, input.candidate) return MetricResult( - scores=[ - MetricScore(name=score_name, value=scores[score_key].fmeasure) + outputs=[ + MetricOutput(name=score_name, value=scores[score_key].fmeasure) for score_name, score_key in self.scores_mapping.items() ] ) diff --git a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/metrics/string_check.py b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/metrics/string_check.py index 8effdd0966..d7a9c916aa 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/metrics/string_check.py +++ b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/metrics/string_check.py @@ -3,13 +3,14 @@ """String-check metric runtime implementation.""" +from nemo_platform.beta.evaluator.metrics.protocol import MetricInput, MetricOutput, MetricOutputSpec, MetricResult from nemo_platform.beta.evaluator.metrics.template_rendering import ( + TemplateSample, build_template_context, render_template_or_raise, template_metric_repr, ) from nemo_platform.beta.evaluator.values.metrics import StringCheck, StringCheckOperation -from nemo_platform.beta.evaluator.values.results import MetricResult, MetricScore __all__ = ["StringCheckMetric", "StringCheckOperation"] @@ -17,12 +18,14 @@ class StringCheckMetric(StringCheck): """String-comparison metric with operator-based checks.""" - def score_names(self) -> list[str]: - """Return score keys emitted by this metric.""" - return [self.type.value] + def output_spec(self) -> list[MetricOutputSpec]: + """Return outputs emitted by this metric.""" + return [MetricOutputSpec.continuous_score(self.type.value)] - async def compute_scores(self, item: dict, sample: dict) -> MetricResult: + async def compute_scores(self, input: MetricInput) -> MetricResult: """Compute the scores for the metric.""" + item = input.row.data + sample: TemplateSample = input.candidate context = build_template_context(item, sample) metric_repr = template_metric_repr(self) left_value = render_template_or_raise( @@ -63,4 +66,4 @@ async def compute_scores(self, item: dict, sample: dict) -> MetricResult: else: raise ValueError(f"Unsupported operation: {self.operation}") - return MetricResult(scores=[MetricScore(name=self.type.value, value=1.0 if score else 0.0)]) + return MetricResult(outputs=[MetricOutput(name=self.type.value, value=1.0 if score else 0.0)]) diff --git a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/metrics/template_rendering.py b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/metrics/template_rendering.py index 539f6b4cf6..e822c54136 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/metrics/template_rendering.py +++ b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/metrics/template_rendering.py @@ -8,17 +8,27 @@ from typing import Any from jinja2 import UndefinedError +from nemo_platform.beta.evaluator.metrics.protocol import CandidateOutput from nemo_platform.beta.evaluator.templates import render_template from pydantic import BaseModel TemplateValue = str | dict[Any, Any] | list[Any] +TemplateSample = dict[str, Any] | CandidateOutput _DICT_ATTRIBUTE_ERROR_RE = re.compile(r"^'dict object' has no attribute '(?P[^']+)'$") _UNDEFINED_NAME_ERROR_RE = re.compile(r"^'(?P[^']+)' is undefined$") -def build_template_context(item: dict[str, Any], sample: dict[str, Any]) -> dict[str, Any]: +def sample_template_payload(sample: TemplateSample) -> dict[str, Any]: + """Return a sample-shaped dictionary for template rendering helpers.""" + if isinstance(sample, CandidateOutput): + return sample.as_sample() + return sample + + +def build_template_context(item: dict[str, Any], sample: TemplateSample) -> dict[str, Any]: """Build the template context shared by item and sample rendering.""" - return {**item, **sample, "item": item, "sample": sample} + sample_payload = sample_template_payload(sample) + return {**item, **sample_payload, "item": item, "sample": sample_payload} def template_metric_repr(metric: BaseModel | object) -> str: @@ -56,12 +66,13 @@ def render_template_or_raise( template: TemplateValue, context: dict[str, Any], item: dict[str, Any], - sample: dict[str, Any], + sample: TemplateSample, metric_repr: str, item_keys_label: str = "item", sample_keys_label: str = "sample", ) -> object: """Render one template and raise a specific validation error on missing keys.""" + sample_payload = sample_template_payload(sample) try: return render_template(template, context) except UndefinedError as exc: @@ -69,7 +80,7 @@ def render_template_or_raise( base_message = ( f"{metric_repr} could not render its '{template_name}' template for this row.\n" f"Available {item_keys_label} keys={sorted(item.keys())}. \n" - f"Available {sample_keys_label} keys={sorted(sample.keys())}.\n" + f"Available {sample_keys_label} keys={sorted(sample_payload.keys())}.\n" ) if missing_key is not None: detail = f"Dataset item has missing_key='{missing_key}' but the '{template_name}' template references it.\n" @@ -80,9 +91,9 @@ def render_template_or_raise( ) from exc -def render_default_output_text_candidate_or_raise(*, sample: dict[str, Any], metric_name: str) -> object: +def render_default_output_text_candidate_or_raise(*, sample: TemplateSample, metric_name: str) -> object: """Return the default output-text candidate or raise a clear guidance error.""" - prediction = sample.get("output_text") + prediction = sample_template_payload(sample).get("output_text") if prediction is None: raise ValueError( f"{metric_name} has missing `candidate` field.\n" @@ -99,7 +110,7 @@ def render_reference_and_candidate( reference_template: str, candidate_template: str | None, item: dict[str, Any], - sample: dict[str, Any], + sample: TemplateSample, ) -> tuple[str, str]: """Render reference and candidate templates, returning validated strings.""" context = build_template_context(item, sample) diff --git a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/metrics/tool_calling.py b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/metrics/tool_calling.py index 4736c6562c..c244fdf802 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/metrics/tool_calling.py +++ b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/metrics/tool_calling.py @@ -8,13 +8,15 @@ from collections.abc import Mapping from typing import ClassVar, cast +from nemo_platform.beta.evaluator.metrics.protocol import MetricInput, MetricOutput, MetricOutputSpec, MetricResult from nemo_platform.beta.evaluator.metrics.template_rendering import ( + TemplateSample, build_template_context, render_template_or_raise, + sample_template_payload, template_metric_repr, ) from nemo_platform.beta.evaluator.values.metrics import ToolCalling -from nemo_platform.beta.evaluator.values.results import MetricResult, MetricScore __all__ = ["ToolCallingMetric"] @@ -41,12 +43,13 @@ class ToolCallingMetric(ToolCalling): _score_names: ClassVar[list[str]] = ["function_name_accuracy", "function_name_and_args_accuracy"] - def score_names(self) -> list[str]: - """Return score keys emitted by this metric.""" - return list(self._score_names) + def output_spec(self) -> list[MetricOutputSpec]: + """Return outputs emitted by this metric.""" + return [MetricOutputSpec.continuous_score(score_name) for score_name in self._score_names] - def _metric(self, item: dict, sample: dict) -> dict[str, float]: + def _metric(self, item: dict, sample: TemplateSample) -> dict[str, float]: """Compute raw tool-calling scores for one item/sample pair.""" + sample_payload = sample_template_payload(sample) context = build_template_context(item, sample) ground_truth = render_template_or_raise( template_name="reference", @@ -90,7 +93,7 @@ def _metric(self, item: dict, sample: dict) -> dict[str, float]: ) from e # Parse tool calls: check sample (online) first, then item (offline). - response_data = sample.get("response") or item.get("response") + response_data = sample_payload.get("response") or item.get("response") if not response_data: raise ValueError("No response found in sample or item - tool-calling metric requires model response data") if not isinstance(response_data, dict): @@ -107,7 +110,7 @@ def _metric(self, item: dict, sample: dict) -> dict[str, float]: message = first_choice["message"] if not message.get("tool_calls"): - _logger.info("No tool calls found in %s", sample) + _logger.info("No tool calls found in %s", sample_payload) message["tool_calls"] = [] tool_calls = message["tool_calls"] @@ -160,7 +163,11 @@ def _metric(self, item: dict, sample: dict) -> dict[str, float]: "function_name_and_args_accuracy": fn_name_and_args_accuracy_score, } - async def compute_scores(self, item: dict, sample: dict) -> MetricResult: + async def compute_scores(self, input: MetricInput) -> MetricResult: """Compute the scores for the metric.""" + item = input.row.data + sample = input.candidate scores = self._metric(item, sample) - return MetricResult(scores=[MetricScore(name=score_name, value=score) for score_name, score in scores.items()]) + return MetricResult( + outputs=[MetricOutput(name=score_name, value=score) for score_name, score in scores.items()] + ) diff --git a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/metrics/utils.py b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/metrics/utils.py index c33a41b75e..79a7c4b672 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/metrics/utils.py +++ b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/metrics/utils.py @@ -1,10 +1,23 @@ # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -"""Shared helpers for working with runtime metric identifiers.""" +"""Shared helpers for runtime metrics.""" + +import re +import string from nemo_platform.beta.evaluator.enums import MetricType -from nemo_platform.beta.evaluator.metrics.base import Metric +from nemo_platform.beta.evaluator.metrics.protocol import Metric + + +def normalize_text(s: str) -> str: + """Normalize free-form text for token/equality-based metric comparisons.""" + if not s: + return "" + s = s.lower() + s = "".join(ch for ch in s if ch not in set(string.punctuation)) + s = re.sub(r"\b(a|an|the)\b", " ", s) + return " ".join(s.split()) def metric_type_name(metric: Metric) -> str: diff --git a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/values/__init__.py b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/values/__init__.py index 3bc497172e..67685fc413 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/values/__init__.py +++ b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/values/__init__.py @@ -3,6 +3,19 @@ """Public value types for evaluator SDK runtime.""" +from nemo_platform.beta.evaluator.metrics.protocol import ( + BooleanValue, + CandidateOutput, + ContinuousScore, + DatasetRow, + DiscreteScore, + Label, + MetricDescriptor, + MetricInput, + MetricOutput, + MetricOutputSpec, + MetricResult, +) from nemo_platform.beta.evaluator.values.agents import Agent from nemo_platform.beta.evaluator.values.common import SecretRef, SupportedJobTypes from nemo_platform.beta.evaluator.values.dataset_schemas import ( @@ -53,7 +66,6 @@ EvaluationResult, Histogram, HistogramBin, - MetricResult, MetricScore, Percentiles, RowScore, @@ -81,8 +93,13 @@ "AggregateRubricScore", "AggregateScore", "AggregateScoreBase", + "BooleanValue", + "CandidateOutput", + "ContinuousScore", + "DatasetRow", "DatasetRows", "DefaultAggregateFieldName", + "DiscreteScore", "RunConfig", "RunConfigOnline", "RunConfigOnlineModel", @@ -91,6 +108,11 @@ "HistogramBin", "InferenceParams", "JSONScoreParser", + "Label", + "MetricDescriptor", + "MetricInput", + "MetricOutput", + "MetricOutputSpec", "MetricResult", "MetricScore", "Model", diff --git a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/values/multi_metric_results.py b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/values/multi_metric_results.py index 182fced86b..e15eaa2c4b 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/values/multi_metric_results.py +++ b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/values/multi_metric_results.py @@ -10,11 +10,11 @@ from pydantic import BaseModel +from nemo_platform.beta.evaluator.metrics.protocol import MetricOutput from nemo_platform.beta.evaluator.values.results import ( AggregatedMetricResult, AggregateFieldName, EvaluationResult, - MetricScore, ResultView, RowScore, flatten_dict, @@ -49,8 +49,8 @@ def _filter_aggregate_fields( return AggregatedMetricResult(scores=filtered_scores) -def _extract_metric_scores(row_score: RowScore, expected_key: str) -> list[MetricScore]: - """Resolve the score list for one metric from a row result. +def _extract_metric_outputs(row_score: RowScore, expected_key: str) -> list[MetricOutput]: + """Resolve the output list for one metric from a row result. This exists because local and remote execution paths may return row scores either already keyed by the final metric key or as a single unnamed metric @@ -61,7 +61,7 @@ def _extract_metric_scores(row_score: RowScore, expected_key: str) -> list[Metri expected_key: Metric key the caller expects to find on the row. Returns: - The score list for the requested metric key, or an empty list when the + The output list for the requested metric key, or an empty list when the row has no metric output because evaluation failed. Raises: @@ -74,7 +74,7 @@ def _extract_metric_scores(row_score: RowScore, expected_key: str) -> list[Metri return [] if len(row_score.metrics) == 1: return next(iter(row_score.metrics.values())) - raise ValueError(f"Unable to resolve row metric scores for key {expected_key!r}") + raise ValueError(f"Unable to resolve row metric outputs for key {expected_key!r}") def _extract_metric_error(row_score: RowScore, expected_key: str) -> str | None: @@ -123,7 +123,7 @@ def namespace_result( row_index=row_score.row_index, item=row_score.item, sample=row_score.sample, - metrics={metric_key: _extract_metric_scores(row_score, metric_key)}, + metrics={metric_key: _extract_metric_outputs(row_score, metric_key)}, requests=row_score.requests, metric_errors={metric_key: error} if (error := _extract_metric_error(row_score, metric_key)) else None, ) @@ -176,12 +176,12 @@ def collapse_results( combined_rows: list[RowScore] = [] for index in range(row_count): first_row = results_by_key[ordered_keys[0]].row_scores[index] - metrics: dict[str, list[MetricScore]] = {} + metrics: dict[str, list[MetricOutput]] = {} requests: list[dict[str, Any]] = [] metric_errors: dict[str, str] = {} for metric_key in ordered_keys: row_score = results_by_key[metric_key].row_scores[index] - metrics[metric_key] = _extract_metric_scores(row_score, metric_key) + metrics[metric_key] = _extract_metric_outputs(row_score, metric_key) requests.extend(row_score.requests) if row_score.metric_errors: metric_errors.update(row_score.metric_errors) @@ -252,8 +252,8 @@ def to_records(self, view: ResultView = "rows") -> list[dict[str, Any]]: if error_text := row_error_text(row_score): record["error"] = error_text for metric_key, metric_scores in row_score.metrics.items(): - for score in metric_scores: - record[f"score.{metric_key}.{score.name}"] = serialize_value(score.value) + for output in metric_scores: + record[f"output.{metric_key}.{output.name}"] = serialize_value(output.value) records.append(record) return records diff --git a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/values/results.py b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/values/results.py index c46945a0e9..7aa4c33f47 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/values/results.py +++ b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/values/results.py @@ -15,6 +15,8 @@ import pyarrow as pa from pydantic import BaseModel, ConfigDict, Field, field_serializer, field_validator, model_serializer +from nemo_platform.beta.evaluator.metrics.protocol import MetricOutput, MetricResult + ResultView = Literal["rows", "aggregate"] AggregateFieldName = Literal[ # Base statistics @@ -240,12 +242,6 @@ def serialize_nan(self, v): return v -class MetricResult(BaseModel): - """Evaluation results for the metric""" - - scores: list[MetricScore] - - class Percentiles(BaseModel): """Percentile distribution of scores.""" @@ -364,7 +360,7 @@ class RowScore(BaseModel): row_index: int | None = Field(default=None, description="Stable row position used for result alignment.", ge=0) item: dict[str, Any] = Field(description="Input item metadata for the evaluated row.") sample: dict[str, Any] = Field(description="Sample output payload for the evaluated row.") - metrics: dict[str, list[MetricScore]] = Field(description="Metric-level row scores by metric key.") + metrics: dict[str, list[MetricOutput]] = Field(description="Metric-level row outputs by metric key.") requests: list[dict[str, Any]] = Field(description="Request details captured during evaluation.") metric_errors: dict[str, str] | None = Field( default=None, @@ -593,8 +589,8 @@ def to_records(self, view: ResultView = "rows") -> list[dict[str, Any]]: if error_text := row_error_text(row_score): record["error"] = error_text for metric_scores in row_score.metrics.values(): - for score in metric_scores: - record[f"score.{score.name}"] = serialize_value(score.value) + for output in metric_scores: + record[f"output.{output.name}"] = serialize_value(output.value) records.append(record) return records @@ -658,8 +654,8 @@ def format_summary(self, max_rows: int = 10, *, max_error_rows: int | None = Non for index, row_score in enumerate(self.row_scores[:max_rows]): record = summary_row_base_record(row_score, index) for metric_scores in row_score.metrics.values(): - for score in metric_scores: - record[f"score.{score.name}"] = serialize_value(score.value) + for output in metric_scores: + record[f"output.{output.name}"] = serialize_value(output.value) preview_records.append(record) parts = [ summary_header("EvaluationResult", self.row_scores, len(self.aggregate_scores.scores)), diff --git a/sdk/python/nemo-platform/src/nemo_platform/resources/evaluation/api.md b/sdk/python/nemo-platform/src/nemo_platform/resources/evaluation/api.md index 4c144c7a5f..66ae0d9b97 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/resources/evaluation/api.md +++ b/sdk/python/nemo-platform/src/nemo_platform/resources/evaluation/api.md @@ -33,8 +33,8 @@ from nemo_platform.types.evaluation import ( JsonScoreParser, LLMJudgeMetric, LLMJudgeMetricParam, + MetricOutput, MetricRef, - MetricScore, Model, NeMoAgentToolkitRemoteMetric, NeMoAgentToolkitRemoteMetricParam, @@ -61,7 +61,6 @@ from nemo_platform.types.evaluation import ( RunConfig, RunConfigOnline, RunConfigOnlineModel, - ScoreStats, StringCheckMetric, StringCheckMetricParam, ToolCallAccuracyMetric, diff --git a/sdk/python/nemo-platform/src/nemo_platform/types/evaluation/__init__.py b/sdk/python/nemo-platform/src/nemo_platform/types/evaluation/__init__.py index d96602c479..61ed7ef4ac 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/types/evaluation/__init__.py +++ b/sdk/python/nemo-platform/src/nemo_platform/types/evaluation/__init__.py @@ -36,9 +36,8 @@ from .model_param import ModelParam as ModelParam from .percentiles import Percentiles as Percentiles from .range_score import RangeScore as RangeScore -from .score_stats import ScoreStats as ScoreStats from .dataset_rows import DatasetRows as DatasetRows -from .metric_score import MetricScore as MetricScore +from .metric_output import MetricOutput as MetricOutput from .remote_score import RemoteScore as RemoteScore from .rouge_metric import RougeMetric as RougeMetric from .rubric_param import RubricParam as RubricParam diff --git a/sdk/python/nemo-platform/src/nemo_platform/types/evaluation/metric_score.py b/sdk/python/nemo-platform/src/nemo_platform/types/evaluation/metric_output.py similarity index 71% rename from sdk/python/nemo-platform/src/nemo_platform/types/evaluation/metric_score.py rename to sdk/python/nemo-platform/src/nemo_platform/types/evaluation/metric_output.py index 5a08cb563d..64aeb785e2 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/types/evaluation/metric_score.py +++ b/sdk/python/nemo-platform/src/nemo_platform/types/evaluation/metric_output.py @@ -15,23 +15,16 @@ # File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. -from typing import Optional +from typing import Any from ..._models import BaseModel -from .score_stats import ScoreStats -__all__ = ["MetricScore"] +__all__ = ["MetricOutput"] -class MetricScore(BaseModel): - """A computed score for the metric""" +class MetricOutput(BaseModel): + """One named value emitted by a metric.""" name: str - value: float - - stats: Optional[ScoreStats] = None - """Stats for a score. - - Fields that are NaN are serialized as the string "NaN" in the API response. - """ + value: Any diff --git a/sdk/python/nemo-platform/src/nemo_platform/types/evaluation/row_score.py b/sdk/python/nemo-platform/src/nemo_platform/types/evaluation/row_score.py index db0b716377..e94e9afb2e 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/types/evaluation/row_score.py +++ b/sdk/python/nemo-platform/src/nemo_platform/types/evaluation/row_score.py @@ -20,7 +20,7 @@ from pydantic import Field as FieldInfo from ..._models import BaseModel -from .metric_score import MetricScore +from .metric_output import MetricOutput __all__ = ["RowScore"] @@ -31,8 +31,8 @@ class RowScore(BaseModel): item: Dict[str, object] """Input item metadata for the evaluated row.""" - metrics: Dict[str, List[MetricScore]] - """Metric-level row scores by metric key.""" + metrics: Dict[str, List[MetricOutput]] + """Metric-level row outputs by metric key.""" requests: List[Dict[str, object]] """Request details captured during evaluation.""" diff --git a/sdk/python/nemo-platform/src/nemo_platform/types/evaluation/score_stats.py b/sdk/python/nemo-platform/src/nemo_platform/types/evaluation/score_stats.py deleted file mode 100644 index 570df2a2c2..0000000000 --- a/sdk/python/nemo-platform/src/nemo_platform/types/evaluation/score_stats.py +++ /dev/null @@ -1,66 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. - -from typing import List, Union, Optional - -from ..._models import BaseModel -from .rubric_score_stat import RubricScoreStat - -__all__ = ["ScoreStats"] - - -class ScoreStats(BaseModel): - """Stats for a score. - - Fields that are NaN are serialized as the string "NaN" in the API response. - """ - - count: Optional[int] = None - """The number of values used for computing the score.""" - - max: Union[float, str, None] = None - """The maximum of all values used for computing the score.""" - - mean: Union[float, str, None] = None - """The mean of all values used for computing the score.""" - - min: Union[float, str, None] = None - """The minimum of all values used for computing the score.""" - - nan_count: Optional[int] = None - """ - The number of values that are not a number (NaN) and are excluded from the score - stats calculations. - """ - - rubric_distribution: Optional[List[RubricScoreStat]] = None - """The distribution of the rubric grading criteria for the score.""" - - stddev: Union[float, str, None] = None - """The population standard deviation, (note: not the sample standard deviation).""" - - stderr: Union[float, str, None] = None - """The standard error.""" - - sum: Union[float, str, None] = None - """The sum of all values used for computing the score.""" - - sum_squared: Union[float, str, None] = None - """The sum of the square of all values used for computing the score.""" - - variance: Union[float, str, None] = None - """The population variance, (note: not the sample variance).""" diff --git a/services/evaluator/src/nmp/evaluator/api/v2/metrics/manager.py b/services/evaluator/src/nmp/evaluator/api/v2/metrics/manager.py index 1354b19b49..48784ef72e 100644 --- a/services/evaluator/src/nmp/evaluator/api/v2/metrics/manager.py +++ b/services/evaluator/src/nmp/evaluator/api/v2/metrics/manager.py @@ -14,7 +14,7 @@ ) from nemo_evaluator_sdk.execution.scoring import finalize_evaluation_result from nemo_evaluator_sdk.execution.values import EvaluationError -from nemo_evaluator_sdk.metrics.base import SecretResolver +from nemo_evaluator_sdk.metrics.protocol import SecretResolver from nemo_evaluator_sdk.metrics.utils import metric_type_name from nemo_evaluator_sdk.resilience.errors import get_evaluation_error from nemo_evaluator_sdk.values import ( diff --git a/services/evaluator/src/nmp/evaluator/api/v2/metrics/schemas/evaluation.py b/services/evaluator/src/nmp/evaluator/api/v2/metrics/schemas/evaluation.py index 9e451e0658..bebe31b363 100644 --- a/services/evaluator/src/nmp/evaluator/api/v2/metrics/schemas/evaluation.py +++ b/services/evaluator/src/nmp/evaluator/api/v2/metrics/schemas/evaluation.py @@ -89,11 +89,13 @@ def from_row_score(cls, row_score: RowScore, row: dict[str, Any], index: int) -> """ if row_score.error is not None: return cls(index=index, row=row, scores=None, error=row_score.error) - scores: dict[str, float | None] = { - ms.name: ms.value if math.isfinite(ms.value) else None - for metric_scores in row_score.metrics.values() - for ms in metric_scores - } + scores: dict[str, float | None] = {} + for metric_outputs in row_score.metrics.values(): + for output in metric_outputs: + if isinstance(output.value, bool): + scores[output.name] = 1.0 if output.value else 0.0 + elif isinstance(output.value, int | float): + scores[output.name] = float(output.value) if math.isfinite(output.value) else None return cls(index=index, row=row, scores=scores, error=None) diff --git a/services/evaluator/src/nmp/evaluator/app/metrics/aggregation.py b/services/evaluator/src/nmp/evaluator/app/metrics/aggregation.py deleted file mode 100644 index 3c69018053..0000000000 --- a/services/evaluator/src/nmp/evaluator/app/metrics/aggregation.py +++ /dev/null @@ -1,302 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""Aggregation data structures and computation for metric results.""" - -import math -from collections import OrderedDict, defaultdict - -from nemo_evaluator_sdk.values import ( - AggregatedMetricResult, - AggregateRangeScore, - AggregateRubricScore, - AggregateScore, - Histogram, - HistogramBin, - MetricResult, - MetricScore, - Percentiles, - RubricScoreStat, - ScoreStats, -) - - -def add_corpus_scores(aggregated_result: AggregatedMetricResult, corpus_result: MetricResult) -> None: - """Add corpus-level metric scores to an aggregated result. - - Corpus-level metrics produce single values (not per-row), so they are - converted to AggregateRangeScore with count=1. - - Args: - aggregated_result: The aggregated result to add scores to (mutated in place). - corpus_result: The corpus-level metric result containing scores to add. - """ - for score in corpus_result.scores: - value = score.value - # Create an AggregateRangeScore for the corpus-level metric - corpus_score = AggregateRangeScore( - name=score.name, - count=1, - nan_count=0, - sum=value, - mean=value, - min=value, - max=value, - std_dev=0.0, - variance=0.0, - percentiles=Percentiles( - p10=value, - p20=value, - p30=value, - p40=value, - p50=value, - p60=value, - p70=value, - p80=value, - p90=value, - p100=value, - ), - histogram=Histogram(bins=[HistogramBin(lower_bound=value, upper_bound=value, count=1)]), - ) - aggregated_result.scores.append(corpus_score) - - -def _compute_percentile(sorted_values: list[float], percentile: float) -> float: - """Compute a percentile from sorted values using linear interpolation.""" - if not sorted_values: - return 0.0 - n = len(sorted_values) - # Use the "exclusive" percentile method (like Excel's PERCENTILE.EXC) - # Position in the data for the given percentile - pos = (percentile / 100.0) * (n + 1) - 1 - if pos <= 0: - return sorted_values[0] - if pos >= n - 1: - return sorted_values[-1] - # Linear interpolation between adjacent values - lower_idx = int(pos) - frac = pos - lower_idx - return sorted_values[lower_idx] + frac * (sorted_values[lower_idx + 1] - sorted_values[lower_idx]) - - -def _compute_percentiles(sorted_values: list[float]) -> Percentiles: - """Compute standard percentiles from sorted values.""" - return Percentiles( - p10=_compute_percentile(sorted_values, 10), - p20=_compute_percentile(sorted_values, 20), - p30=_compute_percentile(sorted_values, 30), - p40=_compute_percentile(sorted_values, 40), - p50=_compute_percentile(sorted_values, 50), - p60=_compute_percentile(sorted_values, 60), - p70=_compute_percentile(sorted_values, 70), - p80=_compute_percentile(sorted_values, 80), - p90=_compute_percentile(sorted_values, 90), - p100=_compute_percentile(sorted_values, 100), - ) - - -def _compute_histogram(values: list[float], num_bins: int = 10) -> Histogram: - """Compute a histogram from values with the specified number of bins.""" - if not values: - return Histogram(bins=[]) - - min_val = min(values) - max_val = max(values) - - # Handle edge case where all values are the same - if min_val == max_val: - return Histogram(bins=[HistogramBin(lower_bound=min_val, upper_bound=max_val, count=len(values))]) - - bin_width = (max_val - min_val) / num_bins - bins: list[HistogramBin] = [] - - for i in range(num_bins): - lower = min_val + i * bin_width - upper = min_val + (i + 1) * bin_width - # Count values in this bin (last bin includes upper bound) - if i == num_bins - 1: - count = sum(1 for v in values if lower <= v <= upper) - else: - count = sum(1 for v in values if lower <= v < upper) - bins.append(HistogramBin(lower_bound=lower, upper_bound=upper, count=count)) - - return Histogram(bins=bins) - - -def aggregate_metrics(items: list[MetricResult]) -> AggregatedMetricResult: - """Aggregate metrics and compute full statistics. - - For range scores: computes percentiles and histogram. - For rubric scores: computes category distribution and mode. - """ - # Collect all values per score name for computing distribution stats - score_values: dict[str, list[float]] = defaultdict(list) - - aggregated_results: dict[str, MetricScore] = {} - rubric_distribution: dict[str, dict[str, dict]] = defaultdict(lambda: defaultdict(OrderedDict)) - has_rubric: dict[str, bool] = {} - - # First pass: collect values and compute running stats - for item in items: - for score in item.scores: - if score.name not in aggregated_results: - # Initialize with all stats fields - aggregated_results[score.name] = MetricScore( - name=score.name, - value=0.0, - stats=ScoreStats( - count=0, - sum=0, - sum_squared=0, - min=None, - max=None, - mean=0, - variance=None, - stddev=None, - nan_count=0, - ), - ) - # Track if this score has rubric distribution - has_rubric[score.name] = bool(score.stats and score.stats.rubric_distribution) - if score.stats and score.stats.rubric_distribution: - for rubric_stat in score.stats.rubric_distribution: - rubric_distribution[score.name][rubric_stat.label] = { - "label": rubric_stat.label, - "value": rubric_stat.value, - "count": 0, - } - - # Increment rubric label per metric score - if score.stats and score.stats.rubric_distribution: - for rubric_stat in score.stats.rubric_distribution: - if rubric_stat.count: - rubric_distribution[score.name][rubric_stat.label]["count"] += 1 - - results = aggregated_results[score.name] - assert results is not None - assert results.stats is not None - - # Skip NaN values from stats, but track them - if math.isnan(score.value): - results.stats.nan_count = (results.stats.nan_count or 0) + 1 - continue - - # Collect value for distribution stats - score_values[score.name].append(score.value) - - # Update running statistics - results.stats.count = (results.stats.count or 0) + 1 - results.stats.sum = (results.stats.sum or 0) + score.value - results.stats.sum_squared = (results.stats.sum_squared or 0) + score.value**2 - results.stats.mean = results.stats.sum / results.stats.count - - # Update min/max - if results.stats.min is None or score.value < results.stats.min: - results.stats.min = score.value - if results.stats.max is None or score.value > results.stats.max: - results.stats.max = score.value - - # Record mean as the value for backward compatibility - results.value = results.stats.mean - - # Second pass: compute variance and stddev from collected values - for score_name, values in score_values.items(): - if not values: - continue - - results = aggregated_results[score_name] - assert results.stats is not None - - n = len(values) - mean = results.stats.mean or 0 - - # Population variance: sum((x - mean)^2) / n - variance = sum((v - mean) ** 2 for v in values) / n if n > 0 else 0 - results.stats.variance = variance - results.stats.stddev = math.sqrt(variance) - - # Build the full aggregated result with appropriate type - aggregated_scores: list[AggregateScore] = [] - for score_name, metric_score in aggregated_results.items(): - stats = metric_score.stats - assert stats is not None - - values = score_values[score_name] - - # Common fields - base_name = metric_score.name - base_count = stats.count or 0 - base_nan_count = stats.nan_count or 0 - base_mean = stats.mean - base_sum = stats.sum if stats.sum is not None else (None if base_mean is None else (base_mean * base_count)) - base_min = stats.min if stats.min is not None else base_mean - base_max = stats.max if stats.max is not None else base_mean - base_variance = stats.variance if stats.variance is not None else (None if base_mean is None else 0.0) - base_std_dev = stats.stddev if stats.stddev is not None else (None if base_mean is None else 0.0) - - if base_count == 0: - base_sum = None - base_mean = None - base_min = None - base_max = None - base_variance = None - base_std_dev = None - - if has_rubric.get(score_name): - # Rubric score - compute distribution and mode - rubric_dist = [ - RubricScoreStat(label=r["label"], value=r["value"], count=r["count"]) - for r in rubric_distribution[score_name].values() - ] - # Find mode (most frequent category) - mode_category = None - if rubric_dist: - max_count = 0 - for r in rubric_dist: - if r.count > max_count: - max_count = r.count - mode_category = r.label - - aggregated_scores.append( - AggregateRubricScore( - name=base_name, - count=base_count, - nan_count=base_nan_count, - sum=base_sum, - mean=base_mean, - min=base_min, - max=base_max, - variance=base_variance, - std_dev=base_std_dev, - rubric_distribution=rubric_dist, - mode_category=mode_category, - ) - ) - else: - # Range score - compute percentiles and histogram - if values: - sorted_values = sorted(values) - percentiles = _compute_percentiles(sorted_values) - histogram = _compute_histogram(values) - else: - # Empty results - percentile distribution is undefined when no valid samples were scored. - percentiles = None - histogram = Histogram(bins=[]) - - aggregated_scores.append( - AggregateRangeScore( - name=base_name, - count=base_count, - nan_count=base_nan_count, - sum=base_sum, - mean=base_mean, - min=base_min, - max=base_max, - variance=base_variance, - std_dev=base_std_dev, - percentiles=percentiles, - histogram=histogram, - ) - ) - - return AggregatedMetricResult(scores=aggregated_scores) diff --git a/services/evaluator/src/nmp/evaluator/app/metrics/metric.py b/services/evaluator/src/nmp/evaluator/app/metrics/metric.py index d03a1fce22..808fdb8102 100644 --- a/services/evaluator/src/nmp/evaluator/app/metrics/metric.py +++ b/services/evaluator/src/nmp/evaluator/app/metrics/metric.py @@ -15,7 +15,7 @@ ) from nemo_evaluator_sdk.enums import MetricType from nemo_evaluator_sdk.inference import InferenceFn -from nemo_evaluator_sdk.metrics.base import Metric, MetricWithPreflight, MetricWithSecrets, SecretResolver +from nemo_evaluator_sdk.metrics.protocol import Metric, MetricWithPreflight, MetricWithSecrets, SecretResolver from nemo_evaluator_sdk.metrics.ragas.metrics import RAGAS_METRIC_CLASSES from nemo_evaluator_sdk.values import MetricBase, SupportedJobTypes from nmp.evaluator.app import inference as app_inference diff --git a/services/evaluator/src/nmp/evaluator/app/metrics/remote.py b/services/evaluator/src/nmp/evaluator/app/metrics/remote.py index 49789aad42..196dc98b7f 100644 --- a/services/evaluator/src/nmp/evaluator/app/metrics/remote.py +++ b/services/evaluator/src/nmp/evaluator/app/metrics/remote.py @@ -11,12 +11,13 @@ import nemo_evaluator_sdk.metrics.remote as _sdk_remote from nemo_evaluator_sdk import inference from nemo_evaluator_sdk.resilience.api import run_with_resilience +from nemo_evaluator_sdk.values import MetricInput, MetricResult def _sync_sdk_remote_bindings() -> None: """Mirror patchable service symbols into SDK module globals.""" _sdk_remote.requests_log_var = inference.requests_log_var - _sdk_remote.httpx = httpx + setattr(_sdk_remote, "httpx", httpx) _sdk_remote.run_with_resilience = run_with_resilience @@ -40,15 +41,15 @@ async def _post_to_remote_endpoint( class RemoteMetric(_sdk_remote.RemoteMetric): - async def compute_scores(self, item: dict[str, Any], sample: dict[str, Any]): + async def compute_scores(self, input: MetricInput) -> MetricResult: _sync_sdk_remote_bindings() - return await super().compute_scores(item, sample) + return await super().compute_scores(input) class NemoAgentToolkitRemoteMetric(_sdk_remote.NemoAgentToolkitRemoteMetric): - async def compute_scores(self, item: dict[str, Any], sample: dict[str, Any]): + async def compute_scores(self, input: MetricInput) -> MetricResult: _sync_sdk_remote_bindings() - return await super().compute_scores(item, sample) + return await super().compute_scores(input) __all__ = ["NemoAgentToolkitRemoteMetric", "RemoteMetric", "_post_to_remote_endpoint", "httpx", "run_with_resilience"] diff --git a/services/evaluator/tests/app/metrics/test_metric.py b/services/evaluator/tests/app/metrics/test_metric.py index 4e2ff35b6a..26f4391fcb 100644 --- a/services/evaluator/tests/app/metrics/test_metric.py +++ b/services/evaluator/tests/app/metrics/test_metric.py @@ -2,9 +2,11 @@ # SPDX-License-Identifier: Apache-2.0 import math +from typing import cast import nmp.evaluator.entities as entities import pytest +from nemo_evaluator_sdk.metrics.aggregation import aggregate_metrics from nemo_evaluator_sdk.metrics.bleu import BLEUMetric from nemo_evaluator_sdk.metrics.exact_match import ExactMatchMetric from nemo_evaluator_sdk.metrics.f1 import F1Metric @@ -13,26 +15,36 @@ from nemo_evaluator_sdk.metrics.tool_calling import ToolCallingMetric from nemo_evaluator_sdk.values import ( AggregateRangeScore, - AggregateRubricScore, + MetricOutput, + MetricOutputSpec, MetricResult, - MetricScore, - RubricScoreStat, - ScoreStats, ) -from nmp.evaluator.app.metrics.aggregation import aggregate_metrics from nmp.evaluator.app.metrics.metric import new_metric +from nmp.evaluator.app.values.metrics import Metric as MetricParams + + +def _metric_result(name: str, value: float | int | bool) -> MetricResult: + return MetricResult(outputs=[MetricOutput(name=name, value=value)]) + + +def _output_specs(*names: str) -> list[MetricOutputSpec]: + return [MetricOutputSpec.continuous_score(name) for name in names] + + +async def _new_metric(params: object): + return await new_metric(cast(MetricParams, params)) def test_aggregate_metrics(): """Test basic aggregation computes all statistics correctly.""" metric_results = [ - MetricResult(scores=[MetricScore(name="my-score", value=0)]), - MetricResult(scores=[MetricScore(name="my-score", value=2)]), - MetricResult(scores=[MetricScore(name="my-score", value=5)]), - MetricResult(scores=[MetricScore(name="my-score", value=15)]), + _metric_result("my-score", 0), + _metric_result("my-score", 2), + _metric_result("my-score", 5), + _metric_result("my-score", 15), ] # Expected: sum=22, mean=5.5, min=0, max=15, variance=33.25, stddev=5.766... - results = aggregate_metrics(metric_results) + results = aggregate_metrics(metric_results, _output_specs("my-score")) assert len(results.scores) == 1 score = results.scores[0] @@ -43,6 +55,8 @@ def test_aggregate_metrics(): assert score.sum == 22.0 assert score.min == 0.0 assert score.max == 15.0 + assert score.variance is not None + assert score.std_dev is not None assert math.isclose(score.variance, 33.25) # population variance assert math.isclose(score.std_dev, 5.766281297335398) assert score.nan_count == 0 @@ -51,14 +65,14 @@ def test_aggregate_metrics(): def test_aggregate_metrics_nan(): """Test that NaN values are excluded from statistics but counted.""" metric_results = [ - MetricResult(scores=[MetricScore(name="my-score", value=0)]), - MetricResult(scores=[MetricScore(name="my-score", value=float("nan"))]), - MetricResult(scores=[MetricScore(name="my-score", value=2)]), - MetricResult(scores=[MetricScore(name="my-score", value=5)]), - MetricResult(scores=[MetricScore(name="my-score", value=float("nan"))]), - MetricResult(scores=[MetricScore(name="my-score", value=15)]), + _metric_result("my-score", 0), + _metric_result("my-score", float("nan")), + _metric_result("my-score", 2), + _metric_result("my-score", 5), + _metric_result("my-score", float("nan")), + _metric_result("my-score", 15), ] - results = aggregate_metrics(metric_results) + results = aggregate_metrics(metric_results, _output_specs("my-score")) assert len(results.scores) == 1 score = results.scores[0] @@ -74,11 +88,11 @@ def test_aggregate_metrics_nan(): def test_aggregate_metrics_all_nan_returns_null_aggregates(): metric_results = [ - MetricResult(scores=[MetricScore(name="my-score", value=float("nan"))]), - MetricResult(scores=[MetricScore(name="my-score", value=float("nan"))]), + _metric_result("my-score", float("nan")), + _metric_result("my-score", float("nan")), ] - results = aggregate_metrics(metric_results) + results = aggregate_metrics(metric_results, _output_specs("my-score")) assert len(results.scores) == 1 score = results.scores[0] @@ -95,117 +109,11 @@ def test_aggregate_metrics_all_nan_returns_null_aggregates(): assert score.percentiles is None -def test_aggregate_metrics_rubrics(): - """Test rubric score aggregation with distribution tracking.""" - metric_results = [ - MetricResult( - scores=[ - MetricScore( - name="length", - value=2, - stats=ScoreStats( - rubric_distribution=[ - RubricScoreStat(label="short", value=0, count=0), - RubricScoreStat(label="medium", value=1, count=0), - RubricScoreStat(label="long", value=2, count=1), - ] - ), - ) - ] - ), - MetricResult( - scores=[ - MetricScore( - name="length", - value=1, - stats=ScoreStats( - rubric_distribution=[ - RubricScoreStat(label="short", value=0, count=0), - RubricScoreStat(label="medium", value=1, count=1), - RubricScoreStat(label="long", value=2, count=0), - ] - ), - ) - ] - ), - MetricResult( - scores=[ - MetricScore( - name="length", - value=2, - stats=ScoreStats( - rubric_distribution=[ - RubricScoreStat(label="short", value=0, count=0), - RubricScoreStat(label="medium", value=1, count=0), - RubricScoreStat(label="long", value=2, count=1), - ] - ), - ) - ] - ), - MetricResult( - scores=[ - MetricScore( - name="length", - value=0, - stats=ScoreStats( - rubric_distribution=[ - RubricScoreStat(label="short", value=0, count=1), - RubricScoreStat(label="medium", value=1, count=0), - RubricScoreStat(label="long", value=2, count=0), - ] - ), - ) - ] - ), - MetricResult( - scores=[ - MetricScore( - name="quality", - value=10, - stats=ScoreStats( - rubric_distribution=[ - RubricScoreStat(label="high", value=10, count=1), - RubricScoreStat(label="low", value=0, count=0), - ] - ), - ) - ] - ), - ] - - results = aggregate_metrics(metric_results) - - # Check "length" score - values [2, 1, 2, 0] - length_score = next(s for s in results.scores if s.name == "length") - assert isinstance(length_score, AggregateRubricScore) - assert length_score.mean == 1.25 - assert length_score.count == 4 - assert length_score.sum == 5.0 - assert length_score.min == 0.0 - assert length_score.max == 2.0 - assert length_score.nan_count == 0 - # Rubric distribution should be aggregated - rubric_dict = {r.label: r.count for r in length_score.rubric_distribution} - assert rubric_dict == {"short": 1, "medium": 1, "long": 2} - # Mode category should be "long" (count=2) - assert length_score.mode_category == "long" - - # Check "quality" score - single value [10] - quality_score = next(s for s in results.scores if s.name == "quality") - assert isinstance(quality_score, AggregateRubricScore) - assert quality_score.mean == 10.0 - assert quality_score.count == 1 - assert quality_score.sum == 10.0 - assert quality_score.variance == 0.0 # Single value has 0 variance - assert len(quality_score.rubric_distribution) > 0 - - def test_aggregate_metrics_returns_distribution(): """Test that aggregate_metrics returns percentiles and histogram for range scores.""" - metric_results = [MetricResult(scores=[MetricScore(name="my-score", value=i)]) for i in range(10)] # values 0-9 + metric_results = [_metric_result("my-score", i) for i in range(10)] # values 0-9 - result = aggregate_metrics(metric_results) + result = aggregate_metrics(metric_results, _output_specs("my-score")) assert len(result.scores) == 1 agg_score = result.scores[0] @@ -217,6 +125,7 @@ def test_aggregate_metrics_returns_distribution(): assert agg_score.max == 9.0 # Check percentiles (approximate for 10 values) + assert agg_score.percentiles is not None assert agg_score.percentiles.p50 == 4.5 # median assert agg_score.percentiles.p100 == 9.0 # max @@ -237,7 +146,7 @@ async def test_new_metric_creates_bleu_metric(self): references=["{{item.reference}}"], ) - metric = await new_metric(params) + metric = await _new_metric(params) assert isinstance(metric, BLEUMetric) assert metric.references == ["{{item.reference}}"] @@ -256,7 +165,7 @@ async def test_new_metric_creates_exact_match_metric(self): reference="{{item.reference}}", ) - metric = await new_metric(params) + metric = await _new_metric(params) assert isinstance(metric, ExactMatchMetric) assert metric.reference == "{{item.reference}}" @@ -275,7 +184,7 @@ async def test_new_metric_creates_f1_metric(self): reference="{{item.reference}}", ) - metric = await new_metric(params) + metric = await _new_metric(params) assert isinstance(metric, F1Metric) assert metric.reference == "{{item.reference}}" @@ -294,7 +203,7 @@ async def test_new_metric_creates_rouge_metric(self): reference="{{item.reference}}", ) - metric = await new_metric(params) + metric = await _new_metric(params) assert isinstance(metric, ROUGEMetric) assert metric.reference == "{{item.reference}}" @@ -315,7 +224,7 @@ async def test_new_metric_creates_string_check_metric(self): right_template="{{sample.output_text}}", ) - metric = await new_metric(params) + metric = await _new_metric(params) assert isinstance(metric, StringCheckMetric) assert metric.operation == "equals" @@ -334,7 +243,7 @@ async def test_new_metric_creates_tool_calling_metric(self): reference="{{item.expected_tool_calls}}", ) - metric = await new_metric(params) + metric = await _new_metric(params) assert isinstance(metric, ToolCallingMetric) assert metric.reference == "{{item.expected_tool_calls}}" diff --git a/services/evaluator/tests/data/metric-jobs/llm-judge-offline-results.json b/services/evaluator/tests/data/metric-jobs/llm-judge-offline-results.json index 537a86ebe9..29b0dc89f5 100644 --- a/services/evaluator/tests/data/metric-jobs/llm-judge-offline-results.json +++ b/services/evaluator/tests/data/metric-jobs/llm-judge-offline-results.json @@ -31,4 +31,4 @@ "mode_category": "short" } ] -} \ No newline at end of file +} diff --git a/services/evaluator/tests/integration/tasks/test_agent_inference.py b/services/evaluator/tests/integration/tasks/test_agent_inference.py index ca7fdf4c0e..e92fb3f93f 100644 --- a/services/evaluator/tests/integration/tasks/test_agent_inference.py +++ b/services/evaluator/tests/integration/tasks/test_agent_inference.py @@ -15,9 +15,8 @@ from nemo_evaluator_sdk.agent_inference import make_agent_inference_request from nemo_evaluator_sdk.enums import AgentFormat from nemo_evaluator_sdk.execution.metric_execution import ComputeMetricPipeline, generate_online_sample_agent -from nemo_evaluator_sdk.metrics.base import Metric +from nemo_evaluator_sdk.metrics.protocol import Metric, MetricInput, MetricOutputSpec, MetricResult from nemo_evaluator_sdk.values.agents import Agent -from nemo_evaluator_sdk.values.results import MetricResult from nmp.evaluator.app.values import BenchmarkOnlineAgentJob from nmp.evaluator.app.values.metrics_job import MetricOnlineAgentJob @@ -48,12 +47,12 @@ def metric(self, item: dict, sample: dict, trace=None) -> float: del item, sample, trace return 1.0 - async def compute_scores(self, item: dict, sample: dict) -> MetricResult: - del item, sample + async def compute_scores(self, input: MetricInput) -> MetricResult: + del input raise AssertionError("compute_scores is not used in these generation-only tests") - def score_names(self) -> list[str]: - return ["exact-match"] + def output_spec(self) -> list[MetricOutputSpec]: + return [MetricOutputSpec.continuous_score("exact-match")] def _test_metric() -> Metric: diff --git a/services/evaluator/tests/integration/tasks/test_evaluate_benchmark.py b/services/evaluator/tests/integration/tasks/test_evaluate_benchmark.py index 3cc484e788..6a9f973266 100644 --- a/services/evaluator/tests/integration/tasks/test_evaluate_benchmark.py +++ b/services/evaluator/tests/integration/tasks/test_evaluate_benchmark.py @@ -9,12 +9,20 @@ from jinja2.exceptions import UndefinedError from nemo_evaluator_sdk import inference from nemo_evaluator_sdk.execution.values import EvaluationError, EvaluationPhase -from nemo_evaluator_sdk.values import MetricResult, MetricScore, Model +from nemo_evaluator_sdk.values import MetricInput, MetricOutput, MetricOutputSpec, MetricResult, Model from nmp.evaluator.app.values import BenchmarkOfflineJob, BenchmarkOnlineJob from nmp.evaluator.tasks.evaluate_benchmark import __main__ as benchmark_task from pytest_mock import MockerFixture +def _metric_result(name: str, value: float) -> MetricResult: + return MetricResult(outputs=[MetricOutput(name=name, value=value)]) + + +def _output_spec(name: str) -> list[MetricOutputSpec]: + return [MetricOutputSpec.continuous_score(name)] + + @pytest.fixture def test_offline_job() -> BenchmarkOfflineJob: return BenchmarkOfflineJob.model_validate( @@ -113,13 +121,13 @@ class _FakeMetric: def __init__(self, metric_name: str): self._metric_name = metric_name - def score_names(self) -> list[str]: - return [self._metric_name] + def output_spec(self) -> list[MetricOutputSpec]: + return _output_spec(self._metric_name) - async def compute_scores(self, item: dict, sample: dict) -> MetricResult: + async def compute_scores(self, input: MetricInput) -> MetricResult: requests_log = inference.requests_log_var.get() - requests_log.append({"metric": self._metric_name, "item_id": item["id"]}) - return MetricResult(scores=[MetricScore(name=self._metric_name, value=1.0)]) + requests_log.append({"metric": self._metric_name, "item_id": input.row.data["id"]}) + return _metric_result(self._metric_name, 1.0) async def _fake_new_metric(metric_config, *args, **kwargs): return _FakeMetric(str(metric_config.type.value)) @@ -148,15 +156,15 @@ class _FakeMetric: def __init__(self, metric_name: str): self._metric_name = metric_name - def score_names(self) -> list[str]: - return [self._metric_name] + def output_spec(self) -> list[MetricOutputSpec]: + return _output_spec(self._metric_name) - async def compute_scores(self, item: dict, sample: dict) -> MetricResult: + async def compute_scores(self, input: MetricInput) -> MetricResult: requests_log = inference.requests_log_var.get() - requests_log.append({"metric": self._metric_name, "item_id": item["id"]}) + requests_log.append({"metric": self._metric_name, "item_id": input.row.data["id"]}) if self._metric_name == "string-check": raise RuntimeError("boom") - return MetricResult(scores=[MetricScore(name=self._metric_name, value=1.0)]) + return _metric_result(self._metric_name, 1.0) async def _fake_new_metric(metric_config, *args, **kwargs): return _FakeMetric(str(metric_config.type.value)) @@ -180,15 +188,15 @@ class _FakeMetric: def __init__(self, metric_name: str): self._metric_name = metric_name - def score_names(self) -> list[str]: - return [self._metric_name] + def output_spec(self) -> list[MetricOutputSpec]: + return _output_spec(self._metric_name) - async def compute_scores(self, item: dict, sample: dict) -> MetricResult: + async def compute_scores(self, input: MetricInput) -> MetricResult: requests_log = inference.requests_log_var.get() - requests_log.append({"metric": self._metric_name, "item_id": item["id"]}) + requests_log.append({"metric": self._metric_name, "item_id": input.row.data["id"]}) if self._metric_name == "string-check": raise RuntimeError("boom") - return MetricResult(scores=[MetricScore(name=self._metric_name, value=1.0)]) + return _metric_result(self._metric_name, 1.0) async def _fake_new_metric(metric_config, *args, **kwargs): return _FakeMetric(str(metric_config.type.value)) @@ -222,11 +230,12 @@ async def test_evaluate_benchmark_offline_progress_tracking_completes( ) class _FakeMetric: - def score_names(self) -> list[str]: - return ["exact-match"] + def output_spec(self) -> list[MetricOutputSpec]: + return _output_spec("exact-match") - async def compute_scores(self, _item: dict, _sample: dict) -> MetricResult: - return MetricResult(scores=[MetricScore(name="exact-match", value=1.0)]) + async def compute_scores(self, input: MetricInput) -> MetricResult: + del input + return _metric_result("exact-match", 1.0) async def _fake_new_metric(*_args, **_kwargs): return _FakeMetric() @@ -276,11 +285,12 @@ async def test_evaluate_benchmark_duplicate_metric_types_use_unique_metric_refs( ) class _FakeMetric: - def score_names(self) -> list[str]: - return ["score"] + def output_spec(self) -> list[MetricOutputSpec]: + return _output_spec("score") - async def compute_scores(self, _item: dict, _sample: dict) -> MetricResult: - return MetricResult(scores=[MetricScore(name="score", value=1.0)]) + async def compute_scores(self, input: MetricInput) -> MetricResult: + del input + return _metric_result("score", 1.0) async def _fake_new_metric(*_args, **_kwargs): return _FakeMetric() @@ -314,13 +324,13 @@ async def test_evaluate_benchmark_partial_failures_keep_nan_under_metric_score_n ) class _FlakyMetric: - def score_names(self) -> list[str]: - return ["f1_score"] + def output_spec(self) -> list[MetricOutputSpec]: + return _output_spec("f1_score") - async def compute_scores(self, item: dict, _sample: dict) -> MetricResult: - if item["id"] == 2: + async def compute_scores(self, input: MetricInput) -> MetricResult: + if input.row.data["id"] == 2: raise RuntimeError("transient") - return MetricResult(scores=[MetricScore(name="f1_score", value=1.0)]) + return _metric_result("f1_score", 1.0) async def _fake_new_metric(*_args, **_kwargs): return _FlakyMetric() @@ -349,10 +359,11 @@ async def test_evaluate_benchmark_all_failures_use_declared_score_names(tmp_path ) class _AlwaysFailMetric: - def score_names(self) -> list[str]: - return ["f1_score"] + def output_spec(self) -> list[MetricOutputSpec]: + return _output_spec("f1_score") - async def compute_scores(self, _item: dict, _sample: dict) -> MetricResult: + async def compute_scores(self, input: MetricInput) -> MetricResult: + del input raise RuntimeError("transient") async def _fake_new_metric(*_args, **_kwargs): @@ -387,11 +398,12 @@ class _FakeMetric: def __init__(self, metric_name: str): self._metric_name = metric_name - def score_names(self) -> list[str]: - return [self._metric_name] + def output_spec(self) -> list[MetricOutputSpec]: + return _output_spec(self._metric_name) - async def compute_scores(self, _item: dict, _sample: dict) -> MetricResult: - return MetricResult(scores=[MetricScore(name=self._metric_name, value=1.0)]) + async def compute_scores(self, input: MetricInput) -> MetricResult: + del input + return _metric_result(self._metric_name, 1.0) async def _fake_new_metric(metric_config, *_args, **_kwargs): return _FakeMetric(str(metric_config.type.value)) @@ -431,12 +443,13 @@ async def _fake_inference_fn(*_args: Any, **_kwargs: Any) -> dict[str, Any]: return {"choices": [{"message": {"role": "assistant", "content": "answer"}}]} class _FakeMetric: - def score_names(self) -> list[str]: - return ["exact-match"] + def output_spec(self) -> list[MetricOutputSpec]: + return _output_spec("exact-match") - async def compute_scores(self, _item: dict, _sample: dict) -> MetricResult: + async def compute_scores(self, input: MetricInput) -> MetricResult: + del input metric_started.set() - return MetricResult(scores=[MetricScore(name="exact-match", value=1.0)]) + return _metric_result("exact-match", 1.0) async def _fake_new_metric(*_args, **_kwargs): return _FakeMetric() @@ -457,10 +470,11 @@ async def test_evaluate_offline_benchmark_surfaces_strict_metric_error_context(t monkeypatch.setattr(benchmark_task, "_load_dataset_items", lambda *args, **kwargs: [{"id": 1}]) class _FailingMetric: - def score_names(self) -> list[str]: - return ["exact-match"] + def output_spec(self) -> list[MetricOutputSpec]: + return _output_spec("exact-match") - async def compute_scores(self, _item: dict, _sample: dict) -> MetricResult: + async def compute_scores(self, input: MetricInput) -> MetricResult: + del input raise ValueError("metric exploded") async def _fake_new_metric(*_args, **_kwargs): diff --git a/services/evaluator/tests/nmp/evaluator/api/v2/metrics/schemas/test_evaluation.py b/services/evaluator/tests/nmp/evaluator/api/v2/metrics/schemas/test_evaluation.py index 7931074028..48570141d1 100644 --- a/services/evaluator/tests/nmp/evaluator/api/v2/metrics/schemas/test_evaluation.py +++ b/services/evaluator/tests/nmp/evaluator/api/v2/metrics/schemas/test_evaluation.py @@ -2,7 +2,7 @@ # SPDX-License-Identifier: Apache-2.0 import pytest -from nemo_evaluator_sdk.values import DatasetRows, MetricScore, Model, RowScore +from nemo_evaluator_sdk.values import DatasetRows, MetricOutput, Model, RowScore from nmp.evaluator.api.v2.metrics.schemas.evaluation import ( EvaluateDatasetRows, MetricEvaluationRequest, @@ -348,7 +348,7 @@ def test_serialization_round_trip_inline_metric(self): def _row_score( *, index: int = 0, - metrics: dict[str, list[MetricScore]] | None = None, + metrics: dict[str, list[MetricOutput]] | None = None, metric_errors: dict[str, str] | None = None, ) -> RowScore: """Build a minimal RowScore with only the fields `from_row_score` reads.""" @@ -374,7 +374,7 @@ def test_success_preserves_finite_scores(self) -> None: """Finite metric values pass through unchanged into the scores dict.""" row_score = _row_score( index=3, - metrics={"exact_match": [MetricScore(name="score", value=1.0)]}, + metrics={"exact_match": [MetricOutput(name="score", value=1.0)]}, ) result = MetricEvaluationRowScore.from_row_score(row_score, row=ROW, index=3) @@ -387,7 +387,7 @@ def test_success_preserves_finite_scores(self) -> None: def test_non_finite_scores_become_none(self, bad_value: float) -> None: """NaN and ±inf are serialized as None for JSON compatibility.""" row_score = _row_score( - metrics={"m": [MetricScore(name="score", value=bad_value)]}, + metrics={"m": [MetricOutput(name="score", value=bad_value)]}, ) result = MetricEvaluationRowScore.from_row_score(row_score, row=ROW, index=0) @@ -404,18 +404,33 @@ def test_errored_row_emits_null_scores_and_error(self) -> None: assert "boom" in result.error def test_multiple_metrics_flatten_into_single_scores_dict(self) -> None: - """All MetricScore entries across metric keys flatten into one dict keyed by name.""" + """All MetricOutput entries across metric keys flatten into one dict keyed by name.""" row_score = _row_score( index=2, metrics={ - "a": [MetricScore(name="precision", value=0.5)], - "b": [MetricScore(name="recall", value=0.75)], + "a": [MetricOutput(name="precision", value=0.5)], + "b": [MetricOutput(name="recall", value=0.75)], }, ) result = MetricEvaluationRowScore.from_row_score(row_score, row=ROW, index=2) assert result.scores == {"precision": 0.5, "recall": 0.75} + def test_non_numeric_outputs_are_excluded_from_scores_dict(self) -> None: + """Labels and structured outputs stay in row artifacts but not the live scores response.""" + row_score = _row_score( + metrics={ + "judge": [ + MetricOutput(name="quality", value=0.75), + MetricOutput(name="quality.label", value="good"), + MetricOutput(name="details", value={"rationale": "clear"}), + ] + }, + ) + result = MetricEvaluationRowScore.from_row_score(row_score, row=ROW, index=0) + + assert result.scores == {"quality": 0.75} + def test_empty_metrics_emit_empty_scores_dict(self) -> None: """Successful rows with no metrics still emit a (possibly empty) scores dict.""" row_score = _row_score(index=4) diff --git a/services/evaluator/tests/nmp/evaluator/api/v2/metrics/test_service_evaluate.py b/services/evaluator/tests/nmp/evaluator/api/v2/metrics/test_service_evaluate.py index a0dd25fa4b..093a97345d 100644 --- a/services/evaluator/tests/nmp/evaluator/api/v2/metrics/test_service_evaluate.py +++ b/services/evaluator/tests/nmp/evaluator/api/v2/metrics/test_service_evaluate.py @@ -14,8 +14,8 @@ AggregatedMetricResult, DatasetRows, EvaluationResult, + MetricOutput, MetricResult, - MetricScore, Model, RangeScore, RowScore, @@ -400,23 +400,23 @@ async def test_evaluate_ignored_failures_do_not_leak_synthetic_metric_scores( return_value=[ ( 0, - MetricResult(scores=[MetricScore(name="quality", value=0.8)]), + MetricResult(outputs=[MetricOutput(name="quality", value=0.8)]), RowScore( row_index=0, item={"output": "good"}, sample={}, - metrics={"quality": [MetricScore(name="quality", value=0.8)]}, + metrics={"quality": [MetricOutput(name="quality", value=0.8)]}, requests=[], ), ), ( 1, - MetricResult(scores=[MetricScore(name="llm-judge", value=float("nan"))]), + MetricResult(outputs=[MetricOutput(name="llm-judge", value=float("nan"))]), RowScore( row_index=1, item={"output": "bad"}, sample={}, - metrics={"llm-judge": [MetricScore(name="llm-judge", value=float("nan"))]}, + metrics={"llm-judge": [MetricOutput(name="llm-judge", value=float("nan"))]}, requests=[], metric_errors={"llm-judge": "request timed out"}, ), @@ -466,7 +466,7 @@ def test_row_scores_serializes_non_finite_values_as_null(self): row_index=0, item={"expected": "x", "output": "x"}, sample={}, - metrics={"string-check": [MetricScore(name="string-check", value=float("nan"))]}, + metrics={"string-check": [MetricOutput(name="string-check", value=float("nan"))]}, requests=[], ) @@ -484,7 +484,7 @@ def test_failed_row_scores_remain_null_when_pipeline_includes_placeholder_metric row_index=1, item={"expected": "x", "output": "x"}, sample={}, - metrics={"llm-judge": [MetricScore(name="llm-judge", value=float("nan"))]}, + metrics={"llm-judge": [MetricOutput(name="llm-judge", value=float("nan"))]}, requests=[], metric_errors={"llm-judge": "request timed out"}, ) @@ -508,12 +508,17 @@ async def test_evaluate_raises_on_out_of_bounds_row_index(self, metrics_service, left_template="{{expected}}", right_template="{{output}}", ) + runtime_metric = StringCheckMetric( + operation="equals", + left_template="{{expected}}", + right_template="{{output}}", + ) dataset = DatasetRows(rows=[{"expected": "a", "output": "a"}]) mocker.patch( "nmp.evaluator.api.v2.metrics.manager.new_metric", new_callable=AsyncMock, - return_value=metric, + return_value=runtime_metric, ) mocker.patch( "nmp.evaluator.api.v2.metrics.manager.run_generated_sample_scoring_pipeline", @@ -521,12 +526,12 @@ async def test_evaluate_raises_on_out_of_bounds_row_index(self, metrics_service, return_value=[ ( 0, - MetricResult(scores=[MetricScore(name="string-check", value=1.0)]), + MetricResult(outputs=[MetricOutput(name="string-check", value=1.0)]), RowScore( row_index=999, # out-of-bounds for a 1-row dataset item={"expected": "a", "output": "a"}, sample={}, - metrics={"string-check": [MetricScore(name="string-check", value=1.0)]}, + metrics={"string-check": [MetricOutput(name="string-check", value=1.0)]}, requests=[], ), ), @@ -544,7 +549,7 @@ def test_row_scores_fall_back_to_list_position_when_sdk_row_index_is_missing(sel row_index=None, item={"expected": "x", "output": "x"}, sample={}, - metrics={"string-check": [MetricScore(name="string-check", value=1.0)]}, + metrics={"string-check": [MetricOutput(name="string-check", value=1.0)]}, requests=[], ) @@ -568,8 +573,8 @@ def test_multiple_metrics_flattened(self): item={}, sample={}, metrics={ - "precision": [MetricScore(name="precision", value=0.9)], - "recall": [MetricScore(name="recall", value=0.8)], + "precision": [MetricOutput(name="precision", value=0.9)], + "recall": [MetricOutput(name="recall", value=0.8)], }, requests=[], ) @@ -585,7 +590,7 @@ def test_all_nan_scores_become_all_null(self): row_index=0, item={}, sample={}, - metrics={"m": [MetricScore(name="m", value=float("nan"))]}, + metrics={"m": [MetricOutput(name="m", value=float("nan"))]}, requests=[], ) @@ -600,8 +605,8 @@ def test_inf_score_becomes_null(self): item={}, sample={}, metrics={ - "pos": [MetricScore(name="pos", value=float("inf"))], - "neg": [MetricScore(name="neg", value=float("-inf"))], + "pos": [MetricOutput(name="pos", value=float("inf"))], + "neg": [MetricOutput(name="neg", value=float("-inf"))], }, requests=[], ) @@ -651,7 +656,7 @@ def test_error_with_multiple_metric_errors(self): row_index=0, item={}, sample={}, - metrics={"a": [MetricScore(name="a", value=float("nan"))]}, + metrics={"a": [MetricOutput(name="a", value=float("nan"))]}, requests=[], metric_errors={"a": "timeout", "b": "rate limited"}, ) @@ -893,7 +898,7 @@ async def test_post_pipeline_from_row_score_failure_is_wrapped(self, metrics_ser row_index=0, item={"expected": "a", "output": "a"}, sample={}, - metrics={"string-check": [MetricScore(name="string-check", value=1.0)]}, + metrics={"string-check": [MetricOutput(name="string-check", value=1.0)]}, requests=[], ) ], diff --git a/services/evaluator/tests/nmp/evaluator/app/metrics/test_metric_protocol.py b/services/evaluator/tests/nmp/evaluator/app/metrics/test_metric_protocol.py index e6bf409bfb..ec4159e534 100644 --- a/services/evaluator/tests/nmp/evaluator/app/metrics/test_metric_protocol.py +++ b/services/evaluator/tests/nmp/evaluator/app/metrics/test_metric_protocol.py @@ -6,12 +6,12 @@ import pytest from nemo_evaluator_sdk.enums import MetricType, ModelFormat -from nemo_evaluator_sdk.metrics.base import Metric from nemo_evaluator_sdk.metrics.bleu import BLEUMetric from nemo_evaluator_sdk.metrics.exact_match import ExactMatchMetric from nemo_evaluator_sdk.metrics.f1 import F1Metric from nemo_evaluator_sdk.metrics.llm_judge import LLMJudgeMetric from nemo_evaluator_sdk.metrics.number_check import NumberCheckMetric +from nemo_evaluator_sdk.metrics.protocol import Metric from nemo_evaluator_sdk.metrics.rouge import ROUGEMetric from nemo_evaluator_sdk.metrics.string_check import StringCheckMetric from nemo_evaluator_sdk.metrics.tool_calling import ToolCallingMetric diff --git a/services/evaluator/tests/tasks/test_evaluate_benchmark.py b/services/evaluator/tests/tasks/test_evaluate_benchmark.py index d8fda6a054..4075de250a 100644 --- a/services/evaluator/tests/tasks/test_evaluate_benchmark.py +++ b/services/evaluator/tests/tasks/test_evaluate_benchmark.py @@ -5,9 +5,9 @@ import pytest from nemo_evaluator_sdk.execution.values import EvaluationError, EvaluationPhase +from nemo_evaluator_sdk.metrics.protocol import MetricInput, MetricOutput, MetricOutputSpec, MetricResult from nemo_evaluator_sdk.values import AggregateRangeScore from nemo_evaluator_sdk.values.multi_metric_results import BenchmarkEvaluationResult as SDKBenchmarkEvaluationResult -from nemo_evaluator_sdk.values.results import MetricResult, MetricScore from nmp.common.jobs.constants import NEMO_JOB_STEP_CONFIG_FILE_PATH_ENVVAR, PERSISTENT_JOB_STORAGE_PATH_ENVVAR from nmp.evaluator.app.values import ( BenchmarkEvaluationResult, @@ -252,14 +252,14 @@ async def test_agent_benchmark_passes_platform_headers_to_agent_inference( class _FakeMetric: """Metric test double that returns a fixed score.""" - def score_names(self) -> list[str]: - """Return the score names exposed by this metric.""" - return ["score"] + def output_spec(self) -> list[MetricOutputSpec]: + """Return the outputs exposed by this metric.""" + return [MetricOutputSpec.continuous_score("score")] - async def compute_scores(self, item: dict, sample: dict) -> MetricResult: + async def compute_scores(self, input: MetricInput) -> MetricResult: """Return one fixed metric score.""" - del item, sample - return MetricResult(scores=[MetricScore(name="score", value=1.0)]) + del input + return MetricResult(outputs=[MetricOutput(name="score", value=1.0)]) async def _agent_inference( agent, diff --git a/web/packages/common/src/components/AssistantChat/index.spec.tsx b/web/packages/common/src/components/AssistantChat/index.spec.tsx index 6e712d29a5..79c65ed3b8 100644 --- a/web/packages/common/src/components/AssistantChat/index.spec.tsx +++ b/web/packages/common/src/components/AssistantChat/index.spec.tsx @@ -79,6 +79,7 @@ const createHangingStream = (content: string): Stream => { }; const completion = createCompletion('Hello from inference gateway.'); +const interactionTimeoutMs = 10_000; interface CompletionRequestWithSignal { signal?: AbortSignal; @@ -97,138 +98,161 @@ describe('AssistantChat', () => { mocks.createChatCompletion.mockResolvedValue(completion); }); - it('sends text prompts through useChatCompletion', async () => { - renderAssistantChat(); + it( + 'sends text prompts through useChatCompletion', + async () => { + renderAssistantChat(); + + await userEvent.type(screen.getByRole('textbox', { name: /Task prompt/i }), 'Hello model'); + await userEvent.click(screen.getByRole('button', { name: /Submit/i })); + + expect(await screen.findByText('Hello from inference gateway.')).toBeInTheDocument(); + expect(screen.getByText('Hello model')).toBeInTheDocument(); + + await waitFor(() => + expect(mocks.createChatCompletion).toHaveBeenCalledWith( + expect.objectContaining({ + model: 'test-model', + workspace: 'default', + stream: true, + messages: [{ role: 'user', content: 'Hello model' }], + }) + ) + ); + }, + interactionTimeoutMs + ); + + it( + 'clears the current thread', + async () => { + renderAssistantChat( + + ); + + expect(screen.getByText('Existing message')).toBeInTheDocument(); + + await userEvent.click(screen.getByRole('button', { name: /Reset/i })); + + expect(screen.queryByText('Existing message')).not.toBeInTheDocument(); + expect(screen.getByText('Ready')).toBeInTheDocument(); + }, + interactionTimeoutMs + ); + + it( + 'edits a user message and re-runs inference with the edited prompt', + async () => { + mocks.createChatCompletion + .mockResolvedValueOnce(createCompletion('Original response.')) + .mockResolvedValueOnce(createCompletion('Edited response.')); - await userEvent.type(screen.getByRole('textbox', { name: /Task prompt/i }), 'Hello model'); - await userEvent.click(screen.getByRole('button', { name: /Submit/i })); + renderAssistantChat(); - expect(await screen.findByText('Hello from inference gateway.')).toBeInTheDocument(); - expect(screen.getByText('Hello model')).toBeInTheDocument(); + await userEvent.type( + screen.getByRole('textbox', { name: /Task prompt/i }), + 'Original prompt' + ); + await userEvent.click(screen.getByRole('button', { name: /Submit/i })); - await waitFor(() => - expect(mocks.createChatCompletion).toHaveBeenCalledWith( + expect(await screen.findByText('Original response.')).toBeInTheDocument(); + + await userEvent.click(screen.getByRole('button', { name: /Edit message/i })); + const editInput = screen.getByRole('textbox', { name: /Edit message/i }); + expect(editInput).toHaveValue('Original prompt'); + expect(editInput.tagName).toBe('TEXTAREA'); + + await userEvent.clear(editInput); + await userEvent.type(editInput, 'Edited prompt'); + await userEvent.click(screen.getByRole('button', { name: /Save edit/i })); + + expect(await screen.findByText('Edited response.')).toBeInTheDocument(); + expect(screen.getByText('Edited prompt')).toBeInTheDocument(); + expect(screen.queryByText('Original prompt')).not.toBeInTheDocument(); + expect(screen.queryByText('Original response.')).not.toBeInTheDocument(); + + await waitFor(() => expect(mocks.createChatCompletion).toHaveBeenCalledTimes(2)); + expect(mocks.createChatCompletion).toHaveBeenLastCalledWith( expect.objectContaining({ model: 'test-model', workspace: 'default', stream: true, - messages: [{ role: 'user', content: 'Hello model' }], + messages: [{ role: 'user', content: 'Edited prompt' }], }) - ) - ); - }); - - it('clears the current thread', async () => { - renderAssistantChat( - - ); - - expect(screen.getByText('Existing message')).toBeInTheDocument(); - - await userEvent.click(screen.getByRole('button', { name: /Reset/i })); - - expect(screen.queryByText('Existing message')).not.toBeInTheDocument(); - expect(screen.getByText('Ready')).toBeInTheDocument(); - }); - - it('edits a user message and re-runs inference with the edited prompt', async () => { - mocks.createChatCompletion - .mockResolvedValueOnce(createCompletion('Original response.')) - .mockResolvedValueOnce(createCompletion('Edited response.')); - - renderAssistantChat(); - - await userEvent.type(screen.getByRole('textbox', { name: /Task prompt/i }), 'Original prompt'); - await userEvent.click(screen.getByRole('button', { name: /Submit/i })); - - expect(await screen.findByText('Original response.')).toBeInTheDocument(); - - await userEvent.click(screen.getByRole('button', { name: /Edit message/i })); - const editInput = screen.getByRole('textbox', { name: /Edit message/i }); - expect(editInput).toHaveValue('Original prompt'); - expect(editInput.tagName).toBe('TEXTAREA'); - - await userEvent.clear(editInput); - await userEvent.type(editInput, 'Edited prompt'); - await userEvent.click(screen.getByRole('button', { name: /Save edit/i })); - - expect(await screen.findByText('Edited response.')).toBeInTheDocument(); - expect(screen.getByText('Edited prompt')).toBeInTheDocument(); - expect(screen.queryByText('Original prompt')).not.toBeInTheDocument(); - expect(screen.queryByText('Original response.')).not.toBeInTheDocument(); - - await waitFor(() => expect(mocks.createChatCompletion).toHaveBeenCalledTimes(2)); - expect(mocks.createChatCompletion).toHaveBeenLastCalledWith( - expect.objectContaining({ - model: 'test-model', - workspace: 'default', - stream: true, - messages: [{ role: 'user', content: 'Edited prompt' }], - }) - ); - }); + ); + }, + interactionTimeoutMs + ); - it('stops a hanging stream when stop is clicked', async () => { - const stream = createHangingStream('0 this is an example response'); - const abortSpy = vi.spyOn(stream.controller, 'abort'); - mocks.createChatCompletion.mockResolvedValueOnce(stream); + it( + 'stops a hanging stream when stop is clicked', + async () => { + const stream = createHangingStream('0 this is an example response'); + const abortSpy = vi.spyOn(stream.controller, 'abort'); + mocks.createChatCompletion.mockResolvedValueOnce(stream); - renderAssistantChat(); + renderAssistantChat(); - await userEvent.type(screen.getByRole('textbox', { name: /Task prompt/i }), 'Hang forever'); - await userEvent.click(screen.getByRole('button', { name: /Submit/i })); + await userEvent.type(screen.getByRole('textbox', { name: /Task prompt/i }), 'Hang forever'); + await userEvent.click(screen.getByRole('button', { name: /Submit/i })); - expect(await screen.findByText('0 this is an example response')).toBeInTheDocument(); - expect(screen.getByRole('button', { name: /Stop/i })).toBeEnabled(); + expect(await screen.findByText('0 this is an example response')).toBeInTheDocument(); + expect(screen.getByRole('button', { name: /Stop/i })).toBeEnabled(); - await userEvent.click(screen.getByRole('button', { name: /Stop/i })); + await userEvent.click(screen.getByRole('button', { name: /Stop/i })); - await waitFor(() => expect(abortSpy).toHaveBeenCalledTimes(1)); - await waitFor(() => - expect(screen.queryByRole('button', { name: /Stop/i })).not.toBeInTheDocument() - ); - expect(screen.getByRole('button', { name: /Submit/i })).toBeInTheDocument(); - expect(screen.getByText('0 this is an example response')).toBeInTheDocument(); - }); + await waitFor(() => expect(abortSpy).toHaveBeenCalledTimes(1)); + await waitFor(() => + expect(screen.queryByRole('button', { name: /Stop/i })).not.toBeInTheDocument() + ); + expect(screen.getByRole('button', { name: /Submit/i })).toBeInTheDocument(); + expect(screen.getByText('0 this is an example response')).toBeInTheDocument(); + }, + interactionTimeoutMs + ); - it('aborts a pending completion request when stop is clicked', async () => { - let requestSignal: AbortSignal | undefined; - const abortError = new Error('aborted'); - abortError.name = 'AbortError'; + it( + 'aborts a pending completion request when stop is clicked', + async () => { + let requestSignal: AbortSignal | undefined; + const abortError = new Error('aborted'); + abortError.name = 'AbortError'; - mocks.createChatCompletion.mockImplementationOnce((request: CompletionRequestWithSignal) => { - requestSignal = request.signal; + mocks.createChatCompletion.mockImplementationOnce((request: CompletionRequestWithSignal) => { + requestSignal = request.signal; - return new Promise((_resolve, reject) => { - request.signal?.addEventListener('abort', () => reject(abortError), { once: true }); + return new Promise((_resolve, reject) => { + request.signal?.addEventListener('abort', () => reject(abortError), { once: true }); + }); }); - }); - renderAssistantChat(); + renderAssistantChat(); - await userEvent.type( - screen.getByRole('textbox', { name: /Task prompt/i }), - 'Hang before stream' - ); - await userEvent.click(screen.getByRole('button', { name: /Submit/i })); + await userEvent.type( + screen.getByRole('textbox', { name: /Task prompt/i }), + 'Hang before stream' + ); + await userEvent.click(screen.getByRole('button', { name: /Submit/i })); - await waitFor(() => expect(requestSignal).toBeDefined()); - expect(requestSignal?.aborted).toBe(false); + await waitFor(() => expect(requestSignal).toBeDefined()); + expect(requestSignal?.aborted).toBe(false); - await userEvent.click(screen.getByRole('button', { name: /Stop/i })); + await userEvent.click(screen.getByRole('button', { name: /Stop/i })); - await waitFor(() => expect(requestSignal?.aborted).toBe(true)); - await waitFor(() => - expect(screen.queryByRole('button', { name: /Stop/i })).not.toBeInTheDocument() - ); - expect(screen.queryByText('aborted')).not.toBeInTheDocument(); - }); + await waitFor(() => expect(requestSignal?.aborted).toBe(true)); + await waitFor(() => + expect(screen.queryByRole('button', { name: /Stop/i })).not.toBeInTheDocument() + ); + expect(screen.queryByText('aborted')).not.toBeInTheDocument(); + }, + interactionTimeoutMs + ); }); diff --git a/web/packages/sdk/generated/platform/schema/MetricScore.ts b/web/packages/sdk/generated/platform/schema/MetricOutput.ts similarity index 55% rename from web/packages/sdk/generated/platform/schema/MetricScore.ts rename to web/packages/sdk/generated/platform/schema/MetricOutput.ts index 095e9a4f0a..105d03190f 100644 --- a/web/packages/sdk/generated/platform/schema/MetricScore.ts +++ b/web/packages/sdk/generated/platform/schema/MetricOutput.ts @@ -6,14 +6,11 @@ * Do not edit manually. * Nemo Platform API */ -import type { ScoreStats } from './ScoreStats'; /** - * A computed score for the metric + * One named value emitted by a metric. */ -export interface MetricScore { +export interface MetricOutput { name: string; - value: number; - /** Computed score statistics for the score. */ - stats?: ScoreStats; + value: unknown; } diff --git a/web/packages/sdk/generated/platform/schema/RowScore.ts b/web/packages/sdk/generated/platform/schema/RowScore.ts index 003d3d2023..34f86fd57b 100644 --- a/web/packages/sdk/generated/platform/schema/RowScore.ts +++ b/web/packages/sdk/generated/platform/schema/RowScore.ts @@ -25,7 +25,7 @@ export interface RowScore { item: RowScoreItem; /** Sample output payload for the evaluated row. */ sample: RowScoreSample; - /** Metric-level row scores by metric key. */ + /** Metric-level row outputs by metric key. */ metrics: RowScoreMetrics; /** Request details captured during evaluation. */ requests: RowScoreRequestsItem[]; diff --git a/web/packages/sdk/generated/platform/schema/RowScoreMetrics.ts b/web/packages/sdk/generated/platform/schema/RowScoreMetrics.ts index 02fe1eadec..276fe03ce1 100644 --- a/web/packages/sdk/generated/platform/schema/RowScoreMetrics.ts +++ b/web/packages/sdk/generated/platform/schema/RowScoreMetrics.ts @@ -6,9 +6,9 @@ * Do not edit manually. * Nemo Platform API */ -import type { MetricScore } from './MetricScore'; +import type { MetricOutput } from './MetricOutput'; /** - * Metric-level row scores by metric key. + * Metric-level row outputs by metric key. */ -export type RowScoreMetrics = { [key: string]: MetricScore[] }; +export type RowScoreMetrics = { [key: string]: MetricOutput[] }; diff --git a/web/packages/sdk/generated/platform/schema/ScoreStats.ts b/web/packages/sdk/generated/platform/schema/ScoreStats.ts deleted file mode 100644 index 461fd7c7eb..0000000000 --- a/web/packages/sdk/generated/platform/schema/ScoreStats.ts +++ /dev/null @@ -1,37 +0,0 @@ -/** - * SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - * - * Generated by Orval 🍺 - * Do not edit manually. - * Nemo Platform API - */ -import type { RubricScoreStat } from './RubricScoreStat'; - -/** - * Stats for a score. Fields that are NaN are serialized as the string "NaN" in the API response. - */ -export interface ScoreStats { - /** The number of values used for computing the score. */ - count?: number; - /** The sum of all values used for computing the score. */ - sum?: number | string; - /** The sum of the square of all values used for computing the score. */ - sum_squared?: number | string; - /** The minimum of all values used for computing the score. */ - min?: number | string; - /** The maximum of all values used for computing the score. */ - max?: number | string; - /** The mean of all values used for computing the score. */ - mean?: number | string; - /** The population variance, (note: not the sample variance). */ - variance?: number | string; - /** The population standard deviation, (note: not the sample standard deviation). */ - stddev?: number | string; - /** The standard error. */ - stderr?: number | string; - /** The number of values that are not a number (NaN) and are excluded from the score stats calculations. */ - nan_count?: number; - /** The distribution of the rubric grading criteria for the score. */ - rubric_distribution?: RubricScoreStat[]; -} diff --git a/web/packages/sdk/generated/platform/schema/index.ts b/web/packages/sdk/generated/platform/schema/index.ts index 946a081372..4f8a0bf417 100644 --- a/web/packages/sdk/generated/platform/schema/index.ts +++ b/web/packages/sdk/generated/platform/schema/index.ts @@ -575,10 +575,10 @@ export * from './MetricOnlineAgentJobPromptTemplate'; export * from './MetricOnlineJob'; export * from './MetricOnlineJobMetricParams'; export * from './MetricOnlineJobPromptTemplate'; +export * from './MetricOutput'; export * from './MetricRef'; export * from './MetricRetrieverJob'; export * from './MetricRetrieverJobMetricParams'; -export * from './MetricScore'; export * from './MetricsListResponse'; export * from './MetricsListResponseFilter'; export * from './MetricType'; @@ -883,7 +883,6 @@ export * from './SafeSynthesizerListJobsParams'; export * from './SafeSynthesizerParameters'; export * from './SafeSynthesizerSummary'; export * from './SafeSynthesizerTiming'; -export * from './ScoreStats'; export * from './SecretRef'; export * from './SecretsListSecretsParams'; export * from './SensitiveDataDetection';