feat(evaluator): add metric type name alias

packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/__init__.py

@@ -19,6 +19,7 @@
 from nemo_evaluator_sdk.metrics.number_check import NumberCheckMetric
 from nemo_evaluator_sdk.metrics.protocol import (
     Metric,
+    MetricTypeName,
     validate_metric_result,
 )
 from nemo_evaluator_sdk.metrics.remote import NemoAgentToolkitRemoteMetric, RemoteMetric
@@ -85,6 +86,7 @@
     "InferenceStructuredOutput",
     "JSONScoreParser",
     "Metric",
+    "MetricTypeName",
     "MetricDescriptor",
     "MetricInput",
     "MetricOutput",

packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/metrics/protocol.py

@@ -7,12 +7,13 @@
 
 import math
 from collections.abc import Awaitable, Callable
-from typing import Any, Protocol, runtime_checkable
+from typing import Annotated, Any, Protocol, runtime_checkable
 
 from nemo_evaluator_sdk.values.common import SecretRef
-from pydantic import BaseModel, ConfigDict, Field, RootModel, field_serializer, field_validator
+from pydantic import BaseModel, ConfigDict, Field, RootModel, StringConstraints, field_serializer, field_validator
 
 SecretResolver = Callable[[str], Awaitable[str | None]]
+MetricTypeName = Annotated[str, StringConstraints(min_length=1)]
 
 
 class DatasetRow(BaseModel):
@@ -126,16 +127,9 @@ class MetricDescriptor(BaseModel):
 
     model_config = ConfigDict(extra="forbid")
 
-    type: str
+    type: MetricTypeName
     outputs: list[MetricOutputSpec] = Field(min_length=1)
 
-    @field_validator("type")
-    @classmethod
-    def _type_must_not_be_empty(cls, value: str) -> str:
-        if not value:
-            raise ValueError("metric type must not be empty")
-        return value
-
     @field_validator("outputs")
     @classmethod
     def _output_names_must_be_unique(cls, value: list[MetricOutputSpec]) -> list[MetricOutputSpec]:
@@ -174,7 +168,7 @@ class Metric(Protocol):
     """Shared row-scoring primitive for SDK runtime metrics."""
 
     @property
-    def type(self) -> str:
+    def type(self) -> MetricTypeName:
         """Return the public metric key/type identifier."""
         ...
 

packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/values/__init__.py

@@ -15,6 +15,7 @@
     MetricOutput,
     MetricOutputSpec,
     MetricResult,
+    MetricTypeName,
 )
 from nemo_evaluator_sdk.values.agents import Agent
 from nemo_evaluator_sdk.values.common import SecretRef, SupportedJobTypes
@@ -114,6 +115,7 @@
     "MetricOutput",
     "MetricOutputSpec",
     "MetricResult",
+    "MetricTypeName",
     "MetricScore",
     "Model",
     "DatasetInput",

packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/values/metrics.py

@@ -20,6 +20,7 @@
 from nemo_evaluator_sdk.dataset_schemas.compatibility import merge_metric_required_schemas
 from nemo_evaluator_sdk.dataset_schemas.templates import infer_required_schema_from_template
 from nemo_evaluator_sdk.enums import MetricType
+from nemo_evaluator_sdk.metrics.protocol import MetricTypeName
 from nemo_evaluator_sdk.values.common import SecretRef, SupportedJobTypes
 from nemo_evaluator_sdk.values.dataset_schemas import InputSchema
 from nemo_evaluator_sdk.values.models import Model, ReasoningParams
@@ -129,7 +130,7 @@ class MetricBase(BaseModel):
 
     __entity_type__: ClassVar[str] = "metric"
 
-    type: str = Field(description="The type of metric. Used as a discriminator for the metric type.")
+    type: MetricTypeName = Field(description="The type of metric. Used as a discriminator for the metric type.")
     description: str | None = Field(default=None, description="Human-readable description of the metric.")
     labels: dict[str, str] = Field(
         default_factory=dict, description="Labels are key-value pairs that can be used for grouping and filtering."

packages/nemo_evaluator_sdk/tests/metrics/test_metric_contract.py

@@ -85,6 +85,11 @@ def test_metric_descriptor_rejects_duplicate_outputs() -> None:
         )
 
 
+def test_metric_descriptor_rejects_empty_type() -> None:
+    with pytest.raises(ValidationError):
+        MetricDescriptor(type="", outputs=[MetricOutputSpec.continuous_score("reward")])
+
+
 def test_validate_metric_result_accepts_declared_outputs() -> None:
     outputs = [
         MetricOutputSpec.continuous_score("reward"),