feat(evaluator): add metric model resolvers

packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/__init__.py

@@ -7,6 +7,7 @@
 from importlib.metadata import version as _package_version
 
 from nemo_evaluator_sdk.datasets import DatasetLoadError, load_dataset, load_dataset_as_dicts
+from nemo_evaluator_sdk.execution.backends.local.backend import LocalBackend
 from nemo_evaluator_sdk.execution.evaluator import Evaluator
 from nemo_evaluator_sdk.execution.values import (
     EvaluationError,
@@ -26,6 +27,8 @@
 from nemo_evaluator_sdk.metrics.rouge import ROUGEMetric
 from nemo_evaluator_sdk.metrics.string_check import StringCheckMetric
 from nemo_evaluator_sdk.metrics.tool_calling import ToolCallingMetric
+from nemo_evaluator_sdk.resolver_protocols import ModelResolver, SecretResolver
+from nemo_evaluator_sdk.resolvers import LocalModelResolver, LocalSecretResolver
 from nemo_evaluator_sdk.structured_output import (
     InferenceFn,
     InferenceStructuredOutput,
@@ -52,6 +55,7 @@
     MetricOutputSpec,
     MetricResult,
     Model,
+    ModelRef,
     RangeScore,
     ReasoningParams,
     RemoteScore,
@@ -99,7 +103,12 @@
     "DatasetRow",
     "DiscreteScore",
     "Label",
+    "LocalBackend",
+    "LocalModelResolver",
+    "LocalSecretResolver",
     "Model",
+    "ModelRef",
+    "ModelResolver",
     "NemoAgentToolkitRemoteMetric",
     "NumberCheckMetric",
     "RangeScore",
@@ -109,6 +118,7 @@
     "ROUGEMetric",
     "RubricScore",
     "SecretRef",
+    "SecretResolver",
     "StringCheckMetric",
     "StructuredOutput",
     "StructuredOutputMode",

packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/execution/backends/local/backend.py

@@ -13,9 +13,10 @@
 from nemo_evaluator_sdk.execution.benchmark_execution import evaluate_benchmark as sdk_evaluate_benchmark
 from nemo_evaluator_sdk.execution.config import EvaluationRequest
 from nemo_evaluator_sdk.execution.metric_execution import _merge_online_hooks, evaluate_metric
-from nemo_evaluator_sdk.execution.utils import prepare_metric_for_local_execution, unique_metric_keys
+from nemo_evaluator_sdk.execution.utils import prepare_metric_for_execution, unique_metric_keys
 from nemo_evaluator_sdk.metrics.protocol import Metric
 from nemo_evaluator_sdk.metrics.utils import metric_type_name
+from nemo_evaluator_sdk.resolvers import LocalModelResolver, LocalSecretResolver
 from nemo_evaluator_sdk.values.multi_metric_results import BenchmarkEvaluationResult, namespace_result
 from nemo_evaluator_sdk.values.results import EvaluationResult
 
@@ -25,6 +26,11 @@
 class LocalBackend:
     """Local backend that executes metrics in-process."""
 
+    def __init__(self) -> None:
+        """Create a local backend with local resolver defaults."""
+        self.secret_resolver = LocalSecretResolver()
+        self.model_resolver = LocalModelResolver()
+
     async def _evaluate_one(
         self,
         *,
@@ -44,8 +50,15 @@ async def _evaluate_one(
         Returns:
             A namespaced single-metric evaluation result.
         """
+        prepared_metric = await prepare_metric_for_execution(
+            metric,
+            params=request.params,
+            model_resolver=self.model_resolver,
+            secret_resolver=self.secret_resolver,
+        )
+
         result = await evaluate_metric(
-            metric=metric,
+            metric=prepared_metric,
             target=request.target,
             rows=rows,
             prompt_template=request.prompt_template,
@@ -107,7 +120,15 @@ async def evaluate_benchmark(
             request.params.limit_samples if request.params else None,
         )
         metric_keys = unique_metric_keys(metrics)
-        prepared_metrics = [await prepare_metric_for_local_execution(metric, request.params) for metric in metrics]
+        prepared_metrics = [
+            await prepare_metric_for_execution(
+                metric,
+                params=request.params,
+                model_resolver=self.model_resolver,
+                secret_resolver=self.secret_resolver,
+            )
+            for metric in metrics
+        ]
         metrics_built: list[tuple[str, Metric]] = list(zip(metric_keys, prepared_metrics, strict=True))
         if request.target is not None:
             preprocess_hooks, postprocess_hooks = _merge_online_hooks(

packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/execution/metric_execution.py

@@ -35,10 +35,12 @@
     nan_metric_result,
     score_row,
 )
-from nemo_evaluator_sdk.execution.utils import prepare_metric_for_local_execution
 from nemo_evaluator_sdk.execution.values import EvaluationError, EvaluationPhase
 from nemo_evaluator_sdk.inference import InferenceMetricBase
-from nemo_evaluator_sdk.metrics.protocol import Metric, MetricResult
+from nemo_evaluator_sdk.metrics.protocol import (
+    Metric,
+    MetricResult,
+)
 from nemo_evaluator_sdk.metrics.utils import metric_type_name
 from nemo_evaluator_sdk.resilience.api import run_indexed_tasks, use_resilience_session
 from nemo_evaluator_sdk.resilience.errors import get_evaluation_error
@@ -792,15 +794,13 @@ async def evaluate_metric(
     preprocess_hooks: Sequence[inference.PreprocessRequest] | None = None,
     postprocess_hooks: Sequence[inference.PostprocessResponse] | None = None,
 ) -> EvaluationResult:
-    """Generate model outputs for prepared rows and evaluate the metric online."""
+    """Generate model outputs for prepared rows and evaluate a prepared metric."""
     if not rows:
         log.warning("No rows found in dataset, returning empty evaluation result")
         return empty_evaluation_result()
 
     params = normalize_params(params, target)
 
-    metric = await prepare_metric_for_local_execution(metric, params)
-
     client_close_fn = None
 
     merged_preprocess_hooks, merged_postprocess_hooks = _merge_online_hooks(

packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/execution/utils.py

@@ -6,14 +6,14 @@
 from __future__ import annotations
 
 import copy
-import os
 from collections.abc import Sequence
 from typing import cast
 
 from nemo_evaluator_sdk.execution._protocols import JobParamsConfigurableMetric
-from nemo_evaluator_sdk.metrics.protocol import Metric, MetricWithPreflight, MetricWithSecrets
+from nemo_evaluator_sdk.metrics.protocol import Metric, MetricWithModels, MetricWithPreflight, MetricWithSecrets
 from nemo_evaluator_sdk.metrics.utils import metric_type_name
-from nemo_evaluator_sdk.values.params import RunConfig
+from nemo_evaluator_sdk.resolver_protocols import ModelResolver, SecretResolver
+from nemo_evaluator_sdk.values.params import RunConfig, RunConfigOnline, RunConfigOnlineModel
 from pydantic import BaseModel
 
 
@@ -37,15 +37,12 @@ def unique_metric_keys(metrics: Sequence[Metric]) -> list[str]:
     return keys
 
 
-def _copy_metric(metric: Metric) -> Metric:
+def copy_metric(metric: Metric) -> Metric:
     """Create a best-effort isolated copy of a metric instance.
 
-    Preparation mutates metrics in-place (run-config overrides, secret
-    resolution, preflight) so we copy first to avoid side-effects on the
-    caller's original instance. Pydantic models are copied via
-    ``model_copy(deep=True)``; regular Python metric classes are also
-    supported when ``copy.deepcopy()`` works, for example by implementing
-    ``__deepcopy__``.
+    Preparation mutates metrics in place (runtime params, resolver hydration,
+    preflight state), so backends copy first to avoid side effects on the
+    caller's original metric object.
     """
     if isinstance(metric, BaseModel):
         return cast(Metric, metric.model_copy(deep=True))
@@ -59,58 +56,31 @@ def _copy_metric(metric: Metric) -> Metric:
         ) from exc
 
 
-def _candidate_env_names(secret_name: str) -> list[str]:
-    """Generate environment variable names that may contain one secret."""
-    names = [secret_name, secret_name.upper()]
-    normalized = secret_name.replace("-", "_").replace("/", "_")
-    names.extend([normalized, normalized.upper()])
-    if normalized and normalized[0].isdigit():
-        prefixed = f"_{normalized}"
-        names.extend([prefixed, prefixed.upper()])
-    return list(dict.fromkeys(names))
-
-
-async def _resolve_secret_from_env(secret_name: str) -> str | None:
-    """Resolve one secret value from environment variables.
-
-    Async to satisfy the ``SecretResolver`` protocol expected by metrics.
-    """
-    for candidate in _candidate_env_names(secret_name):
-        value = os.getenv(candidate)
-        if value:
-            return value
-    return None
-
-
-def _apply_runtime_params(metric: Metric, params: RunConfig) -> None:
-    """Apply runtime execution params to one prepared metric.
-
-    Args:
-        metric: Copied metric instance to mutate for this run only.
-        params: Materialized execution params for this run.
-
-    Returns:
-        None.
-    """
-    if isinstance(metric, JobParamsConfigurableMetric):
-        metric.apply_evaluation_job_params(params)
-
-
-async def prepare_metric_for_local_execution(metric: Metric, params: RunConfig) -> Metric:
-    """Prepare one metric for execution.
+async def prepare_metric_for_execution(
+    metric: Metric,
+    *,
+    params: RunConfig | RunConfigOnline | RunConfigOnlineModel,
+    model_resolver: ModelResolver,
+    secret_resolver: SecretResolver,
+) -> Metric:
+    """Copy and prepare one metric for execution.
 
     Args:
         metric: User-provided metric instance.
         params: Materialized execution params for this run.
+        model_resolver: Resolver used for any ``ModelRef`` fields.
+        secret_resolver: Resolver used for any ``SecretRef`` fields.
 
     Returns:
-        A metric object ready for execution in the selected backend.
+        A copied metric ready for execution.
     """
-    prepared = _copy_metric(metric)
-    _apply_runtime_params(prepared, params)
-
-    if isinstance(prepared, MetricWithSecrets):
-        await prepared.resolve_secrets(_resolve_secret_from_env)
-    if isinstance(prepared, MetricWithPreflight):
-        await prepared.preflight()
-    return prepared
+    prepared_metric = copy_metric(metric)
+    if isinstance(prepared_metric, JobParamsConfigurableMetric):
+        prepared_metric.apply_evaluation_job_params(params)
+    if isinstance(prepared_metric, MetricWithModels):
+        await prepared_metric.resolve_models(model_resolver)
+    if isinstance(prepared_metric, MetricWithSecrets):
+        await prepared_metric.resolve_secrets(secret_resolver)
+    if isinstance(prepared_metric, MetricWithPreflight):
+        await prepared_metric.preflight()
+    return prepared_metric