refactor(evaluator): clarify SDK ref handling contracts

docs/evaluator/sdk-resources.md

@@ -36,29 +36,27 @@ Use `submit` when you want to create a durable remote platform job and manage th
 | `plugin_status()` | Returns Evaluator plugin health information from the service. | `dict[str, object]` |
 | `get_job_resource(job_name: str, workspace: str \| None = None)` | Returns a resource for an existing Evaluator plugin job. | `EvaluatorJobResource` |
 
-The `dataset` argument accepts inline rows, local dataset paths, and fileset references. Use `config` for evaluator runtime settings, `aggregate_fields` on result-returning calls to shape aggregate scores, and `target` plus `prompt_template` when the evaluator should generate model or agent responses before scoring.
+The `dataset` argument accepts inline rows, local dataset paths, local glob paths, and fileset references with optional fragment selectors. Use `config` for evaluator runtime settings, `aggregate_fields` on result-returning calls to shape aggregate scores, and `target` plus `prompt_template` when the evaluator should generate model or agent responses before scoring.
 
 ### `run()` arguments
 
 | Argument | Type | Required | Description |
 |----------|------|----------|-------------|
 | `metric` | `Metric` | Yes | Metric configuration used to score each row. |
-| `dataset` | `PluginDatasetInput` | Yes | Inline rows, a local dataset path, or a fileset reference. |
+| `dataset` | `PluginDatasetInput` | Yes | Inline rows, local dataset paths, local glob paths, or fileset references with optional fragment selectors. |
 | `config` | `RunConfig \| RunConfigOnline \| RunConfigOnlineModel \| None` | No | Runtime settings such as sample limits, parallelism, timeouts, and retry behavior. |
 | `aggregate_fields` | `tuple[AggregateFieldName, ...] \| None` | No | Aggregate score fields to include in the returned result. |
 | `target` | `Model \| Agent \| None` | No | Model or agent target used when the evaluator should generate outputs before scoring. |
-| `dataset_glob_pattern` | `str \| None` | No | Pattern used to select files from a dataset path or fileset reference. |
 | `prompt_template` | `str \| dict[str, Any] \| None` | No | Prompt template used with `target` for online model or agent evaluation. |
 
 ### `submit()` arguments
 
 | Argument | Type | Required | Description |
 |----------|------|----------|-------------|
 | `metric` | `Metric` | Yes | Metric configuration serialized into the durable platform job. |
-| `dataset` | `PluginDatasetInput` | Yes | Inline rows, a local dataset path, or a fileset reference. |
+| `dataset` | `PluginDatasetInput` | Yes | Inline rows, local dataset paths, local glob paths, or fileset references with optional fragment selectors. |
 | `config` | `RunConfig \| RunConfigOnline \| RunConfigOnlineModel \| None` | No | Runtime settings applied when the submitted job executes. |
-| `target` | `Model \| Agent \| None` | No | Model or agent target used when the submitted job should generate outputs before scoring. |
-| `dataset_glob_pattern` | `str \| None` | No | Pattern used to select files from a dataset path or fileset reference. |
+| `target` | `Model \| ModelRef \| Agent \| None` | No | Model, model reference, or agent target used when the submitted job should generate outputs before scoring. |
 | `prompt_template` | `str \| dict[str, Any] \| None` | No | Prompt template used with `target` for online model or agent evaluation. |
 
 ### Run locally

packages/nemo_evaluator_sdk/examples/examples.py

@@ -25,6 +25,7 @@
     Model,
     RangeScore,
     RunConfig,
+    RunConfigOnlineModel,
     SecretRef,
 )
 
@@ -325,7 +326,8 @@ async def run_online_local_exact_match_example() -> None:
         metrics=exact_match,
         target=model,
         dataset=ONLINE_EXACT_MATCH_DATASET,
-        config=RunConfig(parallelism=4),
+        prompt_template=ONLINE_CHAT_PROMPT_TEMPLATE,
+        config=RunConfigOnlineModel(parallelism=4),
     )
     exact_match_result.print_summary()
 
@@ -409,7 +411,7 @@ async def run_online_local_benchmark_example() -> None:
         target=model,
         dataset=ONLINE_BENCHMARK_DATASET,
         prompt_template=ONLINE_CHAT_PROMPT_TEMPLATE,
-        config=RunConfig(parallelism=4),
+        config=RunConfigOnlineModel(parallelism=4),
     )
     benchmark_result.print_summary()
     print(f"Online benchmark metric keys: {list(benchmark_result.per_metric)}")
@@ -517,7 +519,8 @@ async def run_online_local_llm_judge_example() -> None:
         metrics=llm_judge_metric,
         target=model_with_custom_headers,
         dataset=ONLINE_JUDGE_DATASET,
-        config=RunConfig(parallelism=2),
+        prompt_template=ONLINE_CHAT_PROMPT_TEMPLATE,
+        config=RunConfigOnlineModel(parallelism=2),
     )
     llm_judge_result.print_summary()
 
@@ -531,7 +534,7 @@ def run_sync_example() -> None:
 
     evaluator = Evaluator()
     result = evaluator.run_sync(
-        metrics=ExactMatchMetric(reference="{{item.reference}}"),
+        metrics=ExactMatchMetric(reference="{{item.reference}}", candidate="{{item.actual}}"),
         dataset=OFFLINE_EXACT_MATCH_DATASET[:1],  # Only run the first sample
         config=RunConfig(parallelism=1),
     )
@@ -548,12 +551,14 @@ async def run_examples() -> None:
     #### Local backend examples ####
     await run_offline_local_exact_match_example()
     await run_online_local_exact_match_example()
+    await run_offline_local_multi_metric_example()
     await run_offline_local_llm_judge_example()
     await run_online_local_llm_judge_example()
     await run_offline_local_benchmark_example()
     await run_online_local_benchmark_example()
     await run_local_benchmark_with_metric_failure_example()
     await run_local_metric_with_template_failure_example()
+    run_sync_example()
 
 
 if __name__ == "__main__":

packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/datasets/loader.py

@@ -83,6 +83,9 @@ def normalize_dataset(
 
     path = Path(dataset)
     if not path.exists():
+        if pattern is None and is_glob_pattern(str(path)):
+            base_path, path_pattern = split_glob_path(path)
+            return load_dataset_as_dicts(base_path, path_pattern)
         raise FileNotFoundError(f"Dataset path does not exist: {path}")
 
     if path.is_dir():
@@ -128,6 +131,30 @@ def is_glob_pattern(pattern: str) -> bool:
     return any(c in pattern for c in glob_chars)
 
 
+def split_glob_path(path: Path) -> tuple[Path, str]:
+    """Split a glob path into a concrete base directory and relative glob pattern.
+
+    Args:
+        path: File path that contains at least one glob metacharacter.
+
+    Returns:
+        A base directory before the first glob segment and the remaining relative
+        glob pattern.
+
+    Raises:
+        ValueError: If ``path`` does not contain glob metacharacters.
+    """
+    parts = path.parts
+    for index, part in enumerate(parts):
+        if is_glob_pattern(part):
+            if index == 0:
+                base_path = Path(path.anchor) if path.is_absolute() else Path(".")
+            else:
+                base_path = Path(*parts[:index])
+            return base_path, str(Path(*parts[index:]))
+    raise ValueError(f"Path does not contain a glob pattern: {path}")
+
+
 def discover_files(base_path: Path, pattern: str | None) -> list[Path]:
     """Resolve dataset files under a base path.
 
@@ -150,18 +177,19 @@ def discover_files(base_path: Path, pattern: str | None) -> list[Path]:
             raise DatasetLoadError(f"No files found in {base_path}")
         return files
 
+    file_path = base_path / pattern
+    if file_path.exists():
+        if not file_path.is_file():
+            raise DatasetLoadError(f"Path is not a file: {file_path}")
+        return [file_path]
+
     if is_glob_pattern(pattern):
         files = list(base_path.glob(pattern))
         if not files:
             raise DatasetLoadError(f"No files found matching pattern '{pattern}' in {base_path}")
         return [f for f in files if f.is_file()]
 
-    file_path = base_path / pattern
-    if not file_path.exists():
-        raise DatasetLoadError(f"File not found: {file_path}")
-    if not file_path.is_file():
-        raise DatasetLoadError(f"Path is not a file: {file_path}")
-    return [file_path]
+    raise DatasetLoadError(f"File not found: {file_path}")
 
 
 def _discover_files(base_path: Path, pattern: str | None) -> list[Path]:

packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/execution/README.md

@@ -13,8 +13,6 @@ The execution package exposes a single public entrypoint:
 - `RunConfig`
   A Pydantic config type for execution-only settings such as parallelism and
   sample limits.
-- `EvaluationRequest`
-  The normalized request object passed into evaluation backends.
 - `EvaluationBackend`
   The protocol implemented by result-returning execution backends.
 

packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/execution/backends/base.py

@@ -6,26 +6,52 @@
 from __future__ import annotations
 
 from collections.abc import Sequence
-from typing import Protocol
+from pathlib import Path
+from typing import Any, Protocol
 
-from nemo_evaluator_sdk.execution.config import EvaluationRequest
+from nemo_evaluator_sdk.inference import PostprocessResponse, PreprocessRequest
 from nemo_evaluator_sdk.metrics.protocol import Metric
+from nemo_evaluator_sdk.values import (
+    Agent,
+    DatasetInput,
+    FieldMapping,
+    Model,
+    RunConfig,
+    RunConfigOnline,
+    RunConfigOnlineModel,
+)
 from nemo_evaluator_sdk.values.multi_metric_results import BenchmarkEvaluationResult
-from nemo_evaluator_sdk.values.results import EvaluationResult
+from nemo_evaluator_sdk.values.results import AggregateFieldName, EvaluationResult
+
+BackendParams = RunConfig | RunConfigOnline | RunConfigOnlineModel
 
 
 class EvaluationBackend(Protocol):
     async def evaluate(
         self,
         *,
         metric: Metric,
-        request: EvaluationRequest,
+        dataset: DatasetInput | str | Path,
+        params: BackendParams,
+        target: Model | Agent | None = None,
+        field_mapping: FieldMapping | None = None,
+        prompt_template: str | dict[str, Any] | None = None,
+        aggregate_fields: tuple[AggregateFieldName, ...] | None = None,
+        preprocess_hooks: tuple[PreprocessRequest, ...] | None = None,
+        postprocess_hooks: tuple[PostprocessResponse, ...] | None = None,
     ) -> EvaluationResult:
         """Evaluate one metric directly and return the completed result.
 
         Args:
-            metric: Metric to execute.
-            request: Normalized evaluator request shared across backends.
+            metric: Metric to prepare and execute.
+            dataset: Inline dataset rows, a dataset file, or a dataset directory/glob path.
+            params: Validated run configuration for the selected target mode.
+            target: Optional model or agent used to generate candidate responses before scoring.
+            field_mapping: Optional mapping from canonical evaluator fields to dataset columns.
+            prompt_template: Optional prompt template for online target generation.
+            aggregate_fields: Optional aggregate score fields to keep in the returned result.
+            preprocess_hooks: Optional request preprocess hooks for online execution.
+            postprocess_hooks: Optional response postprocess hooks for online execution.
 
         Returns:
             The completed single-metric evaluation result.
@@ -36,13 +62,27 @@ async def evaluate_benchmark(
         self,
         *,
         metrics: Sequence[Metric],
-        request: EvaluationRequest,
+        dataset: DatasetInput | str | Path,
+        params: BackendParams,
+        target: Model | Agent | None = None,
+        field_mapping: FieldMapping | None = None,
+        prompt_template: str | dict[str, Any] | None = None,
+        aggregate_fields: tuple[AggregateFieldName, ...] | None = None,
+        preprocess_hooks: tuple[PreprocessRequest, ...] | None = None,
+        postprocess_hooks: tuple[PostprocessResponse, ...] | None = None,
     ) -> BenchmarkEvaluationResult:
         """Evaluate multiple metrics directly and return the completed result.
 
         Args:
-            metrics: Metrics to execute together.
-            request: Normalized evaluator request shared across backends.
+            metrics: Metrics to prepare and execute together.
+            dataset: Inline dataset rows, a dataset file, or a dataset directory/glob path.
+            params: Validated run configuration for the selected target mode.
+            target: Optional model or agent used to generate candidate responses before scoring.
+            field_mapping: Optional mapping from canonical evaluator fields to dataset columns.
+            prompt_template: Optional prompt template for online target generation.
+            aggregate_fields: Optional aggregate score fields to keep in the returned result.
+            preprocess_hooks: Optional request preprocess hooks for online execution.
+            postprocess_hooks: Optional response postprocess hooks for online execution.
 
         Returns:
             The completed multi-metric evaluation result.
@@ -55,13 +95,27 @@ def evaluate(
         self,
         *,
         metric: Metric,
-        request: EvaluationRequest,
+        dataset: DatasetInput | str | Path,
+        params: BackendParams,
+        target: Model | Agent | None = None,
+        field_mapping: FieldMapping | None = None,
+        prompt_template: str | dict[str, Any] | None = None,
+        aggregate_fields: tuple[AggregateFieldName, ...] | None = None,
+        preprocess_hooks: tuple[PreprocessRequest, ...] | None = None,
+        postprocess_hooks: tuple[PostprocessResponse, ...] | None = None,
     ) -> EvaluationResult:
         """Evaluate one metric directly and return the completed result.
 
         Args:
-            metric: Metric to execute.
-            request: Normalized evaluator request shared across backends.
+            metric: Metric to prepare and execute.
+            dataset: Inline dataset rows, a dataset file, or a dataset directory/glob path.
+            params: Validated run configuration for the selected target mode.
+            target: Optional model or agent used to generate candidate responses before scoring.
+            field_mapping: Optional mapping from canonical evaluator fields to dataset columns.
+            prompt_template: Optional prompt template for online target generation.
+            aggregate_fields: Optional aggregate score fields to keep in the returned result.
+            preprocess_hooks: Optional request preprocess hooks for online execution.
+            postprocess_hooks: Optional response postprocess hooks for online execution.
 
         Returns:
             The completed single-metric evaluation result.
@@ -72,13 +126,27 @@ def evaluate_benchmark(
         self,
         *,
         metrics: Sequence[Metric],
-        request: EvaluationRequest,
+        dataset: DatasetInput | str | Path,
+        params: BackendParams,
+        target: Model | Agent | None = None,
+        field_mapping: FieldMapping | None = None,
+        prompt_template: str | dict[str, Any] | None = None,
+        aggregate_fields: tuple[AggregateFieldName, ...] | None = None,
+        preprocess_hooks: tuple[PreprocessRequest, ...] | None = None,
+        postprocess_hooks: tuple[PostprocessResponse, ...] | None = None,
     ) -> BenchmarkEvaluationResult:
         """Evaluate multiple metrics directly and return the completed result.
 
         Args:
-            metrics: Metrics to execute together.
-            request: Normalized evaluator request shared across backends.
+            metrics: Metrics to prepare and execute together.
+            dataset: Inline dataset rows, a dataset file, or a dataset directory/glob path.
+            params: Validated run configuration for the selected target mode.
+            target: Optional model or agent used to generate candidate responses before scoring.
+            field_mapping: Optional mapping from canonical evaluator fields to dataset columns.
+            prompt_template: Optional prompt template for online target generation.
+            aggregate_fields: Optional aggregate score fields to keep in the returned result.
+            preprocess_hooks: Optional request preprocess hooks for online execution.
+            postprocess_hooks: Optional response postprocess hooks for online execution.
 
         Returns:
             The completed multi-metric result.