cp: fix(cli): fail fast when gym eval run --split has no matching dataset (2911) into r0.6.0

fern/versions/latest/pages/reference/cli-commands.mdx

@@ -618,7 +618,7 @@ Collate data, start the servers, and collect rollouts. This is the main evaluati
 | `--num-repeats` | Rollouts per task (for mean@k metrics). Pass an int to apply to every task, or a dict keyed by `agent_ref.name` for per-agent counts (e.g. `'{simple_agent: 32, swe_agent: 1}'`) when one input file mixes agents. In dict form, the special key `_default` is the fallback for agents not explicitly listed; without it, any unlisted row's agent raises a single consolidated error. |
 | `--prompt-config` | Prompt template YAML to apply. |
 | `--concurrency` | Maximum number of concurrent samples. |
-| `--split` | Dataset split to use (`train`, `validation`, or `benchmark`). |
+| `--split` | Dataset split to use (`train`, `validation`, or `benchmark`). A dataset of the matching `type` must be declared in the loaded configs. `example` datasets are smoke-test samples, not a runnable split — run them with `--no-serve` and `--input` as shown in the [Quickstart](/get-started/quickstart). |
 | `--model`, `-m` | Served model identifier. |
 | `--model-url` | Model server base URL. |
 | `--model-api-key` | Model server API key. |

nemo_gym/cli/eval.py

@@ -40,14 +40,21 @@
     print_rich_table,
     render_component_inspection,
 )
-from nemo_gym.config_types import BaseNeMoGymCLIConfig, BenchmarkDatasetConfig, ConfigError, ConfigPathNotFoundError
+from nemo_gym.config_types import (
+    BaseNeMoGymCLIConfig,
+    BenchmarkDatasetConfig,
+    ConfigError,
+    ConfigPathNotFoundError,
+    ServerInstanceConfig,
+)
 from nemo_gym.discovery import read_config_metadata
 from nemo_gym.global_config import (
     COMPONENT_NAME_KEY_NAME,
     JSON_OUTPUT_KEY_NAME,
     QUERY_KEY_NAME,
     ROLLOUT_INDEX_KEY_NAME,
     TASK_INDEX_KEY_NAME,
+    GlobalConfigDictParser,
     GlobalConfigDictParserConfig,
     get_first_server_config_dict,
     get_global_config_dict,
@@ -337,6 +344,55 @@ def prepare_benchmark() -> None:
         list(tqdm(results, total=len(validated)))
 
 
+def _validate_split_datasets_declared(split: str, server_instance_configs: Sequence[ServerInstanceConfig]) -> None:
+    """Fail fast when no config declares a dataset of the requested split's type.
+
+    Data preparation silently produces nothing for such a split, so without this check the run
+    walks the entire preparation sequence (including its success banners) and only dies later
+    trying to read the collated split file.
+    """
+    declared_lines: List[str] = []
+    declared_types: set = set()
+    example_fpaths: List[str] = []
+    for c in server_instance_configs:
+        if c.SERVER_TYPE not in ("responses_api_agents", "resources_servers"):
+            continue
+        for d in c.datasets or []:
+            declared_types.add(d.type)
+            declared_lines.append(f"- {c.name}: {d.name} (type: {d.type})")
+            if d.type == "example":
+                example_fpaths.append(str(d.jsonl_fpath))
+    if split in declared_types:
+        return
+
+    declared_str = "\n".join(declared_lines) if declared_lines else "- (none)"
+    message = (
+        f"No dataset of type `{split}` is declared in this config, so `--split {split}` has nothing to run.\n"
+        f"Declared datasets:\n{declared_str}"
+    )
+    if example_fpaths:
+        example_fpaths_str = "\n".join(
+            f"  gym eval run --no-serve --input {fpath} --output <out>.jsonl" for fpath in example_fpaths
+        )
+        message += (
+            "\nExample datasets are committed smoke-test samples and are not runnable via --split. "
+            "To run one, start the servers (gym env start ...) and collect against the file directly:\n"
+            f"{example_fpaths_str}"
+        )
+    raise ConfigError(message)
+
+
+def _validate_prepared_split_file_exists(input_jsonl_fpath: Path, split: str, output_dirpath: Path) -> None:
+    """Explicit check (not an assert: user-facing, and must survive `python -O`)."""
+    if input_jsonl_fpath.exists():
+        return
+    prepared = sorted(p.name for p in output_dirpath.glob("*.jsonl")) if output_dirpath.exists() else []
+    raise ConfigError(
+        f"Data preparation did not produce `{input_jsonl_fpath}` for split `{split}`. "
+        f"Files prepared under `{output_dirpath}`: {prepared if prepared else 'none'}."
+    )
+
+
 @exit_cleanly_on_config_error
 def e2e_rollout_collection():  # pragma: no cover
     from nemo_gym.rollout_collection import (
@@ -361,6 +417,9 @@ def e2e_rollout_collection():  # pragma: no cover
         data_process_output_dir = output_fpath.with_suffix("") / "preprocessed_datasets"
         data_processor_config_dict["output_dirpath"] = str(data_process_output_dir)
 
+    server_instance_configs = GlobalConfigDictParser().filter_for_server_instance_configs(global_config_dict)
+    _validate_split_datasets_declared(e2e_rollout_collection_config.split, server_instance_configs)
+
     input_jsonl_fpath = data_process_output_dir / f"{e2e_rollout_collection_config.split}.jsonl"
     should_skip_data_processing = (
         e2e_rollout_collection_config.reuse_existing_data_preparation and input_jsonl_fpath.exists()
@@ -381,7 +440,9 @@ def e2e_rollout_collection():  # pragma: no cover
     # Convert to RolloutCollectionConfig
     rollout_collection_config_dict = deepcopy(global_config_dict)
     with open_dict(rollout_collection_config_dict):
-        assert input_jsonl_fpath.exists(), input_jsonl_fpath
+        _validate_prepared_split_file_exists(
+            input_jsonl_fpath, e2e_rollout_collection_config.split, data_process_output_dir
+        )
         rollout_collection_config_dict["input_jsonl_fpath"] = str(input_jsonl_fpath)
 
     rollout_collection_config = RolloutCollectionConfig.model_validate(

nemo_gym/rollout_collection.py

@@ -20,6 +20,7 @@
 import warnings
 from asyncio import Future, Semaphore
 from collections import Counter, defaultdict
+from collections.abc import Mapping
 from contextlib import nullcontext
 from datetime import timedelta
 from difflib import get_close_matches
@@ -636,8 +637,9 @@ class E2ERolloutCollectionConfig(SharedRolloutCollectionConfig):
     def _reject_input_jsonl_fpath(cls, data):
         # This config has no input_jsonl_fpath field, so pydantic would silently drop it and
         # e2e collection would overwrite it with the prepared split path — the user's file
-        # would be ignored without any indication.
-        if isinstance(data, dict) and "input_jsonl_fpath" in data:
+        # would be ignored without any indication. Match on Mapping, not dict: the CLI passes
+        # an OmegaConf DictConfig, which is a Mapping but not a dict.
+        if isinstance(data, Mapping) and "input_jsonl_fpath" in data:
             raise ConfigError(
                 "`input_jsonl_fpath` (-i/--input) is not supported when serving end-to-end: the input is "
                 "always the prepared dataset for the requested split. Either add --no-serve to collect "
@@ -646,6 +648,25 @@ def _reject_input_jsonl_fpath(cls, data):
             )
         return data
 
+    @model_validator(mode="before")
+    @classmethod
+    def _reject_example_split(cls, data):
+        # `example` is a real dataset type but deliberately not a runnable split: example
+        # datasets are the committed smoke-test samples the PR data gate validates, and they
+        # are excluded from prepared splits so they never leak into training or eval data.
+        # Catch it before the Literal check so the user gets the documented recipe instead of
+        # a bare "Input should be 'train'".
+        if isinstance(data, Mapping) and data.get("split") == "example":
+            raise ConfigError(
+                "`--split example` is not runnable end-to-end: example datasets are committed "
+                "smoke-test samples, not prepared train/validation/benchmark splits. To run one, "
+                "start the servers and point at the example file directly:\n"
+                "  gym env start --resources-server <server> ...\n"
+                "  gym eval run --no-serve --agent <agent> --input <server_dir>/data/example.jsonl --output <out>.jsonl\n"
+                "See the Quickstart: https://docs.nvidia.com/nemo/gym/latest/get-started/quickstart"
+            )
+        return data
+
 
 class RolloutCollectionConfig(SharedRolloutCollectionConfig):
     """

tests/unit_tests/test_cli_eval.py

@@ -0,0 +1,104 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+from pathlib import Path
+
+import pytest
+from omegaconf import DictConfig
+
+from nemo_gym.cli.eval import _validate_prepared_split_file_exists, _validate_split_datasets_declared
+from nemo_gym.config_types import ConfigError, ResponsesAPIAgentServerInstanceConfig
+
+
+def _make_agent_instance_config(name: str, dataset_specs: list) -> ResponsesAPIAgentServerInstanceConfig:
+    server_type_config_dict = {
+        "responses_api_agents": {
+            "simple_agent": {
+                "host": "127.0.0.1",
+                "port": 12345,
+                "entrypoint": "app.py",
+                "datasets": [
+                    {
+                        "name": d["name"],
+                        "type": d["type"],
+                        "jsonl_fpath": d.get("jsonl_fpath", f"path/{d['name']}.jsonl"),
+                        "license": None if d["type"] == "example" else "Apache 2.0",
+                    }
+                    for d in dataset_specs
+                ],
+                "resources_server": {
+                    "type": "resources_servers",
+                    "name": f"{name}_resources_server",
+                },
+                "model_server": {
+                    "type": "responses_api_models",
+                    "name": "policy_model",
+                },
+            }
+        }
+    }
+    return ResponsesAPIAgentServerInstanceConfig(
+        name=name,
+        server_type_config_dict=DictConfig(server_type_config_dict),
+        responses_api_agents=server_type_config_dict["responses_api_agents"],
+    )
+
+
+class TestValidateSplitDatasetsDeclared:
+    def test_passes_when_a_dataset_of_the_split_type_is_declared(self) -> None:
+        configs = [_make_agent_instance_config("my_agent", [{"name": "train_data", "type": "train"}])]
+        _validate_split_datasets_declared("train", configs)
+
+    def test_fails_fast_when_only_example_data_is_declared(self) -> None:
+        configs = [
+            _make_agent_instance_config(
+                "example_agent",
+                [{"name": "example", "type": "example", "jsonl_fpath": "resources_servers/x/data/example.jsonl"}],
+            )
+        ]
+        with pytest.raises(ConfigError) as exc_info:
+            _validate_split_datasets_declared("train", configs)
+        message = str(exc_info.value)
+        # The error must name the requested split, list what is declared, and give the
+        # copy-pasteable --no-serve recipe for the example file.
+        assert "No dataset of type `train`" in message
+        assert "example_agent: example (type: example)" in message
+        assert "--no-serve --input resources_servers/x/data/example.jsonl" in message
+
+    def test_fails_when_no_datasets_are_declared_at_all(self) -> None:
+        configs = [_make_agent_instance_config("bare_agent", [])]
+        with pytest.raises(ConfigError, match=r"- \(none\)"):
+            _validate_split_datasets_declared("validation", configs)
+
+    def test_mismatched_split_lists_declared_types(self) -> None:
+        configs = [_make_agent_instance_config("val_agent", [{"name": "val_data", "type": "validation"}])]
+        with pytest.raises(ConfigError, match=r"val_agent: val_data \(type: validation\)"):
+            _validate_split_datasets_declared("train", configs)
+
+
+class TestValidatePreparedSplitFileExists:
+    def test_passes_when_the_file_exists(self, tmp_path: Path) -> None:
+        fpath = tmp_path / "train.jsonl"
+        fpath.write_text("{}\n")
+        _validate_prepared_split_file_exists(fpath, "train", tmp_path)
+
+    def test_fails_with_the_split_and_the_files_actually_prepared(self, tmp_path: Path) -> None:
+        (tmp_path / "validation.jsonl").write_text("{}\n")
+        with pytest.raises(ConfigError, match=r"split `train`.*\['validation.jsonl'\]"):
+            _validate_prepared_split_file_exists(tmp_path / "train.jsonl", "train", tmp_path)
+
+    def test_fails_with_none_when_the_output_dir_is_missing(self, tmp_path: Path) -> None:
+        missing_dir = tmp_path / "does_not_exist"
+        with pytest.raises(ConfigError, match=r"none"):
+            _validate_prepared_split_file_exists(missing_dir / "train.jsonl", "train", missing_dir)

tests/unit_tests/test_rollout_collection.py

@@ -28,7 +28,8 @@
 import pytest
 import yaml
 from aiohttp import ClientConnectorError, ClientResponseError, ServerDisconnectedError
-from omegaconf import OmegaConf
+from omegaconf import DictConfig, OmegaConf
+from pydantic import ValidationError
 
 import nemo_gym.rollout_collection
 import nemo_gym.token_id_capture.delivery
@@ -3290,6 +3291,20 @@ def test_e2e_config_rejects_input_jsonl_fpath(self) -> None:
                 }
             )
 
+    def test_e2e_config_rejects_input_jsonl_fpath_from_dictconfig(self) -> None:
+        # The CLI passes an OmegaConf DictConfig (a Mapping, not a dict). An isinstance(dict)
+        # check silently let input_jsonl_fpath through on the real path — pin the Mapping match.
+        with pytest.raises(ConfigError, match=r"not supported when serving end-to-end"):
+            E2ERolloutCollectionConfig.model_validate(
+                DictConfig(
+                    {
+                        "output_jsonl_fpath": "out.jsonl",
+                        "split": "train",
+                        "input_jsonl_fpath": "my_data.jsonl",
+                    }
+                )
+            )
+
     def test_e2e_config_accepts_without_input_jsonl_fpath(self) -> None:
         config = E2ERolloutCollectionConfig.model_validate({"output_jsonl_fpath": "out.jsonl", "split": "train"})
         assert config.split == "train"
@@ -3301,6 +3316,17 @@ def test_no_serve_config_still_accepts_input_jsonl_fpath(self) -> None:
         assert config.input_jsonl_fpath == "my_data.jsonl"
 
 
+class TestE2EExampleSplitRejected:
+    @pytest.mark.parametrize("wrap", [dict, DictConfig])
+    def test_example_split_gets_actionable_error_not_literal_error(self, wrap) -> None:
+        with pytest.raises(ConfigError, match=r"--no-serve --agent <agent> --input"):
+            E2ERolloutCollectionConfig.model_validate(wrap({"output_jsonl_fpath": "out.jsonl", "split": "example"}))
+
+    def test_other_invalid_splits_still_fail_literal_validation(self) -> None:
+        with pytest.raises(ValidationError, match=r"split"):
+            E2ERolloutCollectionConfig.model_validate({"output_jsonl_fpath": "out.jsonl", "split": "test"})
+
+
 class TestAgentMapRouting:
     """Pins the agent_map / agent_name routing contract (see dataset-decoupling RFC).