diff --git a/fern/versions/latest/pages/reference/cli-commands.mdx b/fern/versions/latest/pages/reference/cli-commands.mdx index 599be23c20..14e12b6d99 100644 --- a/fern/versions/latest/pages/reference/cli-commands.mdx +++ b/fern/versions/latest/pages/reference/cli-commands.mdx @@ -625,7 +625,7 @@ Collate data, start the servers, and collect rollouts. This is the main evaluati | `--num-repeats` | Rollouts per task (for mean@k metrics). Pass an int to apply to every task, or a dict keyed by `agent_ref.name` for per-agent counts (e.g. `'{simple_agent: 32, swe_agent: 1}'`) when one input file mixes agents. In dict form, the special key `_default` is the fallback for agents not explicitly listed; without it, any unlisted row's agent raises a single consolidated error. | | `--prompt-config` | Prompt template YAML to apply. | | `--concurrency` | Maximum number of concurrent samples. | -| `--split` | Dataset split to use (`train`, `validation`, or `benchmark`). | +| `--split` | Dataset split to use (`train`, `validation`, or `benchmark`). A dataset of the matching `type` must be declared in the loaded configs. `example` datasets are smoke-test samples, not a runnable split — run them with `--no-serve` and `--input` as shown in the [Quickstart](/get-started/quickstart). | | `--model`, `-m` | Served model identifier. | | `--model-url` | Model server base URL. | | `--model-api-key` | Model server API key. | diff --git a/nemo_gym/cli/eval.py b/nemo_gym/cli/eval.py index 6a679effcc..fbb0fc8958 100644 --- a/nemo_gym/cli/eval.py +++ b/nemo_gym/cli/eval.py @@ -40,7 +40,13 @@ print_rich_table, render_component_inspection, ) -from nemo_gym.config_types import BaseNeMoGymCLIConfig, BenchmarkDatasetConfig, ConfigError, ConfigPathNotFoundError +from nemo_gym.config_types import ( + BaseNeMoGymCLIConfig, + BenchmarkDatasetConfig, + ConfigError, + ConfigPathNotFoundError, + ServerInstanceConfig, +) from nemo_gym.discovery import read_config_metadata from nemo_gym.global_config import ( COMPONENT_NAME_KEY_NAME, @@ -48,6 +54,7 @@ QUERY_KEY_NAME, ROLLOUT_INDEX_KEY_NAME, TASK_INDEX_KEY_NAME, + GlobalConfigDictParser, GlobalConfigDictParserConfig, get_first_server_config_dict, get_global_config_dict, @@ -337,6 +344,55 @@ def prepare_benchmark() -> None: list(tqdm(results, total=len(validated))) +def _validate_split_datasets_declared(split: str, server_instance_configs: Sequence[ServerInstanceConfig]) -> None: + """Fail fast when no config declares a dataset of the requested split's type. + + Data preparation silently produces nothing for such a split, so without this check the run + walks the entire preparation sequence (including its success banners) and only dies later + trying to read the collated split file. + """ + declared_lines: List[str] = [] + declared_types: set = set() + example_fpaths: List[str] = [] + for c in server_instance_configs: + if c.SERVER_TYPE not in ("responses_api_agents", "resources_servers"): + continue + for d in c.datasets or []: + declared_types.add(d.type) + declared_lines.append(f"- {c.name}: {d.name} (type: {d.type})") + if d.type == "example": + example_fpaths.append(str(d.jsonl_fpath)) + if split in declared_types: + return + + declared_str = "\n".join(declared_lines) if declared_lines else "- (none)" + message = ( + f"No dataset of type `{split}` is declared in this config, so `--split {split}` has nothing to run.\n" + f"Declared datasets:\n{declared_str}" + ) + if example_fpaths: + example_fpaths_str = "\n".join( + f" gym eval run --no-serve --input {fpath} --output .jsonl" for fpath in example_fpaths + ) + message += ( + "\nExample datasets are committed smoke-test samples and are not runnable via --split. " + "To run one, start the servers (gym env start ...) and collect against the file directly:\n" + f"{example_fpaths_str}" + ) + raise ConfigError(message) + + +def _validate_prepared_split_file_exists(input_jsonl_fpath: Path, split: str, output_dirpath: Path) -> None: + """Explicit check (not an assert: user-facing, and must survive `python -O`).""" + if input_jsonl_fpath.exists(): + return + prepared = sorted(p.name for p in output_dirpath.glob("*.jsonl")) if output_dirpath.exists() else [] + raise ConfigError( + f"Data preparation did not produce `{input_jsonl_fpath}` for split `{split}`. " + f"Files prepared under `{output_dirpath}`: {prepared if prepared else 'none'}." + ) + + @exit_cleanly_on_config_error def e2e_rollout_collection(): # pragma: no cover from nemo_gym.rollout_collection import ( @@ -361,6 +417,9 @@ def e2e_rollout_collection(): # pragma: no cover data_process_output_dir = output_fpath.with_suffix("") / "preprocessed_datasets" data_processor_config_dict["output_dirpath"] = str(data_process_output_dir) + server_instance_configs = GlobalConfigDictParser().filter_for_server_instance_configs(global_config_dict) + _validate_split_datasets_declared(e2e_rollout_collection_config.split, server_instance_configs) + input_jsonl_fpath = data_process_output_dir / f"{e2e_rollout_collection_config.split}.jsonl" should_skip_data_processing = ( e2e_rollout_collection_config.reuse_existing_data_preparation and input_jsonl_fpath.exists() @@ -381,7 +440,9 @@ def e2e_rollout_collection(): # pragma: no cover # Convert to RolloutCollectionConfig rollout_collection_config_dict = deepcopy(global_config_dict) with open_dict(rollout_collection_config_dict): - assert input_jsonl_fpath.exists(), input_jsonl_fpath + _validate_prepared_split_file_exists( + input_jsonl_fpath, e2e_rollout_collection_config.split, data_process_output_dir + ) rollout_collection_config_dict["input_jsonl_fpath"] = str(input_jsonl_fpath) rollout_collection_config = RolloutCollectionConfig.model_validate( diff --git a/nemo_gym/rollout_collection.py b/nemo_gym/rollout_collection.py index 7cd1025aa1..fc42fbc190 100644 --- a/nemo_gym/rollout_collection.py +++ b/nemo_gym/rollout_collection.py @@ -20,6 +20,7 @@ import warnings from asyncio import Future, Semaphore from collections import Counter, defaultdict +from collections.abc import Mapping from contextlib import nullcontext from datetime import timedelta from difflib import get_close_matches @@ -645,8 +646,9 @@ class E2ERolloutCollectionConfig(SharedRolloutCollectionConfig): def _reject_input_jsonl_fpath(cls, data): # This config has no input_jsonl_fpath field, so pydantic would silently drop it and # e2e collection would overwrite it with the prepared split path — the user's file - # would be ignored without any indication. - if isinstance(data, dict) and "input_jsonl_fpath" in data: + # would be ignored without any indication. Match on Mapping, not dict: the CLI passes + # an OmegaConf DictConfig, which is a Mapping but not a dict. + if isinstance(data, Mapping) and "input_jsonl_fpath" in data: raise ConfigError( "`input_jsonl_fpath` (-i/--input) is not supported when serving end-to-end: the input is " "always the prepared dataset for the requested split. Either add --no-serve to collect " @@ -655,6 +657,25 @@ def _reject_input_jsonl_fpath(cls, data): ) return data + @model_validator(mode="before") + @classmethod + def _reject_example_split(cls, data): + # `example` is a real dataset type but deliberately not a runnable split: example + # datasets are the committed smoke-test samples the PR data gate validates, and they + # are excluded from prepared splits so they never leak into training or eval data. + # Catch it before the Literal check so the user gets the documented recipe instead of + # a bare "Input should be 'train'". + if isinstance(data, Mapping) and data.get("split") == "example": + raise ConfigError( + "`--split example` is not runnable end-to-end: example datasets are committed " + "smoke-test samples, not prepared train/validation/benchmark splits. To run one, " + "start the servers and point at the example file directly:\n" + " gym env start --resources-server ...\n" + " gym eval run --no-serve --agent --input /data/example.jsonl --output .jsonl\n" + "See the Quickstart: https://docs.nvidia.com/nemo/gym/latest/get-started/quickstart" + ) + return data + class RolloutCollectionConfig(SharedRolloutCollectionConfig): """ diff --git a/tests/unit_tests/test_cli_eval.py b/tests/unit_tests/test_cli_eval.py new file mode 100644 index 0000000000..c7af7e35fb --- /dev/null +++ b/tests/unit_tests/test_cli_eval.py @@ -0,0 +1,104 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from pathlib import Path + +import pytest +from omegaconf import DictConfig + +from nemo_gym.cli.eval import _validate_prepared_split_file_exists, _validate_split_datasets_declared +from nemo_gym.config_types import ConfigError, ResponsesAPIAgentServerInstanceConfig + + +def _make_agent_instance_config(name: str, dataset_specs: list) -> ResponsesAPIAgentServerInstanceConfig: + server_type_config_dict = { + "responses_api_agents": { + "simple_agent": { + "host": "127.0.0.1", + "port": 12345, + "entrypoint": "app.py", + "datasets": [ + { + "name": d["name"], + "type": d["type"], + "jsonl_fpath": d.get("jsonl_fpath", f"path/{d['name']}.jsonl"), + "license": None if d["type"] == "example" else "Apache 2.0", + } + for d in dataset_specs + ], + "resources_server": { + "type": "resources_servers", + "name": f"{name}_resources_server", + }, + "model_server": { + "type": "responses_api_models", + "name": "policy_model", + }, + } + } + } + return ResponsesAPIAgentServerInstanceConfig( + name=name, + server_type_config_dict=DictConfig(server_type_config_dict), + responses_api_agents=server_type_config_dict["responses_api_agents"], + ) + + +class TestValidateSplitDatasetsDeclared: + def test_passes_when_a_dataset_of_the_split_type_is_declared(self) -> None: + configs = [_make_agent_instance_config("my_agent", [{"name": "train_data", "type": "train"}])] + _validate_split_datasets_declared("train", configs) + + def test_fails_fast_when_only_example_data_is_declared(self) -> None: + configs = [ + _make_agent_instance_config( + "example_agent", + [{"name": "example", "type": "example", "jsonl_fpath": "resources_servers/x/data/example.jsonl"}], + ) + ] + with pytest.raises(ConfigError) as exc_info: + _validate_split_datasets_declared("train", configs) + message = str(exc_info.value) + # The error must name the requested split, list what is declared, and give the + # copy-pasteable --no-serve recipe for the example file. + assert "No dataset of type `train`" in message + assert "example_agent: example (type: example)" in message + assert "--no-serve --input resources_servers/x/data/example.jsonl" in message + + def test_fails_when_no_datasets_are_declared_at_all(self) -> None: + configs = [_make_agent_instance_config("bare_agent", [])] + with pytest.raises(ConfigError, match=r"- \(none\)"): + _validate_split_datasets_declared("validation", configs) + + def test_mismatched_split_lists_declared_types(self) -> None: + configs = [_make_agent_instance_config("val_agent", [{"name": "val_data", "type": "validation"}])] + with pytest.raises(ConfigError, match=r"val_agent: val_data \(type: validation\)"): + _validate_split_datasets_declared("train", configs) + + +class TestValidatePreparedSplitFileExists: + def test_passes_when_the_file_exists(self, tmp_path: Path) -> None: + fpath = tmp_path / "train.jsonl" + fpath.write_text("{}\n") + _validate_prepared_split_file_exists(fpath, "train", tmp_path) + + def test_fails_with_the_split_and_the_files_actually_prepared(self, tmp_path: Path) -> None: + (tmp_path / "validation.jsonl").write_text("{}\n") + with pytest.raises(ConfigError, match=r"split `train`.*\['validation.jsonl'\]"): + _validate_prepared_split_file_exists(tmp_path / "train.jsonl", "train", tmp_path) + + def test_fails_with_none_when_the_output_dir_is_missing(self, tmp_path: Path) -> None: + missing_dir = tmp_path / "does_not_exist" + with pytest.raises(ConfigError, match=r"none"): + _validate_prepared_split_file_exists(missing_dir / "train.jsonl", "train", missing_dir) diff --git a/tests/unit_tests/test_rollout_collection.py b/tests/unit_tests/test_rollout_collection.py index eee9f6a718..e40b367504 100644 --- a/tests/unit_tests/test_rollout_collection.py +++ b/tests/unit_tests/test_rollout_collection.py @@ -28,7 +28,8 @@ import pytest import yaml from aiohttp import ClientConnectorError, ClientResponseError, ServerDisconnectedError -from omegaconf import OmegaConf +from omegaconf import DictConfig, OmegaConf +from pydantic import ValidationError import nemo_gym.rollout_collection import nemo_gym.token_id_capture.delivery @@ -3415,6 +3416,20 @@ def test_e2e_config_rejects_input_jsonl_fpath(self) -> None: } ) + def test_e2e_config_rejects_input_jsonl_fpath_from_dictconfig(self) -> None: + # The CLI passes an OmegaConf DictConfig (a Mapping, not a dict). An isinstance(dict) + # check silently let input_jsonl_fpath through on the real path — pin the Mapping match. + with pytest.raises(ConfigError, match=r"not supported when serving end-to-end"): + E2ERolloutCollectionConfig.model_validate( + DictConfig( + { + "output_jsonl_fpath": "out.jsonl", + "split": "train", + "input_jsonl_fpath": "my_data.jsonl", + } + ) + ) + def test_e2e_config_accepts_without_input_jsonl_fpath(self) -> None: config = E2ERolloutCollectionConfig.model_validate({"output_jsonl_fpath": "out.jsonl", "split": "train"}) assert config.split == "train" @@ -3426,6 +3441,17 @@ def test_no_serve_config_still_accepts_input_jsonl_fpath(self) -> None: assert config.input_jsonl_fpath == "my_data.jsonl" +class TestE2EExampleSplitRejected: + @pytest.mark.parametrize("wrap", [dict, DictConfig]) + def test_example_split_gets_actionable_error_not_literal_error(self, wrap) -> None: + with pytest.raises(ConfigError, match=r"--no-serve --agent --input"): + E2ERolloutCollectionConfig.model_validate(wrap({"output_jsonl_fpath": "out.jsonl", "split": "example"})) + + def test_other_invalid_splits_still_fail_literal_validation(self) -> None: + with pytest.raises(ValidationError, match=r"split"): + E2ERolloutCollectionConfig.model_validate({"output_jsonl_fpath": "out.jsonl", "split": "test"}) + + class TestAgentMapRouting: """Pins the agent_map / agent_name routing contract (see dataset-decoupling RFC).