From ee3c521d2316f176f74fb7b574c3381c73ca046c Mon Sep 17 00:00:00 2001 From: Wojciech Prazuch Date: Mon, 29 Jun 2026 14:40:38 +0200 Subject: [PATCH 1/2] fix: make gym env test + dataset collate work from a wheel install Completes the run-from-wheel story (epic #1205 C5). `gym env start` of a built-in already works once nemo-gym is published (setup_command's non-editable path installs `nemo-gym==` from the index); this fixes the two remaining tooling gaps: - gym env test: the per-server venv installed bare `nemo-gym` from the index, dropping the `[dev]` extra, so `&& pytest` failed (exit 127). Add an opt-in include_dev_extra to setup_env_command; _test_single passes it so the test venv gets pytest. gym env start stays lean (no test deps). - gym dataset collate: the per-dataset artifact writes (_metrics.json / _prepare.jsonl, derived from the raw cwd-relative jsonl_fpath) crashed with FileNotFoundError when the dir didn't exist in the cwd (e.g. a built-in dataset resolved from the install root via #1806). mkdir(parents) the artifact dir before writing. Tests: include_dev_extra installs nemo-gym[dev]; collate creates a missing parent dir. Signed-off-by: Wojciech Prazuch --- nemo_gym/cli/env.py | 6 ++++- nemo_gym/cli/setup_command.py | 12 ++++++--- nemo_gym/train_data_utils.py | 6 +++++ tests/unit_tests/test_cli_setup_command.py | 31 ++++++++++++++++++++++ tests/unit_tests/test_train_data_utils.py | 20 ++++++++++++++ 5 files changed, 71 insertions(+), 4 deletions(-) diff --git a/nemo_gym/cli/env.py b/nemo_gym/cli/env.py index 5513f9e621..84a6a727b4 100644 --- a/nemo_gym/cli/env.py +++ b/nemo_gym/cli/env.py @@ -533,7 +533,11 @@ def _validate_data_single(test_config: TestConfig) -> None: # pragma: no cover def _test_single(test_config: TestConfig, global_config_dict: DictConfig) -> Popen: # pragma: no cover # Eventually we may want more sophisticated testing here, but this is sufficient for now. prefix = test_config.entrypoint.replace("/", "\\/") - command = f"""{setup_env_command(test_config.dir_path, global_config_dict, prefix)} && pytest""" + # include_dev_extra=True so the per-server venv gets pytest (nemo-gym's `[dev]` extra) when + # nemo-gym is installed from an index rather than editable — `gym env test` runs `&& pytest`. + command = ( + f"""{setup_env_command(test_config.dir_path, global_config_dict, prefix, include_dev_extra=True)} && pytest""" + ) return run_command(command, test_config.dir_path) diff --git a/nemo_gym/cli/setup_command.py b/nemo_gym/cli/setup_command.py index c1940d4933..0df17e4d8e 100644 --- a/nemo_gym/cli/setup_command.py +++ b/nemo_gym/cli/setup_command.py @@ -100,8 +100,14 @@ def _get_nemo_gym_version_spec(is_editable_install: bool) -> str: return "" -def setup_env_command(dir_path: Path, global_config_dict: DictConfig, prefix: str) -> str: +def setup_env_command( + dir_path: Path, global_config_dict: DictConfig, prefix: str, include_dev_extra: bool = False +) -> str: head_server_deps = global_config_dict[HEAD_SERVER_DEPS_KEY_NAME] + # `gym env test` opts in to nemo-gym's `[dev]` extra so the per-server venv gets pytest when + # nemo-gym is installed from an index (wheel/PyPI install); `gym env start` stays lean. Editable + # installs already pull `[dev]` via the server's `-e nemo-gym[dev] @ ../../` requirement. + nemo_gym_pkg = "nemo-gym[dev]" if include_dev_extra else "nemo-gym" root_venv_path = global_config_dict[UV_VENV_DIR_KEY_NAME] if Path(root_venv_path).resolve() != PARENT_DIR.resolve(): @@ -146,7 +152,7 @@ def setup_env_command(dir_path: Path, global_config_dict: DictConfig, prefix: st install_flags = _get_nemo_gym_install_flags() version_spec = _get_nemo_gym_version_spec(is_editable_install) install_cmd = ( - f"""uv pip install {verbose_flag}{uv_pip_python_flag}{install_flags}nemo-gym{version_spec} && """ + f"""uv pip install {verbose_flag}{uv_pip_python_flag}{install_flags}{nemo_gym_pkg}{version_spec} && """ f"""uv pip install {verbose_flag}{uv_pip_python_flag}--no-sources '-e .' {" ".join(head_server_deps)}""" ) elif has_requirements_txt: @@ -158,7 +164,7 @@ def setup_env_command(dir_path: Path, global_config_dict: DictConfig, prefix: st install_flags = _get_nemo_gym_install_flags() version_spec = _get_nemo_gym_version_spec(is_editable_install) install_cmd = ( - f"""(echo 'nemo-gym{version_spec}' && grep -v -F '../..' requirements.txt) | """ + f"""(echo '{nemo_gym_pkg}{version_spec}' && grep -v -F '../..' requirements.txt) | """ f"""uv pip install {verbose_flag}{uv_pip_python_flag}{install_flags}-r /dev/stdin {" ".join(head_server_deps)}""" ) else: diff --git a/nemo_gym/train_data_utils.py b/nemo_gym/train_data_utils.py index 054654ecc5..ae48c45d50 100644 --- a/nemo_gym/train_data_utils.py +++ b/nemo_gym/train_data_utils.py @@ -664,6 +664,9 @@ def validate_samples_and_aggregate_metrics( else: continue + # Ensure the artifact dir exists: the metrics file is written next to the dataset's + # (cwd-relative) jsonl_fpath, which may not exist yet when collating from a fresh cwd. + metrics_fpath.parent.mkdir(parents=True, exist_ok=True) with open(metrics_fpath, "w") as f: json.dump(aggregate_metrics_dict, f, indent=4) @@ -709,6 +712,9 @@ def _collate_samples_single_type( data_path = Path(d.jsonl_fpath) prepare_path = data_path.with_name(f"{data_path.stem}_prepare.jsonl") + # Create the artifact dir if needed (the prepared file is written next to the + # cwd-relative jsonl_fpath, which may not exist when collating from a fresh cwd). + prepare_path.parent.mkdir(parents=True, exist_ok=True) with open(prepare_path, "w") as target: for line in self._iter_dataset_lines(d): d = json.loads(line) diff --git a/tests/unit_tests/test_cli_setup_command.py b/tests/unit_tests/test_cli_setup_command.py index 4f531eedc1..39d34d895c 100644 --- a/tests/unit_tests/test_cli_setup_command.py +++ b/tests/unit_tests/test_cli_setup_command.py @@ -199,6 +199,37 @@ def test_installs_from_pypi_when_not_editable( expected_command = f"cd {server_dir} && uv venv --seed --allow-existing --python test python version {server_dir}/.venv > >(sed 's/^/(my server name) /') 2> >(sed 's/^/(my server name) /' >&2) && source {server_dir}/.venv/bin/activate && (echo 'nemo-gym=={version}' && grep -v -F '../..' requirements.txt) | uv pip install -r /dev/stdin ray[default]==test ray version openai==test openai version > >(sed 's/^/(my server name) /') 2> >(sed 's/^/(my server name) /' >&2)" assert expected_command == actual_command + def test_include_dev_extra_installs_nemo_gym_dev_when_not_editable( + self, tmp_path: Path, monkeypatch: MonkeyPatch + ) -> None: + # `gym env test` passes include_dev_extra=True so the index-installed nemo-gym carries the + # `[dev]` extra (pytest); otherwise the per-server test venv would lack pytest. + server_dir = (tmp_path / "first_level" / "second_level").absolute() + server_dir.mkdir(parents=True) + (server_dir / "requirements.txt").write_text("-e nemo-gym[dev] @ ../../\n") + monkeypatch.delenv("NEMO_GYM_ALLOW_PRERELEASE", raising=False) + monkeypatch.delenv("UV_INDEX_URL", raising=False) + monkeypatch.delenv("UV_EXTRA_INDEX_URL", raising=False) + monkeypatch.delenv("UV_INDEX_STRATEGY", raising=False) + + with patch("importlib.metadata.version", return_value="0.3.0"): + actual_command = setup_env_command( + dir_path=server_dir, + global_config_dict=self._debug_global_config_dict(tmp_path), + prefix="my server name", + include_dev_extra=True, + ) + assert "echo 'nemo-gym[dev]==0.3.0'" in actual_command + # The default (include_dev_extra=False) must NOT pull the dev extra. + with patch("importlib.metadata.version", return_value="0.3.0"): + default_command = setup_env_command( + dir_path=server_dir, + global_config_dict=self._debug_global_config_dict(tmp_path), + prefix="my server name", + ) + assert "echo 'nemo-gym==0.3.0'" in default_command + assert "[dev]" not in default_command + @pytest.mark.parametrize("version", ["0.3.0", "0.3.0rc0", "1.0.0", "2.1.3rc1"]) def test_installs_from_pypi_when_not_editable_pyproject( self, tmp_path: Path, version: str, monkeypatch: MonkeyPatch diff --git a/tests/unit_tests/test_train_data_utils.py b/tests/unit_tests/test_train_data_utils.py index 3e3bce0a89..cad0a0e5f0 100644 --- a/tests/unit_tests/test_train_data_utils.py +++ b/tests/unit_tests/test_train_data_utils.py @@ -1120,6 +1120,26 @@ def custom_open(filename, mode="r"): Path("example.jsonl"), ] + def test_collate_creates_missing_parent_dir(self, tmp_path: Path, monkeypatch: MonkeyPatch) -> None: + # The prepared-output file is written next to the dataset's jsonl_fpath. When that dir does not + # exist in the cwd (e.g. a built-in dataset resolved from the install root while collating from + # a fresh cwd), the write must create the parent instead of crashing with FileNotFoundError. + missing_dir = tmp_path / "does" / "not" / "exist" + assert not missing_dir.exists() + cfg = _make_agent_instance_config( + "ex", [{"name": "example", "type": "example", "jsonl_fpath": str(missing_dir / "data.jsonl")}] + ) + processor = TrainDataProcessor() + # Bypass the dataset read so the source file isn't needed; we're exercising the write path. + monkeypatch.setattr(processor, "_iter_dataset_lines", lambda d: iter(['{"foo": "bar"}'])) + + paths = processor._collate_samples_single_type("example", [cfg]) + + prepare_path = missing_dir / "data_prepare.jsonl" + assert paths == [prepare_path] + assert prepare_path.exists() # parent dir auto-created; write did not crash + assert json.loads(prepare_path.read_text().strip())["foo"] == "bar" + def test_collate_samples_metrics_conflict_raises_ValueError(self, monkeypatch: MonkeyPatch) -> None: write_filenames_to_mock = dict() From 6cefd973db58289f3d342e47eae5c981527c4729 Mon Sep 17 00:00:00 2001 From: Wojciech Prazuch Date: Mon, 29 Jun 2026 14:51:29 +0200 Subject: [PATCH 2/2] Drop the env-test [dev] change; keep only the collate mkdir fix Per review: pulling nemo-gym's [dev] extra (pre-commit/mypy/ruff + the pytest deps) into every per-server test venv is too coarse just to get pytest for `gym env test` from a wheel. Reverting the setup_command/_test_single `include_dev_extra` change; `gym env test` from a wheel (a contributor-runs-in-repo edge case) stays a documented limitation. PR #6 now only fixes the clear `gym dataset collate` write crash (mkdir). Signed-off-by: Wojciech Prazuch --- nemo_gym/cli/env.py | 6 +---- nemo_gym/cli/setup_command.py | 12 +++------ tests/unit_tests/test_cli_setup_command.py | 31 ---------------------- 3 files changed, 4 insertions(+), 45 deletions(-) diff --git a/nemo_gym/cli/env.py b/nemo_gym/cli/env.py index 84a6a727b4..5513f9e621 100644 --- a/nemo_gym/cli/env.py +++ b/nemo_gym/cli/env.py @@ -533,11 +533,7 @@ def _validate_data_single(test_config: TestConfig) -> None: # pragma: no cover def _test_single(test_config: TestConfig, global_config_dict: DictConfig) -> Popen: # pragma: no cover # Eventually we may want more sophisticated testing here, but this is sufficient for now. prefix = test_config.entrypoint.replace("/", "\\/") - # include_dev_extra=True so the per-server venv gets pytest (nemo-gym's `[dev]` extra) when - # nemo-gym is installed from an index rather than editable — `gym env test` runs `&& pytest`. - command = ( - f"""{setup_env_command(test_config.dir_path, global_config_dict, prefix, include_dev_extra=True)} && pytest""" - ) + command = f"""{setup_env_command(test_config.dir_path, global_config_dict, prefix)} && pytest""" return run_command(command, test_config.dir_path) diff --git a/nemo_gym/cli/setup_command.py b/nemo_gym/cli/setup_command.py index 0df17e4d8e..c1940d4933 100644 --- a/nemo_gym/cli/setup_command.py +++ b/nemo_gym/cli/setup_command.py @@ -100,14 +100,8 @@ def _get_nemo_gym_version_spec(is_editable_install: bool) -> str: return "" -def setup_env_command( - dir_path: Path, global_config_dict: DictConfig, prefix: str, include_dev_extra: bool = False -) -> str: +def setup_env_command(dir_path: Path, global_config_dict: DictConfig, prefix: str) -> str: head_server_deps = global_config_dict[HEAD_SERVER_DEPS_KEY_NAME] - # `gym env test` opts in to nemo-gym's `[dev]` extra so the per-server venv gets pytest when - # nemo-gym is installed from an index (wheel/PyPI install); `gym env start` stays lean. Editable - # installs already pull `[dev]` via the server's `-e nemo-gym[dev] @ ../../` requirement. - nemo_gym_pkg = "nemo-gym[dev]" if include_dev_extra else "nemo-gym" root_venv_path = global_config_dict[UV_VENV_DIR_KEY_NAME] if Path(root_venv_path).resolve() != PARENT_DIR.resolve(): @@ -152,7 +146,7 @@ def setup_env_command( install_flags = _get_nemo_gym_install_flags() version_spec = _get_nemo_gym_version_spec(is_editable_install) install_cmd = ( - f"""uv pip install {verbose_flag}{uv_pip_python_flag}{install_flags}{nemo_gym_pkg}{version_spec} && """ + f"""uv pip install {verbose_flag}{uv_pip_python_flag}{install_flags}nemo-gym{version_spec} && """ f"""uv pip install {verbose_flag}{uv_pip_python_flag}--no-sources '-e .' {" ".join(head_server_deps)}""" ) elif has_requirements_txt: @@ -164,7 +158,7 @@ def setup_env_command( install_flags = _get_nemo_gym_install_flags() version_spec = _get_nemo_gym_version_spec(is_editable_install) install_cmd = ( - f"""(echo '{nemo_gym_pkg}{version_spec}' && grep -v -F '../..' requirements.txt) | """ + f"""(echo 'nemo-gym{version_spec}' && grep -v -F '../..' requirements.txt) | """ f"""uv pip install {verbose_flag}{uv_pip_python_flag}{install_flags}-r /dev/stdin {" ".join(head_server_deps)}""" ) else: diff --git a/tests/unit_tests/test_cli_setup_command.py b/tests/unit_tests/test_cli_setup_command.py index 39d34d895c..4f531eedc1 100644 --- a/tests/unit_tests/test_cli_setup_command.py +++ b/tests/unit_tests/test_cli_setup_command.py @@ -199,37 +199,6 @@ def test_installs_from_pypi_when_not_editable( expected_command = f"cd {server_dir} && uv venv --seed --allow-existing --python test python version {server_dir}/.venv > >(sed 's/^/(my server name) /') 2> >(sed 's/^/(my server name) /' >&2) && source {server_dir}/.venv/bin/activate && (echo 'nemo-gym=={version}' && grep -v -F '../..' requirements.txt) | uv pip install -r /dev/stdin ray[default]==test ray version openai==test openai version > >(sed 's/^/(my server name) /') 2> >(sed 's/^/(my server name) /' >&2)" assert expected_command == actual_command - def test_include_dev_extra_installs_nemo_gym_dev_when_not_editable( - self, tmp_path: Path, monkeypatch: MonkeyPatch - ) -> None: - # `gym env test` passes include_dev_extra=True so the index-installed nemo-gym carries the - # `[dev]` extra (pytest); otherwise the per-server test venv would lack pytest. - server_dir = (tmp_path / "first_level" / "second_level").absolute() - server_dir.mkdir(parents=True) - (server_dir / "requirements.txt").write_text("-e nemo-gym[dev] @ ../../\n") - monkeypatch.delenv("NEMO_GYM_ALLOW_PRERELEASE", raising=False) - monkeypatch.delenv("UV_INDEX_URL", raising=False) - monkeypatch.delenv("UV_EXTRA_INDEX_URL", raising=False) - monkeypatch.delenv("UV_INDEX_STRATEGY", raising=False) - - with patch("importlib.metadata.version", return_value="0.3.0"): - actual_command = setup_env_command( - dir_path=server_dir, - global_config_dict=self._debug_global_config_dict(tmp_path), - prefix="my server name", - include_dev_extra=True, - ) - assert "echo 'nemo-gym[dev]==0.3.0'" in actual_command - # The default (include_dev_extra=False) must NOT pull the dev extra. - with patch("importlib.metadata.version", return_value="0.3.0"): - default_command = setup_env_command( - dir_path=server_dir, - global_config_dict=self._debug_global_config_dict(tmp_path), - prefix="my server name", - ) - assert "echo 'nemo-gym==0.3.0'" in default_command - assert "[dev]" not in default_command - @pytest.mark.parametrize("version", ["0.3.0", "0.3.0rc0", "1.0.0", "2.1.3rc1"]) def test_installs_from_pypi_when_not_editable_pyproject( self, tmp_path: Path, version: str, monkeypatch: MonkeyPatch