From 84b83289c85481fa43f3577ffc20f6e87d4900f5 Mon Sep 17 00:00:00 2001 From: Wojciech Prazuch Date: Wed, 17 Jun 2026 12:59:15 +0200 Subject: [PATCH 1/4] =?UTF-8?q?feat(cli):=20run=20an=20environment=20by=20?= =?UTF-8?q?name=20=E2=80=94=20'gym=20env=20run=20--env=20'?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Builds on the environment registry: --env resolves a name to its config via resolve_environment_config_paths and merges it with any --config into a single +config_paths, so an environment can be run by name instead of an internal path (epic #1205 friction #8 / FEP-1022). Model flags pass through; an unknown name exits cleanly with the registry's 'did you mean?' message. Pair --env with --config / --model-* for the model the environment references. env run becomes a callable router target (_env_run); config-only behavior is unchanged. Signed-off-by: Wojciech Prazuch --- nemo_gym/cli/main.py | 38 +++++++++++++++++++++++-- tests/unit_tests/test_cli_main.py | 47 +++++++++++++++++++++++++++++++ 2 files changed, 82 insertions(+), 3 deletions(-) diff --git a/nemo_gym/cli/main.py b/nemo_gym/cli/main.py index 3e19677810..58657df3e3 100644 --- a/nemo_gym/cli/main.py +++ b/nemo_gym/cli/main.py @@ -108,6 +108,16 @@ def _bool_flag(name: str, hydra_key: str, flag_help: str) -> Flag: translate_to_hydra=lambda args: [f"+config_paths=[{','.join(args.config)}]"] if args.config else [], ) +# Shared flag: select an environment by name (run-by-name). Resolved to its config via the registry +# in `_env_run` rather than statically, then merged with any --config into a single +config_paths. +ENV = Flag( + register=lambda p: p.add_argument( + "--env", + metavar="NAME", + help="Environment to run, by name (see `gym list environments`); combined with --config / --model-* for the model.", + ), +) + # Shared flag: select the storage backend. Reused by `dataset upload` and `dataset download`. STORAGE = Flag( register=lambda p: p.add_argument( @@ -256,6 +266,28 @@ def _eval_run(args: argparse.Namespace, overrides: list[str]) -> None: dispatch(target, overrides) +def _env_run(args: argparse.Namespace, overrides: list[str]) -> None: + # Run an environment by name: resolve --env to its config via the registry and merge it with any + # --config files into a single +config_paths token (the env references its model server, so pair + # it with --config / --model-* for the model). Other overrides (model flags, passthrough) flow + # through unchanged. + config_paths: list[str] = [] + if getattr(args, "env", None): + from nemo_gym.registry import EnvironmentNotFoundError, resolve_environment_config_paths + + try: + config_paths += resolve_environment_config_paths(args.env) + except EnvironmentNotFoundError as error: + raise SystemExit(f"error: {error}") + if args.config: + config_paths += list(args.config) + + # CONFIG already emitted its own +config_paths token; drop it and emit the merged list instead. + rest = [token for token in overrides if not token.startswith("+config_paths=")] + merged = ([f"+config_paths=[{','.join(config_paths)}]"] if config_paths else []) + rest + dispatch("nemo_gym.cli.env:run", merged) + + def _env_test(args: argparse.Namespace, overrides: list[str]) -> None: # Run a single server's tests if +entrypoint was passed. No need to check for # --resource-server because it is translated to +entrypoint in the flag definition. @@ -409,9 +441,9 @@ def _dataset_download(args: argparse.Namespace, overrides: list[str]) -> None: flags=(RESOURCE_SERVER,), ), "env run": Command( - target="nemo_gym.cli.env:run", - summary="Start the servers.", - flags=(CONFIG, BENCHMARK, RESOURCE_SERVER_CONFIG, MODEL_TYPE, SEARCH_DIR, MODEL, MODEL_URL, MODEL_API_KEY), + target=_env_run, + summary="Start the servers (by --env name and/or --config).", + flags=(ENV, CONFIG, BENCHMARK, RESOURCE_SERVER_CONFIG, MODEL_TYPE, SEARCH_DIR, MODEL, MODEL_URL, MODEL_API_KEY), ), "env status": Command(target="nemo_gym.cli.env:status", summary="Print the server status.", flags=(JSON,)), "eval prepare": Command( diff --git a/tests/unit_tests/test_cli_main.py b/tests/unit_tests/test_cli_main.py index 4ee74912ce..7a5a08ef62 100644 --- a/tests/unit_tests/test_cli_main.py +++ b/tests/unit_tests/test_cli_main.py @@ -866,3 +866,50 @@ def test_list_environments_dispatches(self, monkeypatch: MonkeyPatch) -> None: target, overrides = _dispatch_for(monkeypatch, ["list", "environments"]) assert target == "nemo_gym.cli.env:list_environments" assert overrides == [] + + +class TestEnvRunByName: + def _patch_resolve(self, monkeypatch: MonkeyPatch) -> None: + import nemo_gym.registry + + monkeypatch.setattr( + nemo_gym.registry, + "resolve_environment_config_paths", + lambda name, *a, **k: [f"environments/{name}/config.yaml"], + ) + + def test_env_name_resolves_to_config_path(self, monkeypatch: MonkeyPatch) -> None: + self._patch_resolve(monkeypatch) + target, overrides = _dispatch_for(monkeypatch, ["env", "run", "--env", "alpha"]) + assert target == "nemo_gym.cli.env:run" + assert overrides == ["+config_paths=[environments/alpha/config.yaml]"] + + def test_env_and_config_merge_into_one_config_paths(self, monkeypatch: MonkeyPatch) -> None: + self._patch_resolve(monkeypatch) + _, overrides = _dispatch_for(monkeypatch, ["env", "run", "--env", "alpha", "--config", "model.yaml"]) + config_paths = [o for o in overrides if o.startswith("+config_paths=")] + assert config_paths == ["+config_paths=[environments/alpha/config.yaml,model.yaml]"] + + def test_model_flags_pass_through(self, monkeypatch: MonkeyPatch) -> None: + self._patch_resolve(monkeypatch) + _, overrides = _dispatch_for(monkeypatch, ["env", "run", "--env", "alpha", "--model-name", "gpt"]) + assert "+config_paths=[environments/alpha/config.yaml]" in overrides + assert "+policy_model_name=gpt" in overrides + + def test_config_only_is_unchanged(self, monkeypatch: MonkeyPatch) -> None: + target, overrides = _dispatch_for(monkeypatch, ["env", "run", "--config", "a.yaml"]) + assert target == "nemo_gym.cli.env:run" + assert overrides == ["+config_paths=[a.yaml]"] + + def test_unknown_env_exits_cleanly(self, monkeypatch: MonkeyPatch) -> None: + import nemo_gym.registry + from nemo_gym.registry import EnvironmentNotFoundError + + def boom(name, *a, **k): + raise EnvironmentNotFoundError(f"No environment named '{name}'") + + monkeypatch.setattr(nemo_gym.registry, "resolve_environment_config_paths", boom) + monkeypatch.setattr(cli_main, "dispatch", lambda target, overrides: None) + monkeypatch.setattr(sys, "argv", ["gym", "env", "run", "--env", "nope"]) + with pytest.raises(SystemExit): + main() From f213008ce7b39fe9c91398d209fb5b97f51cc604 Mon Sep 17 00:00:00 2001 From: Wojciech Prazuch Date: Tue, 23 Jun 2026 11:08:26 +0200 Subject: [PATCH 2/4] fix(cli): preserve asset-selector config_paths in 'gym env run' _env_run stripped every +config_paths token to merge --env/--config into one, which also discarded the token the router emits for an asset-selector name (e.g. 'gym env run mcqa'), yielding empty config_paths. Merge --env's resolved paths with the paths already in the emitted token instead, so asset selectors, --config, and --env coalesce. Fixes TestAssetSelectors regression. Signed-off-by: Wojciech Prazuch --- nemo_gym/cli/main.py | 14 ++++++++++---- tests/unit_tests/test_cli_main.py | 12 ++++++++++++ 2 files changed, 22 insertions(+), 4 deletions(-) diff --git a/nemo_gym/cli/main.py b/nemo_gym/cli/main.py index 58657df3e3..87ebb9663e 100644 --- a/nemo_gym/cli/main.py +++ b/nemo_gym/cli/main.py @@ -279,11 +279,17 @@ def _env_run(args: argparse.Namespace, overrides: list[str]) -> None: config_paths += resolve_environment_config_paths(args.env) except EnvironmentNotFoundError as error: raise SystemExit(f"error: {error}") - if args.config: - config_paths += list(args.config) + # Merge --env's resolved config(s) with any +config_paths the router already emitted — from + # --config or from an asset-selector name (e.g. `gym env run mcqa`) — into a single token, + # preserving order. Pulling paths out of the existing token (rather than re-reading args.config) + # means asset-selector resolution is not discarded. + rest: list[str] = [] + for token in overrides: + if token.startswith("+config_paths=[") and token.endswith("]"): + config_paths += [path for path in token[len("+config_paths=[") : -1].split(",") if path] + else: + rest.append(token) - # CONFIG already emitted its own +config_paths token; drop it and emit the merged list instead. - rest = [token for token in overrides if not token.startswith("+config_paths=")] merged = ([f"+config_paths=[{','.join(config_paths)}]"] if config_paths else []) + rest dispatch("nemo_gym.cli.env:run", merged) diff --git a/tests/unit_tests/test_cli_main.py b/tests/unit_tests/test_cli_main.py index 7a5a08ef62..228b142671 100644 --- a/tests/unit_tests/test_cli_main.py +++ b/tests/unit_tests/test_cli_main.py @@ -901,6 +901,18 @@ def test_config_only_is_unchanged(self, monkeypatch: MonkeyPatch) -> None: assert target == "nemo_gym.cli.env:run" assert overrides == ["+config_paths=[a.yaml]"] + def test_existing_config_paths_token_is_preserved(self, monkeypatch: MonkeyPatch) -> None: + # An asset-selector name (or passthrough) reaches _env_run as a +config_paths token with no + # --env/--config; it must survive rather than be stripped. + target, overrides = _dispatch_for(monkeypatch, ["env", "run", "+config_paths=[foo.yaml]"]) + assert target == "nemo_gym.cli.env:run" + assert overrides == ["+config_paths=[foo.yaml]"] + + def test_env_merges_with_existing_config_paths_token(self, monkeypatch: MonkeyPatch) -> None: + self._patch_resolve(monkeypatch) + _, overrides = _dispatch_for(monkeypatch, ["env", "run", "--env", "alpha", "+config_paths=[foo.yaml]"]) + assert overrides == ["+config_paths=[environments/alpha/config.yaml,foo.yaml]"] + def test_unknown_env_exits_cleanly(self, monkeypatch: MonkeyPatch) -> None: import nemo_gym.registry from nemo_gym.registry import EnvironmentNotFoundError From a64adfb8d47254fde1ef5deb292aa9f5513b5449 Mon Sep 17 00:00:00 2001 From: Wojciech Prazuch Date: Wed, 24 Jun 2026 09:08:12 +0200 Subject: [PATCH 3/4] test(cli): use unified --model flag in env-run-by-name model-flag test Signed-off-by: Wojciech Prazuch --- tests/unit_tests/test_cli_main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit_tests/test_cli_main.py b/tests/unit_tests/test_cli_main.py index 228b142671..5e1ccfb1bd 100644 --- a/tests/unit_tests/test_cli_main.py +++ b/tests/unit_tests/test_cli_main.py @@ -892,7 +892,7 @@ def test_env_and_config_merge_into_one_config_paths(self, monkeypatch: MonkeyPat def test_model_flags_pass_through(self, monkeypatch: MonkeyPatch) -> None: self._patch_resolve(monkeypatch) - _, overrides = _dispatch_for(monkeypatch, ["env", "run", "--env", "alpha", "--model-name", "gpt"]) + _, overrides = _dispatch_for(monkeypatch, ["env", "run", "--env", "alpha", "--model", "gpt"]) assert "+config_paths=[environments/alpha/config.yaml]" in overrides assert "+policy_model_name=gpt" in overrides From 3bd8639bfd02f5df50843b9961e1115dd9898bbf Mon Sep 17 00:00:00 2001 From: Wojciech Prazuch Date: Wed, 24 Jun 2026 10:07:08 +0200 Subject: [PATCH 4/4] style(cli): wrap env-run flags tuple to satisfy ruff-format Signed-off-by: Wojciech Prazuch --- nemo_gym/cli/main.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/nemo_gym/cli/main.py b/nemo_gym/cli/main.py index 87ebb9663e..bd1f8e165c 100644 --- a/nemo_gym/cli/main.py +++ b/nemo_gym/cli/main.py @@ -449,7 +449,17 @@ def _dataset_download(args: argparse.Namespace, overrides: list[str]) -> None: "env run": Command( target=_env_run, summary="Start the servers (by --env name and/or --config).", - flags=(ENV, CONFIG, BENCHMARK, RESOURCE_SERVER_CONFIG, MODEL_TYPE, SEARCH_DIR, MODEL, MODEL_URL, MODEL_API_KEY), + flags=( + ENV, + CONFIG, + BENCHMARK, + RESOURCE_SERVER_CONFIG, + MODEL_TYPE, + SEARCH_DIR, + MODEL, + MODEL_URL, + MODEL_API_KEY, + ), ), "env status": Command(target="nemo_gym.cli.env:status", summary="Print the server status.", flags=(JSON,)), "eval prepare": Command(