diff --git a/nemo_gym/agent_registry.py b/nemo_gym/agent_registry.py new file mode 100644 index 0000000000..9b8958eb9b --- /dev/null +++ b/nemo_gym/agent_registry.py @@ -0,0 +1,160 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Registry of agent harnesses under ``responses_api_agents//``. + +An agent harness is one *component* of an environment (an environment = dataset + agent harness + +resources server [verifier and state] + model). This module maps an agent's short ```` (the +directory name) to its config variant(s) so it can be enumerated by name (``gym list agents``) and +classified by how it composes. Resolving an agent name to a config for *running* belongs to the +config composer (via the CLI's generic asset selectors), so this module is intentionally +discovery-only. The ``self_contained`` flag records only what the agent's *own* config reveals about +its resources-server wiring: + +- **Wires a resources server itself (Pattern A, ``self_contained=False``):** the config sets + ``responses_api_agents..resources_server``, so the registry can see it pairs with a separate + resources server + dataset (e.g. the ``simple_agent`` tool-use pattern). +- **Does not wire one in its own config (Pattern B, ``self_contained=True``):** the agent declares + its own ``agent_framework`` (e.g. ``swe_agents``), drives an external LLM loop (e.g. + ``claude_code_agent`` via its own model key), or ships only an entrypoint whose resources-server + pairing is supplied by a separate paired/benchmark config (e.g. ``gymnasium_agent`` + the + ``blackjack`` resources server). These agents are still composable — but *within* a type + (gymnasium↔gymnasium-style, simple_agent↔simple_agent-style), not across types. Which pairings are + actually compatible is the config composer's call, not inferable from the agent config alone. + +Discovery only reads config files; it never resolves interpolations or missing values and never +starts servers, so it is safe to call when secrets/API keys referenced by a config are unset. +""" + +from dataclasses import dataclass +from pathlib import Path +from typing import Dict, Optional, Tuple + +from omegaconf import OmegaConf + +from nemo_gym import PARENT_DIR + + +AGENTS_DIR = PARENT_DIR / "responses_api_agents" +AGENT_CONFIGS_SUBDIR = "configs" + + +@dataclass(frozen=True) +class AgentEntry: + """A discovered agent: its name, where it lives, its config variants, and how it's wired.""" + + name: str + path: Path + config_paths: Tuple[Path, ...] # variant config files, sorted; empty for "zero-config" agents + # True = bundles its own environment/framework (Pattern B; run standalone). + # False = references a separate resources server (Pattern A; wire to a compatible environment). + self_contained: bool + description: Optional[str] = None + + @property + def variants(self) -> Dict[str, Path]: + """Map variant name (config filename stem) -> config path.""" + return {path.stem: path for path in self.config_paths} + + +def _iter_agent_blocks(config_path: Path): + """Yield each ``responses_api_agents.`` mapping in a config (resolution-safe, best effort).""" + try: + container = OmegaConf.to_container(OmegaConf.load(config_path), resolve=False, throw_on_missing=False) + except Exception: + return + if not isinstance(container, dict): + return + for top_level_value in container.values(): + if not isinstance(top_level_value, dict): + continue + agents = top_level_value.get("responses_api_agents") + if not isinstance(agents, dict): + continue + for agent_block in agents.values(): + if isinstance(agent_block, dict): + yield agent_block + + +def _is_agent_config(config_path: Path) -> bool: + """True if the file is a NeMo Gym agent config (a top-level block with ``responses_api_agents``). + + Filters out non-agent YAML that happens to live in an agent's ``configs/`` dir (e.g. a raw + harness config or an empty stub). + """ + return next(_iter_agent_blocks(config_path), None) is not None + + +def _classify(config_paths: Tuple[Path, ...]) -> Tuple[bool, Optional[str]]: + """Return ``(self_contained, description)`` for an agent from its config variants. + + A harness is NOT self-contained (Pattern A) when some variant references a separate resources + server, none declares an ``agent_framework``, and none drives an external LLM loop (e.g. its own + Anthropic key) — it must be wired to a compatible resources server. Otherwise it is + self-contained (Pattern B). Agents with no parseable config are treated as Pattern A (their + wiring lives in a paired benchmark/resources-server config). + """ + has_resources_server = False + has_agent_framework = False + drives_external_harness = False + description: Optional[str] = None + + saw_block = False + for config_path in config_paths: + for block in _iter_agent_blocks(config_path): + saw_block = True + if "resources_server" in block: + has_resources_server = True + if "agent_framework" in block: + has_agent_framework = True + if "anthropic_api_key" in block: + drives_external_harness = True + if description is None and isinstance(block.get("description"), str): + description = block["description"] + + if not saw_block: + return False, description + references_resources_server = has_resources_server and not has_agent_framework and not drives_external_harness + return not references_resources_server, description + + +def discover_agents(agents_dir: Path = AGENTS_DIR) -> Dict[str, AgentEntry]: + """Map agent name -> :class:`AgentEntry` for every agent dir under ``responses_api_agents/``. + + The name is the directory name. A directory is an agent if it has an ``app.py`` or at least one + agent config. Returns an empty dict if the directory is missing. + """ + agents: Dict[str, AgentEntry] = {} + if not agents_dir.is_dir(): + return agents + + for child in sorted(agents_dir.iterdir()): + if not child.is_dir(): + continue + configs_dir = child / AGENT_CONFIGS_SUBDIR + config_files = sorted(configs_dir.glob("*.yaml")) if configs_dir.is_dir() else [] + agent_configs = tuple(path for path in config_files if _is_agent_config(path)) + if not (child / "app.py").is_file() and not agent_configs: + continue + + self_contained, description = _classify(agent_configs) + agents[child.name] = AgentEntry( + name=child.name, + path=child, + config_paths=agent_configs, + self_contained=self_contained, + description=description, + ) + + return agents diff --git a/nemo_gym/cli/agents.py b/nemo_gym/cli/agents.py new file mode 100644 index 0000000000..982ef440b1 --- /dev/null +++ b/nemo_gym/cli/agents.py @@ -0,0 +1,77 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import json + +import rich +from rich.table import Table + +from nemo_gym.agent_registry import discover_agents +from nemo_gym.config_types import BaseNeMoGymCLIConfig +from nemo_gym.global_config import ( + JSON_OUTPUT_KEY_NAME, + GlobalConfigDictParserConfig, + get_global_config_dict, +) + + +def list_agents() -> None: + """CLI command: list discovered agent harnesses and how each composes (Pattern A vs B). + + Complements ``gym list benchmarks``: the asset selectors resolve a component *by name*, but only + this listing surfaces which agents are freely wireable into a separate environment (Pattern A) + versus self-contained harnesses that run with their own config (Pattern B) — the distinction the + config composer's compatibility guard relies on. + """ + global_config_dict = get_global_config_dict( + global_config_dict_parser_config=GlobalConfigDictParserConfig( + initial_global_config_dict=GlobalConfigDictParserConfig.NO_MODEL_GLOBAL_CONFIG_DICT, + ) + ) + BaseNeMoGymCLIConfig.model_validate(global_config_dict) + + agents = discover_agents() + + if global_config_dict.get(JSON_OUTPUT_KEY_NAME, False): + payload = [ + { + "name": name, + "pattern": "B (self-contained)" if entry.self_contained else "A (composable)", + "self_contained": entry.self_contained, + "variants": sorted(entry.variants), + "description": entry.description, + } + for name, entry in agents.items() + ] + print(json.dumps(payload)) + return + + if not agents: + rich.print("No agents found.") + return + + table = Table(title="NeMo Gym agents") + table.add_column("agent", style="bold") + table.add_column("composition") + table.add_column("variants") + table.add_column("description") + for name, entry in agents.items(): + composition = "self-contained (B)" if entry.self_contained else "composable (A)" + table.add_row( + name, + composition, + ", ".join(sorted(entry.variants)) or "—", + entry.description or "", + ) + rich.print(table) diff --git a/nemo_gym/cli/main.py b/nemo_gym/cli/main.py index fbb21e0493..5792545386 100644 --- a/nemo_gym/cli/main.py +++ b/nemo_gym/cli/main.py @@ -285,7 +285,7 @@ def _dataset_download(args: argparse.Namespace, overrides: list[str]) -> None: # One-line help for each command group, shown in `gym --help`. GROUPS = { - "list": "List available components (benchmarks, environments).", + "list": "List available components (benchmarks, agents, environments).", "dataset": "Manage datasets.", "env": "Develop and run environments.", "eval": "Run evaluations.", @@ -299,6 +299,11 @@ def _dataset_download(args: argparse.Namespace, overrides: list[str]) -> None: "list environments": Command( target="nemo_gym.cli.env:list_environments", summary="List available environments by name.", flags=(JSON,) ), + "list agents": Command( + target="nemo_gym.cli.agents:list_agents", + summary="List agent harnesses and how each composes (Pattern A vs self-contained B).", + flags=(JSON,), + ), "search": Command( target="nemo_gym.cli.eval:list_benchmarks", summary="Search available components (currently benchmarks) by name; like `list` filtered to a query.", diff --git a/tests/unit_tests/test_agent_registry.py b/tests/unit_tests/test_agent_registry.py new file mode 100644 index 0000000000..7c2d8982d3 --- /dev/null +++ b/tests/unit_tests/test_agent_registry.py @@ -0,0 +1,133 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from pathlib import Path + +from nemo_gym.agent_registry import ( + AgentEntry, + discover_agents, +) + + +def _make_agent(agents_dir: Path, name: str, *, app: bool = True, configs: dict = None) -> Path: + agent_dir = agents_dir / name + agent_dir.mkdir(parents=True) + if app: + (agent_dir / "app.py").write_text("# app\n") + if configs: + configs_dir = agent_dir / "configs" + configs_dir.mkdir() + for variant, body in configs.items(): + (configs_dir / f"{variant}.yaml").write_text(body) + return agent_dir + + +def _pattern_a(agent_type: str = "simple_agent") -> str: + # References a separate resources server -> composable. + return ( + f"some_key:\n responses_api_agents:\n {agent_type}:\n entrypoint: app.py\n" + " resources_server:\n type: resources_servers\n name: ???\n" + " description: A composable agent\n" + ) + + +def _pattern_b(agent_type: str = "swe_agent") -> str: + # Self-contained framework agent -> not composable. + return ( + f"some_key:\n responses_api_agents:\n {agent_type}:\n entrypoint: app.py\n" + " agent_framework: openhands\n" + ) + + +class TestDiscoverAgents: + def test_discovers_and_classifies_pattern_a(self, tmp_path: Path) -> None: + _make_agent(tmp_path, "simple_agent", configs={"simple_agent": _pattern_a()}) + + agents = discover_agents(tmp_path) + + assert set(agents) == {"simple_agent"} + entry = agents["simple_agent"] + assert entry.self_contained is False + assert entry.description == "A composable agent" + assert list(entry.variants) == ["simple_agent"] + + def test_classifies_pattern_b_as_not_composable(self, tmp_path: Path) -> None: + _make_agent(tmp_path, "swe_agents", configs={"swebench": _pattern_b()}) + + assert discover_agents(tmp_path)["swe_agents"].self_contained is True + + def test_external_harness_agent_is_not_composable(self, tmp_path: Path) -> None: + body = ( + "k:\n responses_api_agents:\n claude_code_agent:\n entrypoint: app.py\n" + " resources_server:\n name: ???\n anthropic_api_key: ???\n" + ) + _make_agent(tmp_path, "claude_code_agent", configs={"claude_code_agent": body}) + + # Has a resources_server but drives an external LLM harness -> not composable. + assert discover_agents(tmp_path)["claude_code_agent"].self_contained is True + + def test_zero_config_agent_is_discovered_and_defaults_composable(self, tmp_path: Path) -> None: + _make_agent(tmp_path, "aviary_agent", configs=None) # app.py only, no configs + + entry = discover_agents(tmp_path)["aviary_agent"] + assert entry.config_paths == () + assert entry.self_contained is False + + def test_multiple_variants_are_all_recorded(self, tmp_path: Path) -> None: + _make_agent( + tmp_path, + "langgraph_agent", + configs={ + "orchestrator_agent": _pattern_a("langgraph_agent"), + "rewoo_agent": _pattern_a("langgraph_agent"), + }, + ) + + assert set(discover_agents(tmp_path)["langgraph_agent"].variants) == {"orchestrator_agent", "rewoo_agent"} + + def test_non_agent_yaml_is_filtered_out(self, tmp_path: Path) -> None: + # A configs/ file that is not a gym agent config (no responses_api_agents) is ignored; + # the dir still counts as an agent because of app.py. + _make_agent(tmp_path, "swe_agents", configs={"raw_harness": "agent:\n type: openhands\n"}) + + entry = discover_agents(tmp_path)["swe_agents"] + assert entry.config_paths == () + + def test_directory_without_app_or_configs_is_skipped(self, tmp_path: Path) -> None: + (tmp_path / "not_an_agent").mkdir() + (tmp_path / "loose_file.txt").write_text("x") + + assert discover_agents(tmp_path) == {} + + def test_unparseable_config_does_not_crash_discovery(self, tmp_path: Path) -> None: + _make_agent(tmp_path, "broken", configs={"broken": "responses_api_agents: [unclosed\n"}) + + # The bad file is skipped (not an agent config); the dir survives via app.py. + assert discover_agents(tmp_path)["broken"].config_paths == () + + def test_missing_directory_yields_no_agents(self, tmp_path: Path) -> None: + assert discover_agents(tmp_path / "nope") == {} + + +class TestRealAgents: + def test_discovers_real_simple_agent_as_composable(self) -> None: + agents = discover_agents() + # The repo ships a `simple_agent`; it pairs with a separate resources server. + if "simple_agent" in agents: + assert agents["simple_agent"].self_contained is False + + def test_agent_entry_is_hashable(self) -> None: + entry = AgentEntry(name="a", path=Path("a"), config_paths=(Path("a/configs/a.yaml"),), self_contained=True) + assert {entry: 1}[entry] == 1 + assert entry.variants == {"a": Path("a/configs/a.yaml")} diff --git a/tests/unit_tests/test_cli_agents.py b/tests/unit_tests/test_cli_agents.py new file mode 100644 index 0000000000..4a60669b2a --- /dev/null +++ b/tests/unit_tests/test_cli_agents.py @@ -0,0 +1,77 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import json +from pathlib import Path +from unittest.mock import patch + +from omegaconf import OmegaConf + +from nemo_gym.agent_registry import AgentEntry +from nemo_gym.cli.agents import list_agents + + +def _mock_global_config(config: dict = None): + return OmegaConf.create(config or {}) + + +def _entry(name: str, self_contained: bool, variants=(), description=None) -> AgentEntry: + path = Path("responses_api_agents") / name + config_paths = tuple(path / "configs" / f"{v}.yaml" for v in variants) + return AgentEntry( + name=name, + path=path, + config_paths=config_paths, + self_contained=self_contained, + description=description, + ) + + +_AGENTS = { + "simple_agent": _entry("simple_agent", self_contained=False, variants=("simple_agent",)), + "swe_agents": _entry("swe_agents", self_contained=True, variants=("swebench_openhands",), description="SWE tasks"), +} + + +class TestListAgents: + def test_lists_found_agents(self, capsys) -> None: + with ( + patch("nemo_gym.cli.agents.get_global_config_dict", return_value=_mock_global_config()), + patch("nemo_gym.cli.agents.discover_agents", return_value=_AGENTS), + ): + list_agents() + out = capsys.readouterr().out + assert "simple_agent" in out and "swe_agents" in out + assert "composable" in out and "self-contained" in out + + def test_no_agents(self, capsys) -> None: + with ( + patch("nemo_gym.cli.agents.get_global_config_dict", return_value=_mock_global_config()), + patch("nemo_gym.cli.agents.discover_agents", return_value={}), + ): + list_agents() + assert "No agents found" in capsys.readouterr().out + + def test_json_output(self, capsys) -> None: + with ( + patch("nemo_gym.cli.agents.get_global_config_dict", return_value=_mock_global_config({"json": True})), + patch("nemo_gym.cli.agents.discover_agents", return_value=_AGENTS), + ): + list_agents() + payload = json.loads(capsys.readouterr().out) + by_name = {entry["name"]: entry for entry in payload} + assert by_name["simple_agent"]["self_contained"] is False + assert by_name["simple_agent"]["pattern"] == "A (composable)" + assert by_name["swe_agents"]["self_contained"] is True + assert by_name["swe_agents"]["variants"] == ["swebench_openhands"] diff --git a/tests/unit_tests/test_cli_main.py b/tests/unit_tests/test_cli_main.py index 45fca591f8..b47625e0c6 100644 --- a/tests/unit_tests/test_cli_main.py +++ b/tests/unit_tests/test_cli_main.py @@ -571,6 +571,7 @@ class TestJsonFlag: "argv, expected_target", [ (["list", "benchmarks", "--json"], "nemo_gym.cli.eval:list_benchmarks"), + (["list", "agents", "--json"], "nemo_gym.cli.agents:list_agents"), (["env", "status", "--json"], "nemo_gym.cli.env:status"), ], )