feat: agent registry — name-based agent discovery + composability (M3 core)

nemo_gym/agent_registry.py

@@ -0,0 +1,160 @@
+# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Registry of agent harnesses under ``responses_api_agents/<name>/``.
+
+An agent harness is one *component* of an environment (an environment = dataset + agent harness +
+resources server [verifier and state] + model). This module maps an agent's short ``<name>`` (the
+directory name) to its config variant(s) so it can be enumerated by name (``gym list agents``) and
+classified by how it composes. Resolving an agent name to a config for *running* belongs to the
+config composer (via the CLI's generic asset selectors), so this module is intentionally
+discovery-only. The ``self_contained`` flag records only what the agent's *own* config reveals about
+its resources-server wiring:
+
+- **Wires a resources server itself (Pattern A, ``self_contained=False``):** the config sets
+  ``responses_api_agents.<type>.resources_server``, so the registry can see it pairs with a separate
+  resources server + dataset (e.g. the ``simple_agent`` tool-use pattern).
+- **Does not wire one in its own config (Pattern B, ``self_contained=True``):** the agent declares
+  its own ``agent_framework`` (e.g. ``swe_agents``), drives an external LLM loop (e.g.
+  ``claude_code_agent`` via its own model key), or ships only an entrypoint whose resources-server
+  pairing is supplied by a separate paired/benchmark config (e.g. ``gymnasium_agent`` + the
+  ``blackjack`` resources server). These agents are still composable — but *within* a type
+  (gymnasium↔gymnasium-style, simple_agent↔simple_agent-style), not across types. Which pairings are
+  actually compatible is the config composer's call, not inferable from the agent config alone.
+
+Discovery only reads config files; it never resolves interpolations or missing values and never
+starts servers, so it is safe to call when secrets/API keys referenced by a config are unset.
+"""
+
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Dict, Optional, Tuple
+
+from omegaconf import OmegaConf
+
+from nemo_gym import PARENT_DIR
+
+
+AGENTS_DIR = PARENT_DIR / "responses_api_agents"
+AGENT_CONFIGS_SUBDIR = "configs"
+
+
+@dataclass(frozen=True)
+class AgentEntry:
+    """A discovered agent: its name, where it lives, its config variants, and how it's wired."""
+
+    name: str
+    path: Path
+    config_paths: Tuple[Path, ...]  # variant config files, sorted; empty for "zero-config" agents
+    # True  = bundles its own environment/framework (Pattern B; run standalone).
+    # False = references a separate resources server (Pattern A; wire to a compatible environment).
+    self_contained: bool
+    description: Optional[str] = None
+
+    @property
+    def variants(self) -> Dict[str, Path]:
+        """Map variant name (config filename stem) -> config path."""
+        return {path.stem: path for path in self.config_paths}
+
+
+def _iter_agent_blocks(config_path: Path):
+    """Yield each ``responses_api_agents.<type>`` mapping in a config (resolution-safe, best effort)."""
+    try:
+        container = OmegaConf.to_container(OmegaConf.load(config_path), resolve=False, throw_on_missing=False)
+    except Exception:
+        return
+    if not isinstance(container, dict):
+        return
+    for top_level_value in container.values():
+        if not isinstance(top_level_value, dict):
+            continue
+        agents = top_level_value.get("responses_api_agents")
+        if not isinstance(agents, dict):
+            continue
+        for agent_block in agents.values():
+            if isinstance(agent_block, dict):
+                yield agent_block
+
+
+def _is_agent_config(config_path: Path) -> bool:
+    """True if the file is a NeMo Gym agent config (a top-level block with ``responses_api_agents``).
+
+    Filters out non-agent YAML that happens to live in an agent's ``configs/`` dir (e.g. a raw
+    harness config or an empty stub).
+    """
+    return next(_iter_agent_blocks(config_path), None) is not None
+
+
+def _classify(config_paths: Tuple[Path, ...]) -> Tuple[bool, Optional[str]]:
+    """Return ``(self_contained, description)`` for an agent from its config variants.
+
+    A harness is NOT self-contained (Pattern A) when some variant references a separate resources
+    server, none declares an ``agent_framework``, and none drives an external LLM loop (e.g. its own
+    Anthropic key) — it must be wired to a compatible resources server. Otherwise it is
+    self-contained (Pattern B). Agents with no parseable config are treated as Pattern A (their
+    wiring lives in a paired benchmark/resources-server config).
+    """
+    has_resources_server = False
+    has_agent_framework = False
+    drives_external_harness = False
+    description: Optional[str] = None
+
+    saw_block = False
+    for config_path in config_paths:
+        for block in _iter_agent_blocks(config_path):
+            saw_block = True
+            if "resources_server" in block:
+                has_resources_server = True
+            if "agent_framework" in block:
+                has_agent_framework = True
+            if "anthropic_api_key" in block:
+                drives_external_harness = True
+            if description is None and isinstance(block.get("description"), str):
+                description = block["description"]
+
+    if not saw_block:
+        return False, description
+    references_resources_server = has_resources_server and not has_agent_framework and not drives_external_harness
+    return not references_resources_server, description
+
+
+def discover_agents(agents_dir: Path = AGENTS_DIR) -> Dict[str, AgentEntry]:
+    """Map agent name -> :class:`AgentEntry` for every agent dir under ``responses_api_agents/``.
+
+    The name is the directory name. A directory is an agent if it has an ``app.py`` or at least one
+    agent config. Returns an empty dict if the directory is missing.
+    """
+    agents: Dict[str, AgentEntry] = {}
+    if not agents_dir.is_dir():
+        return agents
+
+    for child in sorted(agents_dir.iterdir()):
+        if not child.is_dir():
+            continue
+        configs_dir = child / AGENT_CONFIGS_SUBDIR
+        config_files = sorted(configs_dir.glob("*.yaml")) if configs_dir.is_dir() else []
+        agent_configs = tuple(path for path in config_files if _is_agent_config(path))
+        if not (child / "app.py").is_file() and not agent_configs:
+            continue
+
+        self_contained, description = _classify(agent_configs)
+        agents[child.name] = AgentEntry(
+            name=child.name,
+            path=child,
+            config_paths=agent_configs,
+            self_contained=self_contained,
+            description=description,
+        )
+
+    return agents

nemo_gym/cli/agents.py

@@ -0,0 +1,77 @@
+# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import json
+
+import rich
+from rich.table import Table
+
+from nemo_gym.agent_registry import discover_agents
+from nemo_gym.config_types import BaseNeMoGymCLIConfig
+from nemo_gym.global_config import (
+    JSON_OUTPUT_KEY_NAME,
+    GlobalConfigDictParserConfig,
+    get_global_config_dict,
+)
+
+
+def list_agents() -> None:
+    """CLI command: list discovered agent harnesses and how each composes (Pattern A vs B).
+
+    Complements ``gym list benchmarks``: the asset selectors resolve a component *by name*, but only
+    this listing surfaces which agents are freely wireable into a separate environment (Pattern A)
+    versus self-contained harnesses that run with their own config (Pattern B) — the distinction the
+    config composer's compatibility guard relies on.
+    """
+    global_config_dict = get_global_config_dict(
+        global_config_dict_parser_config=GlobalConfigDictParserConfig(
+            initial_global_config_dict=GlobalConfigDictParserConfig.NO_MODEL_GLOBAL_CONFIG_DICT,
+        )
+    )
+    BaseNeMoGymCLIConfig.model_validate(global_config_dict)
+
+    agents = discover_agents()
+
+    if global_config_dict.get(JSON_OUTPUT_KEY_NAME, False):
+        payload = [
+            {
+                "name": name,
+                "pattern": "B (self-contained)" if entry.self_contained else "A (composable)",
+                "self_contained": entry.self_contained,
+                "variants": sorted(entry.variants),
+                "description": entry.description,
+            }
+            for name, entry in agents.items()
+        ]
+        print(json.dumps(payload))
+        return
+
+    if not agents:
+        rich.print("No agents found.")
+        return
+
+    table = Table(title="NeMo Gym agents")
+    table.add_column("agent", style="bold")
+    table.add_column("composition")
+    table.add_column("variants")
+    table.add_column("description")
+    for name, entry in agents.items():
+        composition = "self-contained (B)" if entry.self_contained else "composable (A)"
+        table.add_row(
+            name,
+            composition,
+            ", ".join(sorted(entry.variants)) or "—",
+            entry.description or "",
+        )
+    rich.print(table)

nemo_gym/cli/main.py

@@ -285,7 +285,7 @@ def _dataset_download(args: argparse.Namespace, overrides: list[str]) -> None:
 
 # One-line help for each command group, shown in `gym --help`.
 GROUPS = {
-    "list": "List available components (benchmarks, environments).",
+    "list": "List available components (benchmarks, agents, environments).",
     "dataset": "Manage datasets.",
     "env": "Develop and run environments.",
     "eval": "Run evaluations.",
@@ -299,6 +299,11 @@ def _dataset_download(args: argparse.Namespace, overrides: list[str]) -> None:
     "list environments": Command(
         target="nemo_gym.cli.env:list_environments", summary="List available environments by name.", flags=(JSON,)
     ),
+    "list agents": Command(
+        target="nemo_gym.cli.agents:list_agents",
+        summary="List agent harnesses and how each composes (Pattern A vs self-contained B).",
+        flags=(JSON,),
+    ),
     "search": Command(
         target="nemo_gym.cli.eval:list_benchmarks",
         summary="Search available components (currently benchmarks) by name; like `list` filtered to a query.",

tests/unit_tests/test_agent_registry.py

@@ -0,0 +1,133 @@
+# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+from pathlib import Path
+
+from nemo_gym.agent_registry import (
+    AgentEntry,
+    discover_agents,
+)
+
+
+def _make_agent(agents_dir: Path, name: str, *, app: bool = True, configs: dict = None) -> Path:
+    agent_dir = agents_dir / name
+    agent_dir.mkdir(parents=True)
+    if app:
+        (agent_dir / "app.py").write_text("# app\n")
+    if configs:
+        configs_dir = agent_dir / "configs"
+        configs_dir.mkdir()
+        for variant, body in configs.items():
+            (configs_dir / f"{variant}.yaml").write_text(body)
+    return agent_dir
+
+
+def _pattern_a(agent_type: str = "simple_agent") -> str:
+    # References a separate resources server -> composable.
+    return (
+        f"some_key:\n  responses_api_agents:\n    {agent_type}:\n      entrypoint: app.py\n"
+        "      resources_server:\n        type: resources_servers\n        name: ???\n"
+        "      description: A composable agent\n"
+    )
+
+
+def _pattern_b(agent_type: str = "swe_agent") -> str:
+    # Self-contained framework agent -> not composable.
+    return (
+        f"some_key:\n  responses_api_agents:\n    {agent_type}:\n      entrypoint: app.py\n"
+        "      agent_framework: openhands\n"
+    )
+
+
+class TestDiscoverAgents:
+    def test_discovers_and_classifies_pattern_a(self, tmp_path: Path) -> None:
+        _make_agent(tmp_path, "simple_agent", configs={"simple_agent": _pattern_a()})
+
+        agents = discover_agents(tmp_path)
+
+        assert set(agents) == {"simple_agent"}
+        entry = agents["simple_agent"]
+        assert entry.self_contained is False
+        assert entry.description == "A composable agent"
+        assert list(entry.variants) == ["simple_agent"]
+
+    def test_classifies_pattern_b_as_not_composable(self, tmp_path: Path) -> None:
+        _make_agent(tmp_path, "swe_agents", configs={"swebench": _pattern_b()})
+
+        assert discover_agents(tmp_path)["swe_agents"].self_contained is True
+
+    def test_external_harness_agent_is_not_composable(self, tmp_path: Path) -> None:
+        body = (
+            "k:\n  responses_api_agents:\n    claude_code_agent:\n      entrypoint: app.py\n"
+            "      resources_server:\n        name: ???\n      anthropic_api_key: ???\n"
+        )
+        _make_agent(tmp_path, "claude_code_agent", configs={"claude_code_agent": body})
+
+        # Has a resources_server but drives an external LLM harness -> not composable.
+        assert discover_agents(tmp_path)["claude_code_agent"].self_contained is True
+
+    def test_zero_config_agent_is_discovered_and_defaults_composable(self, tmp_path: Path) -> None:
+        _make_agent(tmp_path, "aviary_agent", configs=None)  # app.py only, no configs
+
+        entry = discover_agents(tmp_path)["aviary_agent"]
+        assert entry.config_paths == ()
+        assert entry.self_contained is False
+
+    def test_multiple_variants_are_all_recorded(self, tmp_path: Path) -> None:
+        _make_agent(
+            tmp_path,
+            "langgraph_agent",
+            configs={
+                "orchestrator_agent": _pattern_a("langgraph_agent"),
+                "rewoo_agent": _pattern_a("langgraph_agent"),
+            },
+        )
+
+        assert set(discover_agents(tmp_path)["langgraph_agent"].variants) == {"orchestrator_agent", "rewoo_agent"}
+
+    def test_non_agent_yaml_is_filtered_out(self, tmp_path: Path) -> None:
+        # A configs/ file that is not a gym agent config (no responses_api_agents) is ignored;
+        # the dir still counts as an agent because of app.py.
+        _make_agent(tmp_path, "swe_agents", configs={"raw_harness": "agent:\n  type: openhands\n"})
+
+        entry = discover_agents(tmp_path)["swe_agents"]
+        assert entry.config_paths == ()
+
+    def test_directory_without_app_or_configs_is_skipped(self, tmp_path: Path) -> None:
+        (tmp_path / "not_an_agent").mkdir()
+        (tmp_path / "loose_file.txt").write_text("x")
+
+        assert discover_agents(tmp_path) == {}
+
+    def test_unparseable_config_does_not_crash_discovery(self, tmp_path: Path) -> None:
+        _make_agent(tmp_path, "broken", configs={"broken": "responses_api_agents: [unclosed\n"})
+
+        # The bad file is skipped (not an agent config); the dir survives via app.py.
+        assert discover_agents(tmp_path)["broken"].config_paths == ()
+
+    def test_missing_directory_yields_no_agents(self, tmp_path: Path) -> None:
+        assert discover_agents(tmp_path / "nope") == {}
+
+
+class TestRealAgents:
+    def test_discovers_real_simple_agent_as_composable(self) -> None:
+        agents = discover_agents()
+        # The repo ships a `simple_agent`; it pairs with a separate resources server.
+        if "simple_agent" in agents:
+            assert agents["simple_agent"].self_contained is False
+
+    def test_agent_entry_is_hashable(self) -> None:
+        entry = AgentEntry(name="a", path=Path("a"), config_paths=(Path("a/configs/a.yaml"),), self_contained=True)
+        assert {entry: 1}[entry] == 1
+        assert entry.variants == {"a": Path("a/configs/a.yaml")}