From 565a8bc8a3e4968b09b9ee15af55262ecca68ec9 Mon Sep 17 00:00:00 2001 From: Wojciech Prazuch Date: Wed, 24 Jun 2026 09:21:34 +0200 Subject: [PATCH 1/6] =?UTF-8?q?feat:=20agent=20registry=20=E2=80=94=20name?= =?UTF-8?q?-based=20agent=20discovery=20+=20composability=20(M3=20core)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add nemo_gym.agent_registry, mirroring the environment registry: discover agents under responses_api_agents//, resolve an agent name (+ optional variant) to its config — the run-by-name primitive for 'gym run --agent ' — and classify each agent as composable (Pattern A: references a separate resources server) vs self-contained (Pattern B: agent_framework / external harness, e.g. swe_agents, harbor_agent, claude_code_agent). Resolution-safe (reads configs only), with did-you-mean hints, variant selection, and AgentNotComposableError for the composer. Foundation for the M3 config_composer (epic #1205, friction #6); CLI wiring deferred to align with the unified CLI. Signed-off-by: Wojciech Prazuch --- nemo_gym/agent_registry.py | 220 +++++++++++++++++++++++ tests/unit_tests/test_agent_registry.py | 226 ++++++++++++++++++++++++ 2 files changed, 446 insertions(+) create mode 100644 nemo_gym/agent_registry.py create mode 100644 tests/unit_tests/test_agent_registry.py diff --git a/nemo_gym/agent_registry.py b/nemo_gym/agent_registry.py new file mode 100644 index 0000000000..e2178ae015 --- /dev/null +++ b/nemo_gym/agent_registry.py @@ -0,0 +1,220 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Registry of agent harnesses under ``responses_api_agents//``. + +An *agent* is a directory ``responses_api_agents//`` providing an agent harness, with zero or +more ``configs/*.yaml`` variants. This module maps an agent's short ```` (the directory name) +to its config variant(s) so it can be referenced by name — the foundation for ``gym run --agent +`` (run-by-name) — and classifies whether the agent is freely *composable* with an arbitrary +environment. + +- **Composable (Pattern A):** the agent references a *separate* resources server + (``responses_api_agents..resources_server``), so it can be paired with any environment. +- **Not composable (Pattern B):** the agent is self-contained — it declares an ``agent_framework`` + or bakes in its own environment/external LLM harness (e.g. ``swe_agents``, ``harbor_agent``, + ``verifiers_agent``, ``claude_code_agent``) — and cannot be dropped onto an arbitrary environment. + +Discovery only reads config files; it never resolves interpolations or missing values and never +starts servers, so it is safe to call when secrets/API keys referenced by a config are unset. +""" + +from dataclasses import dataclass +from difflib import get_close_matches +from pathlib import Path +from typing import Dict, List, Optional, Tuple + +from omegaconf import OmegaConf + +from nemo_gym import PARENT_DIR + + +AGENTS_DIR = PARENT_DIR / "responses_api_agents" +AGENT_CONFIGS_SUBDIR = "configs" + + +class AgentNotFoundError(ValueError): + """An agent was referenced by a name that is not registered under ``responses_api_agents/``.""" + + +class AgentVariantError(ValueError): + """An agent has no standalone config, or has several and no variant was given to disambiguate.""" + + +class AgentNotComposableError(ValueError): + """A self-contained (Pattern B) agent was requested for free composition with an environment.""" + + +@dataclass(frozen=True) +class AgentEntry: + """A discovered agent: its name, where it lives, its config variants, and composability.""" + + name: str + path: Path + config_paths: Tuple[Path, ...] # variant config files, sorted; empty for "zero-config" agents + composable: bool + description: Optional[str] = None + + @property + def variants(self) -> Dict[str, Path]: + """Map variant name (config filename stem) -> config path.""" + return {path.stem: path for path in self.config_paths} + + +def _iter_agent_blocks(config_path: Path): + """Yield each ``responses_api_agents.`` mapping in a config (resolution-safe, best effort).""" + try: + container = OmegaConf.to_container(OmegaConf.load(config_path), resolve=False, throw_on_missing=False) + except Exception: + return + if not isinstance(container, dict): + return + for top_level_value in container.values(): + if not isinstance(top_level_value, dict): + continue + agents = top_level_value.get("responses_api_agents") + if not isinstance(agents, dict): + continue + for agent_block in agents.values(): + if isinstance(agent_block, dict): + yield agent_block + + +def _is_agent_config(config_path: Path) -> bool: + """True if the file is a NeMo Gym agent config (a top-level block with ``responses_api_agents``). + + Filters out non-agent YAML that happens to live in an agent's ``configs/`` dir (e.g. a raw + harness config or an empty stub). + """ + return next(_iter_agent_blocks(config_path), None) is not None + + +def _classify(config_paths: Tuple[Path, ...]) -> Tuple[bool, Optional[str]]: + """Return ``(composable, description)`` for an agent from its config variants. + + Composable iff some variant references a separate resources server, none declares an + ``agent_framework``, and none drives an external LLM harness (e.g. its own Anthropic key). + Agents with no parseable config default to composable (their wiring lives in a paired + benchmark/resources-server config). + """ + has_resources_server = False + has_agent_framework = False + drives_external_harness = False + description: Optional[str] = None + + saw_block = False + for config_path in config_paths: + for block in _iter_agent_blocks(config_path): + saw_block = True + if "resources_server" in block: + has_resources_server = True + if "agent_framework" in block: + has_agent_framework = True + if "anthropic_api_key" in block: + drives_external_harness = True + if description is None and isinstance(block.get("description"), str): + description = block["description"] + + if not saw_block: + return True, description + composable = has_resources_server and not has_agent_framework and not drives_external_harness + return composable, description + + +def discover_agents(agents_dir: Path = AGENTS_DIR) -> Dict[str, AgentEntry]: + """Map agent name -> :class:`AgentEntry` for every agent dir under ``responses_api_agents/``. + + The name is the directory name. A directory is an agent if it has an ``app.py`` or at least one + agent config. Returns an empty dict if the directory is missing. + """ + agents: Dict[str, AgentEntry] = {} + if not agents_dir.is_dir(): + return agents + + for child in sorted(agents_dir.iterdir()): + if not child.is_dir(): + continue + configs_dir = child / AGENT_CONFIGS_SUBDIR + config_files = sorted(configs_dir.glob("*.yaml")) if configs_dir.is_dir() else [] + agent_configs = tuple(path for path in config_files if _is_agent_config(path)) + if not (child / "app.py").is_file() and not agent_configs: + continue + + composable, description = _classify(agent_configs) + agents[child.name] = AgentEntry( + name=child.name, + path=child, + config_paths=agent_configs, + composable=composable, + description=description, + ) + + return agents + + +def _did_you_mean(name: str, available: List[str], noun: str) -> str: + suggestions = get_close_matches(name, available, n=3, cutoff=0.6) + if suggestions: + return "Did you mean: " + ", ".join(repr(s) for s in suggestions) + "?" + return f"Available {noun}: " + (", ".join(repr(n) for n in available) or "(none)") + + +def resolve_agent_config_path( + name: str, + variant: Optional[str] = None, + agents_dir: Path = AGENTS_DIR, + require_composable: bool = False, +) -> str: + """Return the config path to load to run agent ``name`` — the run-by-name primitive. + + Selection: an explicit ``variant`` wins; otherwise a single config is used directly, and a + variant whose name equals ``name`` is the default when several exist. Raises + :class:`AgentNotFoundError` (with a "did you mean?" hint) for an unknown agent, + :class:`AgentVariantError` for a zero-config or ambiguous-variant agent, and — when + ``require_composable`` is set — :class:`AgentNotComposableError` for a Pattern B agent. + """ + agents = discover_agents(agents_dir) + entry = agents.get(name) + if entry is None: + raise AgentNotFoundError( + f"No agent named '{name}' under {agents_dir}.\n{_did_you_mean(name, sorted(agents), 'agents')}" + ) + + if require_composable and not entry.composable: + raise AgentNotComposableError( + f"Agent '{name}' is self-contained (it bakes in its own environment/framework) and cannot " + "be freely composed with an arbitrary environment; run it with its own config instead." + ) + + variants = entry.variants + if not variants: + raise AgentVariantError( + f"Agent '{name}' ships no standalone config; it is composed via its paired " + "benchmark/resources-server config." + ) + + if variant is not None: + if variant not in variants: + raise AgentVariantError( + f"Agent '{name}' has no variant '{variant}'.\n{_did_you_mean(variant, sorted(variants), 'variants')}" + ) + return str(variants[variant]) + + if len(variants) == 1: + return str(next(iter(variants.values()))) + if name in variants: + return str(variants[name]) + raise AgentVariantError( + f"Agent '{name}' has multiple config variants: {sorted(variants)}; pass a variant to select one." + ) diff --git a/tests/unit_tests/test_agent_registry.py b/tests/unit_tests/test_agent_registry.py new file mode 100644 index 0000000000..15ef002985 --- /dev/null +++ b/tests/unit_tests/test_agent_registry.py @@ -0,0 +1,226 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from pathlib import Path + +from pytest import raises + +from nemo_gym.agent_registry import ( + AGENTS_DIR, + AgentEntry, + AgentNotComposableError, + AgentNotFoundError, + AgentVariantError, + discover_agents, + resolve_agent_config_path, +) + + +def _make_agent(agents_dir: Path, name: str, *, app: bool = True, configs: dict = None) -> Path: + agent_dir = agents_dir / name + agent_dir.mkdir(parents=True) + if app: + (agent_dir / "app.py").write_text("# app\n") + if configs: + configs_dir = agent_dir / "configs" + configs_dir.mkdir() + for variant, body in configs.items(): + (configs_dir / f"{variant}.yaml").write_text(body) + return agent_dir + + +def _pattern_a(agent_type: str = "simple_agent") -> str: + # References a separate resources server -> composable. + return ( + f"some_key:\n responses_api_agents:\n {agent_type}:\n entrypoint: app.py\n" + " resources_server:\n type: resources_servers\n name: ???\n" + " description: A composable agent\n" + ) + + +def _pattern_b(agent_type: str = "swe_agent") -> str: + # Self-contained framework agent -> not composable. + return ( + f"some_key:\n responses_api_agents:\n {agent_type}:\n entrypoint: app.py\n" + " agent_framework: openhands\n" + ) + + +class TestDiscoverAgents: + def test_discovers_and_classifies_pattern_a(self, tmp_path: Path) -> None: + _make_agent(tmp_path, "simple_agent", configs={"simple_agent": _pattern_a()}) + + agents = discover_agents(tmp_path) + + assert set(agents) == {"simple_agent"} + entry = agents["simple_agent"] + assert entry.composable is True + assert entry.description == "A composable agent" + assert list(entry.variants) == ["simple_agent"] + + def test_classifies_pattern_b_as_not_composable(self, tmp_path: Path) -> None: + _make_agent(tmp_path, "swe_agents", configs={"swebench": _pattern_b()}) + + assert discover_agents(tmp_path)["swe_agents"].composable is False + + def test_external_harness_agent_is_not_composable(self, tmp_path: Path) -> None: + body = ( + "k:\n responses_api_agents:\n claude_code_agent:\n entrypoint: app.py\n" + " resources_server:\n name: ???\n anthropic_api_key: ???\n" + ) + _make_agent(tmp_path, "claude_code_agent", configs={"claude_code_agent": body}) + + # Has a resources_server but drives an external LLM harness -> not composable. + assert discover_agents(tmp_path)["claude_code_agent"].composable is False + + def test_zero_config_agent_is_discovered_and_defaults_composable(self, tmp_path: Path) -> None: + _make_agent(tmp_path, "aviary_agent", configs=None) # app.py only, no configs + + entry = discover_agents(tmp_path)["aviary_agent"] + assert entry.config_paths == () + assert entry.composable is True + + def test_multiple_variants_are_all_recorded(self, tmp_path: Path) -> None: + _make_agent( + tmp_path, + "langgraph_agent", + configs={ + "orchestrator_agent": _pattern_a("langgraph_agent"), + "rewoo_agent": _pattern_a("langgraph_agent"), + }, + ) + + assert set(discover_agents(tmp_path)["langgraph_agent"].variants) == {"orchestrator_agent", "rewoo_agent"} + + def test_non_agent_yaml_is_filtered_out(self, tmp_path: Path) -> None: + # A configs/ file that is not a gym agent config (no responses_api_agents) is ignored; + # the dir still counts as an agent because of app.py. + _make_agent(tmp_path, "swe_agents", configs={"raw_harness": "agent:\n type: openhands\n"}) + + entry = discover_agents(tmp_path)["swe_agents"] + assert entry.config_paths == () + + def test_directory_without_app_or_configs_is_skipped(self, tmp_path: Path) -> None: + (tmp_path / "not_an_agent").mkdir() + (tmp_path / "loose_file.txt").write_text("x") + + assert discover_agents(tmp_path) == {} + + def test_unparseable_config_does_not_crash_discovery(self, tmp_path: Path) -> None: + _make_agent(tmp_path, "broken", configs={"broken": "responses_api_agents: [unclosed\n"}) + + # The bad file is skipped (not an agent config); the dir survives via app.py. + assert discover_agents(tmp_path)["broken"].config_paths == () + + def test_missing_directory_yields_no_agents(self, tmp_path: Path) -> None: + assert discover_agents(tmp_path / "nope") == {} + + +class TestResolveAgentConfigPath: + def test_single_config_resolves(self, tmp_path: Path) -> None: + _make_agent(tmp_path, "simple_agent", configs={"simple_agent": _pattern_a()}) + + path = resolve_agent_config_path("simple_agent", agents_dir=tmp_path) + assert path.endswith("simple_agent/configs/simple_agent.yaml") + + def test_explicit_variant_resolves(self, tmp_path: Path) -> None: + _make_agent( + tmp_path, + "langgraph_agent", + configs={ + "orchestrator_agent": _pattern_a("langgraph_agent"), + "rewoo_agent": _pattern_a("langgraph_agent"), + }, + ) + + path = resolve_agent_config_path("langgraph_agent", variant="rewoo_agent", agents_dir=tmp_path) + assert path.endswith("rewoo_agent.yaml") + + def test_variant_matching_name_is_default_when_several(self, tmp_path: Path) -> None: + _make_agent( + tmp_path, + "harbor_agent", + configs={"harbor_agent": _pattern_a("harbor_agent"), "harbor_daytona": _pattern_a("harbor_agent")}, + ) + + assert resolve_agent_config_path("harbor_agent", agents_dir=tmp_path).endswith("harbor_agent.yaml") + + def test_ambiguous_variant_raises(self, tmp_path: Path) -> None: + _make_agent( + tmp_path, + "langgraph_agent", + configs={ + "orchestrator_agent": _pattern_a("langgraph_agent"), + "rewoo_agent": _pattern_a("langgraph_agent"), + }, + ) + + with raises(AgentVariantError, match="multiple config variants"): + resolve_agent_config_path("langgraph_agent", agents_dir=tmp_path) + + def test_unknown_variant_raises_with_suggestion(self, tmp_path: Path) -> None: + _make_agent( + tmp_path, + "langgraph_agent", + configs={ + "orchestrator_agent": _pattern_a("langgraph_agent"), + "rewoo_agent": _pattern_a("langgraph_agent"), + }, + ) + + with raises(AgentVariantError, match="Did you mean"): + resolve_agent_config_path("langgraph_agent", variant="rewoo", agents_dir=tmp_path) + + def test_zero_config_agent_raises(self, tmp_path: Path) -> None: + _make_agent(tmp_path, "aviary_agent", configs=None) + + with raises(AgentVariantError, match="no standalone config"): + resolve_agent_config_path("aviary_agent", agents_dir=tmp_path) + + def test_unknown_agent_raises_with_suggestion(self, tmp_path: Path) -> None: + _make_agent(tmp_path, "simple_agent", configs={"simple_agent": _pattern_a()}) + + with raises(AgentNotFoundError, match="Did you mean"): + resolve_agent_config_path("simple_agnt", agents_dir=tmp_path) + + def test_unknown_agent_without_close_match_lists_available(self, tmp_path: Path) -> None: + _make_agent(tmp_path, "simple_agent", configs={"simple_agent": _pattern_a()}) + + with raises(AgentNotFoundError, match="Available agents"): + resolve_agent_config_path("zzzzz", agents_dir=tmp_path) + + def test_require_composable_rejects_pattern_b(self, tmp_path: Path) -> None: + _make_agent(tmp_path, "swe_agents", configs={"swebench": _pattern_b()}) + + with raises(AgentNotComposableError, match="self-contained"): + resolve_agent_config_path("swe_agents", agents_dir=tmp_path, require_composable=True) + + def test_require_composable_allows_pattern_a(self, tmp_path: Path) -> None: + _make_agent(tmp_path, "simple_agent", configs={"simple_agent": _pattern_a()}) + + path = resolve_agent_config_path("simple_agent", agents_dir=tmp_path, require_composable=True) + assert path.endswith("simple_agent.yaml") + + +class TestRealAgents: + def test_discovers_real_simple_agent_as_composable(self) -> None: + agents = discover_agents() + # The repo ships a `simple_agent`; it pairs with a separate resources server. + if "simple_agent" in agents: + assert agents["simple_agent"].composable is True + + def test_agent_entry_is_hashable(self) -> None: + entry = AgentEntry(name="a", path=Path("a"), config_paths=(Path("a/configs/a.yaml"),), composable=True) + assert {entry: 1}[entry] == 1 + assert entry.path == AGENTS_DIR / "a" or True # AGENTS_DIR import exercised From e095048f993e7f345dc4046b1b7cd23d84b04f76 Mon Sep 17 00:00:00 2001 From: Wojciech Prazuch Date: Wed, 24 Jun 2026 09:57:51 +0200 Subject: [PATCH 2/6] docs(agent-registry): clarify composability per review Address review: an agent harness is one component of an environment, not 'paired with any environment'. Reframe the classification around whether the harness references a separate resources server (Pattern A) vs is self-contained (Pattern B), rename the AgentEntry field composable -> self_contained, and note that cross-pattern (e.g. simple_agent vs gymnasium) compatibility is the config composer's concern, not the registry's. Signed-off-by: Wojciech Prazuch --- nemo_gym/agent_registry.py | 61 ++++++++++++++----------- tests/unit_tests/test_agent_registry.py | 12 ++--- 2 files changed, 40 insertions(+), 33 deletions(-) diff --git a/nemo_gym/agent_registry.py b/nemo_gym/agent_registry.py index e2178ae015..d4a36cfabe 100644 --- a/nemo_gym/agent_registry.py +++ b/nemo_gym/agent_registry.py @@ -14,17 +14,21 @@ # limitations under the License. """Registry of agent harnesses under ``responses_api_agents//``. -An *agent* is a directory ``responses_api_agents//`` providing an agent harness, with zero or -more ``configs/*.yaml`` variants. This module maps an agent's short ```` (the directory name) -to its config variant(s) so it can be referenced by name — the foundation for ``gym run --agent -`` (run-by-name) — and classifies whether the agent is freely *composable* with an arbitrary -environment. - -- **Composable (Pattern A):** the agent references a *separate* resources server - (``responses_api_agents..resources_server``), so it can be paired with any environment. -- **Not composable (Pattern B):** the agent is self-contained — it declares an ``agent_framework`` - or bakes in its own environment/external LLM harness (e.g. ``swe_agents``, ``harbor_agent``, - ``verifiers_agent``, ``claude_code_agent``) — and cannot be dropped onto an arbitrary environment. +An agent harness is one *component* of an environment (an environment = dataset + agent harness + +resources server [verifier and state] + model). This module maps an agent's short ```` (the +directory name) to its config variant(s) so it can be referenced by name — the foundation for +``gym run --agent `` (run-by-name) — and records whether the harness bundles its own +environment or references a separate resources server. + +- **References a separate resources server (Pattern A):** the config sets + ``responses_api_agents..resources_server``, so the harness is reusable and must be wired to a + *compatible* resources server + dataset (e.g. the ``simple_agent`` tool-use pattern, the + ``gymnasium`` pattern). Harnesses compose *within* a pattern, not across it; which + harness↔resources-server pairings are actually compatible is decided by the config composer's + compatibility guard, NOT by this registry. +- **Self-contained (Pattern B):** the harness bundles its own environment/framework or external LLM + loop (``agent_framework``; e.g. ``swe_agents``, ``harbor_agent``, ``verifiers_agent``, + ``claude_code_agent``) and runs with its own config rather than wired to a separate environment. Discovery only reads config files; it never resolves interpolations or missing values and never starts servers, so it is safe to call when secrets/API keys referenced by a config are unset. @@ -53,17 +57,19 @@ class AgentVariantError(ValueError): class AgentNotComposableError(ValueError): - """A self-contained (Pattern B) agent was requested for free composition with an environment.""" + """A self-contained (Pattern B) agent was requested for wiring into a separate environment.""" @dataclass(frozen=True) class AgentEntry: - """A discovered agent: its name, where it lives, its config variants, and composability.""" + """A discovered agent: its name, where it lives, its config variants, and how it's wired.""" name: str path: Path config_paths: Tuple[Path, ...] # variant config files, sorted; empty for "zero-config" agents - composable: bool + # True = bundles its own environment/framework (Pattern B; run standalone). + # False = references a separate resources server (Pattern A; wire to a compatible environment). + self_contained: bool description: Optional[str] = None @property @@ -101,12 +107,13 @@ def _is_agent_config(config_path: Path) -> bool: def _classify(config_paths: Tuple[Path, ...]) -> Tuple[bool, Optional[str]]: - """Return ``(composable, description)`` for an agent from its config variants. + """Return ``(self_contained, description)`` for an agent from its config variants. - Composable iff some variant references a separate resources server, none declares an - ``agent_framework``, and none drives an external LLM harness (e.g. its own Anthropic key). - Agents with no parseable config default to composable (their wiring lives in a paired - benchmark/resources-server config). + A harness is NOT self-contained (Pattern A) when some variant references a separate resources + server, none declares an ``agent_framework``, and none drives an external LLM loop (e.g. its own + Anthropic key) — it must be wired to a compatible resources server. Otherwise it is + self-contained (Pattern B). Agents with no parseable config are treated as Pattern A (their + wiring lives in a paired benchmark/resources-server config). """ has_resources_server = False has_agent_framework = False @@ -127,9 +134,9 @@ def _classify(config_paths: Tuple[Path, ...]) -> Tuple[bool, Optional[str]]: description = block["description"] if not saw_block: - return True, description - composable = has_resources_server and not has_agent_framework and not drives_external_harness - return composable, description + return False, description + references_resources_server = has_resources_server and not has_agent_framework and not drives_external_harness + return not references_resources_server, description def discover_agents(agents_dir: Path = AGENTS_DIR) -> Dict[str, AgentEntry]: @@ -151,12 +158,12 @@ def discover_agents(agents_dir: Path = AGENTS_DIR) -> Dict[str, AgentEntry]: if not (child / "app.py").is_file() and not agent_configs: continue - composable, description = _classify(agent_configs) + self_contained, description = _classify(agent_configs) agents[child.name] = AgentEntry( name=child.name, path=child, config_paths=agent_configs, - composable=composable, + self_contained=self_contained, description=description, ) @@ -191,10 +198,10 @@ def resolve_agent_config_path( f"No agent named '{name}' under {agents_dir}.\n{_did_you_mean(name, sorted(agents), 'agents')}" ) - if require_composable and not entry.composable: + if require_composable and entry.self_contained: raise AgentNotComposableError( - f"Agent '{name}' is self-contained (it bakes in its own environment/framework) and cannot " - "be freely composed with an arbitrary environment; run it with its own config instead." + f"Agent '{name}' is self-contained (it bundles its own environment/framework) and cannot " + "be wired into a separate environment; run it with its own config instead." ) variants = entry.variants diff --git a/tests/unit_tests/test_agent_registry.py b/tests/unit_tests/test_agent_registry.py index 15ef002985..acd93b1823 100644 --- a/tests/unit_tests/test_agent_registry.py +++ b/tests/unit_tests/test_agent_registry.py @@ -65,14 +65,14 @@ def test_discovers_and_classifies_pattern_a(self, tmp_path: Path) -> None: assert set(agents) == {"simple_agent"} entry = agents["simple_agent"] - assert entry.composable is True + assert entry.self_contained is False assert entry.description == "A composable agent" assert list(entry.variants) == ["simple_agent"] def test_classifies_pattern_b_as_not_composable(self, tmp_path: Path) -> None: _make_agent(tmp_path, "swe_agents", configs={"swebench": _pattern_b()}) - assert discover_agents(tmp_path)["swe_agents"].composable is False + assert discover_agents(tmp_path)["swe_agents"].self_contained is True def test_external_harness_agent_is_not_composable(self, tmp_path: Path) -> None: body = ( @@ -82,14 +82,14 @@ def test_external_harness_agent_is_not_composable(self, tmp_path: Path) -> None: _make_agent(tmp_path, "claude_code_agent", configs={"claude_code_agent": body}) # Has a resources_server but drives an external LLM harness -> not composable. - assert discover_agents(tmp_path)["claude_code_agent"].composable is False + assert discover_agents(tmp_path)["claude_code_agent"].self_contained is True def test_zero_config_agent_is_discovered_and_defaults_composable(self, tmp_path: Path) -> None: _make_agent(tmp_path, "aviary_agent", configs=None) # app.py only, no configs entry = discover_agents(tmp_path)["aviary_agent"] assert entry.config_paths == () - assert entry.composable is True + assert entry.self_contained is False def test_multiple_variants_are_all_recorded(self, tmp_path: Path) -> None: _make_agent( @@ -218,9 +218,9 @@ def test_discovers_real_simple_agent_as_composable(self) -> None: agents = discover_agents() # The repo ships a `simple_agent`; it pairs with a separate resources server. if "simple_agent" in agents: - assert agents["simple_agent"].composable is True + assert agents["simple_agent"].self_contained is False def test_agent_entry_is_hashable(self) -> None: - entry = AgentEntry(name="a", path=Path("a"), config_paths=(Path("a/configs/a.yaml"),), composable=True) + entry = AgentEntry(name="a", path=Path("a"), config_paths=(Path("a/configs/a.yaml"),), self_contained=True) assert {entry: 1}[entry] == 1 assert entry.path == AGENTS_DIR / "a" or True # AGENTS_DIR import exercised From 666edf93cb520c18f63cc13d2ffc981e290ab29c Mon Sep 17 00:00:00 2001 From: Wojciech Prazuch Date: Wed, 24 Jun 2026 15:05:59 +0200 Subject: [PATCH 3/6] =?UTF-8?q?feat(cli):=20gym=20list=20agents=20?= =?UTF-8?q?=E2=80=94=20list=20harnesses=20and=20how=20each=20composes?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add a `gym list agents` command (under the existing `list` group, mirroring `list benchmarks`) backed by the agent registry. It surfaces the one thing the generic -- asset selectors cannot: each harness’s composition pattern — freely wireable into a separate environment (Pattern A) vs self-contained, run with its own config (Pattern B) — the distinction the config composer’s compatibility guard relies on. Supports a rich table and --json. This gives the agent registry a user-facing CLI payoff on its own, independent of the composer wiring. Run-by-name selection (an --agent asset type) is deferred: --agent is already taken by `eval run` for the in-config agent instance, so the selector needs a non-colliding name. Signed-off-by: Wojciech Prazuch --- nemo_gym/cli/agents.py | 89 +++++++++++++++++++++++++++++ nemo_gym/cli/main.py | 7 ++- tests/unit_tests/test_cli_agents.py | 86 ++++++++++++++++++++++++++++ tests/unit_tests/test_cli_main.py | 1 + 4 files changed, 182 insertions(+), 1 deletion(-) create mode 100644 nemo_gym/cli/agents.py create mode 100644 tests/unit_tests/test_cli_agents.py diff --git a/nemo_gym/cli/agents.py b/nemo_gym/cli/agents.py new file mode 100644 index 0000000000..04c51cb133 --- /dev/null +++ b/nemo_gym/cli/agents.py @@ -0,0 +1,89 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import json + +import rich +from rich.table import Table + +from nemo_gym.agent_registry import discover_agents +from nemo_gym.config_types import BaseNeMoGymCLIConfig +from nemo_gym.global_config import ( + JSON_OUTPUT_KEY_NAME, + QUERY_KEY_NAME, + GlobalConfigDictParserConfig, + get_global_config_dict, +) + + +def _fuzzy_matches(query: str, *fields: str) -> bool: + needle = query.lower() + return any(needle in (field or "").lower() for field in fields) + + +def list_agents() -> None: + """CLI command: list discovered agent harnesses and how each composes (Pattern A vs B). + + Complements ``gym list benchmarks``: the asset selectors resolve a component *by name*, but only + this listing surfaces which agents are freely wireable into a separate environment (Pattern A) + versus self-contained harnesses that run with their own config (Pattern B) — the distinction the + config composer's compatibility guard relies on. + """ + global_config_dict = get_global_config_dict( + global_config_dict_parser_config=GlobalConfigDictParserConfig( + initial_global_config_dict=GlobalConfigDictParserConfig.NO_MODEL_GLOBAL_CONFIG_DICT, + ) + ) + BaseNeMoGymCLIConfig.model_validate(global_config_dict) + + agents = discover_agents() + + query = global_config_dict.get(QUERY_KEY_NAME) + if query: + agents = { + name: entry for name, entry in agents.items() if _fuzzy_matches(query, name, entry.description or "") + } + + if global_config_dict.get(JSON_OUTPUT_KEY_NAME, False): + payload = [ + { + "name": name, + "pattern": "B (self-contained)" if entry.self_contained else "A (composable)", + "self_contained": entry.self_contained, + "variants": sorted(entry.variants), + "description": entry.description, + } + for name, entry in agents.items() + ] + print(json.dumps(payload)) + return + + if not agents: + rich.print("No agents found." if not query else f"No agents match {query!r}.") + return + + table = Table(title="NeMo Gym agents") + table.add_column("agent", style="bold") + table.add_column("composition") + table.add_column("variants") + table.add_column("description") + for name, entry in agents.items(): + composition = "self-contained (B)" if entry.self_contained else "composable (A)" + table.add_row( + name, + composition, + ", ".join(sorted(entry.variants)) or "—", + entry.description or "", + ) + rich.print(table) diff --git a/nemo_gym/cli/main.py b/nemo_gym/cli/main.py index fbb21e0493..5792545386 100644 --- a/nemo_gym/cli/main.py +++ b/nemo_gym/cli/main.py @@ -285,7 +285,7 @@ def _dataset_download(args: argparse.Namespace, overrides: list[str]) -> None: # One-line help for each command group, shown in `gym --help`. GROUPS = { - "list": "List available components (benchmarks, environments).", + "list": "List available components (benchmarks, agents, environments).", "dataset": "Manage datasets.", "env": "Develop and run environments.", "eval": "Run evaluations.", @@ -299,6 +299,11 @@ def _dataset_download(args: argparse.Namespace, overrides: list[str]) -> None: "list environments": Command( target="nemo_gym.cli.env:list_environments", summary="List available environments by name.", flags=(JSON,) ), + "list agents": Command( + target="nemo_gym.cli.agents:list_agents", + summary="List agent harnesses and how each composes (Pattern A vs self-contained B).", + flags=(JSON,), + ), "search": Command( target="nemo_gym.cli.eval:list_benchmarks", summary="Search available components (currently benchmarks) by name; like `list` filtered to a query.", diff --git a/tests/unit_tests/test_cli_agents.py b/tests/unit_tests/test_cli_agents.py new file mode 100644 index 0000000000..a2a8d63b20 --- /dev/null +++ b/tests/unit_tests/test_cli_agents.py @@ -0,0 +1,86 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import json +from pathlib import Path +from unittest.mock import patch + +from omegaconf import OmegaConf + +from nemo_gym.agent_registry import AgentEntry +from nemo_gym.cli.agents import list_agents + + +def _mock_global_config(config: dict = None): + return OmegaConf.create(config or {}) + + +def _entry(name: str, self_contained: bool, variants=(), description=None) -> AgentEntry: + path = Path("responses_api_agents") / name + config_paths = tuple(path / "configs" / f"{v}.yaml" for v in variants) + return AgentEntry( + name=name, + path=path, + config_paths=config_paths, + self_contained=self_contained, + description=description, + ) + + +_AGENTS = { + "simple_agent": _entry("simple_agent", self_contained=False, variants=("simple_agent",)), + "swe_agents": _entry("swe_agents", self_contained=True, variants=("swebench_openhands",), description="SWE tasks"), +} + + +class TestListAgents: + def test_lists_found_agents(self, capsys) -> None: + with ( + patch("nemo_gym.cli.agents.get_global_config_dict", return_value=_mock_global_config()), + patch("nemo_gym.cli.agents.discover_agents", return_value=_AGENTS), + ): + list_agents() + out = capsys.readouterr().out + assert "simple_agent" in out and "swe_agents" in out + assert "composable" in out and "self-contained" in out + + def test_no_agents(self, capsys) -> None: + with ( + patch("nemo_gym.cli.agents.get_global_config_dict", return_value=_mock_global_config()), + patch("nemo_gym.cli.agents.discover_agents", return_value={}), + ): + list_agents() + assert "No agents found" in capsys.readouterr().out + + def test_json_output(self, capsys) -> None: + with ( + patch("nemo_gym.cli.agents.get_global_config_dict", return_value=_mock_global_config({"json": True})), + patch("nemo_gym.cli.agents.discover_agents", return_value=_AGENTS), + ): + list_agents() + payload = json.loads(capsys.readouterr().out) + by_name = {entry["name"]: entry for entry in payload} + assert by_name["simple_agent"]["self_contained"] is False + assert by_name["simple_agent"]["pattern"] == "A (composable)" + assert by_name["swe_agents"]["self_contained"] is True + assert by_name["swe_agents"]["variants"] == ["swebench_openhands"] + + def test_query_filters(self, capsys) -> None: + with ( + patch("nemo_gym.cli.agents.get_global_config_dict", return_value=_mock_global_config({"query": "swe"})), + patch("nemo_gym.cli.agents.discover_agents", return_value=_AGENTS), + ): + list_agents() + out = capsys.readouterr().out + assert "swe_agents" in out and "simple_agent" not in out diff --git a/tests/unit_tests/test_cli_main.py b/tests/unit_tests/test_cli_main.py index 45fca591f8..b47625e0c6 100644 --- a/tests/unit_tests/test_cli_main.py +++ b/tests/unit_tests/test_cli_main.py @@ -571,6 +571,7 @@ class TestJsonFlag: "argv, expected_target", [ (["list", "benchmarks", "--json"], "nemo_gym.cli.eval:list_benchmarks"), + (["list", "agents", "--json"], "nemo_gym.cli.agents:list_agents"), (["env", "status", "--json"], "nemo_gym.cli.env:status"), ], ) From 1e628704c5e867e5d37e53ae32f705b337e3a4a0 Mon Sep 17 00:00:00 2001 From: Wojciech Prazuch Date: Thu, 25 Jun 2026 14:28:56 +0200 Subject: [PATCH 4/6] =?UTF-8?q?refactor(agent-registry):=20address=20revie?= =?UTF-8?q?w=20=E2=80=94=20drop=20dead=20query=20path,=20fix=20tests,=20cl?= =?UTF-8?q?arify=20gymnasium?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review follow-up: - Remove the unreachable query filter from gym list agents (cli/agents.py): no QUERY flag is registered for the command, so the branch could never run; drop it (and _fuzzy_matches and the mock-only test_query_filters that masked it). search stays benchmarks-only. - test_agent_entry_is_hashable: replace the trivially-true `... or True` assertion with a real check on AgentEntry.variants. - Docstring: stop citing gymnasium as a Pattern A (composable) example — gymnasium-style agents ship their own env and classify as self-contained (Pattern B), matching cmunley1s review point. Signed-off-by: Wojciech Prazuch --- nemo_gym/agent_registry.py | 10 +++++----- nemo_gym/cli/agents.py | 14 +------------- tests/unit_tests/test_agent_registry.py | 3 +-- tests/unit_tests/test_cli_agents.py | 9 --------- 4 files changed, 7 insertions(+), 29 deletions(-) diff --git a/nemo_gym/agent_registry.py b/nemo_gym/agent_registry.py index d4a36cfabe..95a8508c2c 100644 --- a/nemo_gym/agent_registry.py +++ b/nemo_gym/agent_registry.py @@ -22,13 +22,13 @@ - **References a separate resources server (Pattern A):** the config sets ``responses_api_agents..resources_server``, so the harness is reusable and must be wired to a - *compatible* resources server + dataset (e.g. the ``simple_agent`` tool-use pattern, the - ``gymnasium`` pattern). Harnesses compose *within* a pattern, not across it; which - harness↔resources-server pairings are actually compatible is decided by the config composer's - compatibility guard, NOT by this registry. + *compatible* resources server + dataset (e.g. the ``simple_agent`` tool-use pattern). Harnesses + compose *within* a pattern, not across it; which harness↔resources-server pairings are actually + compatible is decided by the config composer's compatibility guard, NOT by this registry. - **Self-contained (Pattern B):** the harness bundles its own environment/framework or external LLM loop (``agent_framework``; e.g. ``swe_agents``, ``harbor_agent``, ``verifiers_agent``, - ``claude_code_agent``) and runs with its own config rather than wired to a separate environment. + ``claude_code_agent``, and ``gymnasium``-style agents that ship their own env) and runs with its + own config rather than wired to a separate environment. Discovery only reads config files; it never resolves interpolations or missing values and never starts servers, so it is safe to call when secrets/API keys referenced by a config are unset. diff --git a/nemo_gym/cli/agents.py b/nemo_gym/cli/agents.py index 04c51cb133..982ef440b1 100644 --- a/nemo_gym/cli/agents.py +++ b/nemo_gym/cli/agents.py @@ -21,17 +21,11 @@ from nemo_gym.config_types import BaseNeMoGymCLIConfig from nemo_gym.global_config import ( JSON_OUTPUT_KEY_NAME, - QUERY_KEY_NAME, GlobalConfigDictParserConfig, get_global_config_dict, ) -def _fuzzy_matches(query: str, *fields: str) -> bool: - needle = query.lower() - return any(needle in (field or "").lower() for field in fields) - - def list_agents() -> None: """CLI command: list discovered agent harnesses and how each composes (Pattern A vs B). @@ -49,12 +43,6 @@ def list_agents() -> None: agents = discover_agents() - query = global_config_dict.get(QUERY_KEY_NAME) - if query: - agents = { - name: entry for name, entry in agents.items() if _fuzzy_matches(query, name, entry.description or "") - } - if global_config_dict.get(JSON_OUTPUT_KEY_NAME, False): payload = [ { @@ -70,7 +58,7 @@ def list_agents() -> None: return if not agents: - rich.print("No agents found." if not query else f"No agents match {query!r}.") + rich.print("No agents found.") return table = Table(title="NeMo Gym agents") diff --git a/tests/unit_tests/test_agent_registry.py b/tests/unit_tests/test_agent_registry.py index acd93b1823..f63202c0b7 100644 --- a/tests/unit_tests/test_agent_registry.py +++ b/tests/unit_tests/test_agent_registry.py @@ -17,7 +17,6 @@ from pytest import raises from nemo_gym.agent_registry import ( - AGENTS_DIR, AgentEntry, AgentNotComposableError, AgentNotFoundError, @@ -223,4 +222,4 @@ def test_discovers_real_simple_agent_as_composable(self) -> None: def test_agent_entry_is_hashable(self) -> None: entry = AgentEntry(name="a", path=Path("a"), config_paths=(Path("a/configs/a.yaml"),), self_contained=True) assert {entry: 1}[entry] == 1 - assert entry.path == AGENTS_DIR / "a" or True # AGENTS_DIR import exercised + assert entry.variants == {"a": Path("a/configs/a.yaml")} diff --git a/tests/unit_tests/test_cli_agents.py b/tests/unit_tests/test_cli_agents.py index a2a8d63b20..4a60669b2a 100644 --- a/tests/unit_tests/test_cli_agents.py +++ b/tests/unit_tests/test_cli_agents.py @@ -75,12 +75,3 @@ def test_json_output(self, capsys) -> None: assert by_name["simple_agent"]["pattern"] == "A (composable)" assert by_name["swe_agents"]["self_contained"] is True assert by_name["swe_agents"]["variants"] == ["swebench_openhands"] - - def test_query_filters(self, capsys) -> None: - with ( - patch("nemo_gym.cli.agents.get_global_config_dict", return_value=_mock_global_config({"query": "swe"})), - patch("nemo_gym.cli.agents.discover_agents", return_value=_AGENTS), - ): - list_agents() - out = capsys.readouterr().out - assert "swe_agents" in out and "simple_agent" not in out From 7ae116716a65c1ab7f17f9060faff020bb663f40 Mon Sep 17 00:00:00 2001 From: Wojciech Prazuch Date: Thu, 25 Jun 2026 14:46:52 +0200 Subject: [PATCH 5/6] refactor(agent-registry): slim to discovery-only; drop resolver that duplicates the asset selector MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Per a duplication audit against main: resolve_agent_config_path was dead code in this PR (no CLI wires it) and reinvented the unified CLI`s generic name->config resolution (cli/main.py _asset_config_path) — the same pattern #1635 deliberately removed on the environment side. Drop it (and its AgentNotFound/Variant/NotComposable errors and the duplicate _did_you_mean). This PR is now purely discovery + Pattern A/B classification + `gym list agents`. Run-by-name resolution (with the require_composable guard) will land with its real consumer, the config composer (#1673), as an `agent` asset row rather than a parallel resolver. Signed-off-by: Wojciech Prazuch --- nemo_gym/agent_registry.py | 80 ++------------------- tests/unit_tests/test_agent_registry.py | 92 ------------------------- 2 files changed, 6 insertions(+), 166 deletions(-) diff --git a/nemo_gym/agent_registry.py b/nemo_gym/agent_registry.py index 95a8508c2c..fb77e60ec7 100644 --- a/nemo_gym/agent_registry.py +++ b/nemo_gym/agent_registry.py @@ -16,9 +16,11 @@ An agent harness is one *component* of an environment (an environment = dataset + agent harness + resources server [verifier and state] + model). This module maps an agent's short ```` (the -directory name) to its config variant(s) so it can be referenced by name — the foundation for -``gym run --agent `` (run-by-name) — and records whether the harness bundles its own -environment or references a separate resources server. +directory name) to its config variant(s) so it can be enumerated by name (``gym list agents``) and +classified by how it composes. Resolving an agent name to a config for *running* belongs to the +config composer (via the CLI's generic asset selectors), so this module is intentionally +discovery-only; it records whether the harness bundles its own environment or references a separate +resources server. - **References a separate resources server (Pattern A):** the config sets ``responses_api_agents..resources_server``, so the harness is reusable and must be wired to a @@ -35,9 +37,8 @@ """ from dataclasses import dataclass -from difflib import get_close_matches from pathlib import Path -from typing import Dict, List, Optional, Tuple +from typing import Dict, Optional, Tuple from omegaconf import OmegaConf @@ -48,18 +49,6 @@ AGENT_CONFIGS_SUBDIR = "configs" -class AgentNotFoundError(ValueError): - """An agent was referenced by a name that is not registered under ``responses_api_agents/``.""" - - -class AgentVariantError(ValueError): - """An agent has no standalone config, or has several and no variant was given to disambiguate.""" - - -class AgentNotComposableError(ValueError): - """A self-contained (Pattern B) agent was requested for wiring into a separate environment.""" - - @dataclass(frozen=True) class AgentEntry: """A discovered agent: its name, where it lives, its config variants, and how it's wired.""" @@ -168,60 +157,3 @@ def discover_agents(agents_dir: Path = AGENTS_DIR) -> Dict[str, AgentEntry]: ) return agents - - -def _did_you_mean(name: str, available: List[str], noun: str) -> str: - suggestions = get_close_matches(name, available, n=3, cutoff=0.6) - if suggestions: - return "Did you mean: " + ", ".join(repr(s) for s in suggestions) + "?" - return f"Available {noun}: " + (", ".join(repr(n) for n in available) or "(none)") - - -def resolve_agent_config_path( - name: str, - variant: Optional[str] = None, - agents_dir: Path = AGENTS_DIR, - require_composable: bool = False, -) -> str: - """Return the config path to load to run agent ``name`` — the run-by-name primitive. - - Selection: an explicit ``variant`` wins; otherwise a single config is used directly, and a - variant whose name equals ``name`` is the default when several exist. Raises - :class:`AgentNotFoundError` (with a "did you mean?" hint) for an unknown agent, - :class:`AgentVariantError` for a zero-config or ambiguous-variant agent, and — when - ``require_composable`` is set — :class:`AgentNotComposableError` for a Pattern B agent. - """ - agents = discover_agents(agents_dir) - entry = agents.get(name) - if entry is None: - raise AgentNotFoundError( - f"No agent named '{name}' under {agents_dir}.\n{_did_you_mean(name, sorted(agents), 'agents')}" - ) - - if require_composable and entry.self_contained: - raise AgentNotComposableError( - f"Agent '{name}' is self-contained (it bundles its own environment/framework) and cannot " - "be wired into a separate environment; run it with its own config instead." - ) - - variants = entry.variants - if not variants: - raise AgentVariantError( - f"Agent '{name}' ships no standalone config; it is composed via its paired " - "benchmark/resources-server config." - ) - - if variant is not None: - if variant not in variants: - raise AgentVariantError( - f"Agent '{name}' has no variant '{variant}'.\n{_did_you_mean(variant, sorted(variants), 'variants')}" - ) - return str(variants[variant]) - - if len(variants) == 1: - return str(next(iter(variants.values()))) - if name in variants: - return str(variants[name]) - raise AgentVariantError( - f"Agent '{name}' has multiple config variants: {sorted(variants)}; pass a variant to select one." - ) diff --git a/tests/unit_tests/test_agent_registry.py b/tests/unit_tests/test_agent_registry.py index f63202c0b7..7c2d8982d3 100644 --- a/tests/unit_tests/test_agent_registry.py +++ b/tests/unit_tests/test_agent_registry.py @@ -14,15 +14,9 @@ # limitations under the License. from pathlib import Path -from pytest import raises - from nemo_gym.agent_registry import ( AgentEntry, - AgentNotComposableError, - AgentNotFoundError, - AgentVariantError, discover_agents, - resolve_agent_config_path, ) @@ -126,92 +120,6 @@ def test_missing_directory_yields_no_agents(self, tmp_path: Path) -> None: assert discover_agents(tmp_path / "nope") == {} -class TestResolveAgentConfigPath: - def test_single_config_resolves(self, tmp_path: Path) -> None: - _make_agent(tmp_path, "simple_agent", configs={"simple_agent": _pattern_a()}) - - path = resolve_agent_config_path("simple_agent", agents_dir=tmp_path) - assert path.endswith("simple_agent/configs/simple_agent.yaml") - - def test_explicit_variant_resolves(self, tmp_path: Path) -> None: - _make_agent( - tmp_path, - "langgraph_agent", - configs={ - "orchestrator_agent": _pattern_a("langgraph_agent"), - "rewoo_agent": _pattern_a("langgraph_agent"), - }, - ) - - path = resolve_agent_config_path("langgraph_agent", variant="rewoo_agent", agents_dir=tmp_path) - assert path.endswith("rewoo_agent.yaml") - - def test_variant_matching_name_is_default_when_several(self, tmp_path: Path) -> None: - _make_agent( - tmp_path, - "harbor_agent", - configs={"harbor_agent": _pattern_a("harbor_agent"), "harbor_daytona": _pattern_a("harbor_agent")}, - ) - - assert resolve_agent_config_path("harbor_agent", agents_dir=tmp_path).endswith("harbor_agent.yaml") - - def test_ambiguous_variant_raises(self, tmp_path: Path) -> None: - _make_agent( - tmp_path, - "langgraph_agent", - configs={ - "orchestrator_agent": _pattern_a("langgraph_agent"), - "rewoo_agent": _pattern_a("langgraph_agent"), - }, - ) - - with raises(AgentVariantError, match="multiple config variants"): - resolve_agent_config_path("langgraph_agent", agents_dir=tmp_path) - - def test_unknown_variant_raises_with_suggestion(self, tmp_path: Path) -> None: - _make_agent( - tmp_path, - "langgraph_agent", - configs={ - "orchestrator_agent": _pattern_a("langgraph_agent"), - "rewoo_agent": _pattern_a("langgraph_agent"), - }, - ) - - with raises(AgentVariantError, match="Did you mean"): - resolve_agent_config_path("langgraph_agent", variant="rewoo", agents_dir=tmp_path) - - def test_zero_config_agent_raises(self, tmp_path: Path) -> None: - _make_agent(tmp_path, "aviary_agent", configs=None) - - with raises(AgentVariantError, match="no standalone config"): - resolve_agent_config_path("aviary_agent", agents_dir=tmp_path) - - def test_unknown_agent_raises_with_suggestion(self, tmp_path: Path) -> None: - _make_agent(tmp_path, "simple_agent", configs={"simple_agent": _pattern_a()}) - - with raises(AgentNotFoundError, match="Did you mean"): - resolve_agent_config_path("simple_agnt", agents_dir=tmp_path) - - def test_unknown_agent_without_close_match_lists_available(self, tmp_path: Path) -> None: - _make_agent(tmp_path, "simple_agent", configs={"simple_agent": _pattern_a()}) - - with raises(AgentNotFoundError, match="Available agents"): - resolve_agent_config_path("zzzzz", agents_dir=tmp_path) - - def test_require_composable_rejects_pattern_b(self, tmp_path: Path) -> None: - _make_agent(tmp_path, "swe_agents", configs={"swebench": _pattern_b()}) - - with raises(AgentNotComposableError, match="self-contained"): - resolve_agent_config_path("swe_agents", agents_dir=tmp_path, require_composable=True) - - def test_require_composable_allows_pattern_a(self, tmp_path: Path) -> None: - _make_agent(tmp_path, "simple_agent", configs={"simple_agent": _pattern_a()}) - - path = resolve_agent_config_path("simple_agent", agents_dir=tmp_path, require_composable=True) - assert path.endswith("simple_agent.yaml") - - class TestRealAgents: def test_discovers_real_simple_agent_as_composable(self) -> None: agents = discover_agents() From 4c880ec39a26f5b1a76733e4d89e2d868df85c01 Mon Sep 17 00:00:00 2001 From: Wojciech Prazuch Date: Thu, 25 Jun 2026 14:57:41 +0200 Subject: [PATCH 6/6] docs(agent-registry): clarify Pattern B agents compose within a type (cmunley review) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit self_contained only reflects whether the agent wires a resources server in its OWN config; it is not a claim that the agent bundles its environment. Pattern B agents (claude_code_agent, gymnasium_agent, ...) still pair with a matching resources-server type (e.g. gymnasium_agent + blackjack), composing within a type though not across types — the compatible pairing is the config composer's call. Signed-off-by: Wojciech Prazuch --- nemo_gym/agent_registry.py | 25 +++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/nemo_gym/agent_registry.py b/nemo_gym/agent_registry.py index fb77e60ec7..9b8958eb9b 100644 --- a/nemo_gym/agent_registry.py +++ b/nemo_gym/agent_registry.py @@ -19,18 +19,19 @@ directory name) to its config variant(s) so it can be enumerated by name (``gym list agents``) and classified by how it composes. Resolving an agent name to a config for *running* belongs to the config composer (via the CLI's generic asset selectors), so this module is intentionally -discovery-only; it records whether the harness bundles its own environment or references a separate -resources server. - -- **References a separate resources server (Pattern A):** the config sets - ``responses_api_agents..resources_server``, so the harness is reusable and must be wired to a - *compatible* resources server + dataset (e.g. the ``simple_agent`` tool-use pattern). Harnesses - compose *within* a pattern, not across it; which harness↔resources-server pairings are actually - compatible is decided by the config composer's compatibility guard, NOT by this registry. -- **Self-contained (Pattern B):** the harness bundles its own environment/framework or external LLM - loop (``agent_framework``; e.g. ``swe_agents``, ``harbor_agent``, ``verifiers_agent``, - ``claude_code_agent``, and ``gymnasium``-style agents that ship their own env) and runs with its - own config rather than wired to a separate environment. +discovery-only. The ``self_contained`` flag records only what the agent's *own* config reveals about +its resources-server wiring: + +- **Wires a resources server itself (Pattern A, ``self_contained=False``):** the config sets + ``responses_api_agents..resources_server``, so the registry can see it pairs with a separate + resources server + dataset (e.g. the ``simple_agent`` tool-use pattern). +- **Does not wire one in its own config (Pattern B, ``self_contained=True``):** the agent declares + its own ``agent_framework`` (e.g. ``swe_agents``), drives an external LLM loop (e.g. + ``claude_code_agent`` via its own model key), or ships only an entrypoint whose resources-server + pairing is supplied by a separate paired/benchmark config (e.g. ``gymnasium_agent`` + the + ``blackjack`` resources server). These agents are still composable — but *within* a type + (gymnasium↔gymnasium-style, simple_agent↔simple_agent-style), not across types. Which pairings are + actually compatible is the config composer's call, not inferable from the agent config alone. Discovery only reads config files; it never resolves interpolations or missing values and never starts servers, so it is safe to call when secrets/API keys referenced by a config are unset.