Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
25 changes: 23 additions & 2 deletions run_agent.py
Original file line number Diff line number Diff line change
Expand Up @@ -2116,12 +2116,33 @@ def _run_review():
with open(_os.devnull, "w") as _devnull, \
contextlib.redirect_stdout(_devnull), \
contextlib.redirect_stderr(_devnull):
# Use the primary model for background review, not the
# current turn's model which may be a cheap/weak model
# from smart routing that lacks tool-calling ability.
_review_model = self.model
_review_provider = self.provider
_review_kwargs = {}
try:
from hermes_cli.config import load_config as _load_review_cfg
_rcfg = _load_review_cfg()
_model_cfg = _rcfg.get("model", {}) or {}
if isinstance(_model_cfg, dict) and _model_cfg.get("default"):
_review_model = _model_cfg["default"]
_review_provider = _model_cfg.get("provider") or self.provider
if _model_cfg.get("base_url"):
_review_kwargs["base_url"] = _model_cfg["base_url"]
if _model_cfg.get("api_key"):
_review_kwargs["api_key"] = _model_cfg["api_key"]
except Exception:
pass # Fall back to current model

review_agent = AIAgent(
model=self.model,
model=_review_model,
max_iterations=8,
quiet_mode=True,
platform=self.platform,
provider=self.provider,
provider=_review_provider,
**_review_kwargs,
)
review_agent._memory_store = self._memory_store
review_agent._memory_enabled = self._memory_enabled
Expand Down
130 changes: 130 additions & 0 deletions tests/run_agent/test_background_review_model.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,130 @@
"""Tests for _spawn_background_review model selection.

When smart model routing is active, the current turn's model may be a cheap/
weak model that lacks tool-calling ability. The background review agent should
use the primary (strong) model from config.yaml instead.
"""

from unittest.mock import MagicMock, patch

import pytest

from run_agent import AIAgent


def _make_tool_defs(*names):
return [
{
"type": "function",
"function": {
"name": n,
"description": f"{n} tool",
"parameters": {"type": "object", "properties": {}},
},
}
for n in names
]


@pytest.fixture()
def cheap_agent():
"""Agent whose self.model is a cheap routing model."""
with (
patch(
"run_agent.get_tool_definitions",
return_value=_make_tool_defs("web_search", "memory"),
),
patch("run_agent.check_toolset_requirements", return_value={}),
patch("run_agent.OpenAI"),
):
a = AIAgent(
api_key="test-key-1234567890",
model="local-cheap-9b",
provider="custom",
quiet_mode=True,
skip_context_files=True,
skip_memory=True,
)
a.client = MagicMock()
a._memory_enabled = True
a._user_profile_enabled = True
a._memory_store = MagicMock()
return a


class TestBackgroundReviewModelSelection:
"""Verify background review reads the primary model from config."""

def test_review_uses_config_primary_model(self, cheap_agent, monkeypatch):
"""When config has model.default, review agent uses it (not self.model)."""
created_agents = []

original_init = AIAgent.__init__

def tracking_init(self, *args, **kwargs):
original_init(self, *args, **kwargs)
created_agents.append({"model": kwargs.get("model"), "provider": kwargs.get("provider")})

monkeypatch.setattr(
"hermes_cli.config.load_config",
lambda: {
"model": {
"default": "strong-cloud-70b",
"provider": "openrouter",
"base_url": "https://openrouter.ai/api/v1",
"api_key": "sk-strong-key",
}
},
)

# Capture AIAgent constructor calls during _spawn_background_review
with patch.object(AIAgent, "__init__", tracking_init):
with patch.object(AIAgent, "run_conversation", return_value=None):
cheap_agent._spawn_background_review(
messages_snapshot=[{"role": "user", "content": "test"}],
review_memory=True,
)

# Wait for the background thread to complete
import threading
for t in threading.enumerate():
if t.name != "MainThread" and t.is_alive():
t.join(timeout=5)

# The review agent should have been created with the config's primary model
assert len(created_agents) >= 1
review = created_agents[-1]
assert review["model"] == "strong-cloud-70b"
assert review["provider"] == "openrouter"

def test_review_falls_back_to_self_model_on_config_error(self, cheap_agent, monkeypatch):
"""When config loading fails, review agent uses self.model as fallback."""
created_agents = []

original_init = AIAgent.__init__

def tracking_init(self, *args, **kwargs):
original_init(self, *args, **kwargs)
created_agents.append({"model": kwargs.get("model"), "provider": kwargs.get("provider")})

monkeypatch.setattr(
"hermes_cli.config.load_config",
lambda: (_ for _ in ()).throw(RuntimeError("config broken")),
)

with patch.object(AIAgent, "__init__", tracking_init):
with patch.object(AIAgent, "run_conversation", return_value=None):
cheap_agent._spawn_background_review(
messages_snapshot=[{"role": "user", "content": "test"}],
review_memory=True,
)

import threading
for t in threading.enumerate():
if t.name != "MainThread" and t.is_alive():
t.join(timeout=5)

assert len(created_agents) >= 1
review = created_agents[-1]
# Falls back to cheap_agent's own model
assert review["model"] == "local-cheap-9b"