Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
76 changes: 66 additions & 10 deletions agent/background_review.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,8 +9,9 @@

The fork inherits the parent's live runtime (provider, model, base_url,
credentials, cached system prompt) so it hits the same prefix cache and
uses the same auth. It runs with a tool whitelist limited to memory and
skill management tools; everything else is denied at runtime.
uses the same auth. It runs behind a runtime tool whitelist that defaults
to memory and skill management tools; operators can add narrowly scoped
toolsets via ``memory.review_toolsets``.

See the ``hermes-agent-dev`` skill (``references/self-improvement-loop.md``)
for invariants and PR review criteria.
Expand All @@ -26,6 +27,45 @@

logger = logging.getLogger(__name__)

_DEFAULT_REVIEW_TOOLSETS = ["memory", "skills"]


def _normalize_review_toolsets(raw: Any) -> List[str]:
"""Return a stable list of configured background-review toolsets."""
if raw is None:
return list(_DEFAULT_REVIEW_TOOLSETS)
if isinstance(raw, str):
items = raw.split(",")
elif isinstance(raw, (list, tuple)):
items = list(raw)
else:
return list(_DEFAULT_REVIEW_TOOLSETS)

normalized: List[str] = []
for item in items:
if not isinstance(item, str):
continue
toolset = item.strip()
if toolset and toolset not in normalized:
normalized.append(toolset)
return normalized


def _resolve_background_review_toolsets() -> List[str]:
"""Load the toolsets the self-improvement review fork may use."""
try:
from hermes_cli.config import load_config

cfg = load_config() or {}
memory_cfg = cfg.get("memory") if isinstance(cfg, dict) else {}
if not isinstance(memory_cfg, dict):
memory_cfg = {}
raw_toolsets = memory_cfg.get("review_toolsets", _DEFAULT_REVIEW_TOOLSETS)
except Exception as exc:
logger.debug("Failed to load background-review toolset config: %s", exc)
raw_toolsets = _DEFAULT_REVIEW_TOOLSETS
return _normalize_review_toolsets(raw_toolsets)


# Review-prompt strings — used by ``spawn_background_review_thread`` to build
# the user-message that the forked review agent receives. AIAgent exposes
Expand Down Expand Up @@ -467,27 +507,43 @@ def _bg_review_auto_deny(command, description, **kwargs):
clear_thread_tool_whitelist,
)

review_toolsets = _resolve_background_review_toolsets()

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Blocking: this replaces main's current memory-disable gate (agent/background_review.py:802-808, commit 437dcacbb). Because the new default contains memory, a profile with both _memory_enabled and _user_profile_enabled false will again whitelist the built-in memory tool. Compose the configured list with that existing gate instead of replacing it.

review_tool_defs = get_tool_definitions(
enabled_toolsets=review_toolsets,
quiet_mode=True,
)
# Include raw pre-Tool-Search schemas in the runtime whitelist too.
# If a configured MCP/plugin toolset is collapsed behind
# tool_search/tool_call, the model first calls the bridge, but the
# executor unwraps tool_call to the underlying tool before plugin
# hooks run. Whitelisting both the visible bridge schemas and the
# raw configured schemas keeps that legitimate path open while
# preserving the sandbox.
raw_review_tool_defs = get_tool_definitions(
enabled_toolsets=review_toolsets,
quiet_mode=True,
skip_tool_search_assembly=True,
)
review_whitelist = {
t["function"]["name"]
for t in get_tool_definitions(
enabled_toolsets=["memory", "skills"],
quiet_mode=True,
)
for t in [*review_tool_defs, *raw_review_tool_defs]
}
review_toolset_desc = ", ".join(review_toolsets) or "none"
set_thread_tool_whitelist(
review_whitelist,
deny_msg_fmt=(
"Background review denied non-whitelisted tool: "
"{tool_name}. Only memory/skill tools are allowed."
"{tool_name}. Only configured background-review toolsets "
f"are allowed ({review_toolset_desc})."
),
)
try:
review_agent.run_conversation(
user_message=(
prompt
+ "\n\nYou can only call memory and skill "
"management tools. Other tools will be denied "
"at runtime — do not attempt them."
+ "\n\nYou can only call tools from the configured "
f"background-review toolsets ({review_toolset_desc}). "
"Other tools will be denied at runtime — do not attempt them."
),
conversation_history=messages_snapshot,
)
Expand Down
4 changes: 4 additions & 0 deletions hermes_cli/config.py
Original file line number Diff line number Diff line change
Expand Up @@ -1730,6 +1730,10 @@ def _ensure_hermes_home_managed(home: Path):
# /memory reject <id>.
# To disable memory entirely, use memory_enabled: false instead.
"write_approval": False,
# Toolsets the background self-improvement review may call. Defaults
# preserve the historic memory+skills sandbox; add narrowly scoped MCP
# or plugin toolsets only when the review fork should persist there.
"review_toolsets": ["memory", "skills"],
"memory_char_limit": 2200, # ~800 tokens at 2.75 chars/token
"user_char_limit": 1375, # ~500 tokens at 2.75 chars/token
# External memory provider plugin (empty = built-in only).
Expand Down
88 changes: 88 additions & 0 deletions tests/run_agent/test_background_review_cache_parity.py
Original file line number Diff line number Diff line change
Expand Up @@ -237,3 +237,91 @@ def close(self):
f"disabled_toolsets mismatch: {captured.get('disabled_toolsets')!r} "
f"vs expected {agent.disabled_toolsets!r}"
)


def test_default_config_keeps_background_review_toolsets_safe():
"""Background review defaults must preserve the existing memory+skills sandbox."""
from hermes_cli.config import DEFAULT_CONFIG

assert DEFAULT_CONFIG["memory"]["review_toolsets"] == ["memory", "skills"]


def test_background_review_whitelist_uses_configured_review_toolsets():
"""Operators can opt review forks into deterministic persistence toolsets."""
import run_agent

agent = _make_agent_stub(run_agent.AIAgent)
captured = {}
tool_def_calls = []

class _Recorder:
def __init__(self, *args, **kwargs):
self._cached_system_prompt = None
self._memory_write_origin = None
self._memory_write_context = None
self._memory_store = None
self._memory_enabled = None
self._user_profile_enabled = None
self._memory_nudge_interval = None
self._skill_nudge_interval = None
self.suppress_status_output = None
self.session_start = None
self.session_id = None

def run_conversation(self, *args, **kwargs):
raise RuntimeError("stop after whitelist setup — don't call the API")

def shutdown_memory_provider(self):
pass

def close(self):
pass

def fake_get_tool_definitions(*, enabled_toolsets=None, quiet_mode=False, skip_tool_search_assembly=False, **kwargs):
tool_def_calls.append((list(enabled_toolsets or []), skip_tool_search_assembly))
if list(enabled_toolsets or []) == ["memory", "skills", "mcp-gbrain"]:
# The raw pass includes the actual MCP persistence tool; the normal
# pass may collapse it behind Tool Search bridge tools.
if skip_tool_search_assembly:
return [
{"type": "function", "function": {"name": "memory"}},
{"type": "function", "function": {"name": "skill_manage"}},
{"type": "function", "function": {"name": "mcp_gbrain_put_page"}},
]
return [
{"type": "function", "function": {"name": "memory"}},
{"type": "function", "function": {"name": "skill_manage"}},
{"type": "function", "function": {"name": "tool_search"}},
{"type": "function", "function": {"name": "tool_describe"}},
{"type": "function", "function": {"name": "tool_call"}},
]
return []

def fake_set_thread_tool_whitelist(allowed, deny_msg_fmt):
captured["allowed"] = set(allowed)
captured["deny_msg_fmt"] = deny_msg_fmt

with patch.object(run_agent, "AIAgent", _Recorder), \

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Blocking coverage gap: this test mocks schema resolution and stops before any tool dispatch. It cannot verify the claimed deferred MCP path, whose real executor scope comes from the inherited agent.enabled_toolsets (agent/tool_executor.py:239-256,402-415), nor catch the memory-disabled-profile regression.

patch("threading.Thread", _SyncThread), \
patch("hermes_cli.config.load_config", return_value={
"memory": {"review_toolsets": ["memory", "skills", "mcp-gbrain"]}
}), \
patch("model_tools.get_tool_definitions", side_effect=fake_get_tool_definitions), \
patch("hermes_cli.plugins.set_thread_tool_whitelist", side_effect=fake_set_thread_tool_whitelist):
agent._spawn_background_review(
messages_snapshot=[],
review_memory=True,
review_skills=True,
)

assert (["memory", "skills", "mcp-gbrain"], True) in tool_def_calls
assert (["memory", "skills", "mcp-gbrain"], False) in tool_def_calls
assert captured["allowed"] == {
"memory",
"skill_manage",
"mcp_gbrain_put_page",
"tool_search",
"tool_describe",
"tool_call",
}
assert "configured background-review toolsets" in captured["deny_msg_fmt"]