Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
117 changes: 93 additions & 24 deletions run_agent.py
Original file line number Diff line number Diff line change
Expand Up @@ -245,6 +245,21 @@ def _is_ephemeral_scaffolding(msg: Any) -> bool:

_MAX_TOOL_WORKERS = 8

# Intrinsic marker stamped on a message dict once it has been written to the
# SQLite session store. Used by ``_flush_messages_to_session_db`` to decide
# what is already durable. An object-identity (``id(msg)``) dedup set cannot be
# trusted across turns: once a flushed message dict is dropped from the live
# list (e.g. by scaffolding rewind or in-place compaction) and garbage-
# collected, CPython is free to hand its address to a brand-new assistant/tool
# message, whose ``id()`` then collides with the stale entry and the real turn
# is silently never persisted. A marker bound to the dict itself cannot be
# aliased that way. The ``_`` prefix is mandatory: the wire sanitizers
# (agent/transports/chat_completions.py, agent/chat_completion_helpers.py) strip
# every top-level ``_``-prefixed key before the request leaves the process, so
# this never reaches a strict OpenAI-compatible gateway.
_DB_PERSISTED_MARKER = "_db_persisted"


# Guard so the OpenRouter metadata pre-warm thread is only spawned once per
# process, not once per AIAgent instantiation. Without this, long-running
# gateway processes leak one OS thread per incoming message and eventually
Expand Down Expand Up @@ -1634,9 +1649,17 @@ def _persist_session(self, messages: List[Dict], conversation_history: List[Dict
"""Save session state to both JSON log and SQLite on any exit path.

Ensures conversations are never lost, even on errors or early returns.

Trailing empty-response scaffolding is dropped from the live list in
place (it is ephemeral junk the real transcript should shed). The
persist user-message *override* is NOT applied here — it is resolved
inside ``_flush_messages_to_session_db`` and written only to the DB row,
never mutating the live message list used by the API call (#48677 is
thus closed for every persist caller, not just this one).
"""
# Scaffolding removal mutates the live list (desired — ephemeral
# retry/failure sentinels must not survive into the real transcript).
self._drop_trailing_empty_response_scaffolding(messages)
self._apply_persist_user_message_override(messages)
self._session_messages = messages
self._save_session_log(messages)
self._flush_messages_to_session_db(messages, conversation_history)
Expand Down Expand Up @@ -1702,10 +1725,19 @@ def _repair_message_sequence(self, messages: List[Dict]) -> int:
def _flush_messages_to_session_db(self, messages: List[Dict], conversation_history: List[Dict] = None):
"""Persist any un-flushed messages to the SQLite session store.

Uses per-session message identity tracking so repeated calls (from
multiple exit paths) only write truly new messages — preventing the
duplicate-write bug (#860) without relying on positional slices that
can drift after message-sequence repair.
Deduplicates via an intrinsic ``_DB_PERSISTED_MARKER`` stamped on each
written message dict, so repeated calls (from multiple exit paths) only
write truly new messages — preventing the duplicate-write bug (#860)
without relying on positional slices that can drift after
message-sequence repair, and without a retained ``id(msg)`` set that
CPython could alias onto a freed-then-reused address (#50372). The
``_flushed_db_message_ids`` attribute is now only a one-shot seed
(translated to markers, then cleared each flush), not a persisted set.

Note: the marker is stamped on the live/shared conversation dict, which
correctly makes re-persistence idempotent across turns. No code path
edits a persisted message's content/role in place expecting a re-write
(in-place compaction resets the seed and re-diffs by identity).
"""
# Persistence-isolated agents (e.g. the background skill/memory review
# fork) must NEVER write into the canonical session store. The fork
Expand All @@ -1718,7 +1750,18 @@ def _flush_messages_to_session_db(self, messages: List[Dict], conversation_histo
return
if not self._session_db:
return
self._apply_persist_user_message_override(messages)
# Persist user-message override (#48677 chokepoint): historically this
# mutated the live `messages` list in place, which — on the early
# crash-resilience persist that runs BEFORE the API call is built —
# stripped observed group-chat context off the live user message and
# silently dropped it. Instead, resolve the override here and apply it
# ONLY to the value written to the DB (see the write loop below); the
# live dict is never mutated, so every caller (early persist, mid-loop
# flush, /resume, /branch) is protected uniformly. Timestamp override is
# metadata and is likewise applied only to the written row.
_ov_idx = getattr(self, "_persist_user_message_idx", None)
_ov_content = getattr(self, "_persist_user_message_override", None)
_ov_timestamp = getattr(self, "_persist_user_message_timestamp", None)
try:
# Retry row creation if the earlier attempt failed transiently.
if not self._session_db_created:
Expand All @@ -1731,25 +1774,36 @@ def _flush_messages_to_session_db(self, messages: List[Dict], conversation_histo
# larger than len(messages); the slice is then empty and delivered
# assistant responses never reach state.db (#46053).
#
# Track object identities instead. `messages` is a shallow copy of
# `conversation_history`, so history dicts are skipped by identity,
# and new dicts appended during this turn are written once even if
# repair compacts the list around them.
# Track persistence with an intrinsic per-message marker rather than
# id(msg). `messages` is a shallow copy of `conversation_history`, so
# history dicts are skipped by identity, and new dicts appended
# during this turn are written once even if repair compacts the list
# around them. Unlike an id()-keyed set, a marker bound to the dict
# cannot be aliased onto a freed-then-reused address, so a real turn
# can never be silently skipped (see _DB_PERSISTED_MARKER).
#
# `self._flushed_db_message_ids` is still honoured as a *one-shot*
# seed: external callers (gateway shutdown, tests) populate it with
# {id(m) for m in already_persisted} immediately before the flush,
# while those objects are alive — so the ids are valid at that
# instant. We translate the seed into durable markers and then clear
# the set, so stale ids can never accumulate across turns and alias a
# future message.
current_session_id = getattr(self, "session_id", None)
flushed_session_id = getattr(self, "_flushed_db_message_session_id", None)
if flushed_session_id != current_session_id or self._last_flushed_db_idx == 0:
self._flushed_db_message_ids = set()
self._flushed_db_message_session_id = current_session_id
flushed_ids = getattr(self, "_flushed_db_message_ids", None)
if not isinstance(flushed_ids, set):
flushed_ids = set()
self._flushed_db_message_ids = flushed_ids
seed_ids = set()
else:
seed_ids = getattr(self, "_flushed_db_message_ids", None)
if not isinstance(seed_ids, set):
seed_ids = set()
self._flushed_db_message_session_id = current_session_id
history_ids = {
id(item) for item in (conversation_history or [])
if isinstance(item, dict)
}

for msg in messages:
for _msg_idx, msg in enumerate(messages):
if not isinstance(msg, dict):
continue
# Never write ephemeral recovery scaffolding to the session
Expand All @@ -1763,14 +1817,26 @@ def _flush_messages_to_session_db(self, messages: List[Dict], conversation_histo
# the synthetic pair buried mid-list, not just at the tail.
if _is_ephemeral_scaffolding(msg):
continue
msg_id = id(msg)
if msg_id in flushed_ids:
if msg.get(_DB_PERSISTED_MARKER):
continue
if msg_id in history_ids:
flushed_ids.add(msg_id)
# Already-durable messages: either carried over from the loaded
# history copy, or seeded by a caller. Stamp them so future
# flushes skip them without consulting any id() set again.
if id(msg) in history_ids or id(msg) in seed_ids:
msg[_DB_PERSISTED_MARKER] = True
continue
role = msg.get("role", "unknown")
content = msg.get("content")
_row_timestamp = msg.get("timestamp")
# Apply the persist override to THIS row's written values only
# (never to the live dict). Match the original guard: text-only
# content is replaced; multimodal (list) content is left intact
# so image/audio blocks aren't clobbered by the text override.
if _ov_idx == _msg_idx and msg.get("role") == "user":
if _ov_content is not None and not isinstance(content, list):
content = _ov_content
if _ov_timestamp is not None:
_row_timestamp = _ov_timestamp
# Persist multimodal tool results as their text summary only —
# base64 images would bloat the session DB and aren't useful
# for cross-session replay.
Expand Down Expand Up @@ -1806,9 +1872,13 @@ def _flush_messages_to_session_db(self, messages: List[Dict], conversation_histo
reasoning_details=msg.get("reasoning_details") if role == "assistant" else None,
codex_reasoning_items=msg.get("codex_reasoning_items") if role == "assistant" else None,
codex_message_items=msg.get("codex_message_items") if role == "assistant" else None,
timestamp=msg.get("timestamp"),
timestamp=_row_timestamp,
)
flushed_ids.add(msg_id)
msg[_DB_PERSISTED_MARKER] = True
# The intrinsic markers are now the sole source of truth. Reset the
# one-shot seed so no id() outlives this flush to alias a message
# allocated next turn at a recycled address.
self._flushed_db_message_ids = set()
self._last_flushed_db_idx = len(messages)
except Exception as e:
logger.warning("Session DB append_message failed: %s", e)
Expand Down Expand Up @@ -5511,7 +5581,6 @@ def _dispatch_delegate_task(self, function_args: dict) -> str:
return _delegate_task(
goal=function_args.get("goal"),
context=function_args.get("context"),
toolsets=function_args.get("toolsets"),
tasks=function_args.get("tasks"),
max_iterations=function_args.get("max_iterations"),
acp_command=function_args.get("acp_command"),
Expand Down
2 changes: 1 addition & 1 deletion skills/autonomous-ai-agents/hermes-agent/SKILL.md
Original file line number Diff line number Diff line change
Expand Up @@ -706,7 +706,7 @@ here; full developer notes live in `AGENTS.md`, user-facing docs under

Spawn a subagent with an isolated context + terminal session.

- **Single:** `delegate_task(goal, context, toolsets)`.
- **Single:** `delegate_task(goal, context)`.
- **Batch:** `delegate_task(tasks=[{goal, ...}, ...])` runs children in
parallel, capped by `delegation.max_concurrent_children` (default 3).
- **Background:** `delegate_task(background=true)` returns a handle
Expand Down
26 changes: 25 additions & 1 deletion tests/tools/test_async_delegation.py
Original file line number Diff line number Diff line change
Expand Up @@ -262,7 +262,7 @@ def slow_child(task_index, goal, child=None, parent_agent=None, **kw):
monkeypatch.setattr(dt, "_run_single_child", slow_child)
monkeypatch.setattr(dt, "_resolve_delegation_credentials", lambda *a, **k: creds)
out = dt.delegate_task(
goal="the real task", context="ctx", toolsets=["web"],
goal="the real task", context="ctx",
background=True, parent_agent=parent,
)

Expand Down Expand Up @@ -422,6 +422,30 @@ def _fake_delegate(**kwargs):
assert captured["background"] is False


def test_dispatch_never_forwards_model_toolsets():
"""The model has no toolsets argument — subagents always inherit the
parent's toolsets. Even if a model smuggles a `toolsets` key into the
tool-call args, the live dispatch path must NOT forward it to
delegate_task (which no longer accepts it) and must not crash."""
from unittest.mock import patch
import run_agent

class _FakeAgent:
_delegate_depth = 0

captured = {}

def _fake_delegate(**kwargs):
captured.update(kwargs)
return "{}"

with patch("tools.delegate_tool.delegate_task", _fake_delegate):
run_agent.AIAgent._dispatch_delegate_task(
_FakeAgent(), {"goal": "x", "toolsets": ["web", "terminal"]}
)
assert "toolsets" not in captured


def test_delegate_task_background_detaches_child_from_parent(monkeypatch):
"""A background child must NOT remain in parent._active_children —
otherwise parent-turn interrupts / cache evicts / session close would
Expand Down
57 changes: 56 additions & 1 deletion tests/tools/test_delegate.py
Original file line number Diff line number Diff line change
Expand Up @@ -69,7 +69,11 @@ def test_schema_valid(self):
self.assertIn("goal", props)
self.assertIn("tasks", props)
self.assertIn("context", props)
self.assertIn("toolsets", props)
# toolsets is intentionally NOT exposed to the model — subagents always
# inherit the parent's toolsets. Letting the model name toolsets was a
# capability-selection surface the model should not control.
self.assertNotIn("toolsets", props)
self.assertNotIn("toolsets", props["tasks"]["items"]["properties"])
# max_iterations is intentionally NOT exposed to the model — it's
# config-authoritative via delegation.max_iterations so users get
# predictable budgets.
Expand Down Expand Up @@ -918,6 +922,31 @@ def test_exit_reason_max_iterations(self):
result = json.loads(delegate_task(goal="Test max iter", parent_agent=parent))
self.assertEqual(result["results"][0]["exit_reason"], "max_iterations")

def test_empty_sentinel_marks_status_failed(self):
"""Regression: a child that returns the literal '(empty)' sentinel
(emitted by run_agent.py when the LLM returns empty responses after
retries — e.g. transport misrouting) must be reported as failed, not
silently accepted as a completed delegation. Otherwise the parent
surfaces an empty string as if the subagent succeeded."""
parent = _make_mock_parent(depth=0)

with patch("run_agent.AIAgent") as MockAgent:
mock_child = MagicMock()
mock_child.model = "claude-sonnet-4-6"
mock_child.session_prompt_tokens = 0
mock_child.session_completion_tokens = 0
mock_child.run_conversation.return_value = {
"final_response": "(empty)",
"completed": True,
"interrupted": False,
"api_calls": 4,
"messages": [],
}
MockAgent.return_value = mock_child

result = json.loads(delegate_task(goal="Test empty sentinel", parent_agent=parent))
self.assertEqual(result["results"][0]["status"], "failed")


class TestSubagentCostRollup(unittest.TestCase):
"""Port of Kilo-Org/kilocode#9448 — parent's session_estimated_cost_usd
Expand Down Expand Up @@ -1341,6 +1370,32 @@ def test_runtime_missing_provider_key_returns_none(self, mock_resolve):
creds = _resolve_delegation_credentials(cfg, parent)
self.assertIsNone(creds["provider"])

@patch("hermes_cli.runtime_provider.resolve_runtime_provider")
def test_bedrock_provider_with_base_url_uses_runtime_resolver(self, mock_resolve):
"""Regression: provider=bedrock + base_url set must NOT fall through the
direct-base_url branch (which would force provider='custom' +
chat_completions and silently misroute OpenAI JSON to the Bedrock
native endpoint, returning empty responses)."""
mock_resolve.return_value = {
"provider": "bedrock",
"base_url": "https://bedrock-runtime.us-west-2.amazonaws.com",
"api_key": "aws-resolved-key",
"api_mode": "bedrock_converse",
}
parent = _make_mock_parent(depth=0)
cfg = {
"model": "us.anthropic.claude-sonnet-4-6",
"provider": "bedrock",
"base_url": "https://bedrock-runtime.us-west-2.amazonaws.com",
}
creds = _resolve_delegation_credentials(cfg, parent)
# Must use Bedrock, not 'custom'
self.assertEqual(creds["provider"], "bedrock")
self.assertEqual(creds["api_mode"], "bedrock_converse")
mock_resolve.assert_called_once()
self.assertEqual(mock_resolve.call_args.kwargs.get("requested"), "bedrock")



class TestDelegationProviderIntegration(unittest.TestCase):
"""Integration tests: delegation config → _run_single_child → AIAgent construction."""
Expand Down
Loading
Loading