Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions cli.py
Original file line number Diff line number Diff line change
Expand Up @@ -977,6 +977,11 @@ def _run_cleanup(*, notify_session_finalize: bool = True):
_cleanup_all_terminals()
except Exception:
pass
try:
from tools.async_delegation import interrupt_all as _interrupt_async_delegations
_interrupt_async_delegations(reason="CLI shutdown")
except Exception:
pass
try:
_cleanup_all_browsers()
except Exception:
Expand Down
136 changes: 126 additions & 10 deletions gateway/run.py
Original file line number Diff line number Diff line change
Expand Up @@ -1921,9 +1921,42 @@ def _format_gateway_process_notification(evt: dict) -> "str | None":
text += "]"
return text

if evt_type == "async_delegation":
# Reuse the shared rich formatter (self-contained task-source block).
from tools.process_registry import format_process_notification
return format_process_notification(evt)

return None


def _drain_gateway_watch_events(completion_queue) -> "list[dict]":
"""Drain gateway-owned watch events without spinning on requeued events.

Watch events are handled by the post-turn gateway drain. Process
completions are owned by their per-process watcher task, and async
delegation completions are owned by ``_async_delegation_watcher``.
Requeueing async events inside ``while not queue.empty()`` would make the
loop non-terminating, so detach the current batch first, then requeue any
events this drain does not own after the queue is empty.
"""
watch_events: list[dict] = []
requeue: list[dict] = []
while not completion_queue.empty():
try:
evt = completion_queue.get_nowait()
except Exception:
break
evt_type = evt.get("type", "completion")
if evt_type in {"watch_match", "watch_disabled"}:
watch_events.append(evt)
elif evt_type == "async_delegation":
requeue.append(evt)
# else: process completion events are handled by the watcher task
for evt in requeue:
completion_queue.put(evt)
return watch_events


# Module-level weak reference to the active GatewayRunner instance.
# Used by tools (e.g. send_message) that need to route through a live
# adapter for plugin platforms. Set in GatewayRunner.__init__().
Expand Down Expand Up @@ -5353,6 +5386,12 @@ async def start(self) -> bool:
# turn so the agent kicks off the new chat.
asyncio.create_task(self._handoff_watcher())

# Start background async-delegation watcher — drains completion events
# from delegate_task(background=true) subagents and injects each
# result back into its originating session as a new turn, covering the
# idle case where the subagent finishes with no agent turn running.
asyncio.create_task(self._async_delegation_watcher())

logger.info("Press Ctrl+C to stop")

return True
Expand Down Expand Up @@ -5989,6 +6028,16 @@ def _kill_tool_subprocesses(phase: str) -> None:
)
except Exception as _e:
logger.debug("process_registry.kill_all (%s) error: %s", phase, _e)
try:
from tools.async_delegation import interrupt_all as _interrupt_async
_async_n = _interrupt_async(reason=f"gateway shutdown ({phase})")
if _async_n:
logger.info(
"Shutdown (%s): interrupted %d background delegation(s)",
phase, _async_n,
)
except Exception as _e:
logger.debug("async interrupt_all (%s) error: %s", phase, _e)
try:
from tools.terminal_tool import cleanup_all_environments
cleanup_all_environments()
Expand Down Expand Up @@ -8992,18 +9041,17 @@ async def _handle_message_with_agent(self, event, source, _quick_key: str, run_g
logger.error("Process watcher setup error: %s", e)

# Drain watch pattern notifications that arrived during the agent run.
# Watch events and completions share the same queue; completions are
# already handled by the per-process watcher task above, so we only
# inject watch-type events here.
# Watch events and completions share the same queue; process
# completions are already handled by the per-process watcher task
# above, so we only inject watch-type events here.
#
# Async-delegation completions ALSO ride this shared queue but are
# owned by the dedicated _async_delegation_watcher (started at
# boot), which covers both the idle and post-turn cases with a
# single consumer — so we leave them on the queue here.
try:
from tools.process_registry import process_registry as _pr
_watch_events = []
while not _pr.completion_queue.empty():
evt = _pr.completion_queue.get_nowait()
evt_type = evt.get("type", "completion")
if evt_type in {"watch_match", "watch_disabled"}:
_watch_events.append(evt)
# else: completion events are handled by the watcher task
_watch_events = _drain_gateway_watch_events(_pr.completion_queue)
for evt in _watch_events:
synth_text = _format_gateway_process_notification(evt)
if synth_text:
Expand Down Expand Up @@ -12262,6 +12310,74 @@ async def _inject_watch_notification(self, synth_text: str, evt: dict) -> None:
except Exception as e:
logger.error("Watch notification injection error: %s", e)

def _enrich_async_delegation_routing(self, evt: dict) -> None:
"""Fill platform/chat_id/thread_id/chat_type on an async-delegation event.

Async-delegation completion events only carry ``session_key`` (the
daemon worker has no access to the per-message routing metadata the
terminal background watcher captures at spawn time). Parse the
session_key into the routing fields ``_build_process_event_source``
expects. Best-effort: a CLI-origin event (empty session_key) is left
as-is and simply won't route on the gateway.
"""
if evt.get("platform"):
return # already enriched
parsed = _parse_session_key(evt.get("session_key", "") or "")
if not parsed:
return
evt["platform"] = parsed.get("platform", "")
evt["chat_type"] = parsed.get("chat_type", "")
evt["chat_id"] = parsed.get("chat_id", "")
if parsed.get("thread_id"):
evt["thread_id"] = parsed["thread_id"]

async def _async_delegation_watcher(self, interval: float = 2.0) -> None:
"""Drain async-delegation completions and inject them as new turns.

Background subagents (``delegate_task(background=true)``) run on the
async-delegation daemon executor — they have no per-process watcher
task, so their completion events would only be seen by the post-turn
queue drain. This watcher covers the IDLE case: when a background
subagent finishes while no agent turn is running, its result still
re-enters the originating session promptly.

Mirrors the CLI's idle ``process_loop`` drain. Stays silent when the
queue has nothing for us; ignores non-async event types (those are
handled by ``_run_process_watcher`` / the post-turn drain).
"""
await asyncio.sleep(3) # let platforms finish connecting
from tools.process_registry import process_registry as _pr
while self._running:
try:
# Peek the queue for async-delegation events. We must NOT
# consume watch/completion events here (other drains own them),
# so requeue anything that isn't ours.
requeue = []
async_events = []
while not _pr.completion_queue.empty():
try:
evt = _pr.completion_queue.get_nowait()
except Exception:
break
if evt.get("type") == "async_delegation":
async_events.append(evt)
else:
requeue.append(evt)
for evt in requeue:
_pr.completion_queue.put(evt)
for evt in async_events:
self._enrich_async_delegation_routing(evt)
synth_text = _format_gateway_process_notification(evt)
if not synth_text:
continue
try:
await self._inject_watch_notification(synth_text, evt)
except Exception as e:
logger.error("Async delegation injection error: %s", e)
except Exception as e:
logger.debug("Async delegation watcher error: %s", e)
await asyncio.sleep(interval)

async def _run_process_watcher(self, watcher: dict) -> None:
"""
Periodically check a background process and push updates to the user.
Expand Down
40 changes: 35 additions & 5 deletions hermes_cli/cli_commands_mixin.py
Original file line number Diff line number Diff line change
Expand Up @@ -225,7 +225,8 @@ def _handle_snapshot_command(self, command: str):
print(" Usage: /snapshot [list|create [label]|restore <id>|prune [N]]")

def _handle_stop_command(self):
"""Handle /stop — kill all running background processes.
"""Handle /stop — kill all running background processes and
background (async) delegations.

Inspired by OpenAI Codex's separation of interrupt (stop current turn)
from /stop (clean up background processes). See openai/codex#14602.
Expand All @@ -235,13 +236,26 @@ def _handle_stop_command(self):
processes = process_registry.list_sessions()
running = [p for p in processes if p.get("status") == "running"]

if not running:
# Background subagents dispatched via delegate_task(background=true)
# live in their own registry, not the process registry.
try:
from tools.async_delegation import active_count, interrupt_all
n_async = active_count()
except Exception:
n_async = 0
interrupt_all = None

if not running and not n_async:
print(" No running background processes.")
return

print(f" Stopping {len(running)} background process(es)...")
killed = process_registry.kill_all()
print(f" ✅ Stopped {killed} process(es).")
if running:
print(f" Stopping {len(running)} background process(es)...")
killed = process_registry.kill_all()
print(f" ✅ Stopped {killed} process(es).")
if n_async and interrupt_all is not None:
stopped = interrupt_all(reason="/stop")
print(f" ✅ Interrupted {stopped} background delegation(s).")

def _handle_agents_command(self):
"""Handle /agents — show background processes and agent status."""
Expand All @@ -261,6 +275,22 @@ def _handle_agents_command(self):
if finished:
_cprint(f" Recently finished: {len(finished)}")

# Background (async) delegations — delegate_task(background=true)
try:
from tools.async_delegation import list_async_delegations
delegations = list_async_delegations()
except Exception:
delegations = []
running_d = [d for d in delegations if d.get("status") == "running"]
if delegations:
_cprint(f" Background delegations: {len(running_d)} running")
for d in delegations:
goal = (d.get("goal") or "")[:60]
_cprint(
f" {d.get('delegation_id', '?')} · "
f"{d.get('status', '?')} · {goal}"
)

agent_running = getattr(self, "_agent_running", False)
_cprint(f" Agent: {'running' if agent_running else 'idle'}")

Expand Down
1 change: 1 addition & 0 deletions hermes_cli/config.py
Original file line number Diff line number Diff line change
Expand Up @@ -1775,6 +1775,7 @@ def _ensure_hermes_home_managed(home: Path):
"reasoning_effort": "", # reasoning effort for subagents: "xhigh", "high", "medium",
# "low", "minimal", "none" (empty = inherit parent's level)
"max_concurrent_children": 3, # max parallel children per batch; floor of 1 enforced, no ceiling
"max_async_children": 3, # max concurrent background (background=true) subagents; new dispatches rejected at capacity
# Orchestrator role controls (see tools/delegate_tool.py:_get_max_spawn_depth
# and _get_orchestrator_enabled). Floored at 1, no upper ceiling —
# raise deliberately, each level multiplies API cost.
Expand Down
Loading
Loading