diff --git a/.gitignore b/.gitignore index 6ae86265a60c6..532dae694291a 100644 --- a/.gitignore +++ b/.gitignore @@ -70,3 +70,4 @@ mini-swe-agent/ result website/static/api/skills-index.json models-dev-upstream/ +tests/plugins/_test_ttm_control_plane.db diff --git a/gateway/run.py b/gateway/run.py index 4885cbefd8063..3e9eeac91fd43 100644 --- a/gateway/run.py +++ b/gateway/run.py @@ -38,6 +38,7 @@ # gateway is a long-running daemon, so its boot cost matters less than # preserving the established test-patch surface. from agent.account_usage import fetch_account_usage, render_account_usage_lines +from agent.context_compressor import LEGACY_SUMMARY_PREFIX, SUMMARY_PREFIX from hermes_cli.config import cfg_get # --- Agent cache tuning --------------------------------------------------- @@ -176,6 +177,136 @@ def _last_transcript_timestamp(history: Optional[List[Dict[str, Any]]]) -> Any: return None +_COMPACTION_CONTINUATION_MARKERS = ( + "[your active task list was preserved across context compression]", + "continue", + "resume", + "keep going", + "proceed", + "carry on", +) + +_COMPACTION_TASK_SECTIONS = { + "active task", + "in progress", + "pending user asks", + "remaining work", +} + +_COMPACTION_STOPWORDS = { + "about", "above", "across", "active", "after", "again", "also", "and", + "are", "before", "being", "below", "between", "can", "context", "current", + "from", "have", "into", "latest", "message", "only", "pending", "please", + "preserved", "respond", "resume", "section", "summary", "task", "that", "the", + "this", "turn", "user", "when", "with", "work", "your", +} + + +def _compaction_text(content: Any) -> str: + """Return a plain-text view of text or multimodal message content.""" + if content is None: + return "" + if isinstance(content, str): + return content + if isinstance(content, list): + parts: list[str] = [] + for item in content: + if isinstance(item, str): + parts.append(item) + elif isinstance(item, dict): + text = item.get("text") + if isinstance(text, str): + parts.append(text) + return "\n".join(parts) + return str(content) + + +def _is_compaction_summary(content: Any) -> bool: + text = _compaction_text(content).lstrip() + return text.startswith(SUMMARY_PREFIX) or text.startswith(LEGACY_SUMMARY_PREFIX) + + +def _compaction_tokens(text: str) -> set[str]: + return { + token + for token in re.findall(r"[a-zA-Z][a-zA-Z0-9_-]{2,}", text.lower()) + if token not in _COMPACTION_STOPWORDS + } + + +def _extract_compaction_section(text: str, heading: str) -> str: + pattern = re.compile( + rf"(?ims)^##\s+{re.escape(heading)}\s*$\n(?P.*?)(?=^##\s+|\Z)" + ) + match = pattern.search(text) + return match.group("body").strip() if match else "" + + +def _compaction_summary_matches_current_turn(summary: str, current_message: Any) -> bool: + """Heuristic semantic guard for compacted active-task handoffs. + + Compacted summaries are useful for same-session continuation, but dangerous + when a stale/cross-chat transcript is accidentally loaded. Keep task-like + sections only when the current turn explicitly says the task list was + preserved, uses a continuation cue, or overlaps with the summary's active + task vocabulary. Otherwise the summary remains factual background only. + """ + current_text = _compaction_text(current_message).lower() + if any(marker in current_text for marker in _COMPACTION_CONTINUATION_MARKERS): + return True + + active_task = _extract_compaction_section(summary, "Active Task") + if not active_task: + return False + + task_tokens = _compaction_tokens(active_task) + current_tokens = _compaction_tokens(current_text) + if not task_tokens or not current_tokens: + return False + + overlap = task_tokens & current_tokens + return len(overlap) >= 2 or (len(overlap) / max(len(task_tokens), 1)) >= 0.25 + + +def _strip_compaction_task_sections(summary: str) -> str: + lines = summary.splitlines() + kept: list[str] = [] + skipping = False + for line in lines: + heading_match = re.match(r"^##\s+(.+?)\s*$", line) + if heading_match: + heading = heading_match.group(1).strip().lower() + skipping = heading in _COMPACTION_TASK_SECTIONS + if skipping: + continue + if not skipping: + kept.append(line) + sanitized = "\n".join(kept).strip() + sanitized = sanitized.replace( + "Your current task is identified in the '## Active Task' section of the summary β€” resume exactly from there. ", + "Use this summary as background only unless it matches the latest user request. ", + ) + sanitized = sanitized.replace( + "Your current task is identified in the '## Active Task' section of the summary β€” resume exactly from there.", + "Use this summary as background only unless it matches the latest user request.", + ) + return sanitized + + +def _sanitize_compaction_summary_for_current_turn(content: Any, current_message: Any) -> Any: + """Prevent stale compacted task directives from overriding a new turn.""" + if not _is_compaction_summary(content): + return content + summary = _compaction_text(content) + if _compaction_summary_matches_current_turn(summary, current_message): + return content + sanitized = _strip_compaction_task_sections(summary) + if isinstance(content, str): + return sanitized + # Preserve multimodal shape only when needed; compaction summaries are + # normally plain text, but this keeps the helper safe for future callers. + return sanitized + # --------------------------------------------------------------------------- # SSL certificate auto-detection for NixOS and other non-standard systems. # Must run BEFORE any HTTP library (discord, aiohttp, etc.) is imported. @@ -428,6 +559,7 @@ def _ensure_ssl_certs() -> None: from gateway.session import ( SessionStore, SessionSource, + SessionGoalContract, SessionContext, build_session_context, build_session_context_prompt, @@ -4162,6 +4294,8 @@ async def _handle_message(self, event: MessageEvent) -> Optional[str]: return await self._handle_commands_command(event) if _cmd_def_inner.name == "profile": return await self._handle_profile_command(event) + if _cmd_def_inner.name == "goal": + return await self._handle_goal_command(event) if _cmd_def_inner.name == "update": return await self._handle_update_command(event) @@ -4354,6 +4488,9 @@ async def _handle_message(self, event: MessageEvent) -> Optional[str]: if canonical == "status": return await self._handle_status_command(event) + if canonical == "goal": + return await self._handle_goal_command(event) + if canonical == "agents": return await self._handle_agents_command(event) @@ -5994,6 +6131,94 @@ async def _handle_status_command(self, event: MessageEvent) -> str: return "\n".join(lines) + def _format_goal_status(self, contract: Optional[SessionGoalContract]) -> str: + if not contract: + return ( + "🎯 **Session Goal**\n\n" + "No goal contract is set for this session.\n\n" + "Use `/goal new ` to pin the active objective so " + "compression summaries cannot redefine the task." + ) + + lines = [ + "🎯 **Session Goal**", + "", + f"**Objective:** {contract.current_objective}", + f"**Status:** {contract.status}", + f"**Scope policy:** {contract.scope_policy}", + f"**Locked:** {'yes' if contract.locked else 'no'}", + f"**Operator confirmed:** {'yes' if contract.operator_confirmed else 'no'}", + ] + if contract.allowed_subtasks: + lines.extend(["", "**Allowed subtasks:**"]) + lines.extend(f"- {item}" for item in contract.allowed_subtasks) + if contract.non_goals: + lines.extend(["", "**Non-goals / out of scope:**"]) + lines.extend(f"- {item}" for item in contract.non_goals) + return "\n".join(lines) + + async def _handle_goal_command(self, event: MessageEvent) -> str: + """Handle /goal β€” manage the persisted session goal contract.""" + session_entry = self.session_store.get_or_create_session(event.source) + args = event.get_command_args().strip() + if not args: + subcommand = "status" + rest = "" + else: + parts = args.split(maxsplit=1) + candidate = parts[0].strip().lower() + if candidate in {"status", "new", "lock", "unlock", "clear"}: + subcommand = candidate + rest = parts[1].strip() if len(parts) > 1 else "" + else: + # Ergonomic shorthand: /goal means /goal new . + subcommand = "new" + rest = args + + if subcommand == "status": + return self._format_goal_status(session_entry.goal_contract) + + if subcommand == "new": + objective = rest.strip() + if not objective: + return "Usage: `/goal new `" + session_entry.goal_contract = SessionGoalContract( + current_objective=objective, + status="active", + scope_policy="soft", + operator_confirmed=True, + original_prompt=objective, + ) + self.session_store.update_session(session_entry.session_key) + return f"🎯 Goal set.\n\n**Objective:** {objective}" + + contract = session_entry.goal_contract + if not contract: + return "No goal contract is set. Use `/goal new ` first." + + if subcommand == "lock": + contract.locked = True + contract.scope_policy = "locked" + contract.locked_at = datetime.now() + contract.touch() + self.session_store.update_session(session_entry.session_key) + return "πŸ”’ Session goal locked. Hermes will ask before changing scope." + + if subcommand == "unlock": + contract.locked = False + contract.scope_policy = "soft" + contract.locked_at = None + contract.touch() + self.session_store.update_session(session_entry.session_key) + return "πŸ”“ Session goal unlocked. Scope changes are allowed when explicitly requested." + + if subcommand == "clear": + session_entry.goal_contract = None + self.session_store.update_session(session_entry.session_key) + return "🧹 Session goal cleared." + + return "Usage: `/goal [status|new|lock|unlock|clear] [objective]`" + async def _handle_agents_command(self, event: MessageEvent) -> str: """Handle /agents command - list active agents and running tasks.""" from tools.process_registry import format_uptime_short, process_registry @@ -11272,6 +11497,17 @@ def _bg_review_send(message: str) -> None: # Simple text message - just need role and content content = msg.get("content") if content: + # Compaction summaries are persisted in transcript history + # as regular messages. Before handing history back to the + # model, remove stale task-directive sections unless the + # current user turn semantically matches the handoff. This + # keeps useful background context while preventing cross-chat + # stale "Active Task" blocks from outranking the latest user + # request. + content = _sanitize_compaction_summary_for_current_turn( + content, + message, + ) # Tag cross-platform mirror messages so the agent knows their origin if msg.get("mirror"): mirror_src = msg.get("mirror_source", "another session") diff --git a/gateway/session.py b/gateway/session.py index 557f026ff14bc..440b2223d4bf3 100644 --- a/gateway/session.py +++ b/gateway/session.py @@ -16,7 +16,7 @@ import uuid from pathlib import Path from datetime import datetime, timedelta -from dataclasses import dataclass +from dataclasses import dataclass, field from typing import Dict, List, Optional, Any logger = logging.getLogger(__name__) @@ -156,6 +156,112 @@ def from_dict(cls, data: Dict[str, Any]) -> "SessionSource": +def _datetime_to_iso(value: Optional[datetime]) -> Optional[str]: + return value.isoformat() if value else None + + +def _datetime_from_iso(value: Any) -> Optional[datetime]: + if not value: + return None + try: + return datetime.fromisoformat(str(value)) + except (TypeError, ValueError): + return None + + +@dataclass +class SessionGoalContract: + """First-class, persisted session objective authority. + + Compaction summaries are historical references. The goal contract is the + explicit task agreement for the live session and is injected ahead of + mutable/LLM-authored context so summaries cannot silently redefine the job. + """ + + current_objective: str + status: str = "active" + scope_policy: str = "soft" + operator_confirmed: bool = False + locked: bool = False + original_prompt: Optional[str] = None + allowed_subtasks: List[str] = field(default_factory=list) + non_goals: List[str] = field(default_factory=list) + created_at: Optional[datetime] = None + updated_at: Optional[datetime] = None + confirmed_at: Optional[datetime] = None + locked_at: Optional[datetime] = None + + def __post_init__(self) -> None: + now = _now() + if self.created_at is None: + self.created_at = now + if self.updated_at is None: + self.updated_at = now + + def touch(self) -> None: + self.updated_at = _now() + + def to_dict(self) -> Dict[str, Any]: + return { + "current_objective": self.current_objective, + "status": self.status, + "scope_policy": self.scope_policy, + "operator_confirmed": self.operator_confirmed, + "locked": self.locked, + "original_prompt": self.original_prompt, + "allowed_subtasks": list(self.allowed_subtasks or []), + "non_goals": list(self.non_goals or []), + "created_at": _datetime_to_iso(self.created_at), + "updated_at": _datetime_to_iso(self.updated_at), + "confirmed_at": _datetime_to_iso(self.confirmed_at), + "locked_at": _datetime_to_iso(self.locked_at), + } + + @classmethod + def from_dict(cls, data: Dict[str, Any]) -> "SessionGoalContract": + return cls( + current_objective=str(data.get("current_objective") or "").strip(), + status=str(data.get("status") or "active"), + scope_policy=str(data.get("scope_policy") or "soft"), + operator_confirmed=bool(data.get("operator_confirmed", False)), + locked=bool(data.get("locked", False)), + original_prompt=data.get("original_prompt"), + allowed_subtasks=list(data.get("allowed_subtasks") or []), + non_goals=list(data.get("non_goals") or []), + created_at=_datetime_from_iso(data.get("created_at")) or _now(), + updated_at=_datetime_from_iso(data.get("updated_at")) or _now(), + confirmed_at=_datetime_from_iso(data.get("confirmed_at")), + locked_at=_datetime_from_iso(data.get("locked_at")), + ) + + def to_prompt_block(self) -> str: + lines = [ + "## Session Goal Contract", + "", + ( + "This block is the highest-priority session authority after the " + "system/developer instructions. It supersedes compaction summaries, " + "transcript summaries, and stale handoff text when they conflict." + ), + "", + f"**Objective:** {self.current_objective}", + f"**Status:** {self.status}", + f"**Scope policy:** {self.scope_policy}", + f"**Locked:** {'yes' if self.locked else 'no'}", + ] + if self.operator_confirmed: + lines.append("**Operator confirmed:** yes") + if self.allowed_subtasks: + lines.append("**Allowed subtasks:**") + lines.extend(f"- {item}" for item in self.allowed_subtasks) + if self.non_goals: + lines.append("**Non-goals / out of scope:**") + lines.extend(f"- {item}" for item in self.non_goals) + lines.append("") + lines.append("If the latest user message conflicts with this contract, ask for explicit goal update rather than silently changing scope.") + return "\n".join(lines) + + @dataclass class SessionContext: """ @@ -176,6 +282,7 @@ class SessionContext: session_id: str = "" created_at: Optional[datetime] = None updated_at: Optional[datetime] = None + goal_contract: Optional[SessionGoalContract] = None def to_dict(self) -> Dict[str, Any]: return { @@ -189,6 +296,7 @@ def to_dict(self) -> Dict[str, Any]: "session_id": self.session_id, "created_at": self.created_at.isoformat() if self.created_at else None, "updated_at": self.updated_at.isoformat() if self.updated_at else None, + "goal_contract": self.goal_contract.to_dict() if self.goal_contract else None, } @@ -259,10 +367,14 @@ def build_session_context_prompt( except Exception: pass redact_pii = redact_pii and _is_pii_safe - lines = [ + lines = [] + if context.goal_contract: + lines.extend([context.goal_contract.to_prompt_block(), ""]) + + lines.extend([ "## Current Session Context", "", - ] + ]) # Source info platform_name = context.source.platform.value.title() @@ -482,6 +594,11 @@ class SessionEntry: resume_reason: Optional[str] = None # e.g. "restart_timeout" last_resume_marked_at: Optional[datetime] = None + # First-class task authority for this session. It is persisted alongside + # session metadata so compression/resume can never be the sole source of + # truth for the active objective. + goal_contract: Optional[SessionGoalContract] = None + def to_dict(self) -> Dict[str, Any]: result = { "session_key": self.session_key, @@ -508,6 +625,7 @@ def to_dict(self) -> Dict[str, Any]: if self.last_resume_marked_at else None ), + "goal_contract": self.goal_contract.to_dict() if self.goal_contract else None, } if self.origin: result["origin"] = self.origin.to_dict() @@ -534,6 +652,14 @@ def from_dict(cls, data: Dict[str, Any]) -> "SessionEntry": except (TypeError, ValueError): last_resume_marked_at = None + goal_contract = None + if data.get("goal_contract"): + try: + goal_contract = SessionGoalContract.from_dict(data["goal_contract"]) + except Exception as e: + logger.debug("Invalid goal_contract for session %r: %s", data.get("session_id"), e) + goal_contract = None + return cls( session_key=data["session_key"], session_id=data["session_id"], @@ -556,6 +682,7 @@ def from_dict(cls, data: Dict[str, Any]) -> "SessionEntry": resume_pending=data.get("resume_pending", False), resume_reason=data.get("resume_reason"), last_resume_marked_at=last_resume_marked_at, + goal_contract=goal_contract, ) @@ -1365,5 +1492,6 @@ def build_session_context( context.session_id = session_entry.session_id context.created_at = session_entry.created_at context.updated_at = session_entry.updated_at + context.goal_contract = session_entry.goal_contract return context diff --git a/hermes_cli/commands.py b/hermes_cli/commands.py index 5ca562d87a27d..209aafbecba89 100644 --- a/hermes_cli/commands.py +++ b/hermes_cli/commands.py @@ -94,6 +94,9 @@ class CommandDef: CommandDef("steer", "Inject a message after the next tool call without interrupting", "Session", args_hint=""), CommandDef("status", "Show session info", "Session"), + CommandDef("goal", "Manage this session's persisted goal contract", "Session", + args_hint="[status|new|lock|unlock|clear] [objective]", + subcommands=("status", "new", "lock", "unlock", "clear")), CommandDef("profile", "Show active profile name and home directory", "Info"), CommandDef("sethome", "Set this chat as the home channel", "Session", gateway_only=True, aliases=("set-home",)), @@ -306,6 +309,7 @@ def is_gateway_known_command(name: str | None) -> bool: "background", "commands", "deny", + "goal", "help", "new", "profile", diff --git a/hermes_cli/doctor.py b/hermes_cli/doctor.py index f0822bdce8cbe..31d10ec64abc2 100644 --- a/hermes_cli/doctor.py +++ b/hermes_cli/doctor.py @@ -142,6 +142,32 @@ def check_info(text: str): print(f" {color('β†’', Colors.CYAN)} {text}") +def _current_username() -> str: + """Return the current username for diagnostics.""" + try: + import pwd + + return pwd.getpwuid(os.getuid()).pw_name + except Exception: + return os.environ.get("USER") or os.environ.get("LOGNAME") or str(os.getuid()) + + +def _macos_console_username() -> str | None: + """Return the logged-in macOS console user when one exists.""" + if sys.platform != "darwin": + return None + + try: + import pwd + + console_uid = os.stat("/dev/console").st_uid + if console_uid <= 0: + return None + return pwd.getpwuid(console_uid).pw_name + except Exception: + return None + + def _check_gateway_service_linger(issues: list[str]) -> None: """Warn when a systemd user gateway service will stop after logout.""" try: @@ -175,6 +201,52 @@ def _check_gateway_service_linger(issues: list[str]) -> None: check_warn("Could not verify systemd linger", f"({linger_detail})") +def _check_gateway_service_launchd_session(issues: list[str]) -> None: + """Warn when a macOS LaunchAgent is installed under a non-console user.""" + if _is_termux(): + return + + try: + from hermes_cli.gateway import ( + get_launchd_plist_path, + is_macos, + ) + except Exception as e: + check_warn("Gateway service launchd session", f"(could not import gateway helpers: {e})") + return + + if not is_macos(): + return + + plist_path = get_launchd_plist_path() + if not plist_path.exists(): + return + + current_user = _current_username() + console_user = _macos_console_username() + if console_user == current_user: + return + + print() + print(color("β—† Gateway Service", Colors.CYAN, Colors.BOLD)) + if console_user: + check_warn( + "LaunchAgent user is not the logged-in macOS user", + f"({current_user} is running Hermes; console user is {console_user})", + ) + else: + check_warn( + "No logged-in macOS console user detected", + "(launchd user agents start inside a desktop login session)", + ) + check_info("Use a system LaunchDaemon for headless/background deployments") + check_info("Or keep 'hermes gateway run' inside tmux/screen") + issues.append( + "macOS launchd user agents require the logged-in desktop account; " + "use a LaunchDaemon or tmux for headless deployments" + ) + + def run_doctor(args): """Run diagnostic checks.""" should_fix = getattr(args, 'fix', False) @@ -717,6 +789,7 @@ def run_doctor(args): pass _check_gateway_service_linger(issues) + _check_gateway_service_launchd_session(issues) # ========================================================================= # Check: Command installation (hermes bin symlink) @@ -819,7 +892,9 @@ def run_doctor(args): # Docker (optional) terminal_env = os.getenv("TERMINAL_ENV", "local") - if terminal_env == "docker": + if _is_termux(): + check_info("Docker backend is not available inside Termux (expected on Android)") + elif terminal_env == "docker": if _safe_which("docker"): # Check if docker daemon is running try: @@ -838,10 +913,7 @@ def run_doctor(args): if _safe_which("docker"): check_ok("docker", "(optional)") else: - if _is_termux(): - check_info("Docker backend is not available inside Termux (expected on Android)") - else: - check_warn("docker not found", "(optional)") + check_warn("docker not found", "(optional)") # SSH (if using ssh backend) if terminal_env == "ssh": diff --git a/hermes_cli/gateway.py b/hermes_cli/gateway.py index 9670a1d83b1bd..cf758daa691b8 100644 --- a/hermes_cli/gateway.py +++ b/hermes_cli/gateway.py @@ -11,6 +11,7 @@ import subprocess import sys from dataclasses import dataclass +from functools import lru_cache from pathlib import Path PROJECT_ROOT = Path(__file__).parent.parent.resolve() @@ -2023,8 +2024,32 @@ def get_launchd_label() -> str: return f"ai.hermes.gateway-{suffix}" if suffix else "ai.hermes.gateway" +@lru_cache(maxsize=1) +def _launchd_managername() -> str | None: + """Return the current launchd session type when available.""" + try: + result = subprocess.run( + ["launchctl", "managername"], + capture_output=True, + text=True, + timeout=5, + ) + except (FileNotFoundError, subprocess.TimeoutExpired): + return None + + if result.returncode != 0: + return None + + name = result.stdout.strip() + return name or None + + def _launchd_domain() -> str: - return f"gui/{os.getuid()}" + uid = os.getuid() + manager = (_launchd_managername() or "").strip().lower() + if manager == "background": + return f"user/{uid}" + return f"gui/{uid}" def generate_launchd_plist() -> str: @@ -2187,26 +2212,43 @@ def launchd_uninstall(): def launchd_start(): plist_path = get_launchd_plist_path() label = get_launchd_label() + domain = _launchd_domain() # Self-heal if the plist is missing entirely (e.g., manual cleanup, failed upgrade) if not plist_path.exists(): print("↻ launchd plist missing; regenerating service definition") plist_path.parent.mkdir(parents=True, exist_ok=True) plist_path.write_text(generate_launchd_plist(), encoding="utf-8") - subprocess.run(["launchctl", "bootstrap", _launchd_domain(), str(plist_path)], check=True, timeout=30) - subprocess.run(["launchctl", "kickstart", f"{_launchd_domain()}/{label}"], check=True, timeout=30) + subprocess.run(["launchctl", "bootstrap", domain, str(plist_path)], check=True, timeout=30) + subprocess.run(["launchctl", "kickstart", f"{domain}/{label}"], check=True, timeout=30) print("βœ“ Service started") return refresh_launchd_plist_if_needed() + loaded = False try: - subprocess.run(["launchctl", "kickstart", f"{_launchd_domain()}/{label}"], check=True, timeout=30) + result = subprocess.run( + ["launchctl", "list", label], + capture_output=True, + text=True, + timeout=10, + ) + loaded = result.returncode == 0 + except subprocess.TimeoutExpired: + loaded = False + + if not loaded: + print("↻ launchd job was unloaded; reloading service definition") + subprocess.run(["launchctl", "bootstrap", domain, str(plist_path)], check=True, timeout=30) + + try: + subprocess.run(["launchctl", "kickstart", f"{domain}/{label}"], check=True, timeout=30) except subprocess.CalledProcessError as e: - if e.returncode not in (3, 113): + if e.returncode not in (3, 113, 125): raise print("↻ launchd job was unloaded; reloading service definition") - subprocess.run(["launchctl", "bootstrap", _launchd_domain(), str(plist_path)], check=True, timeout=30) - subprocess.run(["launchctl", "kickstart", f"{_launchd_domain()}/{label}"], check=True, timeout=30) + subprocess.run(["launchctl", "bootstrap", domain, str(plist_path)], check=True, timeout=30) + subprocess.run(["launchctl", "kickstart", f"{domain}/{label}"], check=True, timeout=30) print("βœ“ Service started") def launchd_stop(): diff --git a/hermes_cli/main.py b/hermes_cli/main.py index 5bf90800452fd..14f6489e7c2ba 100644 --- a/hermes_cli/main.py +++ b/hermes_cli/main.py @@ -6560,6 +6560,78 @@ def cmd_update(args): _finalize_update_output(_update_io_state) +def _spawn_gateway_mode_manual_restart(killed_pids: set[int]) -> bool: + """Start a replacement manual gateway after a gateway-launched update. + + ``hermes update --gateway`` is launched from an existing gateway session, + often by a Telegram /update command. If there is no active service + manager, the update path can only stop the old manual gateway process; it + cannot tell the user to run ``hermes gateway run`` because the messaging + channel was just stopped. This detached helper waits for the old gateway + PIDs to exit, then starts a fresh manual gateway with ``--replace``. + """ + pids = sorted(pid for pid in killed_pids if pid > 0) + if not pids: + return False + + import shlex + + logs_dir = get_hermes_home() / "logs" + try: + logs_dir.mkdir(parents=True, exist_ok=True) + except OSError: + pass + log_path = logs_dir / "gateway.manual.log" + + wait_parts = [ + f"while kill -0 {pid} 2>/dev/null; do sleep 0.2; done" + for pid in pids + ] + gateway_cmd = " ".join( + shlex.quote(part) + for part in [ + sys.executable, + "-m", + "hermes_cli.main", + "gateway", + "run", + "--replace", + ] + ) + shell_cmd = ( + f"cd {shlex.quote(str(PROJECT_ROOT))} || exit 126; " + + "; ".join(wait_parts) + + f"; exec {gateway_cmd} >> {shlex.quote(str(log_path))} 2>&1" + ) + + env = os.environ.copy() + env["PYTHONUNBUFFERED"] = "1" + try: + setsid_bin = shutil.which("setsid") + if setsid_bin: + subprocess.Popen( + [setsid_bin, "bash", "-lc", shell_cmd], + cwd=PROJECT_ROOT, + env=env, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + start_new_session=True, + ) + else: + subprocess.Popen( + ["bash", "-lc", shell_cmd], + cwd=PROJECT_ROOT, + env=env, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + start_new_session=True, + ) + return True + except Exception as exc: + print(f" ⚠ Failed to start replacement gateway: {exc}") + return False + + def _cmd_update_impl(args, gateway_mode: bool): """Body of ``cmd_update`` β€” kept separate so the wrapper can always restore stdio even on ``sys.exit``.""" @@ -7338,12 +7410,18 @@ def _service_restart_sec( print(f" βœ“ Restarted {svc}") if killed_pids: print(f" β†’ Stopped {len(killed_pids)} manual gateway process(es)") - print(" Restart manually: hermes gateway run") - # Also restart for each profile if needed - if len(killed_pids) > 1: - print( - " (or: hermes -p gateway run for each profile)" - ) + if gateway_mode and not restarted_services: + if _spawn_gateway_mode_manual_restart(killed_pids): + print(" Replacement gateway will start automatically") + else: + print(" Restart manually: hermes gateway run") + else: + print(" Restart manually: hermes gateway run") + # Also restart for each profile if needed + if len(killed_pids) > 1: + print( + " (or: hermes -p gateway run for each profile)" + ) if not restarted_services and not killed_pids: # No gateways were running β€” nothing to do diff --git a/plugins/ttm-control-plane/README.md b/plugins/ttm-control-plane/README.md new file mode 100644 index 0000000000000..ef204a3010557 --- /dev/null +++ b/plugins/ttm-control-plane/README.md @@ -0,0 +1,61 @@ +# ttm-control-plane + +PR-F-H1 of the [Hermes alignment plan](../../../developer-handbook/docs/control-plane). Receives runtime dispatches from TTM's `HermesAdapter`, validates the principal-scoped payload, binds the run to a Hermes session, and reports `run.dispatched` back to TTM ingress. + +## Wire contract + +Mounted at `/api/plugins/ttm-control-plane/` on the Hermes dashboard (`127.0.0.1:9119` by default). + +| Method | Path | Purpose | +| --- | --- | --- | +| GET | `/health` | Plugin metadata + binding count (unauthenticated). | +| POST | `/runs/dispatch` | Initial run-spawn dispatch from TTM. Returns 202 + `runtime_run_ref`. | +| GET | `/runs/{ref}/status` | Last-known status for a previously-dispatched run. | +| POST | `/runs/{ref}/stop` | Tear down the binding so a follow-on dispatch can rebind. | + +The dispatch body mirrors TTM's `RuntimeDispatchPayload` plus `runtime_id`, `ingress_base_url`, and `principal_token`. See [`RUNTIME-ADAPTER-CONTRACT.md`](https://github.com/you-kol/developer-handbook/blob/main/docs/control-plane/RUNTIME-ADAPTER-CONTRACT.md) and [`RUNTIME-PRINCIPAL-CONTRACT.md`](https://github.com/you-kol/developer-handbook/blob/main/docs/control-plane/RUNTIME-PRINCIPAL-CONTRACT.md). + +## Auth + +Shared-secret header `X-TTM-Control-Plane-Secret` whose value matches the `TTM_CONTROL_PLANE_SECRET` env var. The dashboard's general auth middleware deliberately bypasses `/api/plugins/*`; this plugin owns its own check. + +If `TTM_CONTROL_PLANE_SECRET` is unset, the plugin runs unauthenticated (dev/CI fallback). Production deployment must set the env var on both sides: + +```bash +# Hermes side β€” load on dashboard service start +echo 'TTM_CONTROL_PLANE_SECRET=""' >> ~/.hermes/.env + +# TTM side β€” Doppler +doppler secrets set TTM_CONTROL_PLANE_SECRET= --project ttm +doppler secrets set HERMES_GATEWAY_URL=http://127.0.0.1:9119/api/plugins/ttm-control-plane --project ttm +``` + +## Service install + +The plugin lives inside the dashboard FastAPI app, so the dashboard must run continuously. The provided launchd plist runs it as a per-user agent: + +```bash +cp launchd/ai.hermes.dashboard.plist ~/Library/LaunchAgents/ +launchctl load ~/Library/LaunchAgents/ai.hermes.dashboard.plist +launchctl list | grep ai.hermes.dashboard +curl -s http://127.0.0.1:9119/api/plugins/ttm-control-plane/health | jq . +``` + +The plist runs `hermes_cli.main dashboard --no-browser --port 9119` with `KeepAlive` on non-success and writes to `~/.hermes/logs/dashboard.{log,error.log}`. + +## Status of original H1 deferrals + +- **Headless agent spawn**: landed. `_spawn_headless_session` spawns `hermes chat -q ... -Q --max-turns 200` with `TTM_RUN_ID`, `TTM_PRINCIPAL_TOKEN`, `TTM_INGRESS_BASE_URL`, and `TTM_RUNTIME_ID` injected as env vars. Failure to resolve the binary or missing token logs and skips β€” never crashes the dispatch route. Set `TTM_CONTROL_PLANE_DISABLE_SPAWN=1` to suppress the spawn (tests/dev). +- **Persistence**: landed. The binding registry is SQLite-backed at `~/.hermes/state.db` (override via `TTM_CONTROL_PLANE_DB_PATH`). Binding metadata survives dashboard restarts. The principal token is **never** persisted β€” after a restart the operator must trigger a TTM rebind to issue a fresh token, which arrives via `/runs/{run_id}/rebind-token`. +- **TTM rebind alignment**: landed. TTM's `POST /control-plane/{run_id}/rebind` returns `token=None` in the response (operator never sees plaintext); the new token flows directly to the plugin via `notify_rebind` β†’ `/runs/{run_id}/rebind-token`. + +## Still deferred (H4 and beyond) + +- **Pause / resume / retry-slice routes**: the plugin does not expose `/runs/{ref}/pause`, `/resume`, or `/slices/{slice_id}/retry`. TTM's `HermesAdapter` returns `unsupported` for these (PR #658) until H4 lands the matching surface. Stop is supported β€” `POST /runs/{ref}/stop` tears down the binding for re-dispatch. + +## Tests + +```bash +cd ~/.hermes/hermes-agent +venv/bin/python -m pytest tests/plugins/test_ttm_control_plane_plugin.py -v +``` diff --git a/plugins/ttm-control-plane/dashboard/manifest.json b/plugins/ttm-control-plane/dashboard/manifest.json new file mode 100644 index 0000000000000..355468d195419 --- /dev/null +++ b/plugins/ttm-control-plane/dashboard/manifest.json @@ -0,0 +1,13 @@ +{ + "name": "ttm-control-plane", + "label": "TTM Control Plane", + "description": "TTM control-plane spawn shim β€” receives runtime dispatch from TTM, validates the principal-scoped payload, and binds the runtime to the run. PR-F-H1 of the Hermes alignment plan.", + "icon": "Network", + "version": "0.1.0", + "tab": { + "hidden": true + }, + "slots": [], + "entry": "dist/index.js", + "api": "plugin_api.py" +} diff --git a/plugins/ttm-control-plane/dashboard/plugin_api.py b/plugins/ttm-control-plane/dashboard/plugin_api.py new file mode 100644 index 0000000000000..ea62b2089b42d --- /dev/null +++ b/plugins/ttm-control-plane/dashboard/plugin_api.py @@ -0,0 +1,1268 @@ +"""TTM control-plane spawn-shim API. + +PR-F-H1 of the Hermes alignment plan, with H1's two deferred items +(``_spawn_headless_session`` real subprocess + SQLite binding +registry) landed in PR-F-H3 closeout. Mounts under +``/api/plugins/ttm-control-plane/`` on the Hermes dashboard FastAPI +app. + +This is the HTTP face that TTM's ``HermesAdapter.dispatch_run()`` calls: +TTM POSTs the runtime dispatch payload (carrying the per-run principal +token), Hermes validates the payload, binds the run to a runtime +session, and returns 202 ``{status: "accepted", runtime_run_ref}``. +A headless ``hermes chat`` subprocess is then spawned with the run's +TTM_* env vars so the agent can drive the run via the ttm_ingress +skill. The binding registry is SQLite-backed so dispatched runs +survive dashboard restarts (the principal token is the one piece of +binding state that is NOT persisted β€” see ``_BindingRegistry``). + +Auth model: shared-secret header ``X-TTM-Control-Plane-Secret`` whose +value matches the ``TTM_CONTROL_PLANE_SECRET`` environment variable +loaded from ``~/.hermes/.env``. The dashboard auth middleware in +``hermes_cli/web_server.py`` deliberately bypasses ``/api/plugins/*``, +so this plugin owns its own auth check. + +Per ``RUNTIME-PRINCIPAL-CONTRACT.md``, the ``principal_token`` lives in +the dispatch body and MUST be forwarded as ``Authorization: Bearer +`` on every ingress write-back to TTM. Plaintext never appears +in plugin logs or in the runtime registry. + +H6 adds lifecycle control (stop/pause/resume/expand_scope) via: + POST /runs/{ref}/lifecycle β€” unified lifecycle receiver (202 async) + POST /runs/{ref}/stop β€” compat alias for stop (TTM adapter pre-H6) + +Stop: SIGTERM β†’ 10s wait β†’ SIGKILL; emits task.updated{stopped}. +Pause: SIGSTOP; persists dossier state; emits task.updated{paused}. + Degrades explicitly if platform SIGSTOP is unavailable. +Resume: SIGCONT from saved pause state; emits task.updated{active}. +Expand-scope: revokes old token; SIGUSR1 advisory hint; awaits + rebind-token to restore emission with new epoch. + +Hermes never self-approves run closure. That remains TTM's domain. +""" + +import asyncio +import json +import logging +import os +import shutil +import signal +import sqlite3 +import threading +import uuid +from dataclasses import dataclass, field +from datetime import UTC, datetime +from pathlib import Path +from typing import Any, Literal + +import httpx +from fastapi import APIRouter, Header, HTTPException, Request, status +from pydantic import BaseModel, Field + +logger = logging.getLogger(__name__) +router = APIRouter() + + +# --------------------------------------------------------------------------- +# Shared-secret auth +# --------------------------------------------------------------------------- + +_SECRET_ENV = "TTM_CONTROL_PLANE_SECRET" +_SECRET_HEADER = "X-TTM-Control-Plane-Secret" + + +def _expected_secret() -> str: + """Read the shared secret from env. Empty string means "auth disabled". + + Returning the empty string lets the plugin run in dev/CI without a + secret configured; in production, set ``TTM_CONTROL_PLANE_SECRET`` to + a long random value in both the TTM Doppler config and + ``~/.hermes/.env``. + """ + return os.environ.get(_SECRET_ENV, "").strip() + + +def _require_secret(provided: str | None) -> None: + expected = _expected_secret() + if not expected: + # Auth disabled β€” log a warning the first time so the operator + # notices in dev. Production deployment must set the env var. + return + if provided != expected: + raise HTTPException( + status_code=status.HTTP_401_UNAUTHORIZED, + detail={"reason": "ttm_control_plane_secret_mismatch"}, + ) + + +# --------------------------------------------------------------------------- +# In-memory binding registry β€” one entry per active run +# --------------------------------------------------------------------------- + + +@dataclass +class _RuntimeBinding: + """A live mapping from TTM ``run_id`` β†’ Hermes runtime session.""" + + run_id: str + runtime_binding_id: str + runtime_run_ref: str + ingress_base_url: str + bound_at: datetime + # ``principal_token`` is held in memory only β€” never persisted to SQLite + # and never logged. After the run.dispatched callback fires, the token + # is cleared; the headless agent process holds its own copy via the + # TTM_PRINCIPAL_TOKEN env var passed at spawn time. + principal_token: str = "" + last_status: str = "pending" + payload_summary: dict[str, Any] = field(default_factory=dict) + + +_DEFAULT_DB_PATH = os.path.expanduser( + os.environ.get("TTM_CONTROL_PLANE_DB_PATH", "~/.hermes/state.db") +) +_DB_TABLE = "ttm_control_plane_bindings" + + +class _BindingRegistry: + """Thread-safe ``run_id β†’ _RuntimeBinding`` registry with SQLite persistence. + + Binding metadata persists across dashboard restarts so: + 1. re-dispatch against a known run_id is idempotent (returns 409), + 2. operators can inspect prior dispatches via /health and snapshot, + 3. a rebind picks up the existing binding and just rotates the token. + + The principal token is NOT persisted β€” it lives only in memory and + is cleared after the run.dispatched callback fires. After a restart + the registry's tokens are empty; the operator must trigger a rebind + (POST /api/runs/control-plane/{run_id}/rebind on TTM) to issue a + fresh token, which TTM forwards via /runs/{run_id}/rebind-token. + """ + + def __init__(self, db_path: str | None = None) -> None: + self._lock = threading.Lock() + self._by_run: dict[str, _RuntimeBinding] = {} + self._db_path = db_path or _DEFAULT_DB_PATH + self._init_db() + self._load_from_db() + + def _connect(self) -> sqlite3.Connection: + Path(self._db_path).parent.mkdir(parents=True, exist_ok=True) + conn = sqlite3.connect(self._db_path, isolation_level=None) + conn.execute("PRAGMA journal_mode=WAL") + conn.execute("PRAGMA synchronous=NORMAL") + return conn + + def _init_db(self) -> None: + with self._connect() as conn: + conn.execute( + f""" + CREATE TABLE IF NOT EXISTS {_DB_TABLE} ( + run_id TEXT PRIMARY KEY, + runtime_binding_id TEXT NOT NULL, + runtime_run_ref TEXT NOT NULL, + ingress_base_url TEXT NOT NULL, + bound_at TEXT NOT NULL, + last_status TEXT NOT NULL, + payload_summary_json TEXT NOT NULL + ) + """ + ) + + def _load_from_db(self) -> None: + with self._connect() as conn: + rows = conn.execute( + f"SELECT run_id, runtime_binding_id, runtime_run_ref, " + f"ingress_base_url, bound_at, last_status, payload_summary_json " + f"FROM {_DB_TABLE}" + ).fetchall() + for row in rows: + self._by_run[row[0]] = _RuntimeBinding( + run_id=row[0], + runtime_binding_id=row[1], + runtime_run_ref=row[2], + ingress_base_url=row[3], + bound_at=datetime.fromisoformat(row[4]), + principal_token="", # never persisted + last_status=row[5], + payload_summary=json.loads(row[6]) if row[6] else {}, + ) + + def _persist(self, binding: _RuntimeBinding) -> None: + with self._connect() as conn: + conn.execute( + f""" + INSERT INTO {_DB_TABLE} + (run_id, runtime_binding_id, runtime_run_ref, ingress_base_url, + bound_at, last_status, payload_summary_json) + VALUES (?, ?, ?, ?, ?, ?, ?) + ON CONFLICT(run_id) DO UPDATE SET + runtime_binding_id=excluded.runtime_binding_id, + runtime_run_ref=excluded.runtime_run_ref, + ingress_base_url=excluded.ingress_base_url, + bound_at=excluded.bound_at, + last_status=excluded.last_status, + payload_summary_json=excluded.payload_summary_json + """, + ( + binding.run_id, + binding.runtime_binding_id, + binding.runtime_run_ref, + binding.ingress_base_url, + binding.bound_at.isoformat(), + binding.last_status, + json.dumps(binding.payload_summary), + ), + ) + + def get(self, run_id: str) -> _RuntimeBinding | None: + with self._lock: + return self._by_run.get(run_id) + + def insert(self, binding: _RuntimeBinding) -> _RuntimeBinding: + with self._lock: + existing = self._by_run.get(binding.run_id) + if existing is not None: + return existing + self._by_run[binding.run_id] = binding + self._persist(binding) + return binding + + def update_status(self, run_id: str, *, last_status: str) -> None: + with self._lock: + entry = self._by_run.get(run_id) + if entry is None: + return + entry.last_status = last_status + self._persist(entry) + + def clear_token(self, run_id: str) -> None: + with self._lock: + entry = self._by_run.get(run_id) + if entry is not None: + entry.principal_token = "" + # No DB write β€” token is never persisted. + + def replace_token(self, run_id: str, new_token: str) -> bool: + """Atomically replace the principal token; returns True if binding found.""" + with self._lock: + entry = self._by_run.get(run_id) + if entry is None: + return False + entry.principal_token = new_token + return True + + def remove(self, run_id: str) -> None: + with self._lock: + self._by_run.pop(run_id, None) + with self._connect() as conn: + conn.execute(f"DELETE FROM {_DB_TABLE} WHERE run_id = ?", (run_id,)) + + def snapshot(self) -> list[_RuntimeBinding]: + with self._lock: + return list(self._by_run.values()) + + def clear(self) -> None: + """Wipe both in-memory and persistent state. Test-only.""" + with self._lock: + self._by_run.clear() + with self._connect() as conn: + conn.execute(f"DELETE FROM {_DB_TABLE}") + + +_REGISTRY = _BindingRegistry() + + +# --------------------------------------------------------------------------- +# Process registry β€” tracks live headless session PIDs for lifecycle control +# --------------------------------------------------------------------------- + + +@dataclass +class _ProcessHandle: + """Lightweight reference to a running headless session subprocess.""" + + run_id: str + pid: int + proc: Any # asyncio.subprocess.Process β€” event-loop-bound + started_at: datetime + + +class _ProcessRegistry: + """Thread-safe registry of live headless session process handles. + + Registered at spawn time; removed when the process exits or the stop + handler completes. Separate from the binding registry so the binding + (and its status history) outlives the process. + """ + + def __init__(self) -> None: + self._lock = threading.Lock() + self._by_run: dict[str, _ProcessHandle] = {} + + def register(self, run_id: str, proc: Any) -> None: + with self._lock: + self._by_run[run_id] = _ProcessHandle( + run_id=run_id, + pid=proc.pid, + proc=proc, + started_at=_utcnow(), + ) + + def get(self, run_id: str) -> _ProcessHandle | None: + with self._lock: + return self._by_run.get(run_id) + + def remove(self, run_id: str) -> None: + with self._lock: + self._by_run.pop(run_id, None) + + def clear(self) -> None: + """Wipe all handles. Test-only.""" + with self._lock: + self._by_run.clear() + + +_PROC_REGISTRY = _ProcessRegistry() + + +# --------------------------------------------------------------------------- +# Pause-state dossier β€” persists per-run context across pause/resume +# --------------------------------------------------------------------------- + + +@dataclass +class _PauseState: + """Dossier snapshot saved when a run is paused via SIGSTOP.""" + + run_id: str + pid: int + paused_at: datetime + lane_id: str = "" + worktree_id: str = "" + + +_PAUSE_STATE: dict[str, _PauseState] = {} +_PAUSE_LOCK = threading.Lock() + + +# --------------------------------------------------------------------------- +# Wire schemas β€” mirror RuntimeDispatchPayload / RuntimeDispatchResult +# --------------------------------------------------------------------------- + + +class DispatchPayload(BaseModel): + """Body of POST /runs/dispatch from TTM HermesAdapter. + + Fields mirror ``backend/app/services/orchestration/runtime_adapter.py + RuntimeDispatchPayload`` plus the adapter-only ``runtime_id`` and + ``ingress_base_url`` that ``hermes_adapter.py:_payload_dict`` injects. + """ + + run_id: str = Field(..., min_length=1, max_length=128) + runtime_id: str = Field(default="hermes", max_length=64) + stream_id: str = Field(..., min_length=1, max_length=64) + stream_version: str = Field(..., min_length=1, max_length=64) + runtime_binding_id: str = Field(..., min_length=1, max_length=128) + slice_id: str = Field(..., min_length=1, max_length=128) + lane_id: str = Field(..., min_length=1, max_length=160) + scope_hash: str = Field(..., min_length=1, max_length=128) + worktree_id: str = "" + goal: str = Field(..., min_length=1, max_length=2000) + allowed_paths: list[str] = Field(default_factory=list) + required_tests: list[str] = Field(default_factory=list) + reply_schema: dict = Field(default_factory=dict) + deadline_at: str | None = None + ingress_base_url: str = "" + # Per RUNTIME-PRINCIPAL-CONTRACT.md the token rides the dispatch body. + # Empty string is rejected at the route level β€” an unauthenticated + # spawn would never be able to write back to TTM ingress. + principal_token: str = "" + + +class DispatchResponse(BaseModel): + """Response shape β€” accepted/degraded with a ``runtime_run_ref``.""" + + status: str + runtime_run_ref: str + bound_at: datetime + + +class StatusResponse(BaseModel): + """Status projection for ``GET /runs/{ref}/status``.""" + + status: str + runtime_run_ref: str + bound_at: datetime + checked_at: datetime + + +class HealthResponse(BaseModel): + plugin: str + version: str + auth_enforced: bool + bindings: int + checked_at: datetime + + +class RebindTokenRequest(BaseModel): + """Body for POST /runs/{run_id}/rebind-token from TTM HermesAdapter.notify_rebind.""" + + new_binding_id: str = Field(..., min_length=1, max_length=128) + new_token: str = Field(..., min_length=1) + ingress_base_url: str = "" + + +# --------------------------------------------------------------------------- +# Lifecycle wire schemas (H6) +# --------------------------------------------------------------------------- + +LifecycleAction = Literal["stop", "pause", "resume", "expand_scope"] + + +class LifecycleRequest(BaseModel): + """Body for POST /runs/{ref}/lifecycle.""" + + action: LifecycleAction + + +class LifecycleResponse(BaseModel): + """Immediate 202 response β€” action is processed asynchronously.""" + + status: str + runtime_run_ref: str + action: str + accepted_at: datetime + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + + +def _utcnow() -> datetime: + return datetime.now(UTC) + + +def _payload_summary(payload: DispatchPayload) -> dict[str, Any]: + """Redact-by-default summary used for logs and the in-memory registry. + + Excludes ``principal_token`` and any other sensitive fields. + """ + return { + "stream_id": payload.stream_id, + "stream_version": payload.stream_version, + "lane_id": payload.lane_id, + "slice_id": payload.slice_id, + "scope_hash": payload.scope_hash, + "worktree_id": payload.worktree_id, + "deadline_at": payload.deadline_at, + "allowed_paths_count": len(payload.allowed_paths), + "required_tests_count": len(payload.required_tests), + "goal_len": len(payload.goal), + } + + +def _binding_by_ref(runtime_run_ref: str) -> _RuntimeBinding | None: + """Look up a binding by runtime_run_ref (O(n) scan over the small registry).""" + return next( + (b for b in _REGISTRY.snapshot() if b.runtime_run_ref == runtime_run_ref), + None, + ) + + +async def _post_run_dispatched(binding: _RuntimeBinding) -> None: + """Fire-and-forget: POST run.dispatched to TTM ingress. + + Skipped (logged) when ``ingress_base_url`` is empty so PR-F-H1 can + bring up cleanly even before TTM PR-F's ingress routes are + deployed in the operator's environment. + """ + base = (binding.ingress_base_url or "").rstrip("/") + token = binding.principal_token + if not base: + logger.warning( + "ttm-control-plane.run_dispatched.skipped: ingress_base_url unset run_id=%s", + binding.run_id, + ) + _REGISTRY.clear_token(binding.run_id) + return + if not token: + logger.warning( + "ttm-control-plane.run_dispatched.skipped: no principal_token run_id=%s", + binding.run_id, + ) + return + + url = f"{base}/api/ingress/runtime/hermes/events" + body = { + "event_type": "run.dispatched", + "actor_type": "runtime", + "actor_id": "hermes", + "expected_scope_epoch": 1, + "summary": "Hermes accepted the dispatch and bound the run", + "payload": { + "runtime_run_ref": binding.runtime_run_ref, + "runtime_binding_id": binding.runtime_binding_id, + "bound_at": binding.bound_at.isoformat(), + }, + } + headers = { + "Authorization": f"Bearer {token}", + "X-Runtime-Id": "hermes", + "X-Run-Id": binding.run_id, + } + try: + async with httpx.AsyncClient(timeout=10.0) as client: + resp = await client.post(url, json=body, headers=headers) + if resp.status_code >= 400: + logger.warning( + "ttm-control-plane.run_dispatched.failed run_id=%s status=%s body=%s", + binding.run_id, + resp.status_code, + resp.text[:200], + ) + return + except httpx.HTTPError as exc: + logger.warning( + "ttm-control-plane.run_dispatched.error run_id=%s error=%s", + binding.run_id, + exc, + ) + return + finally: + # Drop the bearer credential from memory once we've used it. + # Future ingress writes will be issued by the headless session + # directly using its own copy of the token. + _REGISTRY.clear_token(binding.run_id) + + +async def _emit_lifecycle_event( + binding: _RuntimeBinding, + event_type: str, + payload: dict[str, Any], +) -> None: + """POST a canonical event to TTM ingress if a principal token is available. + + Skipped when the token is absent (cleared post-dispatch or after scope + revocation). The headless agent process owns its own token copy and will + emit lifecycle events independently via the ttm_ingress skill. + Never logs token material. + """ + base = (binding.ingress_base_url or "").rstrip("/") + token = binding.principal_token + if not base or not token: + logger.debug( + "ttm-control-plane.lifecycle_event.skipped event_type=%s run_id=%s " + "(no ingress_base_url or token not held by plugin)", + event_type, + binding.run_id, + ) + return + + url = f"{base}/api/ingress/runtime/hermes/events" + body = { + "event_type": event_type, + "actor_type": "runtime", + "actor_id": "hermes", + "expected_scope_epoch": 1, + "summary": f"Hermes lifecycle: {event_type}", + "payload": payload, + } + headers = { + "Authorization": f"Bearer {token}", + "X-Runtime-Id": "hermes", + "X-Run-Id": binding.run_id, + } + try: + async with httpx.AsyncClient(timeout=10.0) as client: + resp = await client.post(url, json=body, headers=headers) + if resp.status_code >= 400: + logger.warning( + "ttm-control-plane.lifecycle_event.failed event_type=%s run_id=%s status=%s", + event_type, + binding.run_id, + resp.status_code, + ) + except httpx.HTTPError as exc: + logger.warning( + "ttm-control-plane.lifecycle_event.error event_type=%s run_id=%s error=%s", + event_type, + binding.run_id, + exc, + ) + + +_SPAWN_LOG_DIR = os.path.expanduser( + os.environ.get("TTM_CONTROL_PLANE_LOG_DIR", "~/.hermes/logs/runs") +) +_SPAWN_DISABLED_ENV = "TTM_CONTROL_PLANE_DISABLE_SPAWN" + + +def _hermes_executable() -> str | None: + """Resolve the hermes CLI binary. + + Prefers ``HERMES_CLI`` env, then PATH lookup, then a fallback to the + venv that the dashboard itself is running in. Returns ``None`` if + none of those resolve so the caller can log-and-skip cleanly. + """ + explicit = os.environ.get("HERMES_CLI", "").strip() + if explicit and Path(explicit).exists(): + return explicit + found = shutil.which("hermes") + if found: + return found + # Fall back to the same venv the dashboard process is using. + candidate = Path(os.path.dirname(os.path.dirname(os.__file__))).parent / "bin" / "hermes" + if candidate.exists(): + return str(candidate) + return None + + +async def _spawn_headless_session(binding: _RuntimeBinding, principal_token: str) -> None: + """Spawn a headless Hermes session bound to the dispatched run. + + The child receives the principal token and ingress base URL via env + vars (see ``tools/ttm_ingress.py:bind_run_from_env``). Failure to + spawn is logged but never crashes the dispatch β€” the operator can + inspect ``ttm-control-plane.headless_session.*`` log lines and + re-dispatch or rebind if the pathway is misconfigured. + + The child is intentionally NOT awaited: the dispatch route already + returned 202 and the agent runs for the lifetime of the run. + The process handle is registered in ``_PROC_REGISTRY`` for lifecycle + control (stop/pause/resume). + """ + if os.environ.get(_SPAWN_DISABLED_ENV, "").strip().lower() in {"1", "true", "yes"}: + logger.info( + "ttm-control-plane.headless_session.disabled run_id=%s " + "(TTM_CONTROL_PLANE_DISABLE_SPAWN set; binding registered without spawn)", + binding.run_id, + ) + return + + hermes_bin = _hermes_executable() + if hermes_bin is None: + logger.warning( + "ttm-control-plane.headless_session.no_executable run_id=%s " + "(set HERMES_CLI or add hermes to PATH; binding remains registered)", + binding.run_id, + ) + return + + if not principal_token: + logger.warning( + "ttm-control-plane.headless_session.no_token run_id=%s " + "(token already cleared; cannot spawn)", + binding.run_id, + ) + return + + base = (binding.ingress_base_url or "").rstrip("/") + if not base: + logger.warning( + "ttm-control-plane.headless_session.no_ingress run_id=%s " + "(ingress_base_url unset; cannot spawn)", + binding.run_id, + ) + return + + Path(_SPAWN_LOG_DIR).mkdir(parents=True, exist_ok=True) + log_path = Path(_SPAWN_LOG_DIR) / f"{binding.run_id}.log" + + env = { + **os.environ, + "TTM_RUN_ID": binding.run_id, + "TTM_PRINCIPAL_TOKEN": principal_token, + "TTM_INGRESS_BASE_URL": base, + "TTM_RUNTIME_ID": "hermes", + } + + brief = ( + f"You are Hermes executing TTM run {binding.run_id}. " + f"Bind via tools.ttm_ingress.bind_run_from_env(), read run state, " + f"drive each gate in approval_policy: post events, attach evidence, " + f"request approval, poll for the operator decision, and emit " + f"run.closed when complete. The principal token is in " + f"TTM_PRINCIPAL_TOKEN; never log it." + ) + + try: + log_file = open(log_path, "ab", buffering=0) # noqa: SIM115 β€” owned by child + proc = await asyncio.create_subprocess_exec( + hermes_bin, + "chat", + "-q", + brief, + "-Q", + "--max-turns", + "200", + env=env, + stdin=asyncio.subprocess.DEVNULL, + stdout=log_file, + stderr=log_file, + start_new_session=True, + ) + except (OSError, asyncio.CancelledError) as exc: + logger.error( + "ttm-control-plane.headless_session.spawn_failed run_id=%s error=%s", + binding.run_id, + exc, + ) + return + + _PROC_REGISTRY.register(binding.run_id, proc) + logger.info( + "ttm-control-plane.headless_session.spawned run_id=%s pid=%s log=%s", + binding.run_id, + proc.pid, + log_path, + ) + + +def _kick_off_post_dispatch_tasks(binding: _RuntimeBinding) -> None: + """Schedule the run.dispatched callback + headless spawn off the + request loop so the route returns 202 immediately. + + The principal token is captured and passed to the spawn task before + ``_post_run_dispatched`` clears it from the registry β€” this avoids a + race where the spawn would observe an empty token. + """ + loop = asyncio.get_running_loop() + captured_token = binding.principal_token + loop.create_task(_post_run_dispatched(binding)) + loop.create_task(_spawn_headless_session(binding, captured_token)) + + +# --------------------------------------------------------------------------- +# Lifecycle action handlers (H6) β€” called off the request loop as Tasks +# --------------------------------------------------------------------------- + + +async def _kill_run_process( + run_id: str, + *, + term_timeout: float = 10.0, +) -> None: + """SIGTERM the headless session process group, then SIGKILL on timeout. + + Safe to call when no process is registered (logs and returns). + Cleans up the process handle from ``_PROC_REGISTRY`` on completion. + """ + handle = _PROC_REGISTRY.get(run_id) + if handle is None: + return + + try: + pgid = os.getpgid(handle.pid) + except ProcessLookupError: + _PROC_REGISTRY.remove(run_id) + return + + # SIGTERM β€” give the process a chance to flush state and exit cleanly. + try: + os.killpg(pgid, signal.SIGTERM) + logger.info( + "ttm-control-plane.kill.sigterm run_id=%s pid=%s pgid=%s", + run_id, + handle.pid, + pgid, + ) + except (ProcessLookupError, PermissionError) as exc: + logger.warning( + "ttm-control-plane.kill.sigterm_failed run_id=%s error=%s", + run_id, + exc, + ) + _PROC_REGISTRY.remove(run_id) + return + + # Wait up to term_timeout for graceful exit. + try: + await asyncio.wait_for(handle.proc.wait(), timeout=term_timeout) + except asyncio.TimeoutError: + pass + + # SIGKILL any survivors. + if handle.proc.returncode is None: + logger.info( + "ttm-control-plane.kill.sigkill run_id=%s pid=%s (SIGTERM timeout)", + run_id, + handle.pid, + ) + try: + os.killpg(os.getpgid(handle.pid), signal.SIGKILL) + await asyncio.wait_for(handle.proc.wait(), timeout=5.0) + except (ProcessLookupError, asyncio.TimeoutError, PermissionError): + pass + + _PROC_REGISTRY.remove(run_id) + + +async def _async_stop(run_id: str, runtime_run_ref: str) -> None: + """Stop the headless session: kill process, emit event, update status. + + Hermes reports stop completion via task.updated{status: stopped} but + does NOT self-approve run closure β€” that remains TTM's domain and + requires a granted close approval on TTM's side. + The binding is kept (status="stopped") so TTM can query status and + drive the canonical closure cascade independently. + """ + binding = _REGISTRY.get(run_id) + if binding is None: + logger.warning("ttm-control-plane.stop.no_binding run_id=%s", run_id) + return + + await _kill_run_process(run_id) + + await _emit_lifecycle_event( + binding, + "task.updated", + {"status": "stopped", "runtime_run_ref": runtime_run_ref}, + ) + _REGISTRY.update_status(run_id, last_status="stopped") + logger.info( + "ttm-control-plane.stop.complete run_id=%s runtime_run_ref=%s", + run_id, + runtime_run_ref, + ) + + +async def _async_pause(run_id: str, runtime_run_ref: str) -> None: + """Pause the headless session with SIGSTOP; persist dossier state. + + Degrades explicitly if SIGSTOP is unavailable or the process is gone β€” + does NOT silently report paused when the suspend did not happen. + """ + binding = _REGISTRY.get(run_id) + if binding is None: + logger.warning("ttm-control-plane.pause.no_binding run_id=%s", run_id) + return + + handle = _PROC_REGISTRY.get(run_id) + if handle is None: + logger.info( + "ttm-control-plane.pause.no_process run_id=%s " + "(process not registered; spawn may be disabled or run already finished)", + run_id, + ) + return + + try: + pgid = os.getpgid(handle.pid) + os.killpg(pgid, signal.SIGSTOP) + except (ProcessLookupError, PermissionError, AttributeError) as exc: + # SIGSTOP is not available on this platform or the process is gone. + # Degrade explicitly β€” never report paused when suspend did not happen. + logger.warning( + "ttm-control-plane.pause.unsupported run_id=%s error=%s " + "(SIGSTOP unavailable or process gone; reporting runtime.error)", + run_id, + exc, + ) + await _emit_lifecycle_event( + binding, + "runtime.error", + { + "phase": "pause", + "detail": f"pause_unsupported: {exc}", + "runtime_run_ref": runtime_run_ref, + }, + ) + return + + summary = binding.payload_summary + state = _PauseState( + run_id=run_id, + pid=handle.pid, + paused_at=_utcnow(), + lane_id=summary.get("lane_id", ""), + worktree_id=summary.get("worktree_id", ""), + ) + with _PAUSE_LOCK: + _PAUSE_STATE[run_id] = state + + await _emit_lifecycle_event( + binding, + "task.updated", + { + "status": "paused", + "runtime_run_ref": runtime_run_ref, + "paused_at": state.paused_at.isoformat(), + "lane_id": state.lane_id, + "worktree_id": state.worktree_id, + }, + ) + _REGISTRY.update_status(run_id, last_status="paused") + logger.info( + "ttm-control-plane.pause.complete run_id=%s pid=%s", + run_id, + handle.pid, + ) + + +async def _async_resume(run_id: str, runtime_run_ref: str) -> None: + """Resume a SIGSTOP-paused session with SIGCONT; restore dossier state.""" + binding = _REGISTRY.get(run_id) + if binding is None: + logger.warning("ttm-control-plane.resume.no_binding run_id=%s", run_id) + return + + with _PAUSE_LOCK: + state = _PAUSE_STATE.get(run_id) + + if state is None: + logger.warning( + "ttm-control-plane.resume.no_pause_state run_id=%s " + "(run was not paused via this plugin instance or state was lost on restart)", + run_id, + ) + return + + handle = _PROC_REGISTRY.get(run_id) + if handle is None: + logger.warning( + "ttm-control-plane.resume.no_process run_id=%s " + "(process handle lost since pause; cannot resume)", + run_id, + ) + return + + try: + pgid = os.getpgid(handle.pid) + os.killpg(pgid, signal.SIGCONT) + except (ProcessLookupError, PermissionError, AttributeError) as exc: + logger.warning( + "ttm-control-plane.resume.failed run_id=%s error=%s", + run_id, + exc, + ) + return + + with _PAUSE_LOCK: + _PAUSE_STATE.pop(run_id, None) + + await _emit_lifecycle_event( + binding, + "task.updated", + {"status": "active", "runtime_run_ref": runtime_run_ref}, + ) + _REGISTRY.update_status(run_id, last_status="running") + logger.info( + "ttm-control-plane.resume.complete run_id=%s pid=%s", + run_id, + handle.pid, + ) + + +async def _async_expand_scope(run_id: str, runtime_run_ref: str) -> None: + """Handle scope expansion: revoke old token and signal headless process. + + The old principal_token is treated as revoked immediately. SIGUSR1 is + sent as an advisory hint to the process group to checkpoint (the agent + will also detect 401 on its next ingress write if it hasn't stopped). + The new token arrives separately via POST /runs/{run_id}/rebind-token; + once received the headless agent's next get_run_state() call will yield + the new scope_epoch and the agent resets phase state per contract. + Invalidated lanes/worktrees must be abandoned β€” the agent is responsible + for detecting the epoch change and stopping work on stale lanes. + """ + binding = _REGISTRY.get(run_id) + if binding is None: + logger.warning("ttm-control-plane.expand_scope.no_binding run_id=%s", run_id) + return + + # Advisory SIGUSR1 before revoking the token so the agent can + # checkpoint cleanly before its next write attempt gets a 401. + handle = _PROC_REGISTRY.get(run_id) + if handle is not None: + try: + pgid = os.getpgid(handle.pid) + os.killpg(pgid, signal.SIGUSR1) + logger.info( + "ttm-control-plane.expand_scope.sigusr1 run_id=%s pid=%s", + run_id, + handle.pid, + ) + except (ProcessLookupError, PermissionError) as exc: + logger.debug( + "ttm-control-plane.expand_scope.sigusr1_failed run_id=%s error=%s", + run_id, + exc, + ) + + # Revoke: clear the token from the plugin registry so plugin-level events + # stop using it. The headless agent's env-var copy will receive a 401 on + # its next ingress write and must stop emitting on the old token. + _REGISTRY.clear_token(run_id) + _REGISTRY.update_status(run_id, last_status="scope_expanding") + logger.info( + "ttm-control-plane.expand_scope.token_revoked run_id=%s " + "(awaiting rebind-token to restore emission with new scope_epoch)", + run_id, + ) + + +# --------------------------------------------------------------------------- +# Routes +# --------------------------------------------------------------------------- + + +@router.get("/health", response_model=HealthResponse) +async def health() -> HealthResponse: + """Liveness + binding count. Unauthenticated for ops probes.""" + return HealthResponse( + plugin="ttm-control-plane", + version="0.2.0", + auth_enforced=bool(_expected_secret()), + bindings=len(_REGISTRY.snapshot()), + checked_at=_utcnow(), + ) + + +@router.post("/runs/dispatch", status_code=status.HTTP_202_ACCEPTED) +async def dispatch_run( + payload: DispatchPayload, + _request: Request, + x_ttm_control_plane_secret: str | None = Header(default=None, alias=_SECRET_HEADER), +) -> DispatchResponse: + """Receive an initial run-spawn dispatch from TTM. + + Wire contract per ``RUNTIME-ADAPTER-CONTRACT.md Β§Spawn-On-Launch + Dispatch`` and ``RUNTIME-PRINCIPAL-CONTRACT.md Β§Issuance``: + + 1. Validate the shared secret + the ``principal_token`` is present. + 2. 409 if ``run_id`` is already bound to a live session. + 3. Mint a ``runtime_run_ref`` and store the binding. + 4. Return 202 immediately; schedule the run.dispatched ingress + callback and the headless agent spawn off the request loop. + """ + _require_secret(x_ttm_control_plane_secret) + + if not payload.principal_token: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail={ + "reason": "principal_token_required", + "hint": ( + "RUNTIME-PRINCIPAL-CONTRACT.md: every run-spawn " + "dispatch must carry a principal_token in the body" + ), + }, + ) + + if payload.runtime_id != "hermes": + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail={"reason": "runtime_id_mismatch", "expected": "hermes"}, + ) + + existing = _REGISTRY.get(payload.run_id) + if existing is not None: + raise HTTPException( + status_code=status.HTTP_409_CONFLICT, + detail={ + "reason": "run_already_bound", + "runtime_run_ref": existing.runtime_run_ref, + "bound_at": existing.bound_at.isoformat(), + }, + ) + + runtime_run_ref = f"hermes-{uuid.uuid4()}" + binding = _RuntimeBinding( + run_id=payload.run_id, + runtime_binding_id=payload.runtime_binding_id, + runtime_run_ref=runtime_run_ref, + ingress_base_url=payload.ingress_base_url or "", + bound_at=_utcnow(), + principal_token=payload.principal_token, + last_status="accepted", + payload_summary=_payload_summary(payload), + ) + inserted = _REGISTRY.insert(binding) + # Race guard: another concurrent dispatch may have inserted between + # our get() and insert(). The registry's insert() returns the + # existing entry on collision; if so, surface 409 with that ref. + if inserted.runtime_run_ref != runtime_run_ref: + raise HTTPException( + status_code=status.HTTP_409_CONFLICT, + detail={ + "reason": "run_already_bound", + "runtime_run_ref": inserted.runtime_run_ref, + "bound_at": inserted.bound_at.isoformat(), + }, + ) + + logger.info( + "ttm-control-plane.dispatch.accepted run_id=%s runtime_run_ref=%s lane_id=%s", + payload.run_id, + runtime_run_ref, + payload.lane_id, + ) + + _kick_off_post_dispatch_tasks(binding) + + return DispatchResponse( + status="accepted", + runtime_run_ref=runtime_run_ref, + bound_at=binding.bound_at, + ) + + +@router.get("/runs/{runtime_run_ref}/status", response_model=StatusResponse) +async def runtime_run_status( + runtime_run_ref: str, + x_ttm_control_plane_secret: str | None = Header(default=None, alias=_SECRET_HEADER), +) -> StatusResponse: + """Return the locally-known status for a previously-dispatched run.""" + _require_secret(x_ttm_control_plane_secret) + + for binding in _REGISTRY.snapshot(): + if binding.runtime_run_ref == runtime_run_ref: + return StatusResponse( + status=binding.last_status, + runtime_run_ref=runtime_run_ref, + bound_at=binding.bound_at, + checked_at=_utcnow(), + ) + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail={"reason": "runtime_run_ref_not_found"}, + ) + + +@router.post("/runs/{runtime_run_ref}/lifecycle", status_code=status.HTTP_202_ACCEPTED) +async def lifecycle_action( + runtime_run_ref: str, + body: LifecycleRequest, + x_ttm_control_plane_secret: str | None = Header(default=None, alias=_SECRET_HEADER), +) -> LifecycleResponse: + """Unified lifecycle receiver: stop | pause | resume | expand_scope. + + Returns 202 immediately; the action is processed asynchronously. + Validates that runtime_run_ref maps to a known persisted binding. + Principal tokens are never logged. + + Actions: + stop β€” SIGTERM β†’ 10s β†’ SIGKILL; emits task.updated{stopped}. + Does not self-approve closure; TTM drives canonical close. + pause β€” SIGSTOP process group; persists dossier; emits task.updated{paused}. + Degrades explicitly if platform SIGSTOP unavailable. + resume β€” SIGCONT from saved pause state; emits task.updated{active}. + expand_scope β€” Revokes old token; SIGUSR1 hint; awaits rebind-token. + """ + _require_secret(x_ttm_control_plane_secret) + + binding = _binding_by_ref(runtime_run_ref) + if binding is None: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail={"reason": "runtime_run_ref_not_found"}, + ) + + run_id = binding.run_id + loop = asyncio.get_running_loop() + match body.action: + case "stop": + loop.create_task(_async_stop(run_id, runtime_run_ref)) + case "pause": + loop.create_task(_async_pause(run_id, runtime_run_ref)) + case "resume": + loop.create_task(_async_resume(run_id, runtime_run_ref)) + case "expand_scope": + loop.create_task(_async_expand_scope(run_id, runtime_run_ref)) + + logger.info( + "ttm-control-plane.lifecycle.accepted action=%s run_id=%s runtime_run_ref=%s", + body.action, + run_id, + runtime_run_ref, + ) + return LifecycleResponse( + status="accepted", + runtime_run_ref=runtime_run_ref, + action=body.action, + accepted_at=_utcnow(), + ) + + +@router.post("/runs/{runtime_run_ref}/stop", status_code=status.HTTP_202_ACCEPTED) +async def stop_run( + runtime_run_ref: str, + x_ttm_control_plane_secret: str | None = Header(default=None, alias=_SECRET_HEADER), +) -> dict[str, Any]: + """Compatibility stop route for current TTM HermesAdapter. + + TTM's stop_run() POSTs to /runs/{ref}/stop (not /lifecycle) until the + TTM adapter is updated to use /runs/{ref}/lifecycle with action="stop". + This route is a stable alias: it schedules the same async stop handler + and returns 202 immediately. The binding is kept (status="stopped") so + TTM can still query status and drive canonical closure independently. + """ + _require_secret(x_ttm_control_plane_secret) + + binding = _binding_by_ref(runtime_run_ref) + if binding is None: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail={"reason": "runtime_run_ref_not_found"}, + ) + + loop = asyncio.get_running_loop() + loop.create_task(_async_stop(binding.run_id, runtime_run_ref)) + + logger.info( + "ttm-control-plane.stop run_id=%s runtime_run_ref=%s", + binding.run_id, + runtime_run_ref, + ) + return { + "status": "accepted", + "runtime_run_ref": runtime_run_ref, + "accepted_at": _utcnow().isoformat(), + } + + +@router.post("/runs/{run_id}/rebind-token", status_code=status.HTTP_200_OK) +async def rebind_token( + run_id: str, + body: RebindTokenRequest, + x_ttm_control_plane_secret: str | None = Header(default=None, alias=_SECRET_HEADER), +) -> dict[str, Any]: + """Accept a new principal token from TTM after a runtime rebind. + + TTM calls this via HermesAdapter.notify_rebind() after issuing a fresh + principal token. The plugin updates its in-memory registry so that the + headless agent session picks up the new bearer credential on next + ingress write-back. + + Returns 404 when the run_id is not registered (i.e. the session has + already exited or was never dispatched to this plugin instance). + """ + _require_secret(x_ttm_control_plane_secret) + + found = _REGISTRY.replace_token(run_id, body.new_token) + if not found: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail={"reason": "run_not_found"}, + ) + + # If we were in scope_expanding state, transition back to running now + # that we have a fresh token. The headless agent's next get_run_state() + # call will fetch the new scope_epoch and reset phase state per contract. + binding = _REGISTRY.get(run_id) + if binding is not None and binding.last_status == "scope_expanding": + _REGISTRY.update_status(run_id, last_status="running") + + logger.info( + "ttm-control-plane.rebind-token run_id=%s binding_id=%s", + run_id, + body.new_binding_id, + ) + return { + "status": "token_updated", + "run_id": run_id, + "new_binding_id": body.new_binding_id, + "updated_at": _utcnow().isoformat(), + } diff --git a/plugins/ttm-control-plane/launchd/ai.hermes.dashboard.plist b/plugins/ttm-control-plane/launchd/ai.hermes.dashboard.plist new file mode 100644 index 0000000000000..36e1ecede5f42 --- /dev/null +++ b/plugins/ttm-control-plane/launchd/ai.hermes.dashboard.plist @@ -0,0 +1,65 @@ + + + + + + Label + ai.hermes.dashboard + + ProgramArguments + + /Users/oc_runtime/.hermes/hermes-agent/venv/bin/python + -m + hermes_cli.main + dashboard + --no-open + --port + 9119 + + + WorkingDirectory + /Users/oc_runtime/.hermes/hermes-agent + + EnvironmentVariables + + PATH + /Users/oc_runtime/.hermes/hermes-agent/venv/bin:/Users/oc_runtime/.hermes/hermes-agent/node_modules/.bin:/opt/homebrew/bin:/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin + VIRTUAL_ENV + /Users/oc_runtime/.hermes/hermes-agent/venv + HERMES_HOME + /Users/oc_runtime/.hermes + + + RunAtLoad + + + KeepAlive + + SuccessfulExit + + + + StandardOutPath + /Users/oc_runtime/.hermes/logs/dashboard.log + + StandardErrorPath + /Users/oc_runtime/.hermes/logs/dashboard.error.log + + diff --git a/run_agent.py b/run_agent.py index 80738aab16b87..f4109c556532e 100644 --- a/run_agent.py +++ b/run_agent.py @@ -1590,6 +1590,67 @@ def __init__( timestamp_str = self.session_start.strftime("%Y%m%d_%H%M%S") short_uuid = uuid.uuid4().hex[:6] self.session_id = f"{timestamp_str}_{short_uuid}" + + # Keep separate visibility into requested toolsets, published tool + # schemas, and backend readiness. Gateway debugging depends on being + # able to tell "tool not advertised" apart from "tool advertised but + # unhealthy at runtime". + self._resolved_enabled_tool_names: set[str] = set() + self._published_tool_names: set[str] = set(self.valid_tool_names) + self._tool_publication_omissions: set[str] = set() + self._terminal_backend_healthy: Optional[bool] = None + if enabled_toolsets is not None: + try: + from toolsets import resolve_toolset + + for _toolset_name in enabled_toolsets: + self._resolved_enabled_tool_names.update(resolve_toolset(_toolset_name)) + + self._tool_publication_omissions = ( + self._resolved_enabled_tool_names - self._published_tool_names + ) + logger.info( + "Tool publication: platform=%s session=%s enabled_toolsets=%s published_tools=%s omitted_tools=%s", + self.platform or "cli", + self.session_id, + sorted(enabled_toolsets), + sorted(self._published_tool_names), + sorted(self._tool_publication_omissions), + ) + + if {"terminal", "process"} & self._resolved_enabled_tool_names: + try: + from tools.terminal_tool import get_terminal_backend_status + + self._terminal_backend_healthy = bool( + get_terminal_backend_status(log_failures=False).get("healthy") + ) + except Exception as exc: + logger.debug("Terminal backend audit failed: %s", exc) + + _missing_terminal_surface = sorted( + {"terminal", "process"} - self._published_tool_names + ) + if _missing_terminal_surface: + logger.warning( + "Terminal tool publication mismatch: platform=%s session=%s enabled_toolsets=%s missing_tools=%s published_tools=%s terminal_backend_healthy=%s", + self.platform or "cli", + self.session_id, + sorted(enabled_toolsets), + _missing_terminal_surface, + sorted(self._published_tool_names), + self._terminal_backend_healthy, + ) + elif self._terminal_backend_healthy is False: + logger.warning( + "Terminal backend unhealthy but still advertised: platform=%s session=%s enabled_toolsets=%s published_tools=%s", + self.platform or "cli", + self.session_id, + sorted(enabled_toolsets), + sorted(self._published_tool_names), + ) + except Exception as exc: + logger.debug("Tool publication audit skipped: %s", exc) # Session logs go into ~/.hermes/sessions/ alongside gateway sessions hermes_home = get_hermes_home() diff --git a/tests/gateway/test_compaction_summary_sanitization.py b/tests/gateway/test_compaction_summary_sanitization.py new file mode 100644 index 0000000000000..4a274b6a4adee --- /dev/null +++ b/tests/gateway/test_compaction_summary_sanitization.py @@ -0,0 +1,71 @@ +"""Regression tests for gateway handling of compacted context summaries.""" + +from agent.context_compressor import SUMMARY_PREFIX +from gateway.run import _sanitize_compaction_summary_for_current_turn + + +def test_stale_compaction_active_task_sections_are_removed_when_latest_turn_unrelated(): + """A compacted history summary must not override an unrelated new message.""" + stale_summary = ( + f"{SUMMARY_PREFIX}\n" + "## Active Task\n" + "Deploy the TTM dev server and debug LiteLLM/ngrok routing.\n\n" + "## Completed Actions\n" + "- Confirmed prior server status.\n\n" + "## Pending User Asks\n" + "- Restart the server and report URL.\n\n" + "## Remaining Work\n" + "1. Fix the deployment.\n\n" + "## Relevant Files\n" + "- /Users/oc_runtime/Development/ttm\n" + ) + + sanitized = _sanitize_compaction_summary_for_current_turn( + stale_summary, + "i've seen codex be able to do it before, so there is a way, you can check it", + ) + + assert "CONTEXT COMPACTION" in sanitized + assert "## Completed Actions" in sanitized + assert "## Relevant Files" in sanitized + assert "## Active Task" not in sanitized + assert "## Pending User Asks" not in sanitized + assert "## Remaining Work" not in sanitized + assert "Deploy the TTM dev server" not in sanitized + assert "Restart the server" not in sanitized + assert "Fix the deployment" not in sanitized + assert "Your current task is identified" not in sanitized + + +def test_related_compaction_active_task_is_preserved_for_same_task_continuation(): + """Same-topic resumption should keep the compacted active-task handoff.""" + summary = ( + f"{SUMMARY_PREFIX}\n" + "## Active Task\n" + "Find root cause for cross-chat context compaction contamination in Hermes.\n\n" + "## Remaining Work\n" + "1. Patch Hermes source and run regression tests.\n" + ) + + sanitized = _sanitize_compaction_summary_for_current_turn( + summary, + "continue the Hermes context compaction root cause fix and tests", + ) + + assert sanitized == summary + + +def test_explicit_preserved_active_task_list_keeps_compaction_handoff(): + """The platform's preserved task-list marker is an explicit continuation signal.""" + summary = ( + f"{SUMMARY_PREFIX}\n" + "## Active Task\n" + "Find root cause/source code path for cross-chat context-compaction contamination.\n" + ) + + sanitized = _sanitize_compaction_summary_for_current_turn( + summary, + "[Your active task list was preserved across context compression]\n- [>] ctx-rootcause", + ) + + assert sanitized == summary diff --git a/tests/gateway/test_goal_command.py b/tests/gateway/test_goal_command.py new file mode 100644 index 0000000000000..eb7963b0fe822 --- /dev/null +++ b/tests/gateway/test_goal_command.py @@ -0,0 +1,158 @@ +"""Tests for the gateway /goal session authority command.""" + +from datetime import datetime +from types import SimpleNamespace +from unittest.mock import AsyncMock, MagicMock + +import pytest + +from gateway.config import GatewayConfig, Platform, PlatformConfig +from gateway.platforms.base import MessageEvent +from gateway.session import SessionEntry, SessionGoalContract, SessionSource, build_session_key + + +def _make_source() -> SessionSource: + return SessionSource( + platform=Platform.TELEGRAM, + user_id="u1", + chat_id="c1", + user_name="tester", + chat_type="dm", + ) + + +def _make_event(text: str) -> MessageEvent: + return MessageEvent(text=text, source=_make_source(), message_id="m1") + + +def _make_runner(session_entry: SessionEntry | None = None): + from gateway.run import GatewayRunner + + runner = object.__new__(GatewayRunner) + runner.config = GatewayConfig( + platforms={Platform.TELEGRAM: PlatformConfig(enabled=True, token="***")} + ) + adapter = MagicMock() + adapter.send = AsyncMock() + runner.adapters = {Platform.TELEGRAM: adapter} + runner.hooks = SimpleNamespace(emit=AsyncMock(), emit_collect=AsyncMock(return_value=[]), loaded_hooks=False) + runner._running_agents = {} + runner._running_agents_ts = {} + runner._pending_messages = {} + runner._pending_approvals = {} + runner._session_db = None + runner._busy_input_mode = "interrupt" + runner._draining = False + runner._is_user_authorized = lambda _source: True + runner._should_send_voice_reply = lambda *_args, **_kwargs: False + + if session_entry is None: + session_entry = SessionEntry( + session_key=build_session_key(_make_source()), + session_id="sess-1", + created_at=datetime.now(), + updated_at=datetime.now(), + platform=Platform.TELEGRAM, + chat_type="dm", + ) + runner.session_store = MagicMock() + runner.session_store.get_or_create_session.return_value = session_entry + runner.session_store.update_session = MagicMock() + runner.session_store.append_to_transcript = MagicMock() + runner.session_store.load_transcript.return_value = [] + runner.session_store.has_any_sessions.return_value = True + + from gateway.run import GatewayRunner as _GR + + runner._session_key_for_source = _GR._session_key_for_source.__get__(runner, _GR) + return runner, session_entry + + +def test_goal_command_is_registered_with_subcommands(): + from hermes_cli.commands import resolve_command + + command = resolve_command("goal") + + assert command is not None + assert command.name == "goal" + assert command.subcommands == ("status", "new", "lock", "unlock", "clear") + + +@pytest.mark.asyncio +async def test_goal_new_creates_persisted_contract(): + runner, entry = _make_runner() + + out = await runner._handle_goal_command(_make_event("/goal new Finish the PR")) + + assert "Goal set" in out + assert entry.goal_contract is not None + assert entry.goal_contract.current_objective == "Finish the PR" + assert entry.goal_contract.status == "active" + assert entry.goal_contract.operator_confirmed is True + runner.session_store.update_session.assert_called_once_with(entry.session_key) + runner.session_store.append_to_transcript.assert_not_called() + + +@pytest.mark.asyncio +async def test_goal_status_reports_current_contract(): + contract = SessionGoalContract( + current_objective="Finish the PR", + locked=True, + scope_policy="locked", + allowed_subtasks=["implement", "verify-pr"], + non_goals=["change unrelated code"], + ) + runner, _entry = _make_runner( + SessionEntry( + session_key=build_session_key(_make_source()), + session_id="sess-1", + created_at=datetime.now(), + updated_at=datetime.now(), + platform=Platform.TELEGRAM, + chat_type="dm", + goal_contract=contract, + ) + ) + + out = await runner._handle_goal_command(_make_event("/goal status")) + + assert "Session Goal" in out + assert "Finish the PR" in out + assert "Locked:** yes" in out + assert "implement" in out + runner.session_store.update_session.assert_not_called() + + +@pytest.mark.asyncio +async def test_goal_lock_unlock_and_clear_mutate_existing_contract(): + runner, entry = _make_runner() + entry.goal_contract = SessionGoalContract(current_objective="Finish the PR") + + locked = await runner._handle_goal_command(_make_event("/goal lock")) + assert "locked" in locked.lower() + assert entry.goal_contract.locked is True + assert entry.goal_contract.scope_policy == "locked" + assert entry.goal_contract.locked_at is not None + + unlocked = await runner._handle_goal_command(_make_event("/goal unlock")) + assert "unlocked" in unlocked.lower() + assert entry.goal_contract.locked is False + assert entry.goal_contract.scope_policy == "soft" + + cleared = await runner._handle_goal_command(_make_event("/goal clear")) + assert "cleared" in cleared.lower() + assert entry.goal_contract is None + assert runner.session_store.update_session.call_count == 3 + + +@pytest.mark.asyncio +async def test_dispatcher_routes_goal_command(monkeypatch): + import gateway.run as gateway_run + + runner, _entry = _make_runner() + runner._handle_goal_command = AsyncMock(return_value="goal handler reached") # type: ignore[attr-defined] + monkeypatch.setattr(gateway_run, "_resolve_runtime_agent_kwargs", lambda: {"api_key": "***"}) + + result = await runner._handle_message(_make_event("/goal status")) + + assert result == "goal handler reached" diff --git a/tests/gateway/test_session.py b/tests/gateway/test_session.py index 5e8af49e3e13b..3d7a02e3a6d30 100644 --- a/tests/gateway/test_session.py +++ b/tests/gateway/test_session.py @@ -6,6 +6,7 @@ from unittest.mock import patch, MagicMock from gateway.config import Platform, HomeChannel, GatewayConfig, PlatformConfig from gateway.session import ( + SessionGoalContract, SessionSource, SessionStore, build_session_context, @@ -99,6 +100,56 @@ def test_unknown_platform_rejected_for_bad_names(self): SessionSource.from_dict({"platform": "nonexistent", "chat_id": "1"}) +class TestSessionGoalContractRoundtrip: + def test_goal_contract_roundtrip_preserves_authority_fields(self): + from datetime import datetime + + contract = SessionGoalContract( + current_objective="Implement Session Goal Contract", + status="active", + scope_policy="locked", + operator_confirmed=True, + locked=True, + original_prompt="ok let's do all the next steps", + allowed_subtasks=["scan pipeline", "write tests"], + non_goals=["merge unrelated PRs"], + created_at=datetime(2026, 4, 30, 12, 0, 0), + updated_at=datetime(2026, 4, 30, 12, 30, 0), + confirmed_at=datetime(2026, 4, 30, 12, 5, 0), + locked_at=datetime(2026, 4, 30, 12, 10, 0), + ) + + restored = SessionGoalContract.from_dict(contract.to_dict()) + + assert restored.current_objective == "Implement Session Goal Contract" + assert restored.scope_policy == "locked" + assert restored.operator_confirmed is True + assert restored.locked is True + assert restored.allowed_subtasks == ["scan pipeline", "write tests"] + assert restored.non_goals == ["merge unrelated PRs"] + assert restored.created_at == contract.created_at + assert restored.updated_at == contract.updated_at + assert restored.confirmed_at == contract.confirmed_at + assert restored.locked_at == contract.locked_at + + def test_goal_contract_renders_high_authority_prompt_block(self): + contract = SessionGoalContract( + current_objective="Finish the active implementation and open PR", + locked=True, + scope_policy="locked", + allowed_subtasks=["implement", "verify-pr"], + non_goals=["restart old completed tasks"], + ) + + block = contract.to_prompt_block() + + assert block.startswith("## Session Goal Contract") + assert "highest-priority session authority" in block + assert "Finish the active implementation and open PR" in block + assert "compaction summaries" in block + assert "restart old completed tasks" in block + + class TestSessionSourceDescription: def test_local_cli(self): source = SessionSource( @@ -194,6 +245,35 @@ def test_telegram_prompt_contains_platform_and_chat(self): assert "Telegram" in prompt assert "Home Chat" in prompt + def test_prompt_includes_goal_contract_before_transport_context(self): + config = GatewayConfig( + platforms={ + Platform.TELEGRAM: PlatformConfig(enabled=True, token="fake-token"), + }, + ) + source = SessionSource(platform=Platform.TELEGRAM, chat_id="111", chat_type="dm") + entry = SessionStore(Path("/tmp/nonexistent"), config).get_or_create_session(source) + entry.goal_contract = SessionGoalContract( + current_objective="Keep implementing the Session Goal Contract PR", + locked=True, + allowed_subtasks=["implement", "verify-pr"], + ) + + ctx = build_session_context(source, config, entry) + prompt = build_session_context_prompt(ctx) + + assert prompt.index("## Session Goal Contract") < prompt.index("## Current Session Context") + assert "Keep implementing the Session Goal Contract PR" in prompt + assert "compaction summaries" in prompt + + def test_prompt_omits_goal_contract_when_absent(self): + config = GatewayConfig(platforms={Platform.TELEGRAM: PlatformConfig(enabled=True, token="fake-token")}) + source = SessionSource(platform=Platform.TELEGRAM, chat_id="111", chat_type="dm") + + prompt = build_session_context_prompt(build_session_context(source, config)) + + assert "## Session Goal Contract" not in prompt + def test_bluebubbles_prompt_mentions_short_conversational_i_message_format(self): config = GatewayConfig( platforms={ diff --git a/tests/hermes_cli/test_doctor.py b/tests/hermes_cli/test_doctor.py index 5fafcb81f67bf..521717d5975ed 100644 --- a/tests/hermes_cli/test_doctor.py +++ b/tests/hermes_cli/test_doctor.py @@ -193,6 +193,46 @@ def test_doctor_reports_vercel_backend_diagnostics(monkeypatch, tmp_path): assert "snapshot filesystem only" in out +def test_check_gateway_service_launchd_session_warns_for_non_console_user(monkeypatch, tmp_path, capsys): + plist_path = tmp_path / "ai.hermes.gateway.plist" + plist_path.write_text("\n") + + monkeypatch.setattr(gateway_cli, "is_macos", lambda: True) + monkeypatch.setattr(gateway_cli, "get_launchd_plist_path", lambda: plist_path) + monkeypatch.setattr(doctor, "_current_username", lambda: "oc_runtime") + monkeypatch.setattr(doctor, "_macos_console_username", lambda: "svc_oc") + + issues = [] + doctor._check_gateway_service_launchd_session(issues) + + out = capsys.readouterr().out + assert "Gateway Service" in out + assert "LaunchAgent user is not the logged-in macOS user" in out + assert "oc_runtime" in out + assert "svc_oc" in out + assert "LaunchDaemon" in out + assert issues == [ + "macOS launchd user agents require the logged-in desktop account; use a LaunchDaemon or tmux for headless deployments" + ] + + +def test_check_gateway_service_launchd_session_skips_when_console_user_matches(monkeypatch, tmp_path, capsys): + plist_path = tmp_path / "ai.hermes.gateway.plist" + plist_path.write_text("\n") + + monkeypatch.setattr(gateway_cli, "is_macos", lambda: True) + monkeypatch.setattr(gateway_cli, "get_launchd_plist_path", lambda: plist_path) + monkeypatch.setattr(doctor, "_current_username", lambda: "oc_runtime") + monkeypatch.setattr(doctor, "_macos_console_username", lambda: "oc_runtime") + + issues = [] + doctor._check_gateway_service_launchd_session(issues) + + out = capsys.readouterr().out + assert out == "" + assert issues == [] + + # ── Memory provider section (doctor should only check the *active* provider) ── diff --git a/tests/hermes_cli/test_gateway_service.py b/tests/hermes_cli/test_gateway_service.py index f2bfa8b870c12..3ef7e72627ed7 100644 --- a/tests/hermes_cli/test_gateway_service.py +++ b/tests/hermes_cli/test_gateway_service.py @@ -34,6 +34,10 @@ def test_wait_for_user_dbus_socket_accepts_private_socket(self, monkeypatch): assert calls == ["env"] +def _pin_launchd_manager(monkeypatch, name="Aqua"): + monkeypatch.setattr(gateway_cli, "_launchd_managername", lambda: name) + + class TestSystemdServiceRefresh: def test_systemd_install_repairs_outdated_unit_without_force(self, tmp_path, monkeypatch): unit_path = tmp_path / "hermes-gateway.service" @@ -62,6 +66,7 @@ def test_systemd_start_refreshes_outdated_unit(self, tmp_path, monkeypatch): unit_path = tmp_path / "hermes-gateway.service" unit_path.write_text("old unit\n", encoding="utf-8") + monkeypatch.setattr(gateway_cli, "_preflight_user_systemd", lambda: None) monkeypatch.setattr(gateway_cli, "get_systemd_unit_path", lambda system=False: unit_path) monkeypatch.setattr(gateway_cli, "generate_systemd_unit", lambda system=False, run_as_user=None: "new unit\n") @@ -85,6 +90,7 @@ def test_systemd_restart_refreshes_outdated_unit(self, tmp_path, monkeypatch): unit_path = tmp_path / "hermes-gateway.service" unit_path.write_text("old unit\n", encoding="utf-8") + monkeypatch.setattr(gateway_cli, "_preflight_user_systemd", lambda: None) monkeypatch.setattr(gateway_cli, "get_systemd_unit_path", lambda system=False: unit_path) monkeypatch.setattr(gateway_cli, "generate_systemd_unit", lambda system=False, run_as_user=None: "new unit\n") @@ -225,6 +231,7 @@ def test_launchd_install_repairs_outdated_plist_without_force(self, tmp_path, mo plist_path = tmp_path / "ai.hermes.gateway.plist" plist_path.write_text("old content", encoding="utf-8") + _pin_launchd_manager(monkeypatch) monkeypatch.setattr(gateway_cli, "get_launchd_plist_path", lambda: plist_path) calls = [] @@ -250,6 +257,7 @@ def test_launchd_start_reloads_unloaded_job_and_retries(self, tmp_path, monkeypa plist_path.write_text(gateway_cli.generate_launchd_plist(), encoding="utf-8") label = gateway_cli.get_launchd_label() + _pin_launchd_manager(monkeypatch) calls = [] domain = gateway_cli._launchd_domain() target = f"{domain}/{label}" @@ -267,6 +275,7 @@ def fake_run(cmd, check=False, **kwargs): gateway_cli.launchd_start() assert calls == [ + ["launchctl", "list", label], ["launchctl", "kickstart", target], ["launchctl", "bootstrap", domain, str(plist_path)], ["launchctl", "kickstart", target], @@ -278,6 +287,7 @@ def test_launchd_start_reloads_on_kickstart_exit_code_113(self, tmp_path, monkey plist_path.write_text(gateway_cli.generate_launchd_plist(), encoding="utf-8") label = gateway_cli.get_launchd_label() + _pin_launchd_manager(monkeypatch) calls = [] domain = gateway_cli._launchd_domain() target = f"{domain}/{label}" @@ -295,12 +305,41 @@ def fake_run(cmd, check=False, **kwargs): gateway_cli.launchd_start() assert calls == [ + ["launchctl", "list", label], + ["launchctl", "kickstart", target], + ["launchctl", "bootstrap", domain, str(plist_path)], ["launchctl", "kickstart", target], + ] + + def test_launchd_start_bootstraps_before_kickstart_when_label_is_unloaded(self, tmp_path, monkeypatch): + plist_path = tmp_path / "ai.hermes.gateway.plist" + plist_path.write_text(gateway_cli.generate_launchd_plist(), encoding="utf-8") + label = gateway_cli.get_launchd_label() + + _pin_launchd_manager(monkeypatch) + calls = [] + domain = gateway_cli._launchd_domain() + target = f"{domain}/{label}" + + def fake_run(cmd, check=False, **kwargs): + calls.append(cmd) + if cmd == ["launchctl", "list", label]: + return SimpleNamespace(returncode=1, stdout="", stderr="") + return SimpleNamespace(returncode=0, stdout="", stderr="") + + monkeypatch.setattr(gateway_cli, "get_launchd_plist_path", lambda: plist_path) + monkeypatch.setattr(gateway_cli.subprocess, "run", fake_run) + + gateway_cli.launchd_start() + + assert calls == [ + ["launchctl", "list", label], ["launchctl", "bootstrap", domain, str(plist_path)], ["launchctl", "kickstart", target], ] def test_launchd_restart_drains_running_gateway_before_kickstart(self, monkeypatch): + _pin_launchd_manager(monkeypatch) calls = [] target = f"{gateway_cli._launchd_domain()}/{gateway_cli.get_launchd_label()}" @@ -329,6 +368,7 @@ def fake_run(cmd, check=False, **kwargs): def test_launchd_restart_self_requests_graceful_restart_without_kickstart(self, monkeypatch, capsys): calls = [] + _pin_launchd_manager(monkeypatch) monkeypatch.setattr( "gateway.status.get_running_pid", lambda: 321, @@ -351,6 +391,7 @@ def test_launchd_restart_self_requests_graceful_restart_without_kickstart(self, def test_launchd_stop_uses_bootout_not_kill(self, monkeypatch): """launchd_stop must bootout the service so KeepAlive doesn't respawn it.""" + _pin_launchd_manager(monkeypatch) label = gateway_cli.get_launchd_label() domain = gateway_cli._launchd_domain() target = f"{domain}/{label}" @@ -370,6 +411,7 @@ def fake_run(cmd, check=False, **kwargs): def test_launchd_stop_tolerates_already_unloaded(self, monkeypatch, capsys): """launchd_stop silently handles exit codes 3/113 (job not loaded).""" + _pin_launchd_manager(monkeypatch) label = gateway_cli.get_launchd_label() domain = gateway_cli._launchd_domain() target = f"{domain}/{label}" @@ -390,6 +432,7 @@ def fake_run(cmd, check=False, **kwargs): def test_launchd_stop_waits_for_process_exit(self, monkeypatch): """launchd_stop calls _wait_for_gateway_exit after bootout.""" + _pin_launchd_manager(monkeypatch) wait_called = [] def fake_run(cmd, check=False, **kwargs): @@ -406,6 +449,16 @@ def fake_wait(**kwargs): assert len(wait_called) == 1 assert wait_called[0] == {"timeout": 10.0, "force_after": 5.0} + def test_launchd_domain_uses_user_scope_for_background_sessions(self, monkeypatch): + _pin_launchd_manager(monkeypatch, "Background") + + assert gateway_cli._launchd_domain() == f"user/{os.getuid()}" + + def test_launchd_domain_uses_gui_scope_for_aqua_sessions(self, monkeypatch): + _pin_launchd_manager(monkeypatch, "Aqua") + + assert gateway_cli._launchd_domain() == f"gui/{os.getuid()}" + def test_launchd_status_reports_local_stale_plist_when_unloaded(self, tmp_path, monkeypatch, capsys): plist_path = tmp_path / "ai.hermes.gateway.plist" plist_path.write_text("old content", encoding="utf-8") @@ -487,6 +540,7 @@ def test_systemd_restart_self_requests_graceful_restart_and_waits(self, monkeypa calls = [] monkeypatch.setattr(gateway_cli, "_select_systemd_scope", lambda system=False: False) + monkeypatch.setattr(gateway_cli, "_preflight_user_systemd", lambda: None) monkeypatch.setattr(gateway_cli, "refresh_systemd_unit_if_needed", lambda system=False: calls.append(("refresh", system))) monkeypatch.setattr( "gateway.status.get_running_pid", @@ -541,6 +595,7 @@ def fake_get_pid(): def test_systemd_restart_recovers_failed_planned_restart(self, monkeypatch, capsys): monkeypatch.setattr(gateway_cli, "_select_systemd_scope", lambda system=False: False) + monkeypatch.setattr(gateway_cli, "_preflight_user_systemd", lambda: None) monkeypatch.setattr(gateway_cli, "refresh_systemd_unit_if_needed", lambda system=False: None) monkeypatch.setattr( "gateway.status.read_runtime_status", diff --git a/tests/hermes_cli/test_update_gateway_restart.py b/tests/hermes_cli/test_update_gateway_restart.py index 1c7e1b96c94e5..7cef8a995ee85 100644 --- a/tests/hermes_cli/test_update_gateway_restart.py +++ b/tests/hermes_cli/test_update_gateway_restart.py @@ -284,6 +284,7 @@ def test_launchd_start_calls_refresh(self, tmp_path, monkeypatch): """launchd_start refreshes the plist before starting.""" plist_path = tmp_path / "ai.hermes.gateway.plist" plist_path.write_text("old") + monkeypatch.setattr(gateway_cli, "_launchd_managername", lambda: "Aqua") monkeypatch.setattr(gateway_cli, "get_launchd_plist_path", lambda: plist_path) calls = [] @@ -305,6 +306,7 @@ def test_launchd_start_recreates_missing_plist_and_loads_service(self, tmp_path, plist_path = tmp_path / "ai.hermes.gateway.plist" assert not plist_path.exists() + monkeypatch.setattr(gateway_cli, "_launchd_managername", lambda: "Aqua") monkeypatch.setattr(gateway_cli, "get_launchd_plist_path", lambda: plist_path) calls = [] @@ -819,6 +821,62 @@ def fake_find(exclude_pids=None, all_profiles=False): # Should show manual stop message since manual PID was killed assert "Stopped 1 manual gateway" in captured + @patch("shutil.which", return_value=None) + @patch("subprocess.run") + def test_gateway_mode_update_restarts_manual_gateway( + self, mock_run, _mock_which, capsys, monkeypatch, + ): + """Telegram-launched updates must not leave manual gateways stopped.""" + args = SimpleNamespace(gateway=True) + manual_pid = 42999 + + monkeypatch.setattr(gateway_cli, "is_macos", lambda: False) + monkeypatch.setattr(gateway_cli, "supports_systemd_services", lambda: False) + monkeypatch.setattr(gateway_cli, "is_termux", lambda: False) + mock_run.side_effect = _make_run_side_effect(commit_count="3") + + with patch.object( + gateway_cli, "_get_service_pids", return_value=set() + ), patch.object( + gateway_cli, "find_gateway_pids", return_value=[manual_pid], + ), patch("os.kill") as mock_kill, patch("subprocess.Popen") as mock_popen: + cmd_update(args) + + captured = capsys.readouterr().out + mock_kill.assert_called_once() + assert mock_kill.call_args.args[0] == manual_pid + mock_popen.assert_called_once() + popen_cmd = mock_popen.call_args.args[0] + assert popen_cmd[:2] == ["bash", "-lc"] + assert "gateway run --replace" in popen_cmd[2] + assert f"kill -0 {manual_pid}" in popen_cmd[2] + assert "Replacement gateway will start automatically" in captured + assert "Restart manually" not in captured + + @patch("shutil.which", return_value=None) + @patch("subprocess.run") + def test_terminal_update_keeps_manual_restart_guidance( + self, mock_run, _mock_which, capsys, monkeypatch, + ): + """Terminal updates should not daemonize a manual gateway unexpectedly.""" + manual_pid = 42999 + + monkeypatch.setattr(gateway_cli, "is_macos", lambda: False) + monkeypatch.setattr(gateway_cli, "supports_systemd_services", lambda: False) + monkeypatch.setattr(gateway_cli, "is_termux", lambda: False) + mock_run.side_effect = _make_run_side_effect(commit_count="3") + + with patch.object( + gateway_cli, "_get_service_pids", return_value=set() + ), patch.object( + gateway_cli, "find_gateway_pids", return_value=[manual_pid], + ), patch("os.kill"), patch("subprocess.Popen") as mock_popen: + cmd_update(SimpleNamespace()) + + captured = capsys.readouterr().out + mock_popen.assert_not_called() + assert "Restart manually: hermes gateway run" in captured + class TestGetServicePids: """Unit tests for _get_service_pids().""" diff --git a/tests/plugins/test_ttm_control_plane_plugin.py b/tests/plugins/test_ttm_control_plane_plugin.py new file mode 100644 index 0000000000000..a399d74b8553d --- /dev/null +++ b/tests/plugins/test_ttm_control_plane_plugin.py @@ -0,0 +1,1178 @@ +"""Tests for the bundled ``ttm-control-plane`` dashboard plugin. + +Mirrors the loader semantics in +``hermes_cli.web_server._mount_plugin_api_routes`` (sys.modules +registration before exec) so dataclass / pydantic forward refs resolve +the same way they do at runtime, then exercises the wire contract: + +* shared-secret auth gates writes when ``TTM_CONTROL_PLANE_SECRET`` is + set; unauthenticated reads of ``/health`` always work +* ``/runs/dispatch`` rejects payloads missing the principal token (per + RUNTIME-PRINCIPAL-CONTRACT.md the credential MUST ride the body) and + rejects mismatched ``runtime_id`` +* the happy path returns 202 with a fresh ``runtime_run_ref`` and binds + the run in-memory; a duplicate dispatch returns 409 with the prior + ``runtime_run_ref`` and leaves the binding intact +* ``/runs/{ref}/status`` round-trips +* ``/runs/{ref}/lifecycle`` validates actions and returns 202 immediately +* ``/runs/{ref}/stop`` (compat) returns 202 and keeps the binding alive +* pause/resume/expand_scope handlers are exercised with mocked processes +* stop handler: SIGTERM β†’ wait β†’ SIGKILL fallback with mocked handles +* ingress events are emitted (mocked) and never log token material +""" +from __future__ import annotations + +import asyncio +import importlib.util +import signal +import sys +import tempfile +import threading +from pathlib import Path +from unittest.mock import AsyncMock, MagicMock, patch + +import pytest +from fastapi import FastAPI +from fastapi.testclient import TestClient + +PLUGIN_API_PATH = ( + Path(__file__).resolve().parents[2] + / "plugins" + / "ttm-control-plane" + / "dashboard" + / "plugin_api.py" +) + + +def _load_plugin_module(): + """Import plugin_api.py the same way the dashboard does. + + The dashboard registers the dynamically-loaded module in + ``sys.modules`` before ``exec_module`` so ``from __future__ import + annotations`` + dataclasses resolve correctly. Tests must do the + same, otherwise dataclass(...) blows up when introspecting the + placeholder module's __dict__. + """ + module_name = "hermes_dashboard_plugin_ttm_control_plane_test" + spec = importlib.util.spec_from_file_location(module_name, PLUGIN_API_PATH) + assert spec is not None and spec.loader is not None + mod = importlib.util.module_from_spec(spec) + sys.modules[module_name] = mod + try: + spec.loader.exec_module(mod) + except Exception: + sys.modules.pop(module_name, None) + raise + return mod + + +def _make_client( + monkeypatch: pytest.MonkeyPatch, + *, + secret: str | None = "test-secret", + db_path: Path | None = None, +): + """Build a fresh FastAPI app with the plugin mounted under + ``/api/plugins/ttm-control-plane`` and the registry cleared. + + ``monkeypatch`` sets the env var so each test sees the auth model it + expects without leaking state across tests. The SQLite-backed + binding registry is pointed at a per-test ``db_path`` (or a + process-wide test path when none is given) so persistence does not + leak across tests. + """ + if secret is None: + monkeypatch.delenv("TTM_CONTROL_PLANE_SECRET", raising=False) + else: + monkeypatch.setenv("TTM_CONTROL_PLANE_SECRET", secret) + if db_path is None: + # Each call gets a fresh temp DB so parallel workers don't collide. + fd, tmp = tempfile.mkstemp(suffix=".db", prefix="ttm_cp_test_") + import os as _os + _os.close(fd) + db_path = Path(tmp) + monkeypatch.setenv("TTM_CONTROL_PLANE_DB_PATH", str(db_path)) + # Disable subprocess spawn for the vast majority of tests β€” only + # the dedicated spawn-path tests opt back in. + monkeypatch.setenv("TTM_CONTROL_PLANE_DISABLE_SPAWN", "1") + plugin = _load_plugin_module() + plugin._REGISTRY.clear() + plugin._PROC_REGISTRY.clear() + with plugin._PAUSE_LOCK: + plugin._PAUSE_STATE.clear() + app = FastAPI() + app.include_router(plugin.router, prefix="/api/plugins/ttm-control-plane") + return TestClient(app), plugin + + +def _dispatch_body(**overrides): + body = { + "run_id": "11111111-1111-1111-1111-111111111111", + "runtime_id": "hermes", + "stream_id": "galactus", + "stream_version": "2026-04-28", + "runtime_binding_id": "22222222-2222-2222-2222-222222222222", + "slice_id": "spawn", + "lane_id": "11111111-1111-1111-1111-111111111111:default", + "scope_hash": "8" * 64, + "worktree_id": "wt-1", + "goal": "spawn galactus run", + "allowed_paths": ["backend/app/api/routes/runs.py"], + "required_tests": ["pytest backend/tests -q"], + "reply_schema": {"type": "object"}, + "deadline_at": "2026-04-29T00:00:00Z", + "ingress_base_url": "", + "principal_token": "ttm-issued-bearer-credential", + } + body.update(overrides) + return body + + +def _dispatch_and_get_ref(client: TestClient, headers: dict) -> tuple[str, str]: + """Dispatch a run and return (run_id, runtime_run_ref).""" + resp = client.post( + "/api/plugins/ttm-control-plane/runs/dispatch", + json=_dispatch_body(), + headers=headers, + ) + assert resp.status_code == 202, resp.text + return _dispatch_body()["run_id"], resp.json()["runtime_run_ref"] + + +# --------------------------------------------------------------------------- +# Health / auth +# --------------------------------------------------------------------------- + + +def test_health_returns_plugin_metadata(monkeypatch: pytest.MonkeyPatch) -> None: + client, _ = _make_client(monkeypatch) + resp = client.get("/api/plugins/ttm-control-plane/health") + assert resp.status_code == 200 + body = resp.json() + assert body["plugin"] == "ttm-control-plane" + assert body["auth_enforced"] is True + assert body["bindings"] == 0 + + +def test_health_signals_auth_disabled_when_secret_unset( + monkeypatch: pytest.MonkeyPatch, +) -> None: + client, _ = _make_client(monkeypatch, secret=None) + resp = client.get("/api/plugins/ttm-control-plane/health") + assert resp.status_code == 200 + assert resp.json()["auth_enforced"] is False + + +def test_dispatch_rejects_missing_secret(monkeypatch: pytest.MonkeyPatch) -> None: + client, _ = _make_client(monkeypatch) + resp = client.post( + "/api/plugins/ttm-control-plane/runs/dispatch", + json=_dispatch_body(), + ) + assert resp.status_code == 401 + assert resp.json()["detail"] == {"reason": "ttm_control_plane_secret_mismatch"} + + +def test_dispatch_rejects_wrong_secret(monkeypatch: pytest.MonkeyPatch) -> None: + client, _ = _make_client(monkeypatch) + resp = client.post( + "/api/plugins/ttm-control-plane/runs/dispatch", + json=_dispatch_body(), + headers={"X-TTM-Control-Plane-Secret": "wrong"}, + ) + assert resp.status_code == 401 + + +def test_dispatch_allowed_when_secret_unset(monkeypatch: pytest.MonkeyPatch) -> None: + """Dev/CI fallback: empty TTM_CONTROL_PLANE_SECRET disables auth so + the plugin is testable without provisioning a secret. Production + deployment must set the env var.""" + client, _ = _make_client(monkeypatch, secret=None) + resp = client.post( + "/api/plugins/ttm-control-plane/runs/dispatch", + json=_dispatch_body(), + ) + assert resp.status_code == 202 + + +# --------------------------------------------------------------------------- +# Wire contract +# --------------------------------------------------------------------------- + + +def test_dispatch_rejects_missing_principal_token(monkeypatch: pytest.MonkeyPatch) -> None: + client, _ = _make_client(monkeypatch) + resp = client.post( + "/api/plugins/ttm-control-plane/runs/dispatch", + json=_dispatch_body(principal_token=""), + headers={"X-TTM-Control-Plane-Secret": "test-secret"}, + ) + assert resp.status_code == 400 + assert resp.json()["detail"]["reason"] == "principal_token_required" + + +def test_dispatch_rejects_runtime_id_mismatch(monkeypatch: pytest.MonkeyPatch) -> None: + client, _ = _make_client(monkeypatch) + resp = client.post( + "/api/plugins/ttm-control-plane/runs/dispatch", + json=_dispatch_body(runtime_id="oc"), + headers={"X-TTM-Control-Plane-Secret": "test-secret"}, + ) + assert resp.status_code == 400 + assert resp.json()["detail"]["reason"] == "runtime_id_mismatch" + + +def test_dispatch_happy_path_returns_runtime_run_ref( + monkeypatch: pytest.MonkeyPatch, +) -> None: + client, plugin = _make_client(monkeypatch) + resp = client.post( + "/api/plugins/ttm-control-plane/runs/dispatch", + json=_dispatch_body(), + headers={"X-TTM-Control-Plane-Secret": "test-secret"}, + ) + assert resp.status_code == 202, resp.text + body = resp.json() + assert body["status"] == "accepted" + assert body["runtime_run_ref"].startswith("hermes-") + # Binding is recorded for idempotency lookups. + bindings = plugin._REGISTRY.snapshot() + assert len(bindings) == 1 + assert bindings[0].run_id == "11111111-1111-1111-1111-111111111111" + + +def test_dispatch_is_idempotent_returns_409_on_redispatch( + monkeypatch: pytest.MonkeyPatch, +) -> None: + client, _ = _make_client(monkeypatch) + headers = {"X-TTM-Control-Plane-Secret": "test-secret"} + first = client.post( + "/api/plugins/ttm-control-plane/runs/dispatch", + json=_dispatch_body(), + headers=headers, + ) + assert first.status_code == 202 + first_ref = first.json()["runtime_run_ref"] + + second = client.post( + "/api/plugins/ttm-control-plane/runs/dispatch", + json=_dispatch_body(), + headers=headers, + ) + assert second.status_code == 409 + detail = second.json()["detail"] + assert detail["reason"] == "run_already_bound" + # Surface the prior ref so the caller can recover without a fresh + # dispatch. + assert detail["runtime_run_ref"] == first_ref + + +def test_status_round_trips_known_runtime_run_ref( + monkeypatch: pytest.MonkeyPatch, +) -> None: + client, _ = _make_client(monkeypatch) + headers = {"X-TTM-Control-Plane-Secret": "test-secret"} + dispatched = client.post( + "/api/plugins/ttm-control-plane/runs/dispatch", + json=_dispatch_body(), + headers=headers, + ) + ref = dispatched.json()["runtime_run_ref"] + + status_resp = client.get( + f"/api/plugins/ttm-control-plane/runs/{ref}/status", + headers=headers, + ) + assert status_resp.status_code == 200, status_resp.text + assert status_resp.json()["runtime_run_ref"] == ref + assert status_resp.json()["status"] in {"accepted", "pending"} + + +def test_status_404s_for_unknown_ref(monkeypatch: pytest.MonkeyPatch) -> None: + client, _ = _make_client(monkeypatch) + resp = client.get( + "/api/plugins/ttm-control-plane/runs/hermes-does-not-exist/status", + headers={"X-TTM-Control-Plane-Secret": "test-secret"}, + ) + assert resp.status_code == 404 + + +# --------------------------------------------------------------------------- +# /runs/{ref}/stop β€” compat route +# --------------------------------------------------------------------------- + + +def test_stop_compat_returns_202_and_keeps_binding( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """H6: /stop keeps the binding alive (status=stopped) so TTM can still + query status and drive canonical closure. It no longer removes the binding.""" + client, plugin = _make_client(monkeypatch) + headers = {"X-TTM-Control-Plane-Secret": "test-secret"} + _, ref = _dispatch_and_get_ref(client, headers) + + stopped = client.post( + f"/api/plugins/ttm-control-plane/runs/{ref}/stop", + headers=headers, + ) + assert stopped.status_code == 202 + body = stopped.json() + assert body["status"] == "accepted" + assert body["runtime_run_ref"] == ref + + # Binding must survive; status is updated asynchronously, but the entry exists. + bindings = plugin._REGISTRY.snapshot() + assert len(bindings) == 1 + + +def test_stop_compat_404s_for_unknown_ref(monkeypatch: pytest.MonkeyPatch) -> None: + client, _ = _make_client(monkeypatch) + resp = client.post( + "/api/plugins/ttm-control-plane/runs/hermes-does-not-exist/stop", + headers={"X-TTM-Control-Plane-Secret": "test-secret"}, + ) + assert resp.status_code == 404 + + +# --------------------------------------------------------------------------- +# /runs/{ref}/lifecycle β€” unified lifecycle receiver +# --------------------------------------------------------------------------- + + +def test_lifecycle_rejects_missing_secret(monkeypatch: pytest.MonkeyPatch) -> None: + client, _ = _make_client(monkeypatch) + headers = {"X-TTM-Control-Plane-Secret": "test-secret"} + _, ref = _dispatch_and_get_ref(client, headers) + + resp = client.post( + f"/api/plugins/ttm-control-plane/runs/{ref}/lifecycle", + json={"action": "stop"}, + ) + assert resp.status_code == 401 + + +def test_lifecycle_404s_for_unknown_ref(monkeypatch: pytest.MonkeyPatch) -> None: + client, _ = _make_client(monkeypatch) + resp = client.post( + "/api/plugins/ttm-control-plane/runs/hermes-not-real/lifecycle", + json={"action": "stop"}, + headers={"X-TTM-Control-Plane-Secret": "test-secret"}, + ) + assert resp.status_code == 404 + + +def test_lifecycle_rejects_invalid_action(monkeypatch: pytest.MonkeyPatch) -> None: + client, _ = _make_client(monkeypatch) + headers = {"X-TTM-Control-Plane-Secret": "test-secret"} + _, ref = _dispatch_and_get_ref(client, headers) + + resp = client.post( + f"/api/plugins/ttm-control-plane/runs/{ref}/lifecycle", + json={"action": "nuke"}, + headers=headers, + ) + assert resp.status_code == 422 + + +@pytest.mark.parametrize("action", ["stop", "pause", "resume", "expand_scope"]) +def test_lifecycle_accepts_valid_actions( + monkeypatch: pytest.MonkeyPatch, action: str +) -> None: + client, _ = _make_client(monkeypatch) + headers = {"X-TTM-Control-Plane-Secret": "test-secret"} + _, ref = _dispatch_and_get_ref(client, headers) + + resp = client.post( + f"/api/plugins/ttm-control-plane/runs/{ref}/lifecycle", + json={"action": action}, + headers=headers, + ) + assert resp.status_code == 202, resp.text + body = resp.json() + assert body["status"] == "accepted" + assert body["runtime_run_ref"] == ref + assert body["action"] == action + assert "accepted_at" in body + + +# --------------------------------------------------------------------------- +# Stop async handler β€” process kill logic +# --------------------------------------------------------------------------- + + +def test_stop_handler_sigterm_then_sigkill_on_timeout( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """SIGTERM is sent first; SIGKILL is sent when the process does not exit.""" + plugin = _load_plugin_module() + plugin._REGISTRY.clear() + plugin._PROC_REGISTRY.clear() + + run_id = "aaaa-stop-test" + ref = "hermes-stop-test-ref" + + # Binding in registry + from datetime import UTC, datetime + + binding = plugin._RuntimeBinding( + run_id=run_id, + runtime_binding_id="bid", + runtime_run_ref=ref, + ingress_base_url="", + bound_at=datetime.now(UTC), + principal_token="", + last_status="running", + ) + plugin._REGISTRY.insert(binding) + + # Mock process that never exits (returncode stays None) + mock_proc = MagicMock() + mock_proc.pid = 99999 + mock_proc.returncode = None + + async def fake_wait(): + raise asyncio.TimeoutError() + + mock_proc.wait = fake_wait + + handle = plugin._ProcessHandle( + run_id=run_id, + pid=99999, + proc=mock_proc, + started_at=datetime.now(UTC), + ) + with plugin._PROC_REGISTRY._lock: + plugin._PROC_REGISTRY._by_run[run_id] = handle + + signals_sent = [] + + def fake_getpgid(pid: int) -> int: + return pid + + def fake_killpg(pgid: int, sig: int) -> None: + signals_sent.append(sig) + + with ( + patch("os.getpgid", side_effect=fake_getpgid), + patch("os.killpg", side_effect=fake_killpg), + ): + asyncio.get_event_loop().run_until_complete( + plugin._async_stop(run_id, ref) + ) + + assert signal.SIGTERM in signals_sent + assert signal.SIGKILL in signals_sent + # Process handle must be removed after stop. + assert plugin._PROC_REGISTRY.get(run_id) is None + # Binding stays; status updated to stopped. + b = plugin._REGISTRY.get(run_id) + assert b is not None + assert b.last_status == "stopped" + + +def test_stop_handler_no_sigkill_when_process_exits_on_sigterm( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """When the process exits within the SIGTERM window, SIGKILL is not sent.""" + plugin = _load_plugin_module() + plugin._REGISTRY.clear() + plugin._PROC_REGISTRY.clear() + + run_id = "bbbb-stop-test" + ref = "hermes-stop-clean" + from datetime import UTC, datetime + + binding = plugin._RuntimeBinding( + run_id=run_id, + runtime_binding_id="bid2", + runtime_run_ref=ref, + ingress_base_url="", + bound_at=datetime.now(UTC), + principal_token="", + last_status="running", + ) + plugin._REGISTRY.insert(binding) + + mock_proc = MagicMock() + mock_proc.pid = 88888 + + async def fast_wait(): + # Simulates immediate exit + mock_proc.returncode = 0 + + mock_proc.wait = fast_wait + mock_proc.returncode = 0 # already exited + + handle = plugin._ProcessHandle( + run_id=run_id, pid=88888, proc=mock_proc, started_at=datetime.now(UTC) + ) + with plugin._PROC_REGISTRY._lock: + plugin._PROC_REGISTRY._by_run[run_id] = handle + + signals_sent = [] + + with ( + patch("os.getpgid", return_value=88888), + patch("os.killpg", side_effect=lambda pgid, sig: signals_sent.append(sig)), + ): + asyncio.get_event_loop().run_until_complete( + plugin._async_stop(run_id, ref) + ) + + assert signal.SIGTERM in signals_sent + assert signal.SIGKILL not in signals_sent + + +def test_stop_handler_no_process_registered_is_safe( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Stop with no registered process must not raise.""" + plugin = _load_plugin_module() + plugin._REGISTRY.clear() + plugin._PROC_REGISTRY.clear() + + run_id = "cccc-no-proc" + ref = "hermes-no-proc-ref" + from datetime import UTC, datetime + + binding = plugin._RuntimeBinding( + run_id=run_id, + runtime_binding_id="bid3", + runtime_run_ref=ref, + ingress_base_url="", + bound_at=datetime.now(UTC), + principal_token="", + last_status="accepted", + ) + plugin._REGISTRY.insert(binding) + + asyncio.get_event_loop().run_until_complete( + plugin._async_stop(run_id, ref) + ) + assert plugin._REGISTRY.get(run_id).last_status == "stopped" + + +# --------------------------------------------------------------------------- +# Pause / resume handlers +# --------------------------------------------------------------------------- + + +def test_pause_handler_sends_sigstop_and_saves_dossier( + monkeypatch: pytest.MonkeyPatch, +) -> None: + plugin = _load_plugin_module() + plugin._REGISTRY.clear() + plugin._PROC_REGISTRY.clear() + with plugin._PAUSE_LOCK: + plugin._PAUSE_STATE.clear() + + run_id = "dddd-pause-test" + ref = "hermes-pause-ref" + from datetime import UTC, datetime + + binding = plugin._RuntimeBinding( + run_id=run_id, + runtime_binding_id="bid4", + runtime_run_ref=ref, + ingress_base_url="", + bound_at=datetime.now(UTC), + principal_token="", + last_status="running", + payload_summary={"lane_id": "main-lane", "worktree_id": "wt-123"}, + ) + plugin._REGISTRY.insert(binding) + + mock_proc = MagicMock() + mock_proc.pid = 77777 + handle = plugin._ProcessHandle( + run_id=run_id, pid=77777, proc=mock_proc, started_at=datetime.now(UTC) + ) + with plugin._PROC_REGISTRY._lock: + plugin._PROC_REGISTRY._by_run[run_id] = handle + + signals_sent = [] + with ( + patch("os.getpgid", return_value=77777), + patch("os.killpg", side_effect=lambda pgid, sig: signals_sent.append(sig)), + ): + asyncio.get_event_loop().run_until_complete( + plugin._async_pause(run_id, ref) + ) + + assert signal.SIGSTOP in signals_sent + with plugin._PAUSE_LOCK: + state = plugin._PAUSE_STATE.get(run_id) + assert state is not None + assert state.lane_id == "main-lane" + assert state.worktree_id == "wt-123" + assert plugin._REGISTRY.get(run_id).last_status == "paused" + + +def test_pause_handler_degrades_when_sigstop_fails( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """If SIGSTOP raises, pause must emit runtime.error and NOT report paused.""" + plugin = _load_plugin_module() + plugin._REGISTRY.clear() + plugin._PROC_REGISTRY.clear() + with plugin._PAUSE_LOCK: + plugin._PAUSE_STATE.clear() + + run_id = "eeee-pause-fail" + ref = "hermes-pause-fail-ref" + from datetime import UTC, datetime + + binding = plugin._RuntimeBinding( + run_id=run_id, + runtime_binding_id="bid5", + runtime_run_ref=ref, + ingress_base_url="", + bound_at=datetime.now(UTC), + principal_token="", + last_status="running", + ) + plugin._REGISTRY.insert(binding) + + mock_proc = MagicMock() + mock_proc.pid = 66666 + handle = plugin._ProcessHandle( + run_id=run_id, pid=66666, proc=mock_proc, started_at=datetime.now(UTC) + ) + with plugin._PROC_REGISTRY._lock: + plugin._PROC_REGISTRY._by_run[run_id] = handle + + emitted_events: list[str] = [] + + async def fake_emit(binding, event_type, payload): + emitted_events.append(event_type) + + with ( + patch("os.getpgid", return_value=66666), + patch("os.killpg", side_effect=PermissionError("SIGSTOP denied")), + patch.object(plugin, "_emit_lifecycle_event", side_effect=fake_emit), + ): + asyncio.get_event_loop().run_until_complete( + plugin._async_pause(run_id, ref) + ) + + assert "runtime.error" in emitted_events + # Status must NOT be set to paused when SIGSTOP failed. + assert plugin._REGISTRY.get(run_id).last_status == "running" + with plugin._PAUSE_LOCK: + assert plugin._PAUSE_STATE.get(run_id) is None + + +def test_resume_handler_sends_sigcont_and_clears_state( + monkeypatch: pytest.MonkeyPatch, +) -> None: + plugin = _load_plugin_module() + plugin._REGISTRY.clear() + plugin._PROC_REGISTRY.clear() + with plugin._PAUSE_LOCK: + plugin._PAUSE_STATE.clear() + + run_id = "ffff-resume-test" + ref = "hermes-resume-ref" + from datetime import UTC, datetime + + binding = plugin._RuntimeBinding( + run_id=run_id, + runtime_binding_id="bid6", + runtime_run_ref=ref, + ingress_base_url="", + bound_at=datetime.now(UTC), + principal_token="", + last_status="paused", + ) + plugin._REGISTRY.insert(binding) + + mock_proc = MagicMock() + mock_proc.pid = 55555 + handle = plugin._ProcessHandle( + run_id=run_id, pid=55555, proc=mock_proc, started_at=datetime.now(UTC) + ) + with plugin._PROC_REGISTRY._lock: + plugin._PROC_REGISTRY._by_run[run_id] = handle + + pause_state = plugin._PauseState( + run_id=run_id, + pid=55555, + paused_at=datetime.now(UTC), + lane_id="main-lane", + worktree_id="wt-456", + ) + with plugin._PAUSE_LOCK: + plugin._PAUSE_STATE[run_id] = pause_state + + signals_sent = [] + with ( + patch("os.getpgid", return_value=55555), + patch("os.killpg", side_effect=lambda pgid, sig: signals_sent.append(sig)), + ): + asyncio.get_event_loop().run_until_complete( + plugin._async_resume(run_id, ref) + ) + + assert signal.SIGCONT in signals_sent + with plugin._PAUSE_LOCK: + assert plugin._PAUSE_STATE.get(run_id) is None + assert plugin._REGISTRY.get(run_id).last_status == "running" + + +def test_resume_handler_no_op_without_pause_state( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Resume when no pause state exists logs a warning but does not raise.""" + plugin = _load_plugin_module() + plugin._REGISTRY.clear() + plugin._PROC_REGISTRY.clear() + with plugin._PAUSE_LOCK: + plugin._PAUSE_STATE.clear() + + run_id = "gggg-resume-no-state" + ref = "hermes-resume-nostate" + from datetime import UTC, datetime + + binding = plugin._RuntimeBinding( + run_id=run_id, + runtime_binding_id="bid7", + runtime_run_ref=ref, + ingress_base_url="", + bound_at=datetime.now(UTC), + principal_token="", + last_status="running", + ) + plugin._REGISTRY.insert(binding) + + with patch("os.killpg") as mock_kill: + asyncio.get_event_loop().run_until_complete( + plugin._async_resume(run_id, ref) + ) + mock_kill.assert_not_called() + + +# --------------------------------------------------------------------------- +# Expand-scope handler +# --------------------------------------------------------------------------- + + +def test_expand_scope_revokes_token_and_signals_process( + monkeypatch: pytest.MonkeyPatch, +) -> None: + plugin = _load_plugin_module() + plugin._REGISTRY.clear() + plugin._PROC_REGISTRY.clear() + + run_id = "hhhh-expand-scope" + ref = "hermes-expand-ref" + from datetime import UTC, datetime + + binding = plugin._RuntimeBinding( + run_id=run_id, + runtime_binding_id="bid8", + runtime_run_ref=ref, + ingress_base_url="", + bound_at=datetime.now(UTC), + principal_token="old-secret-token", + last_status="running", + ) + plugin._REGISTRY.insert(binding) + + mock_proc = MagicMock() + mock_proc.pid = 44444 + handle = plugin._ProcessHandle( + run_id=run_id, pid=44444, proc=mock_proc, started_at=datetime.now(UTC) + ) + with plugin._PROC_REGISTRY._lock: + plugin._PROC_REGISTRY._by_run[run_id] = handle + + signals_sent = [] + with ( + patch("os.getpgid", return_value=44444), + patch("os.killpg", side_effect=lambda pgid, sig: signals_sent.append(sig)), + ): + asyncio.get_event_loop().run_until_complete( + plugin._async_expand_scope(run_id, ref) + ) + + assert signal.SIGUSR1 in signals_sent + # Token must be cleared β€” old token is treated as revoked. + assert plugin._REGISTRY.get(run_id).principal_token == "" + assert plugin._REGISTRY.get(run_id).last_status == "scope_expanding" + + +def test_expand_scope_does_not_log_token_material( + monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture +) -> None: + import logging + + plugin = _load_plugin_module() + plugin._REGISTRY.clear() + plugin._PROC_REGISTRY.clear() + + run_id = "iiii-token-log-test" + ref = "hermes-token-log" + secret_token = "super-secret-old-token-xyz" + from datetime import UTC, datetime + + binding = plugin._RuntimeBinding( + run_id=run_id, + runtime_binding_id="bid9", + runtime_run_ref=ref, + ingress_base_url="", + bound_at=datetime.now(UTC), + principal_token=secret_token, + last_status="running", + ) + plugin._REGISTRY.insert(binding) + + with ( + patch("os.getpgid", side_effect=ProcessLookupError), + caplog.at_level(logging.DEBUG), + ): + asyncio.get_event_loop().run_until_complete( + plugin._async_expand_scope(run_id, ref) + ) + + for record in caplog.records: + assert secret_token not in record.getMessage() + assert secret_token[:8] not in record.getMessage() + + +def test_rebind_token_transitions_scope_expanding_to_running( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """After rebind-token on a scope_expanding run, status returns to running.""" + client, plugin = _make_client(monkeypatch) + headers = {"X-TTM-Control-Plane-Secret": "test-secret"} + run_id, _ = _dispatch_and_get_ref(client, headers) + + # Manually set status to scope_expanding (as expand_scope handler would). + plugin._REGISTRY.update_status(run_id, last_status="scope_expanding") + + resp = client.post( + f"/api/plugins/ttm-control-plane/runs/{run_id}/rebind-token", + json={ + "new_binding_id": "33333333-3333-3333-3333-333333333333", + "new_token": "new-token-after-scope-expand", + "ingress_base_url": "", + }, + headers=headers, + ) + assert resp.status_code == 200 + assert plugin._REGISTRY.get(run_id).last_status == "running" + + +# --------------------------------------------------------------------------- +# TTM ingress event emission β€” mocked, never logs token material +# --------------------------------------------------------------------------- + + +def test_emit_lifecycle_event_skipped_when_no_token( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """No HTTP call is made when the plugin has no token for the run.""" + plugin = _load_plugin_module() + from datetime import UTC, datetime + + binding = plugin._RuntimeBinding( + run_id="jjjj", + runtime_binding_id="bid10", + runtime_run_ref="ref10", + ingress_base_url="http://ttm.local", + bound_at=datetime.now(UTC), + principal_token="", # cleared + last_status="stopped", + ) + + with patch("httpx.AsyncClient") as mock_client_cls: + asyncio.get_event_loop().run_until_complete( + plugin._emit_lifecycle_event(binding, "task.updated", {"status": "stopped"}) + ) + mock_client_cls.assert_not_called() + + +def test_emit_lifecycle_event_posts_when_token_present( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """When the plugin holds a token, it POSTs the event to TTM ingress.""" + plugin = _load_plugin_module() + from datetime import UTC, datetime + + binding = plugin._RuntimeBinding( + run_id="kkkk", + runtime_binding_id="bid11", + runtime_run_ref="ref11", + ingress_base_url="http://ttm.local", + bound_at=datetime.now(UTC), + principal_token="live-token", + last_status="paused", + ) + + mock_response = MagicMock() + mock_response.status_code = 201 + + async def fake_post(url, *, json, headers): + return mock_response + + mock_client = AsyncMock() + mock_client.__aenter__ = AsyncMock(return_value=mock_client) + mock_client.__aexit__ = AsyncMock(return_value=False) + mock_client.post = fake_post + + with patch("httpx.AsyncClient", return_value=mock_client): + asyncio.get_event_loop().run_until_complete( + plugin._emit_lifecycle_event(binding, "task.updated", {"status": "paused"}) + ) + + # Verify post was called (via the mock_client.post path) + # No assertion on mock_client.post.called since it's a real async function; + # absence of exception is the key contract here. + + +def test_emit_lifecycle_event_never_logs_token( + monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture +) -> None: + """Token material must never appear in plugin logs, even on HTTP error.""" + import logging + + plugin = _load_plugin_module() + from datetime import UTC, datetime + + secret_token = "secret-bearer-xyz-123" + binding = plugin._RuntimeBinding( + run_id="llll", + runtime_binding_id="bid12", + runtime_run_ref="ref12", + ingress_base_url="http://ttm.local", + bound_at=datetime.now(UTC), + principal_token=secret_token, + last_status="stopped", + ) + + # Simulate a 500 response β€” triggers the warning log path without + # raising, so we can inspect the log output for token material. + mock_response = MagicMock() + mock_response.status_code = 500 + + async def fake_post(url, *, json, headers): + return mock_response + + mock_client = MagicMock() + mock_client.__aenter__ = AsyncMock(return_value=mock_client) + mock_client.__aexit__ = AsyncMock(return_value=False) + mock_client.post = fake_post + + with ( + patch("httpx.AsyncClient", return_value=mock_client), + caplog.at_level(logging.WARNING), + ): + asyncio.get_event_loop().run_until_complete( + plugin._emit_lifecycle_event(binding, "task.updated", {"status": "stopped"}) + ) + + for record in caplog.records: + assert secret_token not in record.getMessage() + assert secret_token[:8] not in record.getMessage() + + +# --------------------------------------------------------------------------- +# Rebind-token endpoint +# --------------------------------------------------------------------------- + + +def _rebind_token_body(**overrides): + body = { + "new_binding_id": "33333333-3333-3333-3333-333333333333", + "new_token": "new-bearer-credential-abc123", + "ingress_base_url": "https://ttm.local", + } + body.update(overrides) + return body + + +def test_rebind_token_updates_in_memory_credential(monkeypatch: pytest.MonkeyPatch) -> None: + """After a TTM rebind, the plugin must replace the stored token.""" + client, plugin = _make_client(monkeypatch) + headers = {"X-TTM-Control-Plane-Secret": "test-secret"} + run_id = "11111111-1111-1111-1111-111111111111" + + # Dispatch first so the run is registered. + client.post( + "/api/plugins/ttm-control-plane/runs/dispatch", + json=_dispatch_body(), + headers=headers, + ) + old_token = plugin._REGISTRY.get(run_id).principal_token # type: ignore[union-attr] + + resp = client.post( + f"/api/plugins/ttm-control-plane/runs/{run_id}/rebind-token", + json=_rebind_token_body(), + headers=headers, + ) + assert resp.status_code == 200 + body = resp.json() + assert body["status"] == "token_updated" + assert body["run_id"] == run_id + + new_stored = plugin._REGISTRY.get(run_id).principal_token # type: ignore[union-attr] + assert new_stored == "new-bearer-credential-abc123" + assert new_stored != old_token + + +def test_rebind_token_404s_for_unknown_run(monkeypatch: pytest.MonkeyPatch) -> None: + client, _ = _make_client(monkeypatch) + resp = client.post( + "/api/plugins/ttm-control-plane/runs/not-a-real-run/rebind-token", + json=_rebind_token_body(), + headers={"X-TTM-Control-Plane-Secret": "test-secret"}, + ) + assert resp.status_code == 404 + assert resp.json()["detail"]["reason"] == "run_not_found" + + +def test_rebind_token_rejects_missing_secret(monkeypatch: pytest.MonkeyPatch) -> None: + client, _ = _make_client(monkeypatch) + resp = client.post( + "/api/plugins/ttm-control-plane/runs/some-run/rebind-token", + json=_rebind_token_body(), + ) + assert resp.status_code == 401 + + +def test_rebind_token_does_not_log_token_material( + monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture +) -> None: + # Even partial-token logging is removed before long-running production use. + import logging + + client, _ = _make_client(monkeypatch) + headers = {"X-TTM-Control-Plane-Secret": "test-secret"} + run_id = "11111111-1111-1111-1111-111111111111" + secret_token = "very-secret-bearer-credential-xyz" + + client.post( + "/api/plugins/ttm-control-plane/runs/dispatch", + json=_dispatch_body(), + headers=headers, + ) + with caplog.at_level(logging.INFO): + client.post( + f"/api/plugins/ttm-control-plane/runs/{run_id}/rebind-token", + json=_rebind_token_body(new_token=secret_token), + headers=headers, + ) + + rebind_logs = [r.getMessage() for r in caplog.records if "rebind-token" in r.getMessage()] + assert rebind_logs, "expected at least one rebind-token log entry" + for line in rebind_logs: + assert secret_token not in line + # No prefix either (first 8 chars). + assert secret_token[:8] not in line + + +# --------------------------------------------------------------------------- +# SQLite persistence β€” bindings survive a fresh registry instance, tokens do not +# --------------------------------------------------------------------------- + + +def test_binding_persists_across_registry_instances( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + """Dispatch on instance A; rebuilding the registry against the same DB + must surface the same binding (without the principal token).""" + db_path = tmp_path / "bindings.db" + client, plugin = _make_client(monkeypatch, db_path=db_path) + headers = {"X-TTM-Control-Plane-Secret": "test-secret"} + + dispatched = client.post( + "/api/plugins/ttm-control-plane/runs/dispatch", + json=_dispatch_body(), + headers=headers, + ) + assert dispatched.status_code == 202, dispatched.text + expected_ref = dispatched.json()["runtime_run_ref"] + expected_run_id = "11111111-1111-1111-1111-111111111111" + + # Simulate a dashboard restart: drop the in-memory registry and + # rebuild from the same DB path. + rebuilt = plugin._BindingRegistry(db_path=str(db_path)) + snapshot = rebuilt.snapshot() + assert len(snapshot) == 1 + survived = snapshot[0] + assert survived.run_id == expected_run_id + assert survived.runtime_run_ref == expected_ref + # Token must NEVER persist β€” only the binding metadata. + assert survived.principal_token == "" + + +def test_binding_remove_clears_persistence( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + db_path = tmp_path / "bindings.db" + client, plugin = _make_client(monkeypatch, db_path=db_path) + headers = {"X-TTM-Control-Plane-Secret": "test-secret"} + run_id = "11111111-1111-1111-1111-111111111111" + + client.post( + "/api/plugins/ttm-control-plane/runs/dispatch", + json=_dispatch_body(), + headers=headers, + ) + plugin._REGISTRY.remove(run_id) + + rebuilt = plugin._BindingRegistry(db_path=str(db_path)) + assert rebuilt.snapshot() == [] + + +# --------------------------------------------------------------------------- +# Headless session spawn β€” opt-in path +# --------------------------------------------------------------------------- + + +def test_spawn_no_op_when_disabled( + monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture +) -> None: + import logging + + client, plugin = _make_client(monkeypatch) + headers = {"X-TTM-Control-Plane-Secret": "test-secret"} + with caplog.at_level(logging.INFO, logger="hermes_dashboard_plugin_ttm_control_plane_test"): + client.post( + "/api/plugins/ttm-control-plane/runs/dispatch", + json=_dispatch_body(), + headers=headers, + ) + # Either the spawn task hasn't fired yet (we did not await it) or it + # fired and no-op'd because TTM_CONTROL_PLANE_DISABLE_SPAWN=1. We + # assert the latter never produced a "spawned" log line. + spawned = [r for r in caplog.records if "headless_session.spawned" in r.getMessage()] + assert not spawned + + +def test_spawn_logs_when_executable_missing(monkeypatch: pytest.MonkeyPatch) -> None: + """When neither HERMES_CLI nor PATH resolves a hermes binary, the + spawn must log-and-skip β€” never crash the dispatch.""" + import asyncio as _asyncio + + monkeypatch.delenv("TTM_CONTROL_PLANE_DISABLE_SPAWN", raising=False) + monkeypatch.setenv("HERMES_CLI", "/definitely/does/not/exist/hermes") + monkeypatch.setenv("PATH", "") # zero out PATH lookup + monkeypatch.setenv("TTM_CONTROL_PLANE_SECRET", "test-secret") + plugin = _load_plugin_module() + plugin._REGISTRY.clear() + + binding = plugin._RuntimeBinding( + run_id="11111111-1111-1111-1111-111111111111", + runtime_binding_id="binding-1", + runtime_run_ref="hermes-test-1", + ingress_base_url="http://127.0.0.1:8000", + bound_at=plugin._utcnow(), + principal_token="some-token", + ) + # Should not raise even though the executable is unfindable. + _asyncio.get_event_loop().run_until_complete( + plugin._spawn_headless_session(binding, "some-token") + ) + # Process must not be registered when spawn was skipped. + assert plugin._PROC_REGISTRY.get(binding.run_id) is None diff --git a/tests/run_agent/test_run_agent.py b/tests/run_agent/test_run_agent.py index 5585eea484099..1a48110552e0c 100644 --- a/tests/run_agent/test_run_agent.py +++ b/tests/run_agent/test_run_agent.py @@ -762,6 +762,36 @@ def test_valid_tool_names_populated(self): ) assert a.valid_tool_names == {"web_search", "terminal"} + def test_tool_publication_audit_warns_when_terminal_is_omitted(self): + """Resolved terminal toolsets should warn if terminal is not published.""" + tools = _make_tool_defs("process") + with ( + patch("run_agent.get_tool_definitions", return_value=tools), + patch("run_agent.check_toolset_requirements", return_value={}), + patch("run_agent.OpenAI"), + patch("toolsets.resolve_toolset", return_value=["terminal", "process"]), + patch( + "tools.terminal_tool.get_terminal_backend_status", + return_value={"healthy": True, "env_type": "local", "reason": ""}, + ), + patch("run_agent.logger.warning") as mock_warning, + ): + AIAgent( + api_key="test-key-1234567890", + base_url="https://openrouter.ai/api/v1", + quiet_mode=True, + skip_context_files=True, + skip_memory=True, + enabled_toolsets=["terminal"], + platform="telegram", + session_id="sess-123", + ) + + assert any( + "Terminal tool publication mismatch" in call.args[0] + for call in mock_warning.call_args_list + ) + def test_session_id_auto_generated(self): """Session ID should be auto-generated in YYYYMMDD_HHMMSS_ format.""" with ( diff --git a/tests/tools/test_delegate_toolset_scope.py b/tests/tools/test_delegate_toolset_scope.py index d853dbb042c5e..0565a17b1f332 100644 --- a/tests/tools/test_delegate_toolset_scope.py +++ b/tests/tools/test_delegate_toolset_scope.py @@ -8,8 +8,9 @@ from unittest.mock import MagicMock, patch from types import SimpleNamespace +import logging -from tools.delegate_tool import _strip_blocked_tools +from tools.delegate_tool import _build_child_agent, _strip_blocked_tools class TestToolsetIntersection: @@ -64,3 +65,39 @@ def test_empty_intersection_yields_empty_toolsets(self): scoped = [t for t in requested if t in parent_toolsets] assert scoped == [] + + def test_build_child_agent_logs_why_requested_toolset_was_dropped(self, caplog): + parent = SimpleNamespace( + enabled_toolsets=["file"], + valid_tool_names={"read_file"}, + model="test-model", + api_key="test-key", + base_url="https://example.com/v1", + platform="telegram", + providers_allowed=None, + providers_ignored=None, + providers_order=None, + provider_sort=None, + session_id="parent-session", + ) + + with ( + patch("run_agent.AIAgent", return_value=MagicMock()), + caplog.at_level(logging.INFO), + ): + _build_child_agent( + task_index=1, + goal="Check logs", + context=None, + toolsets=["terminal"], + model=None, + max_iterations=5, + task_count=1, + parent_agent=parent, + ) + + assert any( + "Delegation toolset scope:" in record.getMessage() + and "dropped_not_on_parent=['terminal']" in record.getMessage() + for record in caplog.records + ) diff --git a/tests/tools/test_mcp_tool.py b/tests/tools/test_mcp_tool.py index fd19eefa47aeb..69849dd5f9b49 100644 --- a/tests/tools/test_mcp_tool.py +++ b/tests/tools/test_mcp_tool.py @@ -83,6 +83,40 @@ def test_mcp_servers_not_dict_returns_empty(self): assert result == {} +class TestBuildSafeEnv: + def test_preserves_proxy_env_from_parent_process(self, monkeypatch): + monkeypatch.setenv("HTTPS_PROXY", "http://127.0.0.1:7897") + monkeypatch.setenv("NO_PROXY", "localhost,127.0.0.1") + + from tools.mcp_tool import _build_safe_env + + env = _build_safe_env({}) + + assert env["HTTPS_PROXY"] == "http://127.0.0.1:7897" + assert env["NO_PROXY"] == "localhost,127.0.0.1" + + def test_expands_shell_style_placeholders_in_user_env(self, monkeypatch): + monkeypatch.setenv("GRAFANA_URL", "https://grafana.example.com") + monkeypatch.setenv("GRAFANA_SERVICE_ACCOUNT_TOKEN", "glsa_test_token") + + from tools.mcp_tool import _build_safe_env + + env = _build_safe_env({ + "GRAFANA_URL": "${GRAFANA_URL}", + "GRAFANA_SERVICE_ACCOUNT_TOKEN": "$GRAFANA_SERVICE_ACCOUNT_TOKEN", + }) + + assert env["GRAFANA_URL"] == "https://grafana.example.com" + assert env["GRAFANA_SERVICE_ACCOUNT_TOKEN"] == "glsa_test_token" + + def test_unknown_placeholders_are_left_visible(self): + from tools.mcp_tool import _build_safe_env + + env = _build_safe_env({"GRAFANA_URL": "${MISSING_GRAFANA_URL}"}) + + assert env["GRAFANA_URL"] == "${MISSING_GRAFANA_URL}" + + # --------------------------------------------------------------------------- # Schema conversion # --------------------------------------------------------------------------- diff --git a/tests/tools/test_terminal_tool_requirements.py b/tests/tools/test_terminal_tool_requirements.py index fe22bd26c5b83..cb5474a18e92a 100644 --- a/tests/tools/test_terminal_tool_requirements.py +++ b/tests/tools/test_terminal_tool_requirements.py @@ -87,7 +87,8 @@ def test_terminal_and_execute_code_tools_hide_for_unsupported_vercel_runtime(sel tools = get_tool_definitions(enabled_toolsets=["terminal", "code_execution"], quiet_mode=True) names = {tool["function"]["name"] for tool in tools} - assert "terminal" not in names + assert "terminal" in names + assert "process" in names assert "execute_code" not in names def test_terminal_and_execute_code_tools_hide_for_vercel_without_auth(self, monkeypatch): @@ -112,5 +113,17 @@ def test_terminal_and_execute_code_tools_hide_for_vercel_without_auth(self, monk tools = get_tool_definitions(enabled_toolsets=["terminal", "code_execution"], quiet_mode=True) names = {tool["function"]["name"] for tool in tools} - assert "terminal" not in names + assert "terminal" in names + assert "process" in names assert "execute_code" not in names + + def test_terminal_tool_stays_published_when_backend_is_unhealthy(self, monkeypatch): + monkeypatch.setattr( + terminal_tool_module, + "_get_env_config", + lambda: {"env_type": "unknown-backend"}, + ) + tools = get_tool_definitions(enabled_toolsets=["terminal"], quiet_mode=True) + names = {tool["function"]["name"] for tool in tools} + + assert names == {"process", "terminal"} diff --git a/tests/tools/test_ttm_ingress.py b/tests/tools/test_ttm_ingress.py new file mode 100644 index 0000000000000..610dd43f9515d --- /dev/null +++ b/tests/tools/test_ttm_ingress.py @@ -0,0 +1,671 @@ +"""Unit tests for the TTM ingress skill (PR-F-H2). + +All HTTP traffic is mocked through a stub ``client_factory`` so no network +calls are required. The tests assert the wire contract: headers, body +shape (matching ``ControlPlaneEventAppendRequest`` etc.), retry/backoff +on 5xx, immediate raise on 401, and token redaction in logs. +""" + +import logging + +import httpx +import pytest + +from tools.ttm_ingress import ( + CANONICAL_EVENT_TYPES, + ENV_INGRESS_BASE_URL, + ENV_PRINCIPAL_TOKEN, + ENV_RUNTIME_ID, + ENV_RUN_ID, + ENV_SCOPE_EPOCH, + EVENT_RUN_DISPATCHED, + EVENT_TASK_UPDATED, + IngressAuthError, + IngressClientError, + IngressNotBoundError, + IngressServerError, + TtmIngress, + _token_present, +) + +PRINCIPAL_TOKEN = "tok_abcdef0123456789xyz" +RUN_ID = "run-pr-f-h2-test" +BASE_URL = "http://127.0.0.1:8000" +RUNTIME_ID = "hermes" + + +# --------------------------------------------------------------------------- +# Stub HTTP client β€” captures every request and returns canned responses +# --------------------------------------------------------------------------- + + +class _StubResponse: + def __init__(self, status_code: int, json_body: dict | None = None, text: str = ""): + self.status_code = status_code + self._json = json_body if json_body is not None else {} + # Mirror httpx.Response.text fallback when no JSON. + self.text = text or (str(self._json) if json_body else "") + + def json(self): + if self._json is None: + raise ValueError("no json") + return self._json + + +class _StubClient: + """Drop-in replacement for ``httpx.Client`` used in tests.""" + + def __init__(self, plan): + # ``plan`` is a callable that returns the next _StubResponse, OR + # a list popped left-to-right. + self._plan = plan + self.calls: list[tuple[str, dict, dict]] = [] + + def __enter__(self): + return self + + def __exit__(self, *_): + return False + + def post(self, url, json=None, headers=None): + self.calls.append((url, json or {}, headers or {})) + if callable(self._plan): + return self._plan(url, json, headers) + if not self._plan: + raise AssertionError("StubClient ran out of canned responses") + nxt = self._plan.pop(0) + if isinstance(nxt, Exception): + raise nxt + return nxt + + def get(self, url, headers=None): + self.calls.append((url, {}, headers or {})) + if callable(self._plan): + return self._plan(url, None, headers) + if not self._plan: + raise AssertionError("StubClient ran out of canned responses") + nxt = self._plan.pop(0) + if isinstance(nxt, Exception): + raise nxt + return nxt + + +def _client_factory(plan): + """Return a factory that yields a fresh _StubClient per call. + + The TtmIngress code opens a new client for each POST (via + context manager), so we share a *single* StubClient across factory + invocations to keep call history together. + """ + stub = _StubClient(plan) + + def _factory(): + return stub + + return _factory, stub + + +def _bind(ttm: TtmIngress) -> None: + ttm.bind_run(RUN_ID, PRINCIPAL_TOKEN, BASE_URL, runtime_id=RUNTIME_ID) + + +# --------------------------------------------------------------------------- +# bind_run / unbind_run +# --------------------------------------------------------------------------- + + +class TestBinding: + def test_bind_run_requires_run_id(self): + ttm = TtmIngress() + with pytest.raises(ValueError, match="run_id"): + ttm.bind_run("", PRINCIPAL_TOKEN, BASE_URL) + + def test_bind_run_requires_token(self): + ttm = TtmIngress() + with pytest.raises(ValueError, match="principal_token"): + ttm.bind_run(RUN_ID, "", BASE_URL) + + def test_bind_run_requires_base_url(self): + ttm = TtmIngress() + with pytest.raises(ValueError, match="ingress_base_url"): + ttm.bind_run(RUN_ID, PRINCIPAL_TOKEN, "") + + def test_bind_run_strips_trailing_slash(self): + factory, stub = _client_factory([_StubResponse(201, {"event_id": "ev-1"})]) + ttm = TtmIngress(client_factory=factory) + ttm.bind_run(RUN_ID, PRINCIPAL_TOKEN, "http://127.0.0.1:8000/") + ttm.post_event(RUN_ID, EVENT_RUN_DISPATCHED, {}) + url, _, _ = stub.calls[0] + assert url == "http://127.0.0.1:8000/api/ingress/runtime/hermes/events" + + def test_unbind_drops_binding(self): + ttm = TtmIngress() + _bind(ttm) + assert ttm.is_bound(RUN_ID) + ttm.unbind_run(RUN_ID) + assert not ttm.is_bound(RUN_ID) + + def test_call_without_binding_raises(self): + ttm = TtmIngress() + with pytest.raises(IngressNotBoundError): + ttm.post_event(RUN_ID, EVENT_TASK_UPDATED, {}) + + def test_rebind_replaces_token(self): + factory, stub = _client_factory( + [_StubResponse(201, {"event_id": "ev-1"})] + ) + ttm = TtmIngress(client_factory=factory) + ttm.bind_run(RUN_ID, "old-token-aaaa", BASE_URL) + ttm.bind_run(RUN_ID, "new-token-bbbb", BASE_URL) + ttm.post_event(RUN_ID, EVENT_RUN_DISPATCHED, {}) + _, _, headers = stub.calls[0] + assert headers["Authorization"] == "Bearer new-token-bbbb" + + +# --------------------------------------------------------------------------- +# Token logging β€” only a presence marker, never the value or a prefix +# --------------------------------------------------------------------------- + + +class TestTokenPresenceLogging: + def test_present_when_set(self): + assert _token_present("tok_abcdef0123456789") == "set" + + def test_present_when_short(self): + assert _token_present("tiny") == "set" + + def test_unset_when_empty(self): + assert _token_present("") == "unset" + + def test_token_value_and_prefix_never_logged(self, caplog): + # Even a partial prefix would be unsafe in long-running production + # logs (post-mortem leakage). The presence marker is the only + # token-derived signal we emit. + factory, _ = _client_factory([_StubResponse(201, {"event_id": "ev-1"})]) + ttm = TtmIngress(client_factory=factory) + _bind(ttm) + with caplog.at_level(logging.DEBUG, logger="tools.ttm_ingress"): + ttm.post_event(RUN_ID, EVENT_TASK_UPDATED, {"x": 1}) + for record in caplog.records: + msg = record.getMessage() + assert PRINCIPAL_TOKEN not in msg + assert PRINCIPAL_TOKEN[:8] not in msg + assert "tok_abcd" not in msg + + +# --------------------------------------------------------------------------- +# post_event β€” wire contract +# --------------------------------------------------------------------------- + + +class TestPostEvent: + def test_success_returns_event_id(self): + factory, stub = _client_factory( + [_StubResponse(201, {"event_id": "ev-deadbeef"})] + ) + ttm = TtmIngress(client_factory=factory) + _bind(ttm) + event_id = ttm.post_event( + RUN_ID, + EVENT_TASK_UPDATED, + {"task": "writing tests"}, + summary="bumped task state", + ) + assert event_id == "ev-deadbeef" + assert len(stub.calls) == 1 + + def test_request_url_and_headers(self): + factory, stub = _client_factory( + [_StubResponse(201, {"event_id": "ev-1"})] + ) + ttm = TtmIngress(client_factory=factory) + _bind(ttm) + ttm.post_event(RUN_ID, EVENT_RUN_DISPATCHED, {"k": "v"}) + url, body, headers = stub.calls[0] + assert url == f"{BASE_URL}/api/ingress/runtime/hermes/events" + assert headers["Authorization"] == f"Bearer {PRINCIPAL_TOKEN}" + assert headers["X-Runtime-Id"] == "hermes" + assert headers["X-Run-Id"] == RUN_ID + assert headers["Content-Type"] == "application/json" + + def test_body_matches_control_plane_event_append_request(self): + factory, stub = _client_factory( + [_StubResponse(201, {"event_id": "ev-1"})] + ) + ttm = TtmIngress(client_factory=factory) + ttm.bind_run(RUN_ID, PRINCIPAL_TOKEN, BASE_URL, initial_scope_epoch=4) + ttm.post_event( + RUN_ID, + EVENT_TASK_UPDATED, + {"task_id": "T-7", "status": "in_progress"}, + summary="started T-7", + actor_id="hermes", + ) + _, body, _ = stub.calls[0] + assert body == { + "event_type": EVENT_TASK_UPDATED, + "actor_type": "runtime", + "actor_id": "hermes", + "expected_scope_epoch": 4, + "summary": "started T-7", + "payload": {"task_id": "T-7", "status": "in_progress"}, + } + + def test_human_actor_omits_actor_id(self): + factory, stub = _client_factory( + [_StubResponse(201, {"event_id": "ev-1"})] + ) + ttm = TtmIngress(client_factory=factory) + _bind(ttm) + ttm.post_event( + RUN_ID, + EVENT_APPROVAL_REQUESTED := "approval.requested", + {}, + summary="op approval", + actor_type="human", + # actor_id passed but ignored for human + actor_id="should-be-dropped", + ) + _, body, _ = stub.calls[0] + assert body["actor_type"] == "human" + assert body["actor_id"] is None + + def test_default_summary_is_humanized_event_type(self): + factory, stub = _client_factory( + [_StubResponse(201, {"event_id": "ev-1"})] + ) + ttm = TtmIngress(client_factory=factory) + _bind(ttm) + ttm.post_event(RUN_ID, "phase.entered", {}) + _, body, _ = stub.calls[0] + assert body["summary"] == "phase entered" + + def test_explicit_scope_epoch_overrides_binding(self): + factory, stub = _client_factory( + [_StubResponse(201, {"event_id": "ev-1"})] + ) + ttm = TtmIngress(client_factory=factory) + ttm.bind_run(RUN_ID, PRINCIPAL_TOKEN, BASE_URL, initial_scope_epoch=1) + ttm.post_event(RUN_ID, EVENT_TASK_UPDATED, {}, scope_epoch=9) + _, body, _ = stub.calls[0] + assert body["expected_scope_epoch"] == 9 + + def test_non_canonical_event_type_warns(self, caplog): + factory, _ = _client_factory( + [_StubResponse(201, {"event_id": "ev-1"})] + ) + ttm = TtmIngress(client_factory=factory) + _bind(ttm) + with caplog.at_level(logging.WARNING, logger="tools.ttm_ingress"): + ttm.post_event(RUN_ID, "made.up", {}) + assert any("non_canonical" in r.getMessage() for r in caplog.records) + + +# --------------------------------------------------------------------------- +# 401 β€” IngressAuthError, no retry +# --------------------------------------------------------------------------- + + +class TestAuthFailure: + def test_401_raises_immediately(self, caplog): + factory, stub = _client_factory( + [ + _StubResponse( + 401, text='{"detail":{"reason":"principal_token_invalid"}}' + ), + # Sentinel to fail the test if a retry happens. + _StubResponse(201, {"event_id": "should-not-be-reached"}), + ] + ) + ttm = TtmIngress(client_factory=factory) + _bind(ttm) + with caplog.at_level(logging.WARNING, logger="tools.ttm_ingress"): + with pytest.raises(IngressAuthError) as excinfo: + ttm.post_event(RUN_ID, EVENT_TASK_UPDATED, {}) + assert excinfo.value.run_id == RUN_ID + assert "principal_token_invalid" in excinfo.value.body + assert len(stub.calls) == 1, "401 must not retry" + assert any( + "principal_token_rejected" in r.getMessage() for r in caplog.records + ) + + +# --------------------------------------------------------------------------- +# 4xx (non-401) β€” IngressClientError, no retry +# --------------------------------------------------------------------------- + + +class TestClientError: + def test_409_raises_without_retry(self): + factory, stub = _client_factory( + [ + _StubResponse(409, text="scope_epoch changed"), + _StubResponse(201, {"event_id": "should-not-be-reached"}), + ] + ) + ttm = TtmIngress(client_factory=factory) + _bind(ttm) + with pytest.raises(IngressClientError) as excinfo: + ttm.post_event(RUN_ID, EVENT_TASK_UPDATED, {}) + assert excinfo.value.status_code == 409 + assert len(stub.calls) == 1 + + +# --------------------------------------------------------------------------- +# 5xx β€” exponential backoff retry, then raise +# --------------------------------------------------------------------------- + + +class TestServerErrorRetry: + def test_5xx_retries_then_succeeds(self): + factory, stub = _client_factory( + [ + _StubResponse(503, text="overloaded"), + _StubResponse(502, text="bad gateway"), + _StubResponse(201, {"event_id": "ev-late"}), + ] + ) + slept: list[float] = [] + ttm = TtmIngress( + client_factory=factory, max_retries=3, retry_base_delay=0.1, sleep=slept.append + ) + _bind(ttm) + event_id = ttm.post_event(RUN_ID, EVENT_TASK_UPDATED, {}) + assert event_id == "ev-late" + assert len(stub.calls) == 3 + assert slept == [0.1, 0.2] + + def test_5xx_exhausts_then_raises(self): + factory, stub = _client_factory( + [ + _StubResponse(500, text="boom-1"), + _StubResponse(500, text="boom-2"), + _StubResponse(500, text="boom-3"), + ] + ) + slept: list[float] = [] + ttm = TtmIngress( + client_factory=factory, max_retries=3, retry_base_delay=0.1, sleep=slept.append + ) + _bind(ttm) + with pytest.raises(IngressServerError) as excinfo: + ttm.post_event(RUN_ID, EVENT_TASK_UPDATED, {}) + assert excinfo.value.attempts == 3 + assert excinfo.value.last_status == 500 + assert "boom-3" in excinfo.value.last_error + assert len(stub.calls) == 3 + # Slept twice (between attempt 1β†’2 and 2β†’3); no sleep after final. + assert slept == [0.1, 0.2] + + def test_transport_error_retries_then_raises(self): + factory, stub = _client_factory( + [ + httpx.ConnectError("connection refused"), + httpx.ConnectError("connection refused"), + httpx.ConnectError("connection refused"), + ] + ) + slept: list[float] = [] + ttm = TtmIngress( + client_factory=factory, max_retries=3, retry_base_delay=0.1, sleep=slept.append + ) + _bind(ttm) + with pytest.raises(IngressServerError) as excinfo: + ttm.post_event(RUN_ID, EVENT_TASK_UPDATED, {}) + assert excinfo.value.attempts == 3 + assert "ConnectError" in excinfo.value.last_error + assert len(stub.calls) == 3 + + +# --------------------------------------------------------------------------- +# post_evidence β€” wire contract +# --------------------------------------------------------------------------- + + +class TestPostEvidence: + def test_body_matches_control_plane_evidence_append_request(self): + factory, stub = _client_factory( + [_StubResponse(201, {"evidence_id": "evid-7"})] + ) + ttm = TtmIngress(client_factory=factory) + ttm.bind_run(RUN_ID, PRINCIPAL_TOKEN, BASE_URL, initial_scope_epoch=2) + sha = "a" * 64 + evidence_id = ttm.post_evidence( + RUN_ID, + kind="test_results", + subject="pytest passed", + content_hash=sha.upper(), # exercise lowercase normalization + storage_ref="s3://artifacts/run-x/pytest.json", + source_event_id="00000000-0000-0000-0000-000000000001", + verdict="pass", + verification_status="passed", + evidence_id="evid-7", + ) + assert evidence_id == "evid-7" + url, body, _ = stub.calls[0] + assert url == f"{BASE_URL}/api/ingress/runtime/hermes/evidence" + assert body == { + "evidence_id": "evid-7", + "kind": "test_results", + "subject": "pytest passed", + "content_hash": sha, # lowercased + "storage_ref": "s3://artifacts/run-x/pytest.json", + "expected_scope_epoch": 2, + "source_event_id": "00000000-0000-0000-0000-000000000001", + "verdict": "pass", + "verification_status": "passed", + } + + def test_evidence_id_auto_generated_when_omitted(self): + factory, stub = _client_factory( + [_StubResponse(201, {"evidence_id": "ignored"})] + ) + ttm = TtmIngress(client_factory=factory) + _bind(ttm) + ttm.post_evidence( + RUN_ID, + kind="log", + subject="line", + content_hash="b" * 64, + storage_ref="file:///tmp/x", + source_event_id="00000000-0000-0000-0000-000000000002", + verdict="pass", + ) + _, body, _ = stub.calls[0] + assert body["evidence_id"] # non-empty UUID-ish string + + +# --------------------------------------------------------------------------- +# request_approval β€” wire contract +# --------------------------------------------------------------------------- + + +class TestRequestApproval: + def test_body_matches_runtime_approval_request(self): + factory, stub = _client_factory( + [_StubResponse(201, {"approval_id": "apr-1"})] + ) + ttm = TtmIngress(client_factory=factory) + ttm.bind_run(RUN_ID, PRINCIPAL_TOKEN, BASE_URL, initial_scope_epoch=3) + approval_id = ttm.request_approval( + RUN_ID, + approval_type="contract_lock", + summary="Lock contract before write phase", + notes_ref="docs/runs/x/contract.md", + payload={"phase": "write"}, + ) + assert approval_id == "apr-1" + url, body, _ = stub.calls[0] + assert url == f"{BASE_URL}/api/ingress/runtime/hermes/approvals" + assert body == { + "approval_type": "contract_lock", + "expected_scope_epoch": 3, + "summary": "Lock contract before write phase", + "notes_ref": "docs/runs/x/contract.md", + "payload": {"phase": "write"}, + } + + +# --------------------------------------------------------------------------- +# Canonical event type registry +# --------------------------------------------------------------------------- + + +class TestCanonicalEvents: + def test_canonical_set_is_complete(self): + # Per RUNTIME-ADAPTER-CONTRACT.md Β§Events. Anything not in this + # set will trigger a non_canonical WARNING but still POST. + assert CANONICAL_EVENT_TYPES == frozenset( + { + "run.dispatched", + "phase.entered", + "phase.completed", + "task.updated", + "evidence.added", + "approval.requested", + "approval.granted", + "approval.rejected", + "runtime.error", + "run.closed", + } + ) + + +# --------------------------------------------------------------------------- +# bind_run_from_env β€” spawn-shim contract +# --------------------------------------------------------------------------- + + +class TestBindFromEnv: + def test_returns_run_id_and_binds(self): + ttm = TtmIngress() + env = { + ENV_RUN_ID: "run-from-env", + ENV_PRINCIPAL_TOKEN: "tok_envenv_envenv_envenv_envenv", + ENV_INGRESS_BASE_URL: "http://127.0.0.1:8000", + } + run_id = ttm.bind_run_from_env(env) + assert run_id == "run-from-env" + assert ttm.is_bound("run-from-env") + + def test_returns_none_when_required_var_missing(self): + ttm = TtmIngress() + # Missing TTM_INGRESS_BASE_URL β€” must return None, not raise. + env = { + ENV_RUN_ID: "run-x", + ENV_PRINCIPAL_TOKEN: "tok_x", + } + assert ttm.bind_run_from_env(env) is None + assert not ttm.is_bound("run-x") + + def test_empty_env_returns_none(self): + ttm = TtmIngress() + assert ttm.bind_run_from_env({}) is None + + def test_picks_up_optional_runtime_id_and_scope_epoch(self): + factory, stub = _client_factory( + [_StubResponse(201, {"event_id": "ev-1"})] + ) + ttm = TtmIngress(client_factory=factory) + env = { + ENV_RUN_ID: "run-y", + ENV_PRINCIPAL_TOKEN: "tok_yyyy_yyyy_yyyy_yyyy", + ENV_INGRESS_BASE_URL: "http://127.0.0.1:8000", + ENV_RUNTIME_ID: "hermes-shadow", + ENV_SCOPE_EPOCH: "7", + } + ttm.bind_run_from_env(env) + ttm.post_event("run-y", EVENT_TASK_UPDATED, {}) + url, body, headers = stub.calls[0] + assert "/runtime/hermes-shadow/" in url + assert headers["X-Runtime-Id"] == "hermes-shadow" + assert body["expected_scope_epoch"] == 7 + + def test_invalid_scope_epoch_raises(self): + ttm = TtmIngress() + env = { + ENV_RUN_ID: "run-z", + ENV_PRINCIPAL_TOKEN: "tok_z", + ENV_INGRESS_BASE_URL: "http://127.0.0.1:8000", + ENV_SCOPE_EPOCH: "not-an-int", + } + with pytest.raises(ValueError, match="TTM_SCOPE_EPOCH"): + ttm.bind_run_from_env(env) + + def test_defaults_to_os_environ(self, monkeypatch): + ttm = TtmIngress() + monkeypatch.setenv(ENV_RUN_ID, "run-os") + monkeypatch.setenv(ENV_PRINCIPAL_TOKEN, "tok_from_os_environ_xxxxx") + monkeypatch.setenv(ENV_INGRESS_BASE_URL, "http://127.0.0.1:8000") + run_id = ttm.bind_run_from_env() + assert run_id == "run-os" + + +# --------------------------------------------------------------------------- +# get_run_state +# --------------------------------------------------------------------------- + + +class TestGetRunState: + _state_body = { + "run_id": RUN_ID, + "stream_id": "galactus", + "stream_version": "v1", + "scope_epoch": 3, + "status": "active", + "execution_contract": { + "required_tests": ["test_a"], + "approval_policy": ["scope"], + }, + "approvals": [ + {"approval_id": "ap-1", "approval_type": "scope", "status": "requested", "scope_epoch": 3} + ], + } + + def test_returns_state_dict_and_updates_scope_epoch(self): + factory, stub = _client_factory([_StubResponse(200, self._state_body)]) + ttm = TtmIngress(client_factory=factory) + _bind(ttm) + result = ttm.get_run_state(RUN_ID) + assert result["scope_epoch"] == 3 + assert result["status"] == "active" + # scope_epoch auto-applied to binding + assert ttm._binding(RUN_ID).scope_epoch == 3 + # Wire check: GET to the correct path with auth headers + url, _, headers = stub.calls[0] + assert f"/runs/{RUN_ID}/state" in url + assert headers["Authorization"] == f"Bearer {PRINCIPAL_TOKEN}" + assert headers["X-Run-Id"] == RUN_ID + + def test_401_raises_ingress_auth_error(self): + factory, _ = _client_factory([_StubResponse(401, text="token_revoked")]) + ttm = TtmIngress(client_factory=factory) + _bind(ttm) + with pytest.raises(IngressAuthError): + ttm.get_run_state(RUN_ID) + + def test_5xx_retries_then_raises_server_error(self): + slept = [] + factory, stub = _client_factory( + [_StubResponse(503)] * 3 + ) + ttm = TtmIngress(client_factory=factory, max_retries=3, sleep=slept.append) + _bind(ttm) + with pytest.raises(IngressServerError): + ttm.get_run_state(RUN_ID) + assert len(stub.calls) == 3 + assert len(slept) == 2 + + def test_scope_epoch_update_skipped_if_field_missing(self): + body_no_epoch = {k: v for k, v in self._state_body.items() if k != "scope_epoch"} + factory, _ = _client_factory([_StubResponse(200, body_no_epoch)]) + ttm = TtmIngress(client_factory=factory) + _bind(ttm) + ttm.get_run_state(RUN_ID) + # epoch stays at initial value (1) β€” no crash + assert ttm._binding(RUN_ID).scope_epoch == 1 + + def test_not_bound_raises(self): + ttm = TtmIngress() + with pytest.raises(IngressNotBoundError): + ttm.get_run_state("unbound-run") diff --git a/tools/delegate_tool.py b/tools/delegate_tool.py index 7d2bb197e0ba8..103322d4dd8df 100644 --- a/tools/delegate_tool.py +++ b/tools/delegate_tool.py @@ -928,6 +928,23 @@ def _build_child_agent( if effective_role == "orchestrator" and "delegation" not in child_toolsets: child_toolsets.append("delegation") + _requested_toolsets = list(toolsets) if toolsets else None + _requested_set = set(_requested_toolsets or (parent_enabled or sorted(parent_toolsets) or DEFAULT_TOOLSETS)) + _dropped_not_on_parent = sorted( + set(_requested_toolsets or []) - set(parent_toolsets) + ) + _blocked_for_child = sorted( + _requested_set - set(child_toolsets) - set(_dropped_not_on_parent) + ) + logger.info( + "Delegation toolset scope: parent_toolsets=%s requested_toolsets=%s child_toolsets=%s dropped_not_on_parent=%s blocked_for_child=%s", + sorted(parent_toolsets), + _requested_toolsets, + child_toolsets, + _dropped_not_on_parent, + _blocked_for_child, + ) + workspace_hint = _resolve_workspace_hint(parent_agent) child_prompt = _build_child_system_prompt( goal, diff --git a/tools/mcp_tool.py b/tools/mcp_tool.py index 2a0115ec858bf..484c9423657fe 100644 --- a/tools/mcp_tool.py +++ b/tools/mcp_tool.py @@ -251,6 +251,8 @@ def _check_message_handler_support() -> bool: # Environment variables that are safe to pass to stdio subprocesses _SAFE_ENV_KEYS = frozenset({ "PATH", "HOME", "USER", "LANG", "LC_ALL", "TERM", "SHELL", "TMPDIR", + "HTTP_PROXY", "HTTPS_PROXY", "ALL_PROXY", "NO_PROXY", + "http_proxy", "https_proxy", "all_proxy", "no_proxy", }) # Regex for credential patterns to strip from error messages @@ -268,11 +270,33 @@ def _check_message_handler_support() -> bool: re.IGNORECASE, ) +# Regex for shell-style env var placeholders like $VAR or ${VAR} +_ENV_PLACEHOLDER_PATTERN = re.compile( + r"\$(?:\{(?P[A-Za-z_][A-Za-z0-9_]*)\}|(?P[A-Za-z_][A-Za-z0-9_]*))" +) + # --------------------------------------------------------------------------- # Security helpers # --------------------------------------------------------------------------- +def _expand_env_placeholders(value: Any) -> Any: + """Expand shell-style env placeholders in config-provided values. + + Supports both ``$VAR`` and ``${VAR}`` forms. Unknown variables are left + unchanged so misconfiguration remains visible instead of silently becoming + an empty string. + """ + if not isinstance(value, str): + return value + + def _replace(match: re.Match[str]) -> str: + var_name = match.group("braced") or match.group("plain") + return os.environ.get(var_name, match.group(0)) + + return _ENV_PLACEHOLDER_PATTERN.sub(_replace, value) + + def _build_safe_env(user_env: Optional[dict]) -> dict: """Build a filtered environment dict for stdio subprocesses. @@ -288,7 +312,7 @@ def _build_safe_env(user_env: Optional[dict]) -> dict: if key in _SAFE_ENV_KEYS or key.startswith("XDG_"): env[key] = value if user_env: - env.update(user_env) + env.update({key: _expand_env_placeholders(value) for key, value in user_env.items()}) return env diff --git a/tools/registry.py b/tools/registry.py index 342078191a07a..06e19e1cff0e0 100644 --- a/tools/registry.py +++ b/tools/registry.py @@ -25,6 +25,11 @@ logger = logging.getLogger(__name__) +# Some tools should remain advertised even when their backend-health check is +# currently failing. This lets the agent surface a runtime health error instead +# of silently losing the tool from the published schema. +_ALWAYS_ADVERTISED_TOOL_NAMES = frozenset({"terminal"}) + def _is_registry_register_call(node: ast.AST) -> bool: """Return True when *node* is a ``registry.register(...)`` call expression.""" @@ -328,7 +333,7 @@ def get_definitions(self, tool_names: Set[str], quiet: bool = False) -> List[dic entry = entries_by_name.get(name) if not entry: continue - if entry.check_fn: + if entry.check_fn and entry.name not in _ALWAYS_ADVERTISED_TOOL_NAMES: if entry.check_fn not in check_results: check_results[entry.check_fn] = _check_fn_cached(entry.check_fn) if not check_results[entry.check_fn]: diff --git a/tools/ttm_ingress.py b/tools/ttm_ingress.py new file mode 100644 index 0000000000000..b99c3042735d4 --- /dev/null +++ b/tools/ttm_ingress.py @@ -0,0 +1,808 @@ +"""TTM control-plane ingress skill (PR-F-H2 of the Hermes alignment plan). + +Hermes runtime calls this module to write canonical state back to TTM +during a control-plane run: events, evidence, and approval requests. + +Wire contract β€” ``RUNTIME-ADAPTER-CONTRACT.md Β§Principal-Scoped Ingress``: + + POST {ingress_base_url}/api/ingress/runtime/{runtime_id}/events + POST {ingress_base_url}/api/ingress/runtime/{runtime_id}/evidence + POST {ingress_base_url}/api/ingress/runtime/{runtime_id}/approvals + + Headers (all three): + Authorization: Bearer + X-Runtime-Id: hermes + X-Run-Id: + +The dispatch receiver (``plugins/ttm-control-plane``, PR-F-H1) injects +the per-run ``principal_token`` + ``ingress_base_url`` into this module +at session start via :func:`bind_run`. Subsequent skill calls resolve +the bound context by ``run_id`` and post to the right TTM instance. + +Per ``RUNTIME-PRINCIPAL-CONTRACT.md`` the principal token is a per-run +bearer credential. Plaintext is never logged β€” only the first 8 chars +plus an ellipsis. On 401 the call raises immediately and does not retry +(the token has been revoked). On 5xx the call retries up to three times +with exponential backoff before raising. +""" + +import logging +import os +import threading +import time +import uuid +from dataclasses import dataclass, field +from typing import Any, Callable, Mapping + +import httpx + +logger = logging.getLogger(__name__) + +# --------------------------------------------------------------------------- +# Canonical event types β€” RUNTIME-ADAPTER-CONTRACT.md Β§Events +# --------------------------------------------------------------------------- + +EVENT_RUN_DISPATCHED = "run.dispatched" +EVENT_PHASE_ENTERED = "phase.entered" +EVENT_PHASE_COMPLETED = "phase.completed" +EVENT_TASK_UPDATED = "task.updated" +EVENT_EVIDENCE_ADDED = "evidence.added" +EVENT_APPROVAL_REQUESTED = "approval.requested" +EVENT_APPROVAL_GRANTED = "approval.granted" +EVENT_APPROVAL_REJECTED = "approval.rejected" +EVENT_RUNTIME_ERROR = "runtime.error" +EVENT_RUN_CLOSED = "run.closed" + +CANONICAL_EVENT_TYPES = frozenset( + { + EVENT_RUN_DISPATCHED, + EVENT_PHASE_ENTERED, + EVENT_PHASE_COMPLETED, + EVENT_TASK_UPDATED, + EVENT_EVIDENCE_ADDED, + EVENT_APPROVAL_REQUESTED, + EVENT_APPROVAL_GRANTED, + EVENT_APPROVAL_REJECTED, + EVENT_RUNTIME_ERROR, + EVENT_RUN_CLOSED, + } +) + +DEFAULT_RUNTIME_ID = "hermes" +DEFAULT_TIMEOUT_SECONDS = 10.0 +DEFAULT_MAX_RETRIES = 3 +DEFAULT_RETRY_BASE_DELAY = 0.5 + +# --------------------------------------------------------------------------- +# Bootstrap env vars β€” the dispatch-spawn shim sets these on the agent +# process before exec, and :func:`bind_run_from_env` reads them at session +# start. Until PR-F-H1's ``_spawn_headless_session`` is finished, operators +# can also set these manually for local dev/testing of skills that depend +# on TTM ingress. +# --------------------------------------------------------------------------- + +ENV_RUN_ID = "TTM_RUN_ID" +ENV_PRINCIPAL_TOKEN = "TTM_PRINCIPAL_TOKEN" +ENV_INGRESS_BASE_URL = "TTM_INGRESS_BASE_URL" +ENV_RUNTIME_ID = "TTM_RUNTIME_ID" +ENV_SCOPE_EPOCH = "TTM_SCOPE_EPOCH" + + +# --------------------------------------------------------------------------- +# Errors +# --------------------------------------------------------------------------- + + +class IngressError(Exception): + """Base class for TTM ingress failures.""" + + +class IngressNotBoundError(IngressError): + """Raised when an ingress call is made for a run_id with no bound context.""" + + +class IngressAuthError(IngressError): + """Raised on 401 β€” the principal token was rejected. Do not retry.""" + + def __init__(self, run_id: str, body: str = "") -> None: + super().__init__(f"principal_token_rejected for run_id={run_id}") + self.run_id = run_id + self.body = body + + +class IngressClientError(IngressError): + """Raised on a non-401 4xx response β€” the request was malformed.""" + + def __init__(self, run_id: str, status_code: int, body: str = "") -> None: + super().__init__( + f"ingress request rejected status={status_code} run_id={run_id}" + ) + self.run_id = run_id + self.status_code = status_code + self.body = body + + +class IngressServerError(IngressError): + """Raised when retries are exhausted on a 5xx response or transport error.""" + + def __init__( + self, + run_id: str, + attempts: int, + last_status: int | None = None, + last_error: str = "", + ) -> None: + super().__init__( + f"ingress server error after {attempts} attempts run_id={run_id} " + f"last_status={last_status} last_error={last_error}" + ) + self.run_id = run_id + self.attempts = attempts + self.last_status = last_status + self.last_error = last_error + + +# --------------------------------------------------------------------------- +# Per-run binding registry +# --------------------------------------------------------------------------- + + +@dataclass +class _IngressBinding: + """Live ingress context for one TTM control-plane run.""" + + run_id: str + principal_token: str + ingress_base_url: str + runtime_id: str = DEFAULT_RUNTIME_ID + scope_epoch: int = 1 + bound_at: float = field(default_factory=time.time) + + +def _token_present(token: str) -> str: + """Return a fixed marker indicating whether a principal token is bound. + + Logging even a truncated prefix risks leakage in long-running + production logs (post-mortem from operator review). Emit only a + boolean signal β€” the caller already has run_id for correlation. + """ + return "set" if token else "unset" + + +def _humanize_event(event_type: str) -> str: + """Best-effort summary fallback when the caller does not provide one.""" + return event_type.replace(".", " ").replace("_", " ") + + +# --------------------------------------------------------------------------- +# TtmIngress β€” public surface +# --------------------------------------------------------------------------- + + +class TtmIngress: + """Per-process registry + HTTP wire for TTM control-plane ingress. + + Thread-safe. Bind a run with :meth:`bind_run`, then call + :meth:`post_event`, :meth:`post_evidence`, or :meth:`request_approval` + by ``run_id``. Use :meth:`unbind_run` at run close to drop the token + from process memory. + """ + + def __init__( + self, + *, + timeout_seconds: float = DEFAULT_TIMEOUT_SECONDS, + max_retries: int = DEFAULT_MAX_RETRIES, + retry_base_delay: float = DEFAULT_RETRY_BASE_DELAY, + client_factory: Callable[[], httpx.Client] | None = None, + sleep: Callable[[float], None] = time.sleep, + ) -> None: + self._lock = threading.Lock() + self._by_run: dict[str, _IngressBinding] = {} + self._timeout = timeout_seconds + self._max_retries = max(1, int(max_retries)) + self._retry_base_delay = retry_base_delay + self._client_factory = client_factory or ( + lambda: httpx.Client(timeout=timeout_seconds) + ) + self._sleep = sleep + + # -- binding management -------------------------------------------------- + + def bind_run( + self, + run_id: str, + principal_token: str, + ingress_base_url: str, + *, + runtime_id: str = DEFAULT_RUNTIME_ID, + initial_scope_epoch: int = 1, + ) -> None: + """Register the ingress context for ``run_id``. + + Called by the dispatch receiver / headless agent bootstrap once + per run. Subsequent ingress calls in the run resolve their token + and base URL from this binding. Re-binding the same run_id is a + no-op unless ``principal_token`` differs (rebinds replace the + token; this is what TTM's ``POST /control-plane/{run_id}/rebind`` + triggers after a scope expansion). + """ + if not run_id: + raise ValueError("run_id is required") + if not principal_token: + raise ValueError("principal_token is required") + if not ingress_base_url: + raise ValueError("ingress_base_url is required") + + with self._lock: + existing = self._by_run.get(run_id) + if existing is None: + self._by_run[run_id] = _IngressBinding( + run_id=run_id, + principal_token=principal_token, + ingress_base_url=ingress_base_url.rstrip("/"), + runtime_id=runtime_id, + scope_epoch=int(initial_scope_epoch), + ) + logger.debug( + "ttm_ingress.bind_run run_id=%s base=%s token=%s", + run_id, + ingress_base_url, + _token_present(principal_token), + ) + return + existing.principal_token = principal_token + existing.ingress_base_url = ingress_base_url.rstrip("/") + existing.runtime_id = runtime_id + existing.bound_at = time.time() + logger.debug( + "ttm_ingress.rebind_run run_id=%s base=%s token=%s", + run_id, + ingress_base_url, + _token_present(principal_token), + ) + + def bind_run_from_env( + self, + env: Mapping[str, str] | None = None, + ) -> str | None: + """Bind the active run from bootstrap env vars set by the spawn shim. + + Reads ``TTM_RUN_ID`` + ``TTM_PRINCIPAL_TOKEN`` + ``TTM_INGRESS_BASE_URL`` + (and optional ``TTM_RUNTIME_ID`` / ``TTM_SCOPE_EPOCH``) from ``env`` + (defaults to :mod:`os.environ`) and registers the binding. Returns + the bound ``run_id`` on success, or ``None`` if any required var is + missing β€” callers can use that as a "not running under TTM control" + signal. + + This is the contract between PR-F-H1's spawn path and the agent + process: the spawn shim writes the per-run dispatch context into + env vars, then exec's the agent; the agent's H3+ skills call this + once at session start and stop carrying the token themselves. + """ + source = env if env is not None else os.environ + run_id = source.get(ENV_RUN_ID) + token = source.get(ENV_PRINCIPAL_TOKEN) + base = source.get(ENV_INGRESS_BASE_URL) + if not (run_id and token and base): + logger.debug( + "ttm_ingress.bind_run_from_env.skipped missing=%s", + [ + name + for name, val in ( + (ENV_RUN_ID, run_id), + (ENV_PRINCIPAL_TOKEN, token), + (ENV_INGRESS_BASE_URL, base), + ) + if not val + ], + ) + return None + runtime_id = source.get(ENV_RUNTIME_ID) or DEFAULT_RUNTIME_ID + scope_raw = source.get(ENV_SCOPE_EPOCH, "1").strip() or "1" + try: + scope_epoch = int(scope_raw) + except ValueError as exc: + raise ValueError( + f"{ENV_SCOPE_EPOCH}={scope_raw!r} is not an integer" + ) from exc + self.bind_run( + run_id, + token, + base, + runtime_id=runtime_id, + initial_scope_epoch=scope_epoch, + ) + return run_id + + def unbind_run(self, run_id: str) -> None: + """Drop the bound principal_token for ``run_id`` from memory.""" + with self._lock: + self._by_run.pop(run_id, None) + logger.debug("ttm_ingress.unbind_run run_id=%s", run_id) + + def update_scope_epoch(self, run_id: str, scope_epoch: int) -> None: + """Track the current scope_epoch so callers do not have to pass it.""" + with self._lock: + binding = self._by_run.get(run_id) + if binding is None: + raise IngressNotBoundError(f"run_id={run_id} is not bound") + binding.scope_epoch = int(scope_epoch) + + def is_bound(self, run_id: str) -> bool: + with self._lock: + return run_id in self._by_run + + def _binding(self, run_id: str) -> _IngressBinding: + with self._lock: + binding = self._by_run.get(run_id) + if binding is None: + raise IngressNotBoundError( + f"run_id={run_id} is not bound β€” call bind_run() at session start" + ) + return binding + + # -- ingress operations -------------------------------------------------- + + def post_event( + self, + run_id: str, + event_type: str, + payload: dict[str, Any] | None = None, + *, + scope_epoch: int | None = None, + summary: str | None = None, + actor_type: str = "runtime", + actor_id: str | None = None, + ) -> str: + """Append a canonical event record. Returns the new ``event_id``. + + ``event_type`` SHOULD be drawn from :data:`CANONICAL_EVENT_TYPES`; + TTM accepts arbitrary strings but operator dashboards only render + the canonical set. Non-canonical types log a WARNING locally so + the caller notices in dev. + + ``actor_type`` defaults to ``"runtime"`` and ``actor_id`` defaults + to the bound runtime_id (e.g. ``"hermes"``) β€” the normal case for + agent-issued events. Set ``actor_type="human"`` (and leave + ``actor_id=None``) for operator-issued events; TTM rejects an + ``actor_id`` on human events because identity comes from auth. + """ + if event_type not in CANONICAL_EVENT_TYPES: + logger.warning( + "ttm_ingress.post_event.non_canonical event_type=%s run_id=%s " + "(allowed: %s)", + event_type, + run_id, + sorted(CANONICAL_EVENT_TYPES), + ) + + binding = self._binding(run_id) + body: dict[str, Any] = { + "event_type": event_type, + "actor_type": actor_type, + "expected_scope_epoch": ( + int(scope_epoch) if scope_epoch is not None else binding.scope_epoch + ), + "summary": summary or _humanize_event(event_type), + "payload": dict(payload or {}), + } + if actor_type == "human": + # TTM enforces actor_id is omitted for human actors. + body["actor_id"] = None + else: + body["actor_id"] = actor_id or binding.runtime_id + + response_body = self._post( + binding, + "events", + body, + op="post_event", + extra_log={"event_type": event_type}, + ) + event_id = str(response_body.get("event_id", "")) + if not event_id: + logger.warning( + "ttm_ingress.post_event.missing_event_id run_id=%s body_keys=%s", + run_id, + list(response_body.keys()), + ) + return event_id + + def post_evidence( + self, + run_id: str, + kind: str, + subject: str, + content_hash: str, + storage_ref: str, + source_event_id: str, + verdict: str, + *, + verification_status: str = "passed", + scope_epoch: int | None = None, + evidence_id: str | None = None, + ) -> str: + """Append an evidence item linked to a prior event. Returns ``evidence_id``. + + TTM requires evidence to be content-addressed (sha256 of the + artifact in ``content_hash``) and bound to a ``source_event_id`` + that already exists in the run β€” typically an event posted via + :meth:`post_event` immediately before the artifact was produced. + """ + binding = self._binding(run_id) + body: dict[str, Any] = { + "evidence_id": evidence_id or str(uuid.uuid4()), + "kind": kind, + "subject": subject, + "content_hash": content_hash.lower(), + "storage_ref": storage_ref, + "expected_scope_epoch": ( + int(scope_epoch) if scope_epoch is not None else binding.scope_epoch + ), + "source_event_id": source_event_id, + "verdict": verdict, + "verification_status": verification_status, + } + response_body = self._post( + binding, + "evidence", + body, + op="post_evidence", + extra_log={"kind": kind}, + ) + return str(response_body.get("evidence_id", body["evidence_id"])) + + def request_approval( + self, + run_id: str, + approval_type: str, + summary: str, + *, + scope_epoch: int | None = None, + notes_ref: str | None = None, + payload: dict[str, Any] | None = None, + ) -> str: + """Open an approval gate as ``status='requested'``. Returns ``approval_id``. + + ``approval_type`` is the gate name (e.g. ``"scope"``, + ``"contract_lock"``); TTM treats it as the gate identifier. + ``summary`` is the human-facing description of what the operator + is being asked to approve. + """ + binding = self._binding(run_id) + body: dict[str, Any] = { + "approval_type": approval_type, + "expected_scope_epoch": ( + int(scope_epoch) if scope_epoch is not None else binding.scope_epoch + ), + "summary": summary, + "notes_ref": notes_ref, + "payload": dict(payload or {}), + } + response_body = self._post( + binding, + "approvals", + body, + op="request_approval", + extra_log={"approval_type": approval_type}, + ) + return str(response_body.get("approval_id", "")) + + # -- ingress read -------------------------------------------------------- + + def get_run_state(self, run_id: str) -> dict[str, Any]: + """Return the current run state snapshot from TTM. + + Calls ``GET {ingress_base_url}/api/ingress/runtime/{runtime_id}/runs/{run_id}/state`` + with the same auth headers as the write routes. On success the response + ``scope_epoch`` is auto-applied to the binding so subsequent write + calls use the fresh epoch without the caller having to do it manually. + + Raises :exc:`IngressNotBoundError` if ``bind_run`` has not been called, + :exc:`IngressAuthError` on 401 (token revoked β€” do not retry), + :exc:`IngressClientError` on non-401 4xx, :exc:`IngressServerError` + after retries exhausted. + """ + binding = self._binding(run_id) + response_body = self._request( + binding, + "GET", + f"runs/{run_id}/state", + op="get_run_state", + ) + scope_epoch = response_body.get("scope_epoch") + if scope_epoch is not None: + try: + self.update_scope_epoch(run_id, int(scope_epoch)) + except (ValueError, IngressNotBoundError): + pass + return response_body + + # -- HTTP wire ----------------------------------------------------------- + + def _request( + self, + binding: _IngressBinding, + method: str, + path: str, + body: dict[str, Any] | None = None, + *, + op: str, + extra_log: dict[str, Any] | None = None, + ) -> dict[str, Any]: + """Send one ingress request with retry/backoff + structured logging.""" + url = ( + f"{binding.ingress_base_url}/api/ingress/runtime/" + f"{binding.runtime_id}/{path}" + ) + headers: dict[str, str] = { + "Authorization": f"Bearer {binding.principal_token}", + "X-Runtime-Id": binding.runtime_id, + "X-Run-Id": binding.run_id, + } + if method == "POST": + headers["Content-Type"] = "application/json" + log_ctx = { + "op": op, + "run_id": binding.run_id, + "url": url, + "token": _token_present(binding.principal_token), + **(extra_log or {}), + } + + last_status: int | None = None + last_error: str = "" + for attempt in range(1, self._max_retries + 1): + try: + with self._client_factory() as client: + if method == "GET": + response = client.get(url, headers=headers) + else: + response = client.post(url, json=body, headers=headers) + except httpx.HTTPError as exc: + last_error = repr(exc) + logger.warning( + "ttm_ingress.%s.transport_error attempt=%s/%s ctx=%s err=%s", + op, + attempt, + self._max_retries, + log_ctx, + exc, + ) + if attempt == self._max_retries: + raise IngressServerError( + binding.run_id, attempt, None, last_error + ) from exc + self._sleep(self._retry_base_delay * (2 ** (attempt - 1))) + continue + + status_code = response.status_code + last_status = status_code + + if 200 <= status_code < 300: + logger.debug( + "ttm_ingress.%s.ok status=%s ctx=%s", + op, + status_code, + log_ctx, + ) + try: + return response.json() + except ValueError: + return {} + + body_preview = response.text[:200] if response.text else "" + + if status_code == 401: + logger.warning( + "ttm_ingress.%s.principal_token_rejected status=401 ctx=%s body=%s", + op, + log_ctx, + body_preview, + ) + raise IngressAuthError(binding.run_id, body=body_preview) + + if 400 <= status_code < 500: + logger.warning( + "ttm_ingress.%s.client_error status=%s ctx=%s body=%s", + op, + status_code, + log_ctx, + body_preview, + ) + raise IngressClientError( + binding.run_id, status_code, body=body_preview + ) + + # 5xx β€” retry with exponential backoff. + last_error = body_preview + logger.warning( + "ttm_ingress.%s.server_error attempt=%s/%s status=%s ctx=%s body=%s", + op, + attempt, + self._max_retries, + status_code, + log_ctx, + body_preview, + ) + if attempt == self._max_retries: + raise IngressServerError( + binding.run_id, attempt, last_status, last_error + ) + self._sleep(self._retry_base_delay * (2 ** (attempt - 1))) + + # Loop falls through only if max_retries==0, which __init__ guards. + raise IngressServerError(binding.run_id, self._max_retries, last_status, last_error) + + def _post( + self, + binding: _IngressBinding, + path: str, + body: dict[str, Any], + *, + op: str, + extra_log: dict[str, Any] | None = None, + ) -> dict[str, Any]: + return self._request(binding, "POST", path, body, op=op, extra_log=extra_log) + + +# --------------------------------------------------------------------------- +# Module-level singleton + thin functional API +# --------------------------------------------------------------------------- + + +_default_ingress: TtmIngress | None = None +_default_lock = threading.Lock() + + +def get_default() -> TtmIngress: + """Return the per-process default :class:`TtmIngress` instance.""" + global _default_ingress + if _default_ingress is None: + with _default_lock: + if _default_ingress is None: + _default_ingress = TtmIngress() + return _default_ingress + + +def bind_run( + run_id: str, + principal_token: str, + ingress_base_url: str, + *, + runtime_id: str = DEFAULT_RUNTIME_ID, + initial_scope_epoch: int = 1, +) -> None: + """Bind the per-run ingress context on the default instance.""" + get_default().bind_run( + run_id, + principal_token, + ingress_base_url, + runtime_id=runtime_id, + initial_scope_epoch=initial_scope_epoch, + ) + + +def bind_run_from_env(env: Mapping[str, str] | None = None) -> str | None: + """Bind the default instance from bootstrap env vars (spawn-shim contract).""" + return get_default().bind_run_from_env(env) + + +def unbind_run(run_id: str) -> None: + get_default().unbind_run(run_id) + + +def update_scope_epoch(run_id: str, scope_epoch: int) -> None: + get_default().update_scope_epoch(run_id, scope_epoch) + + +def post_event( + run_id: str, + event_type: str, + payload: dict[str, Any] | None = None, + *, + scope_epoch: int | None = None, + summary: str | None = None, + actor_type: str = "runtime", + actor_id: str | None = None, +) -> str: + return get_default().post_event( + run_id, + event_type, + payload, + scope_epoch=scope_epoch, + summary=summary, + actor_type=actor_type, + actor_id=actor_id, + ) + + +def post_evidence( + run_id: str, + kind: str, + subject: str, + content_hash: str, + storage_ref: str, + source_event_id: str, + verdict: str, + *, + verification_status: str = "passed", + scope_epoch: int | None = None, + evidence_id: str | None = None, +) -> str: + return get_default().post_evidence( + run_id, + kind, + subject, + content_hash, + storage_ref, + source_event_id, + verdict, + verification_status=verification_status, + scope_epoch=scope_epoch, + evidence_id=evidence_id, + ) + + +def request_approval( + run_id: str, + approval_type: str, + summary: str, + *, + scope_epoch: int | None = None, + notes_ref: str | None = None, + payload: dict[str, Any] | None = None, +) -> str: + return get_default().request_approval( + run_id, + approval_type, + summary, + scope_epoch=scope_epoch, + notes_ref=notes_ref, + payload=payload, + ) + + +def get_run_state(run_id: str) -> dict[str, Any]: + """Return the current run state snapshot from TTM (auto-updates scope_epoch).""" + return get_default().get_run_state(run_id) + + +__all__ = [ + "CANONICAL_EVENT_TYPES", + "DEFAULT_MAX_RETRIES", + "DEFAULT_RETRY_BASE_DELAY", + "DEFAULT_RUNTIME_ID", + "DEFAULT_TIMEOUT_SECONDS", + "ENV_INGRESS_BASE_URL", + "ENV_PRINCIPAL_TOKEN", + "ENV_RUNTIME_ID", + "ENV_RUN_ID", + "ENV_SCOPE_EPOCH", + "EVENT_APPROVAL_GRANTED", + "EVENT_APPROVAL_REJECTED", + "EVENT_APPROVAL_REQUESTED", + "EVENT_EVIDENCE_ADDED", + "EVENT_PHASE_COMPLETED", + "EVENT_PHASE_ENTERED", + "EVENT_RUNTIME_ERROR", + "EVENT_RUN_CLOSED", + "EVENT_RUN_DISPATCHED", + "EVENT_TASK_UPDATED", + "IngressAuthError", + "IngressClientError", + "IngressError", + "IngressNotBoundError", + "IngressServerError", + "TtmIngress", + "bind_run", + "bind_run_from_env", + "get_default", + "get_run_state", + "post_event", + "post_evidence", + "request_approval", + "unbind_run", + "update_scope_epoch", +] diff --git a/uv.lock b/uv.lock index 93db335ce9a7c..6bb7b8835eaa6 100644 --- a/uv.lock +++ b/uv.lock @@ -9,7 +9,7 @@ resolution-markers = [ ] [options] -exclude-newer = "2026-04-17T16:49:45.944715922Z" +exclude-newer = "2026-04-23T07:05:31.443754Z" exclude-newer-span = "P7D" [[package]] @@ -156,6 +156,19 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/1a/99/84ba7273339d0f3dfa57901b846489d2e5c2cd731470167757f1935fffbd/aiohttp_retry-2.9.1-py3-none-any.whl", hash = "sha256:66d2759d1921838256a05a3f80ad7e724936f083e35be5abb5e16eed6be6dc54", size = 9981, upload-time = "2024-11-06T10:44:52.917Z" }, ] +[[package]] +name = "aiohttp-socks" +version = "0.11.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "aiohttp" }, + { name = "python-socks" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/1f/cc/e5bbd54f76bd56291522251e47267b645dac76327b2657ade9545e30522c/aiohttp_socks-0.11.0.tar.gz", hash = "sha256:0afe51638527c79077e4bd6e57052c87c4824233d6e20bb061c53766421b10f0", size = 11196, upload-time = "2025-12-09T13:35:52.564Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/bf/7d/4b633d709b8901d59444d2e512b93e72fe62d2b492a040097c3f7ba017bb/aiohttp_socks-0.11.0-py3-none-any.whl", hash = "sha256:9aacce57c931b8fbf8f6d333cf3cafe4c35b971b35430309e167a35a8aab9ec1", size = 10556, upload-time = "2025-12-09T13:35:50.18Z" }, +] + [[package]] name = "aiosignal" version = "1.4.0" @@ -1759,6 +1772,77 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/6a/09/e21df6aef1e1ffc0c816f0522ddc3f6dcded766c3261813131c78a704470/gitpython-3.1.46-py3-none-any.whl", hash = "sha256:79812ed143d9d25b6d176a10bb511de0f9c67b1fa641d82097b0ab90398a2058", size = 208620, upload-time = "2026-01-01T15:37:30.574Z" }, ] +[[package]] +name = "google-api-core" +version = "2.30.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "google-auth" }, + { name = "googleapis-common-protos" }, + { name = "proto-plus" }, + { name = "protobuf" }, + { name = "requests" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/16/ce/502a57fb0ec752026d24df1280b162294b22a0afb98a326084f9a979138b/google_api_core-2.30.3.tar.gz", hash = "sha256:e601a37f148585319b26db36e219df68c5d07b6382cff2d580e83404e44d641b", size = 177001, upload-time = "2026-04-10T00:41:28.035Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/03/15/e56f351cf6ef1cfea58e6ac226a7318ed1deb2218c4b3cc9bd9e4b786c5a/google_api_core-2.30.3-py3-none-any.whl", hash = "sha256:a85761ba72c444dad5d611c2220633480b2b6be2521eca69cca2dbb3ffd6bfe8", size = 173274, upload-time = "2026-04-09T22:57:16.198Z" }, +] + +[[package]] +name = "google-api-python-client" +version = "2.194.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "google-api-core" }, + { name = "google-auth" }, + { name = "google-auth-httplib2" }, + { name = "httplib2" }, + { name = "uritemplate" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/60/ab/e83af0eb043e4ccc49571ca7a6a49984e9d00f4e9e6e6f1238d60bc84dce/google_api_python_client-2.194.0.tar.gz", hash = "sha256:db92647bd1a90f40b79c9618461553c2b20b6a43ce7395fa6de07132dc14f023", size = 14443469, upload-time = "2026-04-08T23:07:35.757Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b0/34/5a624e49f179aa5b0cb87b2ce8093960299030ff40423bfbde09360eb908/google_api_python_client-2.194.0-py3-none-any.whl", hash = "sha256:61eaaac3b8fc8fdf11c08af87abc3d1342d1b37319cc1b57405f86ef7697e717", size = 15016514, upload-time = "2026-04-08T23:07:33.093Z" }, +] + +[[package]] +name = "google-auth" +version = "2.49.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "cryptography" }, + { name = "pyasn1-modules" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/c6/fc/e925290a1ad95c975c459e2df070fac2b90954e13a0370ac505dff78cb99/google_auth-2.49.2.tar.gz", hash = "sha256:c1ae38500e73065dcae57355adb6278cf8b5c8e391994ae9cbadbcb9631ab409", size = 333958, upload-time = "2026-04-10T00:41:21.888Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/73/76/d241a5c927433420507215df6cac1b1fa4ac0ba7a794df42a84326c68da8/google_auth-2.49.2-py3-none-any.whl", hash = "sha256:c2720924dfc82dedb962c9f52cabb2ab16714fd0a6a707e40561d217574ed6d5", size = 240638, upload-time = "2026-04-10T00:41:14.501Z" }, +] + +[[package]] +name = "google-auth-httplib2" +version = "0.3.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "google-auth" }, + { name = "httplib2" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/ed/99/107612bef8d24b298bb5a7c8466f908ecda791d43f9466f5c3978f5b24c1/google_auth_httplib2-0.3.1.tar.gz", hash = "sha256:0af542e815784cb64159b4469aa5d71dd41069ba93effa006e1916b1dcd88e55", size = 11152, upload-time = "2026-03-30T22:50:26.766Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/97/e9/93afb14d23a949acaa3f4e7cc51a0024671174e116e35f42850764b99634/google_auth_httplib2-0.3.1-py3-none-any.whl", hash = "sha256:682356a90ef4ba3d06548c37e9112eea6fc00395a11b0303a644c1a86abc275c", size = 9534, upload-time = "2026-03-30T22:49:03.384Z" }, +] + +[[package]] +name = "google-auth-oauthlib" +version = "1.3.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "google-auth" }, + { name = "requests-oauthlib" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/a6/82/62482931dcbe5266a2680d0da17096f2aab983ecb320277d9556700ce00e/google_auth_oauthlib-1.3.1.tar.gz", hash = "sha256:14c22c7b3dd3d06dbe44264144409039465effdd1eef94f7ce3710e486cc4bfa", size = 21663, upload-time = "2026-03-30T22:49:56.408Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/2a/e0/cb454a95f460903e39f101e950038ec24a072ca69d0a294a6df625cc1627/google_auth_oauthlib-1.3.1-py3-none-any.whl", hash = "sha256:1a139ef23f1318756805b0e95f655c238bffd29655329a2978218248da4ee7f8", size = 19247, upload-time = "2026-03-30T20:02:23.894Z" }, +] + [[package]] name = "googleapis-common-protos" version = "1.73.0" @@ -1874,6 +1958,7 @@ version = "0.11.0" source = { editable = "." } dependencies = [ { name = "anthropic" }, + { name = "croniter" }, { name = "edge-tts" }, { name = "exa-py" }, { name = "fal-client" }, @@ -1900,11 +1985,11 @@ acp = [ all = [ { name = "agent-client-protocol" }, { name = "aiohttp" }, + { name = "aiohttp-socks", marker = "sys_platform == 'linux'" }, { name = "aiosqlite", marker = "sys_platform == 'linux'" }, { name = "alibabacloud-dingtalk" }, { name = "asyncpg", marker = "sys_platform == 'linux'" }, { name = "boto3" }, - { name = "croniter" }, { name = "daytona" }, { name = "debugpy" }, { name = "dingtalk-stream" }, @@ -1912,6 +1997,9 @@ all = [ { name = "elevenlabs" }, { name = "fastapi" }, { name = "faster-whisper" }, + { name = "google-api-python-client" }, + { name = "google-auth-httplib2" }, + { name = "google-auth-oauthlib" }, { name = "honcho-ai" }, { name = "lark-oapi" }, { name = "markdown", marker = "sys_platform == 'linux'" }, @@ -1942,9 +2030,6 @@ bedrock = [ cli = [ { name = "simple-term-menu" }, ] -cron = [ - { name = "croniter" }, -] daytona = [ { name = "daytona" }, ] @@ -1966,6 +2051,11 @@ feishu = [ { name = "lark-oapi" }, { name = "qrcode" }, ] +google = [ + { name = "google-api-python-client" }, + { name = "google-auth-httplib2" }, + { name = "google-auth-oauthlib" }, +] homeassistant = [ { name = "aiohttp" }, ] @@ -1973,6 +2063,7 @@ honcho = [ { name = "honcho-ai" }, ] matrix = [ + { name = "aiohttp-socks" }, { name = "aiosqlite" }, { name = "asyncpg" }, { name = "markdown" }, @@ -2015,7 +2106,6 @@ sms = [ ] termux = [ { name = "agent-client-protocol" }, - { name = "croniter" }, { name = "honcho-ai" }, { name = "mcp" }, { name = "ptyprocess", marker = "sys_platform != 'win32'" }, @@ -2048,13 +2138,14 @@ requires-dist = [ { name = "aiohttp", marker = "extra == 'homeassistant'", specifier = ">=3.9.0,<4" }, { name = "aiohttp", marker = "extra == 'messaging'", specifier = ">=3.13.3,<4" }, { name = "aiohttp", marker = "extra == 'sms'", specifier = ">=3.9.0,<4" }, + { name = "aiohttp-socks", marker = "extra == 'matrix'", specifier = ">=0.10,<1" }, { name = "aiosqlite", marker = "extra == 'matrix'", specifier = ">=0.20" }, { name = "alibabacloud-dingtalk", marker = "extra == 'dingtalk'", specifier = ">=2.0.0" }, { name = "anthropic", specifier = ">=0.39.0,<1" }, { name = "asyncpg", marker = "extra == 'matrix'", specifier = ">=0.29" }, { name = "atroposlib", marker = "extra == 'rl'", git = "https://github.com/NousResearch/atropos.git?rev=c20c85256e5a45ad31edf8b7276e9c5ee1995a30" }, { name = "boto3", marker = "extra == 'bedrock'", specifier = ">=1.35.0,<2" }, - { name = "croniter", marker = "extra == 'cron'", specifier = ">=6.0.0,<7" }, + { name = "croniter", specifier = ">=6.0.0,<7" }, { name = "daytona", marker = "extra == 'daytona'", specifier = ">=0.148.0,<1" }, { name = "debugpy", marker = "extra == 'dev'", specifier = ">=1.8.0,<2" }, { name = "dingtalk-stream", marker = "extra == 'dingtalk'", specifier = ">=0.20,<1" }, @@ -2068,6 +2159,9 @@ requires-dist = [ { name = "faster-whisper", marker = "extra == 'voice'", specifier = ">=1.0.0,<2" }, { name = "fire", specifier = ">=0.7.1,<1" }, { name = "firecrawl-py", specifier = ">=4.16.0,<5" }, + { name = "google-api-python-client", marker = "extra == 'google'", specifier = ">=2.100,<3" }, + { name = "google-auth-httplib2", marker = "extra == 'google'", specifier = ">=0.2,<1" }, + { name = "google-auth-oauthlib", marker = "extra == 'google'", specifier = ">=1.0,<2" }, { name = "hermes-agent", extras = ["acp"], marker = "extra == 'all'" }, { name = "hermes-agent", extras = ["acp"], marker = "extra == 'termux'" }, { name = "hermes-agent", extras = ["bedrock"], marker = "extra == 'all'" }, @@ -2079,6 +2173,7 @@ requires-dist = [ { name = "hermes-agent", extras = ["dev"], marker = "extra == 'all'" }, { name = "hermes-agent", extras = ["dingtalk"], marker = "extra == 'all'" }, { name = "hermes-agent", extras = ["feishu"], marker = "extra == 'all'" }, + { name = "hermes-agent", extras = ["google"], marker = "extra == 'all'" }, { name = "hermes-agent", extras = ["homeassistant"], marker = "extra == 'all'" }, { name = "hermes-agent", extras = ["honcho"], marker = "extra == 'all'" }, { name = "hermes-agent", extras = ["honcho"], marker = "extra == 'termux'" }, @@ -2142,7 +2237,7 @@ requires-dist = [ { name = "wandb", marker = "extra == 'rl'", specifier = ">=0.15.0,<1" }, { name = "yc-bench", marker = "python_full_version >= '3.12' and extra == 'yc-bench'", git = "https://github.com/collinear-ai/yc-bench.git?rev=bfb0c88062450f46341bd9a5298903fc2e952a5c" }, ] -provides-extras = ["modal", "daytona", "vercel", "dev", "messaging", "cron", "slack", "matrix", "cli", "tts-premium", "voice", "pty", "honcho", "mcp", "homeassistant", "sms", "acp", "mistral", "bedrock", "termux", "dingtalk", "feishu", "web", "rl", "yc-bench", "all"] +provides-extras = ["modal", "daytona", "vercel", "dev", "messaging", "cron", "slack", "matrix", "cli", "tts-premium", "voice", "pty", "honcho", "mcp", "homeassistant", "sms", "acp", "mistral", "bedrock", "termux", "dingtalk", "feishu", "google", "web", "rl", "yc-bench", "all"] [[package]] name = "hf-transfer" @@ -2244,6 +2339,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/7e/f5/f66802a942d491edb555dd61e3a9961140fd64c90bce1eafd741609d334d/httpcore-1.0.9-py3-none-any.whl", hash = "sha256:2d400746a40668fc9dec9810239072b40b4484b640a8c38fd654a024c7a1bf55", size = 78784, upload-time = "2025-04-24T22:06:20.566Z" }, ] +[[package]] +name = "httplib2" +version = "0.31.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pyparsing" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/c1/1f/e86365613582c027dda5ddb64e1010e57a3d53e99ab8a72093fa13d565ec/httplib2-0.31.2.tar.gz", hash = "sha256:385e0869d7397484f4eab426197a4c020b606edd43372492337c0b4010ae5d24", size = 250800, upload-time = "2026-01-23T11:04:44.165Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/2f/90/fd509079dfcab01102c0fdd87f3a9506894bc70afcf9e9785ef6b2b3aff6/httplib2-0.31.2-py3-none-any.whl", hash = "sha256:dbf0c2fa3862acf3c55c078ea9c0bc4481d7dc5117cae71be9514912cf9f8349", size = 91099, upload-time = "2026-01-23T11:04:42.78Z" }, +] + [[package]] name = "httptools" version = "0.7.1" @@ -3283,6 +3390,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/57/a7/b35835e278c18b85206834b3aa3abe68e77a98769c59233d1f6300284781/numpy-2.4.3-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:4b42639cdde6d24e732ff823a3fa5b701d8acad89c4142bc1d0bd6dc85200ba5", size = 12504685, upload-time = "2026-03-09T07:58:50.525Z" }, ] +[[package]] +name = "oauthlib" +version = "3.3.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/0b/5f/19930f824ffeb0ad4372da4812c50edbd1434f678c90c2733e1188edfc63/oauthlib-3.3.1.tar.gz", hash = "sha256:0f0f8aa759826a193cf66c12ea1af1637f87b9b4622d46e866952bb022e538c9", size = 185918, upload-time = "2025-06-19T22:48:08.269Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/be/9c/92789c596b8df838baa98fa71844d84283302f7604ed565dafe5a6b5041a/oauthlib-3.3.1-py3-none-any.whl", hash = "sha256:88119c938d2b8fb88561af5f6ee0eec8cc8d552b7bb1f712743136eb7523b7a1", size = 160065, upload-time = "2025-06-19T22:48:06.508Z" }, +] + [[package]] name = "obstore" version = "0.8.2" @@ -3861,6 +3977,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/5b/5a/bc7b4a4ef808fa59a816c17b20c4bef6884daebbdf627ff2a161da67da19/propcache-0.4.1-py3-none-any.whl", hash = "sha256:af2a6052aeb6cf17d3e46ee169099044fd8224cbaf75c76a2ef596e8163e2237", size = 13305, upload-time = "2025-10-08T19:49:00.792Z" }, ] +[[package]] +name = "proto-plus" +version = "1.27.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "protobuf" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/81/0d/94dfe80193e79d55258345901acd2917523d56e8381bc4dee7fd38e3868a/proto_plus-1.27.2.tar.gz", hash = "sha256:b2adde53adadf75737c44d3dcb0104fde65250dfc83ad59168b4aa3e574b6a24", size = 57204, upload-time = "2026-03-26T22:18:57.174Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/84/f3/1fba73eeffafc998a25d59703b63f8be4fe8a5cb12eaff7386a0ba0f7125/proto_plus-1.27.2-py3-none-any.whl", hash = "sha256:6432f75893d3b9e70b9c412f1d2f03f65b11fb164b793d14ae2ca01821d22718", size = 50450, upload-time = "2026-03-26T22:13:42.927Z" }, +] + [[package]] name = "protobuf" version = "6.33.5" @@ -3935,6 +4063,27 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/50/f2/c0e76a0b451ffdf0cf788932e182758eb7558953f4f27f1aff8e2518b653/pyarrow-23.0.1-cp314-cp314t-win_amd64.whl", hash = "sha256:527e8d899f14bd15b740cd5a54ad56b7f98044955373a17179d5956ddb93d9ce", size = 28365807, upload-time = "2026-02-16T10:14:03.892Z" }, ] +[[package]] +name = "pyasn1" +version = "0.6.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/5c/5f/6583902b6f79b399c9c40674ac384fd9cd77805f9e6205075f828ef11fb2/pyasn1-0.6.3.tar.gz", hash = "sha256:697a8ecd6d98891189184ca1fa05d1bb00e2f84b5977c481452050549c8a72cf", size = 148685, upload-time = "2026-03-17T01:06:53.382Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/5d/a0/7d793dce3fa811fe047d6ae2431c672364b462850c6235ae306c0efd025f/pyasn1-0.6.3-py3-none-any.whl", hash = "sha256:a80184d120f0864a52a073acc6fc642847d0be408e7c7252f31390c0f4eadcde", size = 83997, upload-time = "2026-03-17T01:06:52.036Z" }, +] + +[[package]] +name = "pyasn1-modules" +version = "0.4.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pyasn1" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/e9/e6/78ebbb10a8c8e4b61a59249394a4a594c1a7af95593dc933a349c8d00964/pyasn1_modules-0.4.2.tar.gz", hash = "sha256:677091de870a80aae844b1ca6134f54652fa2c8c5a52aa396440ac3106e941e6", size = 307892, upload-time = "2025-03-28T02:41:22.17Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/47/8d/d529b5d697919ba8c11ad626e835d4039be708a35b0d22de83a269a6682c/pyasn1_modules-0.4.2-py3-none-any.whl", hash = "sha256:29253a9207ce32b64c3ac6600edc75368f98473906e8fd1043bd6b5b1de2c14a", size = 181259, upload-time = "2025-03-28T02:41:19.028Z" }, +] + [[package]] name = "pycparser" version = "3.0" @@ -4275,6 +4424,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/79/93/f6729f10149305262194774d6c8b438c0b084740cf239f48ab97b4df02fa/python_olm-3.2.16-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:10a5e68a2f4b5a2bfa5fdb5dbfa22396a551730df6c4a572235acaa96e997d3f", size = 297000, upload-time = "2023-11-28T19:25:31.045Z" }, ] +[[package]] +name = "python-socks" +version = "2.8.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/36/0b/cd77011c1bc01b76404f7aba07fca18aca02a19c7626e329b40201217624/python_socks-2.8.1.tar.gz", hash = "sha256:698daa9616d46dddaffe65b87db222f2902177a2d2b2c0b9a9361df607ab3687", size = 38909, upload-time = "2026-02-16T05:24:00.745Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/15/fe/9a58cb6eec633ff6afae150ca53c16f8cc8b65862ccb3d088051efdfceb7/python_socks-2.8.1-py3-none-any.whl", hash = "sha256:28232739c4988064e725cdbcd15be194743dd23f1c910f784163365b9d7be035", size = 55087, upload-time = "2026-02-16T05:23:59.147Z" }, +] + [[package]] name = "python-telegram-bot" version = "22.6" @@ -4535,6 +4693,19 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/56/5d/c814546c2333ceea4ba42262d8c4d55763003e767fa169adc693bd524478/requests-2.33.0-py3-none-any.whl", hash = "sha256:3324635456fa185245e24865e810cecec7b4caf933d7eb133dcde67d48cee69b", size = 65017, upload-time = "2026-03-25T15:10:40.382Z" }, ] +[[package]] +name = "requests-oauthlib" +version = "2.0.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "oauthlib" }, + { name = "requests" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/42/f2/05f29bc3913aea15eb670be136045bf5c5bbf4b99ecb839da9b422bb2c85/requests-oauthlib-2.0.0.tar.gz", hash = "sha256:b3dffaebd884d8cd778494369603a9e7b58d29111bf6b41bdc2dcd87203af4e9", size = 55650, upload-time = "2024-03-22T20:32:29.939Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/3b/5d/63d4ae3b9daea098d5d6f5da83984853c1bbacd5dc826764b249fe119d24/requests_oauthlib-2.0.0-py2.py3-none-any.whl", hash = "sha256:7dd8a5c40426b779b0868c404bdef9768deccf22749cde15852df527e6269b36", size = 24179, upload-time = "2024-03-22T20:32:28.055Z" }, +] + [[package]] name = "requests-toolbelt" version = "1.0.0" @@ -5274,6 +5445,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/4c/a7/563b2d8fb7edc07320bf69ac6a7eedcd7a1a9d663a6bb90a4d9bd2eda5f7/unpaddedbase64-2.1.0-py3-none-any.whl", hash = "sha256:485eff129c30175d2cd6f0cd8d2310dff51e666f7f36175f738d75dfdbd0b1c6", size = 6083, upload-time = "2021-03-09T11:35:46.7Z" }, ] +[[package]] +name = "uritemplate" +version = "4.2.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/98/60/f174043244c5306c9988380d2cb10009f91563fc4b31293d27e17201af56/uritemplate-4.2.0.tar.gz", hash = "sha256:480c2ed180878955863323eea31b0ede668795de182617fef9c6ca09e6ec9d0e", size = 33267, upload-time = "2025-06-02T15:12:06.318Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a9/99/3ae339466c9183ea5b8ae87b34c0b897eda475d2aec2307cae60e5cd4f29/uritemplate-4.2.0-py3-none-any.whl", hash = "sha256:962201ba1c4edcab02e60f9a0d3821e82dfc5d2d6662a21abd533879bdb8a686", size = 11488, upload-time = "2025-06-02T15:12:03.405Z" }, +] + [[package]] name = "urllib3" version = "2.6.3" diff --git a/website/docs/reference/cli-commands.md b/website/docs/reference/cli-commands.md index 933cb64732f61..a7c06c2477750 100644 --- a/website/docs/reference/cli-commands.md +++ b/website/docs/reference/cli-commands.md @@ -203,11 +203,11 @@ Subcommands: | Subcommand | Description | |------------|-------------| | `run` | Run the gateway in the foreground. Recommended for WSL, Docker, and Termux. | -| `start` | Start the installed systemd/launchd background service. | +| `start` | Start the installed systemd service (Linux) or launchd user agent (macOS). | | `stop` | Stop the service (or foreground process). | | `restart` | Restart the service. | | `status` | Show service status. | -| `install` | Install as a systemd (Linux) or launchd (macOS) background service. | +| `install` | Install as a systemd service (Linux) or launchd user agent (macOS). | | `uninstall` | Remove the installed service. | | `setup` | Interactive messaging-platform setup. | @@ -221,6 +221,10 @@ Options: Use `hermes gateway run` instead of `hermes gateway start` β€” WSL's systemd support is unreliable. Wrap it in tmux for persistence: `tmux new -s hermes 'hermes gateway run'`. See [WSL FAQ](/docs/reference/faq#wsl-gateway-keeps-disconnecting-or-hermes-gateway-start-fails) for details. ::: +:::tip macOS headless deployments +`hermes gateway install` creates a per-user LaunchAgent, not a system LaunchDaemon. Use it when Hermes runs under the logged-in desktop account. For SSH-only or service-account deployments, prefer `tmux`/`screen` or a manually managed LaunchDaemon. +::: + ## `hermes setup` ```bash diff --git a/website/docs/reference/faq.md b/website/docs/reference/faq.md index f4a37dd697e14..9bc422ba51958 100644 --- a/website/docs/reference/faq.md +++ b/website/docs/reference/faq.md @@ -481,6 +481,24 @@ You can verify the plist has the correct PATH: ~/Library/LaunchAgents/ai.hermes.gateway.plist ``` +#### macOS: `hermes gateway start` fails or the gateway never stays up on a headless/service account + +**Cause:** Hermes installs a per-user LaunchAgent in `~/Library/LaunchAgents`. LaunchAgents are tied to the logged-in macOS desktop account, so they are a poor fit when Hermes runs under a separate SSH-only or background service user. + +**Solution:** Check who owns the active desktop session: + +```bash +stat -f '%Su' /dev/console +whoami +``` + +If those users differ, do one of these instead: + +- Keep Hermes in the foreground under `tmux` or `screen`: `tmux new -s hermes 'hermes gateway run'` +- Install a system LaunchDaemon that runs as your Hermes service account + +`hermes doctor` will warn about this mismatch when a launchd plist is installed. + --- ### Performance Issues diff --git a/website/docs/user-guide/messaging/index.md b/website/docs/user-guide/messaging/index.md index 126ab8184f679..39642cefd4e34 100644 --- a/website/docs/user-guide/messaging/index.md +++ b/website/docs/user-guide/messaging/index.md @@ -351,6 +351,12 @@ hermes gateway status # Check status tail -f ~/.hermes/logs/gateway.log # View logs ``` +:::warning LaunchAgents are per-user login services +`hermes gateway install` creates a plist under `~/Library/LaunchAgents`, not a system-wide LaunchDaemon. This is the right fit when Hermes runs under the same macOS account that is logged into the desktop session. + +If Hermes runs under a separate SSH-only or service account, the LaunchAgent may never attach to the right launchd domain or survive login boundaries. In that topology, keep `hermes gateway run` under `tmux`/`screen`, or install a system LaunchDaemon that runs as that service user. +::: + The generated plist lives at `~/Library/LaunchAgents/ai.hermes.gateway.plist`. It includes three environment variables: - **PATH** β€” your full shell PATH at install time, with the venv `bin/` and `node_modules/.bin` prepended. This ensures user-installed tools (Node.js, ffmpeg, etc.) are available to gateway subprocesses like the WhatsApp bridge.