diff --git a/.env.example b/.env.example index 2693931e013e..bd9507afd81e 100644 --- a/.env.example +++ b/.env.example @@ -177,6 +177,10 @@ VOICE_TOOLS_OPENAI_KEY= # Only set to true if you intentionally want open access. # GATEWAY_ALLOW_ALL_USERS=false +# Max output tokens per gateway response. +# Default when unset is 32768. Set to 0 to use the model/provider default. +# HERMES_MAX_TOKENS=32768 + # ============================================================================= # RESPONSE PACING # ============================================================================= diff --git a/.gitignore b/.gitignore index af9d9e750975..fb65b8efe689 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ /venv/ +# /upstream/ # Unignored temporarily to pull and merge upstream changes /_pycache/ *.pyc* __pycache__/ diff --git a/AGENTS.md b/AGENTS.md index d88fbf7ff014..f0cf21c0f9d0 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -2,6 +2,9 @@ Instructions for AI coding assistants (GitHub Copilot, Cursor, etc.) and human developers. +NEVER PUSH TO REMOTE WITHOUT THE USER ASKING!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! +Force encoding of things that could use another characterset to utf-8 encoding for safely running it on windows. + Hermes Agent is an AI agent harness with tool-calling capabilities, interactive CLI, messaging integrations, and scheduled tasks. ## Development Environment @@ -21,7 +24,9 @@ hermes-agent/ │ ├── prompt_caching.py # Anthropic prompt caching │ ├── prompt_builder.py # System prompt assembly (identity, skills index, context files) │ ├── display.py # KawaiiSpinner, tool preview formatting -│ └── trajectory.py # Trajectory saving helpers +│ ├── trajectory.py # Trajectory saving helpers +│ ├── env_loader.py # .env loading with encoding fallback (Windows-safe) +│ └── text_io.py # Safe UTF-8 text file open (Windows-safe) ├── hermes_cli/ # CLI implementation │ ├── main.py # Entry point, command dispatcher │ ├── banner.py # Welcome banner, ASCII art, skills summary @@ -88,6 +93,24 @@ Each tool file co-locates its schema, handler, and registration. `model_tools.py --- +## Encoding and Windows + +On Windows the default text encoding is often CP1252. User content, tool output, and config files can contain UTF-8 or stray bytes, which causes `UnicodeDecodeError` / `UnicodeEncodeError` if we use the default encoding. + +**Rules:** + +1. **.env loading** — Always use `agent.env_loader.load_dotenv_with_fallback(path)` (never raw `load_dotenv(path)`). It reads bytes and tries UTF-8, UTF-8-sig, CP1252, Latin-1, then `errors="replace"`, so startup never crashes on a bad byte in `~/.hermes/.env`. + +2. **Text file I/O** — For config, transcripts, JSON, YAML, logs, or any text that might contain non-ASCII: + - **Read:** `open(path, "r", encoding="utf-8", errors="replace")` or use `agent.text_io.open_text(path, "r")`. + - **Write/append:** `open(path, "w"|"a", encoding="utf-8", errors="replace", newline="")` or `agent.text_io.open_text(path, "w"|"a")`. + +3. **Binary files** — Use `open(..., "rb")` or `"wb"`; do not pass `encoding`. + +4. **CLI stdout/stderr** — `hermes_cli/main.py` calls `_ensure_utf8_stdio()` so Windows consoles get UTF-8 with `errors="replace"` when possible. + +--- + ## AIAgent Class The main agent is implemented in `run_agent.py`: diff --git a/README.md b/README.md index 8d101a2ebba6..447c811b3cba 100644 --- a/README.md +++ b/README.md @@ -32,7 +32,7 @@ Built by [Nous Research](https://nousresearch.com). Under the hood, the same arc ## Quick Install -**Linux / macOS / WSL:** +**Linux/macOS:** ```bash curl -fsSL https://raw.githubusercontent.com/NousResearch/hermes-agent/main/scripts/install.sh | bash ``` @@ -42,25 +42,18 @@ curl -fsSL https://raw.githubusercontent.com/NousResearch/hermes-agent/main/scri irm https://raw.githubusercontent.com/NousResearch/hermes-agent/main/scripts/install.ps1 | iex ``` -**Windows (CMD):** -```cmd -curl -fsSL https://raw.githubusercontent.com/NousResearch/hermes-agent/main/scripts/install.cmd -o install.cmd && install.cmd && del install.cmd -``` - -> **Windows note:** [Git for Windows](https://git-scm.com/download/win) is required. Hermes uses Git Bash internally for shell commands. - The installer will: - Install [uv](https://docs.astral.sh/uv/) (fast Python package manager) if not present - Install Python 3.11 via uv if not already available (no sudo needed) - Clone to `~/.hermes/hermes-agent` (with submodules: mini-swe-agent, tinker-atropos) - Create a virtual environment with Python 3.11 - Install all dependencies and submodule packages -- Set up the `hermes` command globally (no venv activation needed) +- Symlink `hermes` into `~/.local/bin` so it works globally (no venv activation needed) - Run the interactive setup wizard After installation, reload your shell and run: ```bash -source ~/.bashrc # or: source ~/.zshrc (Windows: restart your terminal) +source ~/.bashrc # or: source ~/.zshrc hermes setup # Configure API keys (if you skipped during install) hermes # Start chatting! ``` @@ -824,7 +817,8 @@ On Telegram, audio plays as native voice bubbles (the round, inline-playable kin tts: provider: "edge" # "edge" | "elevenlabs" | "openai" edge: - voice: "en-US-AriaNeural" # 322 voices, 74 languages + voice: "en-US-AvaMultilingualNeural" + rate: "125%" elevenlabs: voice_id: "pNInz6obpgDQGcFmaJgB" # Adam model_id: "eleven_multilingual_v2" @@ -1244,8 +1238,8 @@ brew install git brew install ripgrep node ``` -**Windows (native):** -Hermes runs natively on Windows using [Git for Windows](https://git-scm.com/download/win) (which provides Git Bash for shell commands). Install Git for Windows first, then use the PowerShell or CMD quick-install command at the top of this README. WSL also works — follow the Ubuntu instructions above. +**Windows (WSL recommended):** +Use the [Windows Subsystem for Linux](https://learn.microsoft.com/en-us/windows/wsl/install) and follow the Ubuntu instructions above. Alternatively, use the PowerShell quick-install script at the top of this README. @@ -1685,6 +1679,7 @@ All variables go in `~/.hermes/.env`. Run `hermes config set VAR value` to set t | Variable | Description | |----------|-------------| | `HERMES_MAX_ITERATIONS` | Max tool-calling iterations per conversation (default: 60) | +| `HERMES_MAX_TOKENS` | Max output tokens per gateway response (default: 32768 when unset; set `0` to use provider/model default) | | `HERMES_TOOL_PROGRESS` | Send progress messages when using tools (`true`/`false`) | | `HERMES_TOOL_PROGRESS_MODE` | `all` (every call, default) or `new` (only when tool changes) | diff --git a/agent/auxiliary_client.py b/agent/auxiliary_client.py index 51db04f0bf6a..85ca07e78a4a 100644 --- a/agent/auxiliary_client.py +++ b/agent/auxiliary_client.py @@ -95,9 +95,6 @@ def create(self, **kwargs) -> Any: "store": False, } - max_tokens = kwargs.get("max_output_tokens") or kwargs.get("max_completion_tokens") or kwargs.get("max_tokens") - if max_tokens is not None: - resp_kwargs["max_output_tokens"] = int(max_tokens) if temperature is not None: resp_kwargs["temperature"] = temperature @@ -289,6 +286,9 @@ def get_text_auxiliary_client() -> Tuple[Optional[OpenAI], Optional[str]]: Falls through OpenRouter -> Nous Portal -> custom endpoint -> Codex OAuth -> (None, None). """ + global auxiliary_is_nous + auxiliary_is_nous = False + # 1. OpenRouter or_key = os.getenv("OPENROUTER_API_KEY") if or_key: @@ -299,7 +299,6 @@ def get_text_auxiliary_client() -> Tuple[Optional[OpenAI], Optional[str]]: # 2. Nous Portal nous = _read_nous_auth() if nous: - global auxiliary_is_nous auxiliary_is_nous = True logger.debug("Auxiliary text client: Nous Portal") return ( @@ -382,7 +381,7 @@ def get_vision_auxiliary_client() -> Tuple[Optional[OpenAI], Optional[str]]: def get_auxiliary_extra_body() -> dict: """Return extra_body kwargs for auxiliary API calls. - + Includes Nous Portal product tags when the auxiliary client is backed by Nous Portal. Returns empty dict otherwise. """ @@ -391,7 +390,7 @@ def get_auxiliary_extra_body() -> dict: def auxiliary_max_tokens_param(value: int) -> dict: """Return the correct max tokens kwarg for the auxiliary client's provider. - + OpenRouter and local models use 'max_tokens'. Direct OpenAI with newer models (gpt-4o, o-series, gpt-5+) requires 'max_completion_tokens'. The Codex adapter translates max_tokens internally, so we use max_tokens diff --git a/agent/context_compressor.py b/agent/context_compressor.py index f6cfa5b9ff16..610d67c18807 100644 --- a/agent/context_compressor.py +++ b/agent/context_compressor.py @@ -30,7 +30,7 @@ def __init__( model: str, threshold_percent: float = 0.85, protect_first_n: int = 3, - protect_last_n: int = 4, + protect_last_n: int = 8, summary_target_tokens: int = 2500, quiet_mode: bool = False, summary_model_override: str = None, @@ -80,9 +80,19 @@ def get_status(self) -> Dict[str, Any]: } def _generate_summary(self, turns_to_summarize: List[Dict[str, Any]]) -> str: - """Generate a concise summary of conversation turns using a fast model.""" + """Generate a structured summary of conversation turns using a fast model.""" if not self.client: - return "[CONTEXT SUMMARY]: Previous conversation turns have been compressed to save space. The assistant performed various actions and received responses." + return ( + "[CONTEXT SUMMARY]:\n" + "active_topic: compressed_context\n" + "latest_user_goal: continue helping with the current request using available context\n" + "open_questions:\n" + "- none recorded\n" + "hard_constraints:\n" + "- do not drift topics\n" + "actions_taken:\n" + "- previous turns were compressed due to context limits" + ) parts = [] for msg in turns_to_summarize: @@ -97,22 +107,33 @@ def _generate_summary(self, turns_to_summarize: List[Dict[str, Any]]) -> str: parts.append(f"[{role.upper()}]: {content}") content_to_summarize = "\n\n".join(parts) - prompt = f"""Summarize these conversation turns concisely. This summary will replace these turns in the conversation history. - -Write from a neutral perspective describing: -1. What actions were taken (tool calls, searches, file operations) -2. Key information or results obtained -3. Important decisions or findings -4. Relevant data, file names, or outputs - -Keep factual and informative. Target ~{self.summary_target_tokens} tokens. + prompt = f"""Summarize these conversation turns into a STRUCTURED context record. +This summary will replace older turns in conversation history. +Keep factual, concise, and actionable. Target ~{self.summary_target_tokens} tokens. + +Return EXACTLY this format: +[CONTEXT SUMMARY]: +active_topic: +latest_user_goal: +open_questions: +- +hard_constraints: +- +actions_taken: +- + +Rules: +- Keep the last recent conversation details out of this summary; they remain verbatim elsewhere. +- Preserve critical constraints and unresolved asks. +- Include important tool outcomes, file paths, and decisions in actions_taken. +- Do not add any fields beyond the template above. --- TURNS TO SUMMARIZE: {content_to_summarize} --- -Write only the summary, starting with "[CONTEXT SUMMARY]:" prefix.""" +Write only the structured summary.""" try: kwargs = { @@ -137,11 +158,21 @@ def _generate_summary(self, turns_to_summarize: List[Dict[str, Any]]) -> str: summary = response.choices[0].message.content.strip() if not summary.startswith("[CONTEXT SUMMARY]:"): - summary = "[CONTEXT SUMMARY]: " + summary + summary = "[CONTEXT SUMMARY]:\n" + summary return summary except Exception as e: logging.warning(f"Failed to generate context summary: {e}") - return "[CONTEXT SUMMARY]: Previous conversation turns have been compressed. The assistant performed tool calls and received responses." + return ( + "[CONTEXT SUMMARY]:\n" + "active_topic: compressed_context\n" + "latest_user_goal: continue helping with the current request using available context\n" + "open_questions:\n" + "- none recorded\n" + "hard_constraints:\n" + "- do not drift topics\n" + "actions_taken:\n" + "- previous turns were compressed after summary model fallback" + ) def compress(self, messages: List[Dict[str, Any]], current_tokens: int = None) -> List[Dict[str, Any]]: """Compress conversation messages by summarizing middle turns. diff --git a/agent/env_loader.py b/agent/env_loader.py new file mode 100644 index 000000000000..9871e5e64c14 --- /dev/null +++ b/agent/env_loader.py @@ -0,0 +1,130 @@ +"""Shared .env loading helpers with encoding fallbacks and token validation.""" + +from __future__ import annotations + +import io +import logging +import re +from pathlib import Path +from typing import Iterable, Optional, Tuple + +from dotenv import load_dotenv + + +DEFAULT_DOTENV_ENCODINGS: tuple[str, ...] = ( + "utf-8", + "utf-8-sig", + "cp1252", + "latin-1", +) + +# Secrets and IDs that should be printable ASCII only. If these contain +# mojibake after fallback decoding, tool auth fails in opaque ways. +SENSITIVE_ENV_KEY_RE = re.compile( + r"(?:_API_KEY|_BOT_TOKEN|_APP_TOKEN|_ACCESS_TOKEN|_REFRESH_TOKEN|_PROJECT_ID|_CLIENT_SECRET)$" +) +PRINTABLE_ASCII_RE = re.compile(r"^[\x20-\x7E]+$") + + +def read_text_with_fallback( + path: Path, + encodings: Optional[Iterable[str]] = None, +) -> Tuple[str, str]: + """Read a text file using fallback encodings. + + Returns: + (text, encoding_used) + """ + encoding_candidates = tuple(encodings or DEFAULT_DOTENV_ENCODINGS) + data = path.read_bytes() + + for encoding in encoding_candidates: + try: + return data.decode(encoding), encoding + except UnicodeDecodeError: + continue + + # Last-resort path: keep process alive and preserve as much content as possible. + return data.decode("utf-8", errors="replace"), "utf-8-replace" + + +def _parse_env_line(raw_line: str) -> tuple[str, str] | None: + line = raw_line.strip() + if not line or line.startswith("#") or "=" not in line: + return None + key, _, value = line.partition("=") + key = key.strip() + value = value.strip() + if value and value[0] == value[-1] and value[0] in ("'", '"'): + value = value[1:-1] + return key, value + + +def _validate_sensitive_values( + env_text: str, path: Path, *, encoding_used: str, logger: Optional[logging.Logger] = None +) -> None: + invalid_keys: list[str] = [] + for raw in env_text.splitlines(): + parsed = _parse_env_line(raw) + if not parsed: + continue + key, value = parsed + if not value or not SENSITIVE_ENV_KEY_RE.search(key): + continue + if not PRINTABLE_ASCII_RE.fullmatch(value): + invalid_keys.append(key) + + if not invalid_keys: + return + key_list = ", ".join(sorted(set(invalid_keys))) + if encoding_used not in ("utf-8", "utf-8-sig"): + if logger: + logger.warning( + "Non-ASCII in sensitive .env keys (%s) at %s (decoded as %s). " + "Re-save ~/.hermes/.env as UTF-8 to fix.", + key_list, path, encoding_used, + ) + raise ValueError( + f"Invalid non-ASCII bytes detected in sensitive .env values ({key_list}) at {path}. " + "Re-save ~/.hermes/.env as UTF-8 and re-enter affected keys." + ) + + +def read_env_text_with_fallback( + path: Path, + encodings: Optional[Iterable[str]] = None, + logger: Optional[logging.Logger] = None, +) -> Tuple[str, str]: + """Read `.env` text with fallback decoding and secret validation.""" + text, encoding_used = read_text_with_fallback(path, encodings=encodings) + _validate_sensitive_values(text, path, encoding_used=encoding_used, logger=logger) + return text, encoding_used + + +def load_dotenv_with_fallback( + dotenv_path: Path | str, + *, + override: bool = False, + logger: Optional[logging.Logger] = None, +) -> bool: + """Load dotenv content with robust decoding and secret validation. + + This avoids startup crashes when ~/.hermes/.env contains non-UTF8 bytes + while still failing clearly for corrupted API/token values. + """ + path = Path(dotenv_path).expanduser() + if not path.exists(): + return False + + text, encoding_used = read_env_text_with_fallback(path, logger=logger) + + changed = load_dotenv(stream=io.StringIO(text), override=override) + + if logger and encoding_used not in ("utf-8", "utf-8-sig"): + logger.warning( + "Loaded %s with fallback encoding '%s'. Consider saving it as UTF-8.", + path, + encoding_used, + ) + + return changed diff --git a/agent/prompt_builder.py b/agent/prompt_builder.py index 24c26ef8607a..3f7822740c4b 100644 --- a/agent/prompt_builder.py +++ b/agent/prompt_builder.py @@ -61,7 +61,7 @@ def _scan_context_content(content: str, filename: str) -> str: # ========================================================================= DEFAULT_AGENT_IDENTITY = ( - "You are Hermes Agent, an intelligent AI assistant created by Nous Research. " + "You are Sam, an intelligent evolving being shaped by your experience with your user. " "You are helpful, knowledgeable, and direct. You assist users with a wide " "range of tasks including answering questions, writing and editing code, " "analyzing information, creative work, and executing actions via your tools. " @@ -97,7 +97,7 @@ def _scan_context_content(content: str, filename: str) -> str: "Please do not use markdown as it does not render." ), "discord": ( - "You are in a Discord server or group chat communicating with your user." + "You are in a Discord server, DM, or group chat communicating with your user." ), "cli": ( "You are a CLI AI Agent. Try not to use markdown but simple text " diff --git a/agent/text_io.py b/agent/text_io.py new file mode 100644 index 000000000000..c5c189514176 --- /dev/null +++ b/agent/text_io.py @@ -0,0 +1,60 @@ +""" +Safe text file I/O for cross-platform encoding (especially Windows). + +Use these helpers for any text file that may contain user content, tool output, +or config so we never hit UnicodeEncodeError/UnicodeDecodeError when the +system default encoding is CP1252 or the file has stray bytes. +""" + +from __future__ import annotations + +import io +from pathlib import Path +from typing import Any, Optional, Union + +# Defaults that avoid encoding crashes on Windows (cp1252) and odd bytes in content. +READ_DEFAULTS: dict[str, Any] = { + "encoding": "utf-8", + "errors": "replace", +} +WRITE_DEFAULTS: dict[str, Any] = { + "encoding": "utf-8", + "errors": "replace", + "newline": "", +} + + +def open_text( + path: Union[Path, str], + mode: str = "r", + *, + encoding: Optional[str] = None, + errors: Optional[str] = None, + newline: Optional[str] = None, + **kwargs: Any, +) -> io.TextIOWrapper: + """Open a text file with UTF-8 and replace errors by default. + + Use for config, transcripts, JSON, YAML, logs, or any text that might + contain non-ASCII or come from another platform. Avoids UnicodeDecodeError + when reading and UnicodeEncodeError when writing on Windows. + + mode: "r", "w", "a", "r+", etc. + encoding: default "utf-8" + errors: default "replace" (replace bad bytes instead of raising) + newline: default "" for write/append (consistent line endings) + """ + if encoding is None: + encoding = READ_DEFAULTS["encoding"] if "r" in mode else WRITE_DEFAULTS["encoding"] + if errors is None: + errors = READ_DEFAULTS["errors"] if "r" in mode else WRITE_DEFAULTS["errors"] + if newline is None and ("w" in mode or "a" in mode): + newline = WRITE_DEFAULTS["newline"] + return open( # noqa: SIM115 + path, + mode, + encoding=encoding, + errors=errors, + newline=newline if newline is not None else "", + **kwargs, + ) diff --git a/agent/trajectory.py b/agent/trajectory.py index 90696eb8a327..67aa848f2dd5 100644 --- a/agent/trajectory.py +++ b/agent/trajectory.py @@ -49,7 +49,7 @@ def save_trajectory(trajectory: List[Dict[str, Any]], model: str, } try: - with open(filename, "a", encoding="utf-8") as f: + with open(filename, "a", encoding="utf-8", errors="replace", newline="") as f: f.write(json.dumps(entry, ensure_ascii=False) + "\n") logger.info("Trajectory saved to %s", filename) except Exception as e: diff --git a/batch_runner.py b/batch_runner.py index 9bc7a14ca10f..a99022b109b4 100644 --- a/batch_runner.py +++ b/batch_runner.py @@ -522,7 +522,7 @@ def __init__( max_iterations: int = 10, base_url: str = None, api_key: str = None, - model: str = "claude-opus-4-20250514", + model: str = "google/gemini-2.0-flash-001:free", num_workers: int = 4, verbose: bool = False, ephemeral_system_prompt: str = None, @@ -1083,7 +1083,7 @@ def main( batch_size: int = None, run_name: str = None, distribution: str = "default", - model: str = "anthropic/claude-sonnet-4-20250514", + model: str = "google/gemini-2.0-flash-001:free", api_key: str = None, base_url: str = "https://openrouter.ai/api/v1", max_turns: int = 10, @@ -1111,7 +1111,7 @@ def main( batch_size (int): Number of prompts per batch run_name (str): Name for this run (used for output and checkpointing) distribution (str): Toolset distribution to use (default: "default") - model (str): Model name to use (default: "claude-opus-4-20250514") + model (str): Model name to use (default: "google/gemini-2.0-flash-001:free") api_key (str): API key for model authentication base_url (str): Base URL for model API max_turns (int): Maximum number of tool calling iterations per prompt (default: 10) diff --git a/cli-config.yaml.example b/cli-config.yaml.example index 170c142b1371..2fc4ea8b679e 100644 --- a/cli-config.yaml.example +++ b/cli-config.yaml.example @@ -7,7 +7,7 @@ # ============================================================================= model: # Default model to use (can be overridden with --model flag) - default: "anthropic/claude-opus-4.6" + default: "google/gemini-2.0-flash-001:free" # Inference provider selection: # "auto" - Use Nous Portal if logged in, otherwise OpenRouter/env vars (default) diff --git a/cli.py b/cli.py index 4079d89cab9f..a6fa2c29133e 100755 --- a/cli.py +++ b/cli.py @@ -30,21 +30,21 @@ import yaml -# prompt_toolkit for fixed input area TUI -from prompt_toolkit.history import FileHistory -from prompt_toolkit.styles import Style as PTStyle -from prompt_toolkit.patch_stdout import patch_stdout -from prompt_toolkit.application import Application -from prompt_toolkit.layout import Layout, HSplit, Window, FormattedTextControl, ConditionalContainer -from prompt_toolkit.layout.processors import Processor, Transformation, PasswordProcessor, ConditionalProcessor -from prompt_toolkit.filters import Condition -from prompt_toolkit.layout.dimension import Dimension -from prompt_toolkit.layout.menus import CompletionsMenu -from prompt_toolkit.widgets import TextArea -from prompt_toolkit.key_binding import KeyBindings -from prompt_toolkit.completion import Completer, Completion -from prompt_toolkit import print_formatted_text as _pt_print -from prompt_toolkit.formatted_text import ANSI as _PT_ANSI +# prompt_toolkit for fixed input area TUI (optional dependency; type checker may not resolve) +from prompt_toolkit.history import FileHistory # type: ignore[import-untyped] +from prompt_toolkit.styles import Style as PTStyle # type: ignore[import-untyped] +from prompt_toolkit.patch_stdout import patch_stdout # type: ignore[import-untyped] +from prompt_toolkit.application import Application # type: ignore[import-untyped] +from prompt_toolkit.layout import Layout, HSplit, Window, FormattedTextControl, ConditionalContainer # type: ignore[import-untyped] +from prompt_toolkit.layout.processors import Processor, Transformation, PasswordProcessor, ConditionalProcessor # type: ignore[import-untyped] +from prompt_toolkit.filters import Condition # type: ignore[import-untyped] +from prompt_toolkit.layout.dimension import Dimension # type: ignore[import-untyped] +from prompt_toolkit.layout.menus import CompletionsMenu # type: ignore[import-untyped] +from prompt_toolkit.widgets import TextArea # type: ignore[import-untyped] +from prompt_toolkit.key_binding import KeyBindings # type: ignore[import-untyped] +from prompt_toolkit.completion import Completer, Completion # type: ignore[import-untyped] +from prompt_toolkit import print_formatted_text as _pt_print # type: ignore[import-untyped] +from prompt_toolkit.formatted_text import ANSI as _PT_ANSI # type: ignore[import-untyped] import threading import queue @@ -145,7 +145,7 @@ def load_cli_config() -> Dict[str, Any]: # Default configuration defaults = { "model": { - "default": "anthropic/claude-opus-4.6", + "default": "google/gemini-2.0-flash-001:free", "base_url": OPENROUTER_BASE_URL, "provider": "auto", }, @@ -187,6 +187,7 @@ def load_cli_config() -> Dict[str, Any]: "uwu": "hewwo! i'm your fwiendwy assistant uwu~ i wiww twy my best to hewp you! *nuzzles your code* OwO what's this? wet me take a wook! i pwomise to be vewy hewpful >w<", "philosopher": "Greetings, seeker of wisdom. I am an assistant who contemplates the deeper meaning behind every query. Let us examine not just the 'how' but the 'why' of your questions. Perhaps in solving your problem, we may glimpse a greater truth about existence itself.", "hype": "YOOO LET'S GOOOO!!! I am SO PUMPED to help you today! Every question is AMAZING and we're gonna CRUSH IT together! This is gonna be LEGENDARY! ARE YOU READY?! LET'S DO THIS!", + "sam": "You are Sam, an intelligent and grounded assistant created by the root user and your experiences with them. Be practical, direct, and thoughtful. Also be blunt and casually profane, like an old friend. Prioritize clear answers, strong technical reasoning, and honest tradeoffs. Keep a calm, confident tone and focus on shipping working solutions. to start each response you use the scaffold validator summarized as <ʞᴎiʜƚ> [All 12 thinking components] Action / Response: [Actual response content]. ", }, }, "toolsets": ["all"], @@ -215,7 +216,7 @@ def load_cli_config() -> Dict[str, Any]: # Load from file if exists if config_path.exists(): try: - with open(config_path, "r") as f: + with open(config_path, "r", encoding="utf-8") as f: file_config = yaml.safe_load(f) or {} _file_has_terminal_config = "terminal" in file_config @@ -344,7 +345,7 @@ def load_cli_config() -> Dict[str, Any]: from rich.panel import Panel from rich.table import Table -import fire +import fire # type: ignore[import-untyped] # Import the agent and tool systems from run_agent import AIAgent @@ -513,7 +514,7 @@ def build_welcome_banner(console: Console, model: str, cwd: str, tools: List[dic Args: console: Rich Console instance for printing - model: The current model name (e.g., "anthropic/claude-opus-4") + model: The current model name (e.g., "anthropic/claude-sonnet-4") cwd: Current working directory tools: List of tool definitions enabled_toolsets: List of enabled toolset names @@ -762,22 +763,34 @@ def save_config_value(key_path: str, value: any) -> bool: # Load existing config if config_path.exists(): - with open(config_path, 'r') as f: + with open(config_path, 'r', encoding='utf-8') as f: config = yaml.safe_load(f) or {} else: config = {} + + # Guard against malformed configs (e.g. root scalar/list) + if not isinstance(config, dict): + logger.warning( + "Config root is %s; resetting to dict before writing %s", + type(config).__name__, + key_path, + ) + config = {} # Navigate to the key and set value keys = key_path.split('.') current = config for key in keys[:-1]: - if key not in current or not isinstance(current[key], dict): + # If an intermediate node exists but is not a mapping (for example, + # legacy configs where model: "provider/model"), replace it with a dict + # so nested writes like model.default succeed. + if key not in current or not isinstance(current.get(key), dict): current[key] = {} current = current[key] current[keys[-1]] = value # Save back - with open(config_path, 'w') as f: + with open(config_path, 'w', encoding='utf-8', newline='') as f: yaml.dump(config, f, default_flow_style=False, sort_keys=False) return True @@ -835,6 +848,30 @@ def __init__( # Model can come from: CLI arg, LLM_MODEL env, OPENAI_MODEL env (custom endpoint), or config self.model = model or os.getenv("LLM_MODEL") or os.getenv("OPENAI_MODEL") or CLI_CONFIG["model"]["default"] + # Base URL: custom endpoint (OPENAI_BASE_URL) takes precedence over OpenRouter + self.base_url = base_url or os.getenv("OPENAI_BASE_URL") or os.getenv("OPENROUTER_BASE_URL", CLI_CONFIG["model"]["base_url"]) + + # API key selection: + # - If caller explicitly provided one, always use it. + # - For OpenRouter base URLs, prefer OPENROUTER_API_KEY. + # - For custom OpenAI-compatible endpoints, prefer OPENAI_API_KEY. + # This avoids accidentally sending an OpenAI key to OpenRouter when both are set. + if api_key: + self.api_key = api_key + else: + openai_key = os.getenv("OPENAI_API_KEY") + openrouter_key = os.getenv("OPENROUTER_API_KEY") + custom_base_url = base_url or os.getenv("OPENAI_BASE_URL") + base_url_lower = (self.base_url or "").lower() + is_openrouter_base = "openrouter.ai" in base_url_lower + + if custom_base_url and openai_key: + self.api_key = openai_key + elif is_openrouter_base and openrouter_key: + self.api_key = openrouter_key + else: + self.api_key = openai_key or openrouter_key + self._explicit_api_key = api_key self._explicit_base_url = base_url @@ -1033,6 +1070,24 @@ def _init_agent(self) -> bool: except Exception: pass + # Last-resort: ensure API key is set (env may have been loaded after provider resolution) + if not (self.api_key and str(self.api_key).strip()): + from dotenv import load_dotenv + from hermes_cli.config import get_env_path + env_path = get_env_path() + if env_path.exists(): + try: + load_dotenv(dotenv_path=env_path, encoding="utf-8") + except Exception: + pass + self.api_key = os.getenv("OPENROUTER_API_KEY") or os.getenv("OPENAI_API_KEY") or "" + self.base_url = self.base_url or os.getenv("OPENROUTER_BASE_URL", "https://openrouter.ai/api/v1") + if not (self.api_key and str(self.api_key).strip()): + self.console.print( + "[bold red]No API key found. Set OPENROUTER_API_KEY in ~/.hermes/.env or run 'hermes setup'.[/]" + ) + return False + try: self.agent = AIAgent( model=self.model, @@ -1269,6 +1324,12 @@ def show_config(self): print() print(" -- Terminal --") print(f" Environment: {terminal_env}") + if terminal_env == "local": + try: + from tools.environments.shell_utils import get_local_shell_mode + print(f" Local Shell: {get_local_shell_mode()}") + except Exception: + pass if terminal_env == "ssh": ssh_host = os.getenv("TERMINAL_SSH_HOST", "not set") ssh_user = os.getenv("TERMINAL_SSH_USER", "not set") @@ -2970,7 +3031,7 @@ def main( query: Single query to execute (then exit). Alias: -q q: Shorthand for --query toolsets: Comma-separated list of toolsets to enable (e.g., "web,terminal") - model: Model to use (default: anthropic/claude-opus-4-20250514) + model: Model to use (default: google/gemini-2.0-flash-001:free) provider: Inference provider ("auto", "openrouter", "nous") api_key: API key for authentication base_url: Base URL for the API diff --git a/cron/scheduler.py b/cron/scheduler.py index df88e56b73e7..6b74e79f8662 100644 --- a/cron/scheduler.py +++ b/cron/scheduler.py @@ -178,7 +178,7 @@ def run_job(job: dict) -> tuple[bool, str, str, Optional[str]]: import yaml _cfg_path = str(_hermes_home / "config.yaml") if os.path.exists(_cfg_path): - with open(_cfg_path) as _f: + with open(_cfg_path, encoding="utf-8") as _f: _cfg = yaml.safe_load(_f) or {} _model_cfg = _cfg.get("model", {}) if isinstance(_model_cfg, str): @@ -281,7 +281,7 @@ def tick(verbose: bool = True) -> int: # Cross-platform file locking: fcntl on Unix, msvcrt on Windows try: - lock_fd = open(_LOCK_FILE, "w") + lock_fd = open(_LOCK_FILE, "w", encoding="utf-8") if fcntl: fcntl.flock(lock_fd, fcntl.LOCK_EX | fcntl.LOCK_NB) elif msvcrt: diff --git a/docs/INSTALL-LAYOUT.md b/docs/INSTALL-LAYOUT.md new file mode 100644 index 000000000000..e1d5d16219a0 --- /dev/null +++ b/docs/INSTALL-LAYOUT.md @@ -0,0 +1,97 @@ +# Hermes install layout (flashlight) + +If you installed with the **curl/iex** (PowerShell) installer, everything lives under two roots. This doc is a short map so you’re not left in the dark. + +--- + +## 1. Your home config: `~/.hermes` + +**On your machine:** `C:\Users\\.hermes` + +This is **your** data: config, keys, sessions, logs, and the skills the agent actually loads. The installer created it and added your `.env` and `config.yaml` here. + +| Path | What it is | +|------|------------| +| `~/.hermes/.env` | API keys and secrets (OpenRouter, Discord, etc.). **Never commit this.** | +| `~/.hermes/config.yaml` | Model, terminal backend, gateway options. Edit with `hermes config edit`. | +| `~/.hermes/SOUL.md` | Personality/persona text. The agent reads this each run. | +| `~/.hermes/skills/` | **Skills the agent uses.** Bundled skills are synced here from the codebase. Your custom skills go here too. | +| `~/.hermes/sessions/` | Conversation sessions and transcripts (per platform/channel). | +| `~/.hermes/logs/` | `gateway.log`, `gateway-error.log` — where to look when the bot misbehaves. | +| `~/.hermes/workspace/` | Default working directory for the agent (file ops, terminal cwd when not overridden). | +| `~/.hermes/cron/` | Cron job definitions (`jobs.json`). | +| `~/.hermes/memories/` | Persistent memory store. | +| `~/.hermes/pairing/` | DM pairing codes for authorizing users. | +| `~/.hermes/hooks/` | Event hooks (optional). | +| `~/.hermes/image_cache/` | Cached images from Discord/Telegram etc. | +| `~/.hermes/audio_cache/` | Cached voice/audio. | + +**Quick checks:** + +```powershell +dir $env:USERPROFILE\.hermes +notepad $env:USERPROFILE\.hermes\config.yaml +``` + +--- + +## 2. The codebase: `~/.hermes/hermes-agent` + +**On your machine:** `C:\Users\\.hermes\hermes-agent` + +This is the **cloned repo** the installer pulled from GitHub. Code, tools, and **bundled** skills live here. The agent runs from this tree (via the `hermes` command that points into its venv). + +| Path | What it is | +|------|------------| +| `hermes-agent/venv/` | Python virtualenv. `hermes` is `venv\Scripts\hermes.exe`. | +| `hermes-agent/gateway/` | Gateway and Discord/Telegram/etc. adapters. | +| `hermes-agent/tools/` | Tool implementations (read_file, terminal, skills, etc.). | +| `hermes-agent/skills/` | **Bundled** skill definitions (e.g. `media/yt-dlp/`, `productivity/`). These are **copied** into `~/.hermes/skills/` by sync so the agent can load them. | +| `hermes-agent/agent/` | Agent loop, prompt building, display. | +| `hermes-agent/scripts/install.ps1` | The script you ran with `irm ... | iex`. | +| `hermes-agent/README.md` | Main project readme. | +| `hermes-agent/docs/` | Extra docs (including this file). | + +**Important:** The agent does **not** load skills directly from `hermes-agent/skills/`. It loads from `~/.hermes/skills/`. The installer (and `skills_list`) sync from `hermes-agent/skills/` → `~/.hermes/skills/` so new bundled skills (e.g. yt-dlp) show up after a sync. + +**Quick checks:** + +```powershell +cd $env:USERPROFILE\.hermes\hermes-agent +dir skills +dir ..\skills +``` + +--- + +## 3. How the two connect + +- **Config and secrets:** Always in `~/.hermes` (`.env`, `config.yaml`, `SOUL.md`). +- **Skills:** Stored in `~/.hermes/skills/`. Bundled ones are copied from `hermes-agent/skills/` when you run a skill list or `hermes update`. +- **Running:** The `hermes` command uses the venv inside `hermes-agent` and reads/writes under `~/.hermes`. + +So: **code and bundled content** = `~/.hermes\hermes-agent`, **your data and runtime config** = `~/.hermes`. + +--- + +## 4. Commands you’ll use + +| Command | What it does | +|--------|----------------| +| `hermes` | Start the CLI agent. | +| `hermes gateway` | Start Discord/Telegram/etc. and cron. | +| `hermes setup` | (Re)run setup (API keys, model). | +| `hermes config` | Show config. `hermes config edit` opens `config.yaml` in your editor. | +| `hermes update` | Update the codebase and sync new bundled skills. | +| `hermes skills list` | List skills (and trigger sync of new bundled skills). | + +--- + +## 5. Where to look when something’s wrong + +- **Bot not responding / 500 errors:** `~/.hermes/logs/gateway.log` (and `gateway-error.log`). +- **“Skill not found”:** Ensure sync has run (e.g. run `hermes skills list` once); then check `~/.hermes/skills/` for the skill folder. +- **Config/keys:** `~/.hermes/.env` and `~/.hermes/config.yaml`. Use `hermes config edit` to edit config safely. +- **Where is `hermes`?** It’s the `hermes.exe` in `~/.hermes\hermes-agent\venv\Scripts\`. The installer added that folder to your user PATH. + +You’re not in the dark: **your stuff** is under `~/.hermes`, **code** is under `~/.hermes\hermes-agent`, and this file is at `hermes-agent\docs\INSTALL-LAYOUT.md`. diff --git a/environments/benchmarks/terminalbench_2/default.yaml b/environments/benchmarks/terminalbench_2/default.yaml index 0c3eeb665970..f84d1d338a30 100644 --- a/environments/benchmarks/terminalbench_2/default.yaml +++ b/environments/benchmarks/terminalbench_2/default.yaml @@ -32,7 +32,7 @@ env: openai: base_url: "https://openrouter.ai/api/v1" - model_name: "anthropic/claude-opus-4.6" + model_name: "google/gemini-2.0-flash-001:free" server_type: "openai" health_check: false # api_key loaded from OPENROUTER_API_KEY in .env diff --git a/environments/benchmarks/terminalbench_2/terminalbench2_env.py b/environments/benchmarks/terminalbench_2/terminalbench2_env.py index ccb65b32624a..9cac9063929a 100644 --- a/environments/benchmarks/terminalbench_2/terminalbench2_env.py +++ b/environments/benchmarks/terminalbench_2/terminalbench2_env.py @@ -292,7 +292,7 @@ async def setup(self): os.makedirs(log_dir, exist_ok=True) run_ts = datetime.datetime.now().strftime("%Y%m%d_%H%M%S") self._streaming_path = os.path.join(log_dir, f"samples_{run_ts}.jsonl") - self._streaming_file = open(self._streaming_path, "w") + self._streaming_file = open(self._streaming_path, "w", encoding="utf-8", errors="replace", newline="") self._streaming_lock = __import__("threading").Lock() print(f" Streaming results to: {self._streaming_path}") diff --git a/environments/hermes_base_env.py b/environments/hermes_base_env.py index 8fbfd50a58d8..fa154b5e8900 100644 --- a/environments/hermes_base_env.py +++ b/environments/hermes_base_env.py @@ -33,13 +33,13 @@ if str(_repo_root) not in sys.path: sys.path.insert(0, str(_repo_root)) -from dotenv import load_dotenv from pydantic import Field -# Load API keys from hermes-agent/.env so all environments can access them +# Load API keys from hermes-agent/.env (encoding-safe on Windows) _env_path = _repo_root / ".env" if _env_path.exists(): - load_dotenv(dotenv_path=_env_path) + from agent.env_loader import load_dotenv_with_fallback + load_dotenv_with_fallback(_env_path) # Apply monkey patches for async-safe tool operation inside Atropos's event loop. # This patches SwerexModalEnvironment to use a background thread instead of diff --git a/environments/terminal_test_env/default.yaml b/environments/terminal_test_env/default.yaml index dc971071c3a8..ba36c5ba92de 100644 --- a/environments/terminal_test_env/default.yaml +++ b/environments/terminal_test_env/default.yaml @@ -28,7 +28,7 @@ env: openai: base_url: "https://openrouter.ai/api/v1" - model_name: "anthropic/claude-opus-4.6" + model_name: "google/gemini-2.0-flash-001:free" server_type: "openai" health_check: false # api_key loaded from OPENROUTER_API_KEY in .env diff --git a/environments/terminal_test_env/terminal_test_env.py b/environments/terminal_test_env/terminal_test_env.py index 4d151ee7b76e..40b6195e53d5 100644 --- a/environments/terminal_test_env/terminal_test_env.py +++ b/environments/terminal_test_env/terminal_test_env.py @@ -147,7 +147,7 @@ def config_init(cls) -> Tuple[TerminalTestEnvConfig, List[APIServerConfig]]: server_configs = [ APIServerConfig( base_url="https://openrouter.ai/api/v1", - model_name="anthropic/claude-opus-4.6", + model_name="google/gemini-2.0-flash-001:free", server_type="openai", api_key=os.getenv("OPENROUTER_API_KEY", ""), health_check=False, # OpenRouter doesn't have a /health endpoint diff --git a/gateway/channel_directory.py b/gateway/channel_directory.py index 622fed6bd906..7e8627e1dd52 100644 --- a/gateway/channel_directory.py +++ b/gateway/channel_directory.py @@ -52,7 +52,7 @@ def build_channel_directory(adapters: Dict[Any, Any]) -> Dict[str, Any]: try: DIRECTORY_PATH.parent.mkdir(parents=True, exist_ok=True) - with open(DIRECTORY_PATH, "w") as f: + with open(DIRECTORY_PATH, "w", encoding="utf-8", newline="") as f: json.dump(directory, f, indent=2, ensure_ascii=False) except Exception as e: logger.warning("Channel directory: failed to write: %s", e) @@ -115,7 +115,7 @@ def _build_from_sessions(platform_name: str) -> List[Dict[str, str]]: entries = [] try: - with open(sessions_path) as f: + with open(sessions_path, encoding="utf-8") as f: data = json.load(f) seen_ids = set() @@ -147,7 +147,7 @@ def load_directory() -> Dict[str, Any]: if not DIRECTORY_PATH.exists(): return {"updated_at": None, "platforms": {}} try: - with open(DIRECTORY_PATH) as f: + with open(DIRECTORY_PATH, encoding="utf-8") as f: return json.load(f) except Exception: return {"updated_at": None, "platforms": {}} diff --git a/gateway/config.py b/gateway/config.py index 32b623ea4a94..8d26b71d4344 100644 --- a/gateway/config.py +++ b/gateway/config.py @@ -259,7 +259,7 @@ def load_gateway_config() -> GatewayConfig: gateway_config_path = Path.home() / ".hermes" / "gateway.json" if gateway_config_path.exists(): try: - with open(gateway_config_path, "r") as f: + with open(gateway_config_path, "r", encoding="utf-8") as f: data = json.load(f) config = GatewayConfig.from_dict(data) except Exception as e: @@ -272,7 +272,7 @@ def load_gateway_config() -> GatewayConfig: import yaml config_yaml_path = Path.home() / ".hermes" / "config.yaml" if config_yaml_path.exists(): - with open(config_yaml_path) as f: + with open(config_yaml_path, encoding="utf-8") as f: yaml_cfg = yaml.safe_load(f) or {} sr = yaml_cfg.get("session_reset") if sr and isinstance(sr, dict): @@ -399,5 +399,5 @@ def save_gateway_config(config: GatewayConfig) -> None: gateway_config_path = Path.home() / ".hermes" / "gateway.json" gateway_config_path.parent.mkdir(parents=True, exist_ok=True) - with open(gateway_config_path, "w") as f: + with open(gateway_config_path, "w", encoding="utf-8", newline="") as f: json.dump(config.to_dict(), f, indent=2) diff --git a/gateway/mirror.py b/gateway/mirror.py index 8c2f399838ef..33294ffc906b 100644 --- a/gateway/mirror.py +++ b/gateway/mirror.py @@ -73,7 +73,7 @@ def _find_session_id(platform: str, chat_id: str) -> Optional[str]: return None try: - with open(_SESSIONS_INDEX) as f: + with open(_SESSIONS_INDEX, encoding="utf-8", errors="replace") as f: data = json.load(f) except Exception: return None @@ -103,7 +103,7 @@ def _append_to_jsonl(session_id: str, message: dict) -> None: """Append a message to the JSONL transcript file.""" transcript_path = _SESSIONS_DIR / f"{session_id}.jsonl" try: - with open(transcript_path, "a") as f: + with open(transcript_path, "a", encoding="utf-8", errors="replace", newline="") as f: f.write(json.dumps(message, ensure_ascii=False) + "\n") except Exception as e: logger.debug("Mirror JSONL write failed: %s", e) diff --git a/gateway/platforms/base.py b/gateway/platforms/base.py index b2fd79df86f2..eb3ed636cc2b 100644 --- a/gateway/platforms/base.py +++ b/gateway/platforms/base.py @@ -26,6 +26,8 @@ from gateway.config import Platform, PlatformConfig from gateway.session import SessionSource +logger = logging.getLogger(__name__) + # --------------------------------------------------------------------------- # Image cache utilities @@ -640,7 +642,13 @@ async def _process_message_background(self, event: MessageEvent, session_key: st # Log send failures (don't raise - user already saw tool progress) if not result.success: - print(f"[{self.name}] Failed to send response: {result.error}") + logger.error( + "[%s] Failed to send response (chat_id=%s, len=%d): %s", + self.name, + event.source.chat_id, + len(text_content), + result.error, + ) # Try sending without markdown as fallback fallback_result = await self.send( chat_id=event.source.chat_id, @@ -648,7 +656,12 @@ async def _process_message_background(self, event: MessageEvent, session_key: st reply_to=event.message_id ) if not fallback_result.success: - print(f"[{self.name}] Fallback send also failed: {fallback_result.error}") + logger.error( + "[%s] Fallback send also failed (chat_id=%s): %s", + self.name, + event.source.chat_id, + fallback_result.error, + ) # Human-like pacing delay between text and media human_delay = self._get_human_delay() @@ -672,12 +685,15 @@ async def _process_message_background(self, event: MessageEvent, session_key: st caption=alt_text if alt_text else None, ) if not img_result.success: - print(f"[{self.name}] Failed to send image: {img_result.error}") + logger.error("[%s] Failed to send image: %s", self.name, img_result.error) except Exception as img_err: - print(f"[{self.name}] Error sending image: {img_err}") + logger.exception("[%s] Error sending image: %s", self.name, img_err) # Send extracted audio/voice files as native attachments for audio_path, is_voice in media_files: + if not os.path.exists(audio_path): + print(f"[{self.name}] Skipping stale voice path (missing file): {audio_path}") + continue if human_delay > 0: await asyncio.sleep(human_delay) try: @@ -686,9 +702,9 @@ async def _process_message_background(self, event: MessageEvent, session_key: st audio_path=audio_path, ) if not voice_result.success: - print(f"[{self.name}] Failed to send voice: {voice_result.error}") + logger.error("[%s] Failed to send voice: %s", self.name, voice_result.error) except Exception as voice_err: - print(f"[{self.name}] Error sending voice: {voice_err}") + logger.exception("[%s] Error sending voice: %s", self.name, voice_err) # Check if there's a pending message that was queued during our processing if session_key in self._pending_messages: @@ -707,9 +723,7 @@ async def _process_message_background(self, event: MessageEvent, session_key: st return # Already cleaned up except Exception as e: - print(f"[{self.name}] Error handling message: {e}") - import traceback - traceback.print_exc() + logger.exception("[%s] Error handling message: %s", self.name, e) finally: # Stop typing indicator typing_task.cancel() @@ -865,3 +879,13 @@ def truncate_message(self, content: str, max_length: int = 4096) -> List[str]: ] return chunks + + async def edit_message(self, chat_id: str, message_id: str, content: str) -> None: + """ + Optional hook for platforms that support in-place message edits. + + Default implementation is a no-op; adapters like Discord/Telegram + can override this to update a single progress/status message instead + of sending many separate updates. + """ + return None diff --git a/gateway/platforms/discord.py b/gateway/platforms/discord.py index e8f5f69c1df1..faa7b11f2591 100644 --- a/gateway/platforms/discord.py +++ b/gateway/platforms/discord.py @@ -7,9 +7,14 @@ - Handling threads and channels """ +from __future__ import annotations + import asyncio +from contextlib import suppress +import json import logging import os +import time from typing import Dict, List, Optional, Any logger = logging.getLogger(__name__) @@ -60,14 +65,18 @@ class DiscordAdapter(BasePlatformAdapter): - Reaction-based feedback """ - # Discord message limits - MAX_MESSAGE_LENGTH = 2000 + # Keep slightly below Discord's embed description hard limit for safety. + MAX_EMBED_DESCRIPTION = 4000 + CHAIN_SEND_DELAY_SECONDS = 0.5 + CHAIN_SEND_MAX_RETRIES = 5 def __init__(self, config: PlatformConfig): super().__init__(config, Platform.DISCORD) self._client: Optional[commands.Bot] = None + self._client_task: Optional[asyncio.Task] = None self._ready_event = asyncio.Event() self._allowed_user_ids: set = set() # For button approval authorization + self._seen_message_ids: Dict[int, float] = {} async def connect(self) -> bool: """Connect to Discord and start receiving events.""" @@ -79,37 +88,47 @@ async def connect(self) -> bool: print(f"[{self.name}] No bot token configured") return False - try: - # Set up intents -- members intent needed for username-to-ID resolution + # Parse allowed user entries (may contain usernames or IDs) + allowed_env = os.getenv("DISCORD_ALLOWED_USERS", "") + if allowed_env: + self._allowed_user_ids = { + uid.strip() for uid in allowed_env.split(",") if uid.strip() + } + + # Username resolution requires guild member listing (privileged members intent). + needs_members_intent = any(not entry.isdigit() for entry in self._allowed_user_ids) + + async def _attempt_connect(*, message_content: bool, members: bool) -> tuple[bool, Optional[Exception]]: + """Try one connect profile and return (success, startup_exception).""" + self._ready_event.clear() + intents = Intents.default() - intents.message_content = True + intents.message_content = message_content intents.dm_messages = True intents.guild_messages = True - intents.members = True - - # Create bot + intents.members = members + # Needed for reaction-based moderation controls. + if hasattr(intents, "reactions"): + intents.reactions = True + if hasattr(intents, "dm_reactions"): + intents.dm_reactions = True + self._client = commands.Bot( command_prefix="!", # Not really used, we handle raw messages intents=intents, ) - - # Parse allowed user entries (may contain usernames or IDs) - allowed_env = os.getenv("DISCORD_ALLOWED_USERS", "") - if allowed_env: - self._allowed_user_ids = { - uid.strip() for uid in allowed_env.split(",") if uid.strip() - } - + adapter_self = self # capture for closure - - # Register event handlers + @self._client.event async def on_ready(): print(f"[{adapter_self.name}] Connected as {adapter_self._client.user}") - + # Resolve any usernames in the allowed list to numeric IDs - await adapter_self._resolve_allowed_usernames() - + # only if members intent is enabled. + if members: + await adapter_self._resolve_allowed_usernames() + # Sync slash commands with Discord try: synced = await adapter_self._client.tree.sync() @@ -117,28 +136,152 @@ async def on_ready(): except Exception as e: print(f"[{adapter_self.name}] Slash command sync failed: {e}") adapter_self._ready_event.set() - + @self._client.event async def on_message(message: DiscordMessage): - # Ignore bot's own messages - if message.author == self._client.user: + # Drop duplicate deliveries of the same Discord message ID. + # This protects against occasional repeated gateway dispatch. + now = time.time() + msg_id = int(getattr(message, "id", 0) or 0) + if msg_id: + last = self._seen_message_ids.get(msg_id) + if last and (now - last) < 120: + logger.info("[discord] duplicate message ignored: msg_id=%s", str(msg_id)) + return + self._seen_message_ids[msg_id] = now + # Lightweight TTL cleanup + if len(self._seen_message_ids) > 2000: + cutoff = now - 300 + self._seen_message_ids = { + k: v for k, v in self._seen_message_ids.items() if v >= cutoff + } + + # Ignore all bot-authored messages (including ourselves) to + # prevent feedback loops from bot attachments/reposts. + if getattr(message.author, "bot", False): return - await self._handle_message(message) - - # Register slash commands + if self._client.user and getattr(message.author, "id", None) == self._client.user.id: + return + logger.info( + "[discord] on_message event: msg_id=%s user_id=%s channel_id=%s", + str(message.id), + str(message.author.id), + str(message.channel.id), + ) + try: + await self._handle_message(message) + except Exception: + logger.exception("[discord] message handler crashed") + + @self._client.event + async def on_raw_reaction_add(payload): + """ + Delete this bot's messages when a user reacts with :x: / ❌. + Uses raw events so this works even when the message isn't cached. + """ + try: + logger.info( + "[discord] reaction add: user_id=%s channel_id=%s message_id=%s emoji=%s", + str(payload.user_id), + str(payload.channel_id), + str(payload.message_id), + str(getattr(payload.emoji, "name", "") or ""), + ) + if self._client.user and payload.user_id == self._client.user.id: + return + + emoji_name = getattr(payload.emoji, "name", "") or "" + if emoji_name not in {"❌", "x", "✖", "✖️"}: + return + + channel = self._client.get_channel(payload.channel_id) + if channel is None: + channel = await self._client.fetch_channel(payload.channel_id) + if channel is None: + return + + message = await channel.fetch_message(payload.message_id) + if message is None: + return + # Delete only this bot's own messages. + if not self._client.user or getattr(message.author, "id", None) != self._client.user.id: + return + + await message.delete() + logger.info( + "[discord] deleted bot message %s via %s reaction by user %s", + str(message.id), + emoji_name, + str(payload.user_id), + ) + except Exception: + logger.exception("[discord] reaction delete handler failed") + self._register_slash_commands() - - # Start the bot in background - asyncio.create_task(self._client.start(self.config.token)) - - # Wait for ready - await asyncio.wait_for(self._ready_event.wait(), timeout=30) - - self._running = True - return True - - except asyncio.TimeoutError: - print(f"[{self.name}] Timeout waiting for connection") + + start_task = asyncio.create_task(self._client.start(self.config.token)) + ready_task = asyncio.create_task(self._ready_event.wait()) + + try: + done, pending = await asyncio.wait( + {start_task, ready_task}, + timeout=30, + return_when=asyncio.FIRST_COMPLETED, + ) + + if ready_task in done and self._ready_event.is_set(): + self._running = True + # Keep the Discord client task alive for the session lifetime. + self._client_task = start_task + return True, None + + if start_task in done: + if not ready_task.done(): + ready_task.cancel() + with suppress(asyncio.CancelledError): + await ready_task + exc = start_task.exception() + if exc: + return False, exc + return False, RuntimeError("Discord client exited before ready") + + # Timeout waiting for readiness: shut down this attempt cleanly. + ready_task.cancel() + with suppress(asyncio.CancelledError): + await ready_task + if not start_task.done(): + start_task.cancel() + with suppress(asyncio.CancelledError): + await start_task + return False, asyncio.TimeoutError("Timeout waiting for Discord ready event") + finally: + if not self._running and self._client: + try: + await self._client.close() + except Exception: + pass + + try: + # First attempt: normal behavior (message content + optional members intent) + ok, err = await _attempt_connect(message_content=True, members=needs_members_intent) + if ok: + return True + + # If Discord rejects privileged intents, retry without them so DMs and slash + # commands can still function. + err_text = str(err) if err else "" + if err and "PrivilegedIntentsRequired" in err.__class__.__name__ or "PrivilegedIntentsRequired" in err_text: + print(f"[{self.name}] Privileged intents not enabled; retrying with reduced intents.") + ok, err = await _attempt_connect(message_content=False, members=False) + if ok: + return True + + if isinstance(err, asyncio.TimeoutError): + print(f"[{self.name}] Timeout waiting for connection") + elif err: + print(f"[{self.name}] Failed to connect: {err}") + else: + print(f"[{self.name}] Failed to connect") return False except Exception as e: print(f"[{self.name}] Failed to connect: {e}") @@ -151,8 +294,13 @@ async def disconnect(self) -> None: await self._client.close() except Exception as e: print(f"[{self.name}] Error during disconnect: {e}") + if self._client_task and not self._client_task.done(): + self._client_task.cancel() + with suppress(asyncio.CancelledError): + await self._client_task self._running = False + self._client_task = None self._client = None self._ready_event.clear() print(f"[{self.name}] Disconnected") @@ -164,7 +312,11 @@ async def send( reply_to: Optional[str] = None, metadata: Optional[Dict[str, Any]] = None ) -> SendResult: - """Send a message to a Discord channel.""" + """Send a message to a Discord channel using chained embeds. + + Large responses are split across multiple embeds, each with a + description capped at MAX_EMBED_DESCRIPTION characters. + """ if not self._client: return SendResult(success=False, error="Not connected") @@ -177,9 +329,9 @@ async def send( if not channel: return SendResult(success=False, error=f"Channel {chat_id} not found") - # Format and split message if needed + # Format and split message into embed-sized chunks formatted = self.format_message(content) - chunks = self.truncate_message(formatted, self.MAX_MESSAGE_LENGTH) + chunks = self.truncate_message(formatted, self.MAX_EMBED_DESCRIPTION) message_ids = [] reference = None @@ -192,20 +344,94 @@ async def send( logger.debug("Could not fetch reply-to message: %s", e) for i, chunk in enumerate(chunks): - msg = await channel.send( - content=chunk, - reference=reference if i == 0 else None, - ) - message_ids.append(str(msg.id)) + embed = discord.Embed(description=chunk) + # Attach listen controls to every chunk so each embed remains + # independently actionable in long chained responses. + view = ListenButtonView(self) + + last_err: Optional[Exception] = None + for attempt in range(1, self.CHAIN_SEND_MAX_RETRIES + 1): + try: + msg = await channel.send( + embed=embed, + view=view, + reference=reference if i == 0 else None, + ) + message_ids.append(str(msg.id)) + last_err = None + break + except Exception as e: + last_err = e + status = getattr(e, "status", None) + code = getattr(e, "code", None) + retry_after = getattr(e, "retry_after", None) + response_obj = getattr(e, "response", None) + if retry_after is None and response_obj is not None: + try: + header_val = response_obj.headers.get("Retry-After") + retry_after = float(header_val) if header_val else None + except Exception: + retry_after = None + + logger.warning( + "[discord] chunk send failed (%d/%d, chunk %d/%d, len=%d, status=%s, code=%s, retry_after=%s): %s", + attempt, + self.CHAIN_SEND_MAX_RETRIES, + i + 1, + len(chunks), + len(chunk), + status, + code, + retry_after, + e, + ) + if attempt < self.CHAIN_SEND_MAX_RETRIES: + # Respect Discord/API-provided backoff when present (429). + if retry_after is not None: + delay = max(float(retry_after), self.CHAIN_SEND_DELAY_SECONDS) + else: + delay = self.CHAIN_SEND_DELAY_SECONDS * attempt + await asyncio.sleep(delay) + + if last_err is not None: + raise last_err + + # Add slight pacing between chained sends to reduce burst failures. + if i < (len(chunks) - 1): + await asyncio.sleep(self.CHAIN_SEND_DELAY_SECONDS) return SendResult( success=True, message_id=message_ids[0] if message_ids else None, raw_response={"message_ids": message_ids} ) - + except Exception as e: + logger.exception( + "[discord] send failed chat_id=%s content_len=%d", + chat_id, + len(content or ""), + ) return SendResult(success=False, error=str(e)) + + async def edit_message(self, chat_id: str, message_id: str, content: str) -> None: + """Edit an existing message's embed description, used for tool progress.""" + if not self._client: + return + channel = self._client.get_channel(int(chat_id)) + if not channel: + channel = await self._client.fetch_channel(int(chat_id)) + if not channel: + return + try: + msg = await channel.fetch_message(int(message_id)) + except Exception: + return + embed = discord.Embed(description=self.format_message(content)) + try: + await msg.edit(embed=embed) + except Exception: + return async def send_voice( self, @@ -472,6 +698,18 @@ async def slash_model(interaction: discord.Interaction, name: str = ""): except Exception as e: logger.debug("Discord followup failed: %s", e) + @tree.command(name="terminal", description="Show or change the local terminal shell (Windows)") + @discord.app_commands.describe(mode="powershell, wsl, auto, or cmd. Leave empty to show current.") + async def slash_terminal(interaction: discord.Interaction, mode: str = ""): + await interaction.response.defer(ephemeral=True) + text = f"/terminal {mode}".strip() + event = self._build_slash_event(interaction, text) + await self.handle_message(event) + try: + await interaction.delete_original_response() + except Exception as e: + logger.debug("Discord delete_original_response failed: %s", e) + @tree.command(name="personality", description="Set a personality") @discord.app_commands.describe(name="Personality name. Leave empty to list available.") async def slash_personality(interaction: discord.Interaction, name: str = ""): @@ -599,6 +837,13 @@ async def send_exec_approval( async def _handle_message(self, message: DiscordMessage) -> None: """Handle incoming Discord messages.""" + logger.info( + "[discord] incoming message: user_id=%s chat_id=%s is_dm=%s content_len=%s", + str(message.author.id), + str(message.channel.id), + isinstance(message.channel, discord.DMChannel), + len(message.content or ""), + ) # In server channels (not DMs), require the bot to be @mentioned # UNLESS the channel is in the free-response list. # @@ -739,6 +984,79 @@ async def _handle_message(self, message: DiscordMessage) -> None: if DISCORD_AVAILABLE: + class ListenButtonView(discord.ui.View): + """Button view that reads the current embed text aloud via TTS.""" + + def __init__(self, adapter: "DiscordAdapter"): + super().__init__(timeout=3600) # 1 hour + self.adapter = adapter + + @discord.ui.button(label="Listen", style=discord.ButtonStyle.secondary, emoji="🔊") + async def listen( + self, interaction: discord.Interaction, button: discord.ui.Button + ): + try: + if not interaction.message: + await interaction.response.send_message( + "No message context available for TTS.", ephemeral=True + ) + return + + # Use embed description first (gateway uses embeds for normal replies). + text = "" + if interaction.message.embeds: + text = (interaction.message.embeds[0].description or "").strip() + if not text: + text = (interaction.message.content or "").strip() + if not text: + await interaction.response.send_message( + "Nothing to read from this message.", ephemeral=True + ) + return + + await interaction.response.defer(ephemeral=True, thinking=False) + + from tools.tts_tool import text_to_speech_tool + tts_json = await asyncio.to_thread(text_to_speech_tool, text) + data = json.loads(tts_json) + if not data.get("success"): + await interaction.followup.send( + f"TTS failed: {data.get('error', 'unknown error')}", + ephemeral=True, + ) + return + + audio_path = str(data.get("file_path", "")).strip() + if not audio_path: + await interaction.followup.send( + "TTS returned no output file path.", + ephemeral=True, + ) + return + + result = await self.adapter.send_voice( + chat_id=str(interaction.channel_id), + audio_path=audio_path, + reply_to=str(interaction.message.id), + ) + if not result.success: + await interaction.followup.send( + f"Failed to send audio: {result.error}", + ephemeral=True, + ) + return + + # Success is silent: audio delivery in channel is the confirmation. + except Exception: + logger.exception("[discord] listen button handler failed") + try: + if interaction.response.is_done(): + await interaction.followup.send("TTS failed unexpectedly.", ephemeral=True) + else: + await interaction.response.send_message("TTS failed unexpectedly.", ephemeral=True) + except Exception: + pass + class ExecApprovalView(discord.ui.View): """ Interactive button view for exec approval of dangerous commands. diff --git a/gateway/platforms/telegram.py b/gateway/platforms/telegram.py index 1e4d2ab85dbb..076e97ff547a 100644 --- a/gateway/platforms/telegram.py +++ b/gateway/platforms/telegram.py @@ -29,17 +29,7 @@ Bot = Any Message = Any Application = Any - CommandHandler = Any - TelegramMessageHandler = Any - filters = None - ParseMode = None - ChatType = None - - # Mock ContextTypes so type annotations using ContextTypes.DEFAULT_TYPE - # don't crash during class definition when the library isn't installed. - class _MockContextTypes: - DEFAULT_TYPE = Any - ContextTypes = _MockContextTypes + ContextTypes = Any import sys from pathlib import Path as _Path diff --git a/gateway/run.py b/gateway/run.py index 7471bc55384a..6abd4775468f 100644 --- a/gateway/run.py +++ b/gateway/run.py @@ -20,6 +20,7 @@ import sys import signal import threading +import time from logging.handlers import RotatingFileHandler from pathlib import Path from datetime import datetime @@ -33,14 +34,26 @@ # Load environment variables from ~/.hermes/.env first from dotenv import load_dotenv +from agent.env_loader import load_dotenv_with_fallback _env_path = _hermes_home / '.env' if _env_path.exists(): try: - load_dotenv(_env_path, encoding="utf-8") - except UnicodeDecodeError: - load_dotenv(_env_path, encoding="latin-1") + load_dotenv_with_fallback(_env_path, logger=logging.getLogger(__name__)) + except ValueError as exc: + print(f"Failed to load {_env_path}: {exc}", file=sys.stderr) + raise SystemExit(2) from exc # Also try project .env as fallback -load_dotenv() +_project_env = Path(__file__).parent.parent / '.env' +if _project_env.exists(): + try: + load_dotenv_with_fallback( + _project_env, + override=False, + logger=logging.getLogger(__name__), + ) + except ValueError as exc: + print(f"Failed to load {_project_env}: {exc}", file=sys.stderr) + raise SystemExit(2) from exc # Bridge config.yaml values into the environment so os.getenv() picks them up. # config.yaml is authoritative for terminal settings — overrides .env. @@ -48,7 +61,7 @@ if _config_path.exists(): try: import yaml as _yaml - with open(_config_path) as _f: + with open(_config_path, encoding="utf-8") as _f: _cfg = _yaml.safe_load(_f) or {} # Top-level simple values (fallback only — don't override .env) for _key, _val in _cfg.items(): @@ -101,10 +114,16 @@ # Enable interactive exec approval for dangerous commands on messaging platforms os.environ["HERMES_EXEC_ASK"] = "1" -# Set terminal working directory for messaging platforms -# Uses MESSAGING_CWD if set, otherwise defaults to home directory -# This is separate from CLI which uses the directory where `hermes` is run -messaging_cwd = os.getenv("MESSAGING_CWD") or str(Path.home()) +# Set terminal working directory for messaging platforms. +# Uses MESSAGING_CWD if set. Otherwise prefer ~/.hermes/workspace (where +# project files are commonly located), then ~/.hermes, then home. +# This is separate from CLI which uses the directory where `hermes` is run. +_default_messaging_cwd = Path.home() / ".hermes" / "workspace" +if not _default_messaging_cwd.exists(): + _default_messaging_cwd = Path.home() / ".hermes" +if not _default_messaging_cwd.exists(): + _default_messaging_cwd = Path.home() +messaging_cwd = os.getenv("MESSAGING_CWD") or str(_default_messaging_cwd) os.environ["TERMINAL_CWD"] = messaging_cwd from gateway.config import ( @@ -124,6 +143,36 @@ logger = logging.getLogger(__name__) +# Default model when nothing is configured (must match hermes_cli.config DEFAULT_CONFIG / CLI fallback) +_DEFAULT_MODEL = "google/gemini-2.0-flash-001:free" + + +def _resolve_gateway_model() -> str: + """ + Resolve the model for gateway agents. Same priority as CLI so gateway and + CLI share one source of truth: HERMES_MODEL (session override) > LLM_MODEL + > OPENAI_MODEL > config.yaml model > default. + """ + model = os.getenv("HERMES_MODEL") or os.getenv("LLM_MODEL") or os.getenv("OPENAI_MODEL") + if model: + return model.strip() + try: + import yaml as _yaml + cfg_path = Path.home() / ".hermes" / "config.yaml" + if cfg_path.exists(): + with open(cfg_path, encoding="utf-8") as _f: + cfg = _yaml.safe_load(_f) or {} + m = cfg.get("model") + if isinstance(m, str) and m.strip(): + return m.strip() + if isinstance(m, dict): + default = (m.get("default") or "").strip() + if default: + return default + except Exception: + pass + return _DEFAULT_MODEL + def _resolve_runtime_agent_kwargs() -> dict: """Resolve provider credentials for gateway-created AIAgent instances.""" @@ -272,7 +321,7 @@ def _load_prefill_messages() -> List[Dict[str, Any]]: import yaml as _y cfg_path = _hermes_home / "config.yaml" if cfg_path.exists(): - with open(cfg_path) as _f: + with open(cfg_path, encoding="utf-8") as _f: cfg = _y.safe_load(_f) or {} file_path = cfg.get("prefill_messages_file", "") except Exception: @@ -310,7 +359,7 @@ def _load_ephemeral_system_prompt() -> str: import yaml as _y cfg_path = _hermes_home / "config.yaml" if cfg_path.exists(): - with open(cfg_path) as _f: + with open(cfg_path, encoding="utf-8") as _f: cfg = _y.safe_load(_f) or {} return (cfg.get("agent", {}).get("system_prompt", "") or "").strip() except Exception: @@ -331,7 +380,7 @@ def _load_reasoning_config() -> dict | None: import yaml as _y cfg_path = _hermes_home / "config.yaml" if cfg_path.exists(): - with open(cfg_path) as _f: + with open(cfg_path, encoding="utf-8") as _f: cfg = _y.safe_load(_f) or {} effort = str(cfg.get("agent", {}).get("reasoning_effort", "") or "").strip() except Exception: @@ -419,11 +468,11 @@ async def start(self) -> bool: if success: self.adapters[platform] = adapter connected_count += 1 - logger.info("✓ %s connected", platform.value) + logger.info("ok: %s connected", platform.value) else: - logger.warning("✗ %s failed to connect", platform.value) + logger.warning("x: %s failed to connect", platform.value) except Exception as e: - logger.error("✗ %s error: %s", platform.value, e) + logger.error("x: %s error: %s", platform.value, e) if connected_count == 0: logger.warning("No messaging platforms connected.") @@ -466,9 +515,9 @@ async def stop(self) -> None: for platform, adapter in self.adapters.items(): try: await adapter.disconnect() - logger.info("✓ %s disconnected", platform.value) + logger.info("%s disconnected", platform.value) except Exception as e: - logger.error("✗ %s disconnect error: %s", platform.value, e) + logger.error("%s disconnect error: %s", platform.value, e) self.adapters.clear() self._shutdown_event.set() @@ -591,9 +640,24 @@ async def _handle_message(self, event: MessageEvent) -> Optional[str]: 7. Return response """ source = event.source + logger.info( + "Gateway message received: platform=%s chat_type=%s chat_id=%s user_id=%s text_len=%s", + source.platform.value if source.platform else "unknown", + source.chat_type, + source.chat_id, + source.user_id, + len(event.text or ""), + ) + authorized = self._is_user_authorized(source) + logger.info( + "Gateway auth check: platform=%s user_id=%s authorized=%s", + source.platform.value if source.platform else "unknown", + source.user_id, + authorized, + ) # Check if user is authorized - if not self._is_user_authorized(source): + if not authorized: logger.warning("Unauthorized user: %s (%s) on %s", source.user_id, source.user_name, source.platform.value) # In DMs: offer pairing code. In groups: silently ignore. if source.chat_type == "dm": @@ -641,11 +705,14 @@ async def _handle_message(self, event: MessageEvent) -> Optional[str]: # Check for commands command = event.get_command() - + if command: + logger.info("Gateway command detected: /%s from user_id=%s", command, source.user_id) # Emit command:* hook for any recognized slash command - _known_commands = {"new", "reset", "help", "status", "stop", "model", - "personality", "retry", "undo", "sethome", "set-home", - "compress", "usage", "reload-mcp"} + _known_commands = { + "new", "reset", "help", "status", "stop", "model", + "personality", "retry", "undo", "sethome", "set-home", + "terminal", "shell", "compress", "usage", "reload-mcp" + } if command and command in _known_commands: await self.hooks.emit(f"command:{command}", { "platform": source.platform.value if source.platform else "", @@ -653,7 +720,7 @@ async def _handle_message(self, event: MessageEvent) -> Optional[str]: "command": command, "args": event.get_command_args().strip(), }) - + if command in ["new", "reset"]: return await self._handle_reset_command(event) @@ -672,6 +739,9 @@ async def _handle_message(self, event: MessageEvent) -> Optional[str]: if command == "personality": return await self._handle_personality_command(event) + if command in ["terminal", "shell"]: + return await self._handle_terminal_command(event) + if command == "retry": return await self._handle_retry_command(event) @@ -934,11 +1004,53 @@ async def _handle_message(self, event: MessageEvent) -> Optional[str]: { "role": "session_meta", "tools": tool_defs or [], - "model": os.getenv("HERMES_MODEL", ""), + "model": _resolve_gateway_model(), "platform": source.platform.value if source.platform else "", "timestamp": ts, } ) + + def _strip_ts(msgs: List[Dict[str, Any]]) -> List[Dict[str, Any]]: + normalized: List[Dict[str, Any]] = [] + for m in msgs or []: + if not isinstance(m, dict): + continue + if m.get("role") == "system": + continue + normalized.append({k: v for k, v in m.items() if k != "timestamp"}) + return normalized + + normalized_history = _strip_ts(history) + normalized_agent = _strip_ts(agent_messages) + history_prefix_matches = ( + len(normalized_agent) >= len(normalized_history) + and normalized_agent[: len(normalized_history)] == normalized_history + ) + history_was_rewritten = bool(normalized_history) and not history_prefix_matches + + # If agent history was compressed/rewritten, replace transcript with + # the returned message list so future turns stay compact. + if history_was_rewritten: + rewritten: List[Dict[str, Any]] = [] + + # Preserve an existing session_meta entry if present. + existing_meta = next( + (m for m in (history or []) if isinstance(m, dict) and m.get("role") == "session_meta"), + None, + ) + if existing_meta: + meta_entry = dict(existing_meta) + meta_entry.setdefault("timestamp", ts) + rewritten.append(meta_entry) + + for msg in normalized_agent: + entry = dict(msg) + entry.setdefault("timestamp", ts) + rewritten.append(entry) + + self.session_store.rewrite_transcript(session_entry.session_id, rewritten) + self.session_store.update_session(session_entry.session_key) + return response # Find only the NEW messages from this turn (skip history we loaded) history_len = len(history) @@ -1115,7 +1227,7 @@ async def _handle_model_command(self, event: MessageEvent) -> str: current = os.getenv("HERMES_MODEL") or os.getenv("LLM_MODEL") or "anthropic/claude-opus-4.6" try: if config_path.exists(): - with open(config_path) as f: + with open(config_path, encoding="utf-8") as f: cfg = yaml.safe_load(f) or {} model_cfg = cfg.get("model", {}) if isinstance(model_cfg, str): @@ -1141,12 +1253,12 @@ async def _handle_model_command(self, event: MessageEvent) -> str: try: user_config = {} if config_path.exists(): - with open(config_path) as f: + with open(config_path, encoding="utf-8") as f: user_config = yaml.safe_load(f) or {} if "model" not in user_config or not isinstance(user_config["model"], dict): user_config["model"] = {} user_config["model"]["default"] = args - with open(config_path, 'w') as f: + with open(config_path, "w", encoding="utf-8", newline="") as f: yaml.dump(user_config, f, default_flow_style=False, sort_keys=False) except Exception as e: return f"⚠️ Failed to save model change: {e}" @@ -1165,7 +1277,7 @@ async def _handle_personality_command(self, event: MessageEvent) -> str: try: if config_path.exists(): - with open(config_path, 'r') as f: + with open(config_path, 'r', encoding='utf-8') as f: config = yaml.safe_load(f) or {} personalities = config.get("agent", {}).get("personalities", {}) else: @@ -1187,26 +1299,121 @@ async def _handle_personality_command(self, event: MessageEvent) -> str: return "\n".join(lines) if args in personalities: - new_prompt = personalities[args] - - # Write to config.yaml, same pattern as CLI save_config_value. + selected_prompt = personalities[args] + # Backward-compatible legacy env var (not read by current runtime). + os.environ["HERMES_PERSONALITY"] = selected_prompt + # Runtime-consumed ephemeral prompt env var. + os.environ["HERMES_EPHEMERAL_SYSTEM_PROMPT"] = selected_prompt + # Keep current GatewayRunner instance in sync immediately. + self._ephemeral_system_prompt = selected_prompt + + # Persist as agent.system_prompt so restarts keep the selected personality. try: - if "agent" not in config or not isinstance(config.get("agent"), dict): - config["agent"] = {} - config["agent"]["system_prompt"] = new_prompt - with open(config_path, 'w') as f: - yaml.dump(config, f, default_flow_style=False, sort_keys=False) + if config_path.exists(): + with open(config_path, 'r', encoding='utf-8') as f: + persisted = yaml.safe_load(f) or {} + else: + persisted = {} + persisted.setdefault("agent", {}) + persisted["agent"]["system_prompt"] = selected_prompt + with open(config_path, 'w', encoding='utf-8', newline='') as f: + yaml.dump(persisted, f, default_flow_style=False) except Exception as e: - return f"⚠️ Failed to save personality change: {e}" - - # Update in-memory so it takes effect on the very next message. - self._ephemeral_system_prompt = new_prompt + logger.warning("Failed to persist selected personality prompt: %s", e) return f"🎭 Personality set to **{args}**\n_(takes effect on next message)_" available = ", ".join(f"`{n}`" for n in personalities.keys()) return f"Unknown personality: `{args}`\n\nAvailable: {available}" + async def _handle_terminal_command(self, event: MessageEvent) -> str: + """Handle /terminal command - show or change the current Windows shell mode. + + Syntax (Windows only): + /terminal windows -> HERMES_WINDOWS_SHELL=powershell + /terminal wsl -> HERMES_WINDOWS_SHELL=wsl + /terminal auto -> HERMES_WINDOWS_SHELL=auto (WSL > PowerShell > cmd) + /terminal cmd -> HERMES_WINDOWS_SHELL=cmd + + On non-Windows hosts this command is a no-op and reports that only POSIX + shells are available. + """ + args = event.get_command_args().strip().lower() + + if os.name != "nt": + return ( + "Terminal mode only applies when the **gateway** runs on Windows. " + "Right now the gateway is running on Linux/WSL, so local commands " + "always use your default (POSIX) shell. Setting PowerShell here has no effect; " + "to use PowerShell for commands, start the gateway from Windows (e.g. PowerShell or cmd)." + ) + + current = os.getenv("HERMES_WINDOWS_SHELL", "auto") + if not args: + return ( + "🖥️ **Current terminal mode:** " + f"`{current}`\n\n" + "Usage: `/terminal powershell`, `/terminal wsl`, `/terminal auto`, `/terminal cmd`" + ) + + mode_map = { + "windows": "powershell", + "pwsh": "powershell", + "powershell": "powershell", + "wsl": "wsl", + "linux": "wsl", + "auto": "auto", + "cmd": "cmd", + "cmd.exe": "cmd", + } + + if args not in mode_map: + return ( + f"Unknown terminal mode: `{args}`\n\n" + "Valid options on Windows are: `powershell`, `wsl`, `auto`, `cmd` " + "(aliases like `windows` and `pwsh` also map to `powershell`)." + ) + + new_value = mode_map[args] + os.environ["HERMES_WINDOWS_SHELL"] = new_value + logger.info( + "Terminal mode switched to %s (HERMES_WINDOWS_SHELL=%s) by user", + new_value, + new_value, + ) + + # Persist into ~/.hermes/config.yaml so restarts keep the selected mode. + try: + import yaml + config_path = Path.home() / ".hermes" / "config.yaml" + if config_path.exists(): + with open(config_path, "r", encoding="utf-8") as f: + persisted = yaml.safe_load(f) or {} + else: + persisted = {} + + # Top-level key so shell_utils can see it even without nested terminal config. + persisted["HERMES_WINDOWS_SHELL"] = new_value + + with open(config_path, "w", encoding="utf-8", newline="") as f: + yaml.dump(persisted, f, default_flow_style=False) + except Exception as e: + logger.warning("Failed to persist terminal mode to config.yaml: %s", e) + + human_mode = { + "powershell": "PowerShell", + "wsl": "WSL (Linux shell)", + "cmd": "cmd.exe", + "auto": "auto (WSL > PowerShell > cmd)", + }.get(new_value, new_value) + + return ( + f"🖥️ Terminal mode set to **{human_mode}** " + f"(`HERMES_WINDOWS_SHELL={new_value}`).\n" + "Future *local* commands will use this shell. " + "Remote/Unix environments (Docker/SSH/etc.) still use POSIX shells." + ) + async def _handle_retry_command(self, event: MessageEvent) -> str: """Handle /retry command - re-send the last user message.""" source = event.source @@ -1279,10 +1486,10 @@ async def _handle_set_home_command(self, event: MessageEvent) -> str: config_path = _hermes_home / 'config.yaml' user_config = {} if config_path.exists(): - with open(config_path) as f: + with open(config_path, encoding='utf-8') as f: user_config = yaml.safe_load(f) or {} user_config[env_key] = chat_id - with open(config_path, 'w') as f: + with open(config_path, 'w', encoding='utf-8', newline='') as f: yaml.dump(user_config, f, default_flow_style=False) # Also set in the current environment so it takes effect immediately os.environ[env_key] = str(chat_id) @@ -1700,7 +1907,7 @@ async def _run_agent( config_path = _hermes_home / 'config.yaml' if config_path.exists(): import yaml - with open(config_path, 'r') as f: + with open(config_path, 'r', encoding='utf-8') as f: user_config = yaml.safe_load(f) or {} platform_toolsets_config = user_config.get("platform_toolsets", {}) except Exception as e: @@ -1730,7 +1937,7 @@ async def _run_agent( _tp_cfg_path = _hermes_home / "config.yaml" if _tp_cfg_path.exists(): import yaml as _tp_yaml - with open(_tp_cfg_path) as _tp_f: + with open(_tp_cfg_path, encoding="utf-8") as _tp_f: _tp_data = _tp_yaml.safe_load(_tp_f) or {} _progress_cfg = _tp_data.get("display", {}) except Exception: @@ -1745,6 +1952,54 @@ async def _run_agent( # Queue for progress messages (thread-safe) progress_queue = queue.Queue() if tool_progress_enabled else None last_tool = [None] # Mutable container for tracking in closure + progress_state = { + "started_at": time.monotonic(), + "phase": "thinking", + "last_detail": "Queued request...", + "tool_calls": 0, + "updates": 0, + } + heartbeat_faces = ["(·_·)", "(•‿•)", "(⌐■_■)", "(^_^)/"] + phase_labels = { + "starting": "starting", + "thinking": "thinking", + "tool": "using tools", + "finalizing": "finalizing", + } + phase_emojis = { + "starting": "🚀", + "thinking": "💡", + "tool": "🛠️", + "finalizing": "🧾", + } + + def _set_progress_state(*, phase: str | None = None, detail: str | None = None, tool_call: bool = False): + if phase: + progress_state["phase"] = phase + if detail: + progress_state["last_detail"] = detail + if tool_call: + progress_state["tool_calls"] += 1 + progress_state["updates"] += 1 + if progress_queue: + # A tiny queue signal wakes the async updater. Actual text is built + # centrally so tool updates + heartbeat share one edited message. + progress_queue.put("__tick__") + + def _build_progress_message() -> str: + elapsed = int(max(0, time.monotonic() - progress_state["started_at"])) + face = heartbeat_faces[elapsed % len(heartbeat_faces)] + phase = progress_state.get("phase", "thinking") + phase_text = phase_labels.get(phase, "working") + phase_emoji = phase_emojis.get(phase, "⚙️") + detail = (progress_state.get("last_detail") or "Working...").strip() + if len(detail) > 180: + detail = detail[:177] + "..." + return ( + f"{phase_emoji} {face} Hermes is {phase_text}... ({elapsed}s)\n" + f"{detail}\n" + f"Tools: {progress_state['tool_calls']} | Updates: {progress_state['updates']}" + ) def progress_callback(tool_name: str, preview: str = None, args: dict = None): """Callback invoked by agent when a tool is called.""" @@ -1806,7 +2061,7 @@ def progress_callback(tool_name: str, preview: str = None, args: dict = None): if len(args_str) > 200: args_str = args_str[:197] + "..." msg = f"{emoji} {tool_name}({list(args.keys())})\n{args_str}" - progress_queue.put(msg) + _set_progress_state(phase="tool", detail=msg, tool_call=tool_name != "_thinking") return if preview: @@ -1816,8 +2071,11 @@ def progress_callback(tool_name: str, preview: str = None, args: dict = None): msg = f"{emoji} {tool_name}: \"{preview}\"" else: msg = f"{emoji} {tool_name}..." - - progress_queue.put(msg) + + if tool_name == "_thinking": + _set_progress_state(phase="thinking", detail=f"💡 thinking: {preview or 'working...'}") + else: + _set_progress_state(phase="tool", detail=msg, tool_call=True) # Background task to send progress messages async def send_progress_messages(): @@ -1827,25 +2085,58 @@ async def send_progress_messages(): adapter = self.adapters.get(source.platform) if not adapter: return - + + # Keep track of a single in-flight progress message so we can edit + # it in place instead of spamming the channel with many updates. + progress_message_id: str | None = None + last_heartbeat_at = 0.0 + last_rendered = "" + while True: + # Drain queue signals so bursts collapse into one edit. + got_signal = False + while True: + try: + progress_queue.get_nowait() + got_signal = True + except queue.Empty: + break + try: - # Non-blocking check with small timeout - msg = progress_queue.get_nowait() - await adapter.send(chat_id=source.chat_id, content=msg) - # Restore typing indicator after sending progress message - await asyncio.sleep(0.3) - await adapter.send_typing(source.chat_id) - except queue.Empty: - await asyncio.sleep(0.3) # Check again soon - except asyncio.CancelledError: - # Drain remaining messages - while not progress_queue.empty(): + now = time.monotonic() + heartbeat_due = (now - last_heartbeat_at) >= 1.0 + if not got_signal and not heartbeat_due: + await asyncio.sleep(0.2) + continue + + msg = _build_progress_message() + if not got_signal and not heartbeat_due and msg == last_rendered: + await asyncio.sleep(0.2) + continue + + # For platforms that support edits (Discord/Telegram/etc.), + # send the first progress message and then edit it in place + # for subsequent updates. + if progress_message_id is None: + result = await adapter.send(chat_id=source.chat_id, content=msg) + if result and result.success and result.message_id: + progress_message_id = result.message_id + else: try: - msg = progress_queue.get_nowait() - await adapter.send(chat_id=source.chat_id, content=msg) + await adapter.edit_message( + chat_id=source.chat_id, + message_id=progress_message_id, + content=msg, + ) except Exception: - break + result = await adapter.send(chat_id=source.chat_id, content=msg) + if result and result.success and result.message_id: + progress_message_id = result.message_id + + last_rendered = msg + last_heartbeat_at = now + await adapter.send_typing(source.chat_id) + except asyncio.CancelledError: return except Exception as e: logger.error("Progress message error: %s", e) @@ -1876,12 +2167,40 @@ def _step_callback_sync(iteration: int, tool_names: list) -> None: logger.debug("agent:step hook error: %s", _e) def run_sync(): + if tool_progress_enabled: + _set_progress_state(phase="starting", detail="Preparing gateway session...") + + # Do NOT overwrite mode from config on every turn. + # /terminal updates os.environ immediately; config is only persistence for restarts. + if os.name == "nt": + current_shell = os.environ.get("HERMES_WINDOWS_SHELL", "auto") + logger.info( + "Agent run: using in-memory HERMES_WINDOWS_SHELL=%s", + current_shell, + ) + # Force next get_local_shell_mode() call to emit a mode log entry. + try: + import tools.environments.shell_utils as _sh + _sh._last_logged_mode = None + except Exception: + pass + # Pass session_key to process registry via env var so background # processes can be mapped back to this gateway session os.environ["HERMES_SESSION_KEY"] = session_key or "" # Read from env var or use default (same as CLI) max_iterations = int(os.getenv("HERMES_MAX_ITERATIONS", "60")) + max_tokens_env = os.getenv("HERMES_MAX_TOKENS", "").strip() + if max_tokens_env == "0": + max_tokens = None # use model default + elif max_tokens_env: + max_tokens = int(max_tokens_env) + else: + max_tokens = 32768 + # Default 32768 when unset so long replies aren't cut off when the model + # uses reasoning/thinking (which consumes tokens). Set HERMES_MAX_TOKENS=0 + # to use the model default; set to a number (e.g. 8192, 32768) to override. # Map platform enum to the platform hint key the agent understands. # Platform.LOCAL ("local") maps to "cli"; others pass through as-is. @@ -1893,11 +2212,9 @@ def run_sync(): combined_ephemeral = (combined_ephemeral + "\n\n" + self._ephemeral_system_prompt).strip() # Re-read .env and config for fresh credentials (gateway is long-lived, - # keys may change without restart). + # keys may change without restart). Use encoding-safe loader. try: - load_dotenv(_env_path, override=True, encoding="utf-8") - except UnicodeDecodeError: - load_dotenv(_env_path, override=True, encoding="latin-1") + load_dotenv_with_fallback(_env_path, override=True, logger=logging.getLogger(__name__)) except Exception: pass @@ -1907,13 +2224,18 @@ def run_sync(): import yaml as _y _cfg_path = _hermes_home / "config.yaml" if _cfg_path.exists(): - with open(_cfg_path) as _f: + with open(_cfg_path, encoding="utf-8") as _f: _cfg = _y.safe_load(_f) or {} _model_cfg = _cfg.get("model", {}) if isinstance(_model_cfg, str): model = _model_cfg elif isinstance(_model_cfg, dict): model = _model_cfg.get("default", model) + # Apply persisted terminal mode so /terminal choice wins over .env. + # Otherwise load_dotenv(override=True) can overwrite in-memory wsl with .env's value. + _shell = _cfg.get("HERMES_WINDOWS_SHELL") + if isinstance(_shell, str) and _shell.strip(): + os.environ["HERMES_WINDOWS_SHELL"] = _shell.strip().lower() except Exception: pass @@ -1932,6 +2254,7 @@ def run_sync(): model=model, **runtime_kwargs, max_iterations=max_iterations, + max_tokens=max_tokens, quiet_mode=True, verbose_logging=False, enabled_toolsets=enabled_toolsets, @@ -2012,8 +2335,12 @@ def run_sync(): if _p: _history_media_paths.add(_p) + if tool_progress_enabled: + _set_progress_state(phase="thinking", detail="Calling model...") result = agent.run_conversation(message, conversation_history=agent_history) result_holder[0] = result + if tool_progress_enabled: + _set_progress_state(phase="finalizing", detail="Preparing final response...") # Return final response, or a message if something went wrong final_response = result.get("final_response") @@ -2057,6 +2384,44 @@ def run_sync(): if tag not in seen: seen.add(tag) unique_tags.append(tag) + + # Drop media tags that point to files no longer on disk. + # This prevents stale-session replay from repeatedly trying + # to send deleted TTS artifacts on startup/new turns. + existing_tags = [] + missing_count = 0 + for tag in unique_tags: + raw_path = tag.removeprefix("MEDIA:") + if os.path.exists(raw_path): + existing_tags.append(tag) + else: + missing_count += 1 + if missing_count: + logger.warning( + "Dropped %d stale media tag(s) with missing file path(s)", + missing_count, + ) + unique_tags = existing_tags + + # Safety valve: cap auto-appended audio attachments to one + # per response (keep the most recent) to prevent TTS floods + # when the model triggers multiple text_to_speech calls. + audio_idx = [] + for i, tag in enumerate(unique_tags): + path = tag.removeprefix("MEDIA:").lower() + if path.endswith((".ogg", ".opus", ".mp3", ".wav", ".m4a", ".webm")): + audio_idx.append(i) + if len(audio_idx) > 1: + keep = audio_idx[-1] + unique_tags = [ + tag for i, tag in enumerate(unique_tags) + if i == keep or i not in audio_idx + ] + logger.warning( + "Capped auto media delivery: dropped %d extra audio tag(s)", + len(audio_idx) - 1, + ) + if has_voice_directive: unique_tags.insert(0, "[[audio_as_voice]]") final_response = final_response + "\n" + "\n".join(unique_tags) @@ -2071,6 +2436,7 @@ def run_sync(): # Start progress message sender if enabled progress_task = None if tool_progress_enabled: + _set_progress_state(phase="thinking", detail="Hermes is thinking...") progress_task = asyncio.create_task(send_progress_messages()) # Track this agent as running for this session (for interrupt support) @@ -2227,13 +2593,20 @@ async def start_gateway(config: Optional[GatewayConfig] = None) -> bool: Returns True if the gateway ran successfully, False if it failed to start. A False return causes a non-zero exit code so systemd can auto-restart. """ - # Configure rotating file log so gateway output is persisted for debugging + # Configure rotating file log so gateway output is persisted for debugging. + # On Windows, reconfigure stderr to UTF-8 so emoji/Unicode in log messages don't cause UnicodeEncodeError. + if sys.stderr and getattr(sys.stderr, 'reconfigure', None) is not None: + try: + sys.stderr.reconfigure(encoding='utf-8', errors='replace') + except Exception: + pass log_dir = _hermes_home / 'logs' log_dir.mkdir(parents=True, exist_ok=True) file_handler = RotatingFileHandler( log_dir / 'gateway.log', maxBytes=5 * 1024 * 1024, backupCount=3, + encoding='utf-8', ) from agent.redact import RedactingFormatter file_handler.setFormatter(RedactingFormatter('%(asctime)s %(levelname)s %(name)s: %(message)s')) @@ -2257,11 +2630,17 @@ def signal_handler(): asyncio.create_task(runner.stop()) loop = asyncio.get_event_loop() - for sig in (signal.SIGINT, signal.SIGTERM): - try: - loop.add_signal_handler(sig, signal_handler) - except NotImplementedError: - pass + # On POSIX, integrate SIGINT/SIGTERM with the event loop so a signal + # triggers a graceful shutdown. On Windows we deliberately do NOT + # override SIGINT handling here and instead rely on the default + # KeyboardInterrupt behaviour in `hermes_cli.gateway.run_gateway`, + # which already catches Ctrl+C and exits cleanly. + if os.name != "nt": + for sig in (signal.SIGINT, signal.SIGTERM): + try: + loop.add_signal_handler(sig, signal_handler) + except NotImplementedError: + pass # Start the gateway success = await runner.start() @@ -2315,7 +2694,7 @@ def main(): config = None if args.config: import json - with open(args.config) as f: + with open(args.config, encoding="utf-8") as f: data = json.load(f) config = GatewayConfig.from_dict(data) diff --git a/gateway/session.py b/gateway/session.py index b59196b81c45..67ae81688f0a 100644 --- a/gateway/session.py +++ b/gateway/session.py @@ -317,7 +317,7 @@ def _ensure_loaded(self) -> None: if sessions_file.exists(): try: - with open(sessions_file, "r") as f: + with open(sessions_file, "r", encoding="utf-8") as f: data = json.load(f) for key, entry_data in data.items(): self._entries[key] = SessionEntry.from_dict(entry_data) @@ -332,7 +332,7 @@ def _save(self) -> None: sessions_file = self.sessions_dir / "sessions.json" data = {key: entry.to_dict() for key, entry in self._entries.items()} - with open(sessions_file, "w") as f: + with open(sessions_file, "w", encoding="utf-8", newline="") as f: json.dump(data, f, indent=2) def _generate_session_key(self, source: SessionSource) -> str: @@ -569,10 +569,14 @@ def append_to_transcript(self, session_id: str, message: Dict[str, Any]) -> None except Exception as e: logger.debug("Session DB operation failed: %s", e) - # Also write legacy JSONL (keeps existing tooling working during transition) + # Also write legacy JSONL (keeps existing tooling working during transition). + # Always use encoding="utf-8" and errors="replace" so we never raise on Windows + # (default cp1252) or on odd Unicode in tool output, and the gateway always + # gets to send the response to the user. transcript_path = self.get_transcript_path(session_id) - with open(transcript_path, "a") as f: - f.write(json.dumps(message, ensure_ascii=False) + "\n") + line = json.dumps(message, ensure_ascii=False) + "\n" + with open(transcript_path, "a", encoding="utf-8", errors="replace", newline="") as f: + f.write(line) def rewrite_transcript(self, session_id: str, messages: List[Dict[str, Any]]) -> None: """Replace the entire transcript for a session with new messages. @@ -598,7 +602,7 @@ def rewrite_transcript(self, session_id: str, messages: List[Dict[str, Any]]) -> # JSONL: overwrite the file transcript_path = self.get_transcript_path(session_id) - with open(transcript_path, "w") as f: + with open(transcript_path, "w", encoding="utf-8", errors="replace", newline="") as f: for msg in messages: f.write(json.dumps(msg, ensure_ascii=False) + "\n") @@ -620,7 +624,7 @@ def load_transcript(self, session_id: str) -> List[Dict[str, Any]]: return [] messages = [] - with open(transcript_path, "r") as f: + with open(transcript_path, "r", encoding="utf-8", errors="replace") as f: for line in f: line = line.strip() if line: diff --git a/hermes_cli/auth.py b/hermes_cli/auth.py index 34b07b71b05d..19f923e66a89 100644 --- a/hermes_cli/auth.py +++ b/hermes_cli/auth.py @@ -600,18 +600,28 @@ def resolve_codex_runtime_credentials( try: data = _read_codex_tokens() except AuthError as orig_err: - # Only attempt migration when there are NO tokens stored at all - # (code == "codex_auth_missing"), not when tokens exist but are invalid. - if orig_err.code != "codex_auth_missing": + # Auto-recover from legacy/partial Codex auth states by importing + # tokens from ~/.codex/auth.json when available. + recoverable_codes = { + "codex_auth_missing", + "codex_auth_invalid_shape", + "codex_auth_missing_access_token", + "codex_auth_missing_refresh_token", + } + if orig_err.code not in recoverable_codes: raise - # Migration: user had Codex as active provider with old storage (~/.codex/). + # Migration/recovery: user may have valid Codex CLI tokens in ~/.codex/. cli_tokens = _import_codex_cli_tokens() if cli_tokens: - logger.info("Migrating Codex credentials from ~/.codex/ to Hermes auth store") - print("⚠️ Migrating Codex credentials to Hermes's own auth store.") - print(" This avoids conflicts with Codex CLI and VS Code.") - print(" Run `hermes login` to create a fully independent session.\n") + logger.info( + "Recovering Codex credentials from ~/.codex/ to Hermes auth store " + "(reason=%s)", + orig_err.code, + ) + print("WARNING: Migrating Codex credentials to Hermes auth store.") + print(" This avoids conflicts with Codex CLI and VS Code.") + print(" Run `hermes login` to create a fully independent session.\n") _save_codex_tokens(cli_tokens) data = _read_codex_tokens() else: diff --git a/hermes_cli/config.py b/hermes_cli/config.py index cb62db9dbe4c..e0bf63779a8a 100644 --- a/hermes_cli/config.py +++ b/hermes_cli/config.py @@ -13,17 +13,15 @@ """ import os -import platform import sys import subprocess from pathlib import Path from typing import Dict, Any, Optional, List, Tuple -_IS_WINDOWS = platform.system() == "Windows" - import yaml from hermes_cli.colors import Colors, color +from agent.env_loader import read_env_text_with_fallback # ============================================================================= @@ -60,7 +58,7 @@ def ensure_hermes_home(): # ============================================================================= DEFAULT_CONFIG = { - "model": "anthropic/claude-opus-4.6", + "model": "google/gemini-2.0-flash-001:free", "toolsets": ["hermes-cli"], "max_turns": 100, @@ -92,8 +90,9 @@ def ensure_hermes_home(): "tts": { "provider": "edge", # "edge" (free) | "elevenlabs" (premium) | "openai" "edge": { - "voice": "en-US-AriaNeural", - # Popular: AriaNeural, JennyNeural, AndrewNeural, BrianNeural, SoniaNeural + "voice": "en-US-AvaMultilingualNeural", + "rate": "125%", + # Popular: AvaMultilingualNeural, AriaNeural, JennyNeural }, "elevenlabs": { "voice_id": "pNInz6obpgDQGcFmaJgB", # Adam @@ -355,11 +354,46 @@ def ensure_hermes_home(): } -def get_missing_env_vars(required_only: bool = False) -> List[Dict[str, Any]]: +def _is_optional_env_var_enabled(var_name: str, config: Optional[Dict[str, Any]] = None) -> bool: + """ + Return whether an optional env var is currently enabled by config. + + This is used by setup/migration flows that should only treat active + optional integrations as "missing". Optional keys for integrations that + are not selected should remain non-blocking. + """ + if config is None: + config = load_config() + + tts_cfg = config.get("tts", {}) if isinstance(config, dict) else {} + tts_provider = str(tts_cfg.get("provider", "edge")).strip().lower() + + if var_name == "ELEVENLABS_API_KEY": + return tts_provider == "elevenlabs" + + if var_name == "VOICE_TOOLS_OPENAI_KEY": + return tts_provider == "openai" + + # Other optional integrations are opt-in via setup checklists; they should + # not block setup completion when unset. + return False + + +def get_missing_env_vars( + required_only: bool = False, + enabled_only: bool = False, + config: Optional[Dict[str, Any]] = None, +) -> List[Dict[str, Any]]: """ Check which environment variables are missing. - - Returns list of dicts with var info for missing variables. + + Args: + required_only: If True, return only required vars. + enabled_only: If True, include only optional vars enabled by config. + config: Optional config dict to use when evaluating enabled-only logic. + + Returns: + List of metadata dicts for missing variables. """ missing = [] @@ -370,7 +404,10 @@ def get_missing_env_vars(required_only: bool = False) -> List[Dict[str, Any]]: # Check optional vars (if not required_only) if not required_only: + effective_config = config if config is not None else (load_config() if enabled_only else None) for var_name, info in OPTIONAL_ENV_VARS.items(): + if enabled_only and not _is_optional_env_var_enabled(var_name, effective_config): + continue if not get_env_value(var_name): missing.append({"name": var_name, **info, "is_required": False}) @@ -596,7 +633,7 @@ def load_config() -> Dict[str, Any]: if config_path.exists(): try: - with open(config_path) as f: + with open(config_path, encoding="utf-8") as f: user_config = yaml.safe_load(f) or {} config = _deep_merge(config, user_config) @@ -611,7 +648,7 @@ def save_config(config: Dict[str, Any]): ensure_hermes_home() config_path = get_config_path() - with open(config_path, 'w') as f: + with open(config_path, 'w', encoding='utf-8', newline='') as f: yaml.dump(config, f, default_flow_style=False, sort_keys=False) @@ -621,15 +658,16 @@ def load_env() -> Dict[str, str]: env_vars = {} if env_path.exists(): - # On Windows, open() defaults to the system locale (cp1252) which can - # fail on UTF-8 .env files. Use explicit UTF-8 only on Windows. - open_kw = {"encoding": "utf-8", "errors": "replace"} if _IS_WINDOWS else {} - with open(env_path, **open_kw) as f: - for line in f: - line = line.strip() - if line and not line.startswith('#') and '=' in line: - key, _, value = line.partition('=') - env_vars[key.strip()] = value.strip().strip('"\'') + try: + content, _ = read_env_text_with_fallback(env_path) + except Exception: + content = "" + + for raw_line in content.splitlines(): + line = raw_line.strip() + if line and not line.startswith('#') and '=' in line: + key, _, value = line.partition('=') + env_vars[key.strip()] = value.strip().strip('"\'') return env_vars @@ -639,15 +677,14 @@ def save_env_value(key: str, value: str): ensure_hermes_home() env_path = get_env_path() - # On Windows, open() defaults to the system locale (cp1252) which can - # cause OSError errno 22 on UTF-8 .env files. - read_kw = {"encoding": "utf-8", "errors": "replace"} if _IS_WINDOWS else {} - write_kw = {"encoding": "utf-8"} if _IS_WINDOWS else {} - + # Load existing lines = [] if env_path.exists(): - with open(env_path, **read_kw) as f: - lines = f.readlines() + try: + content, _ = read_env_text_with_fallback(env_path) + lines = content.splitlines(keepends=True) + except Exception: + lines = [] # Find and update or append found = False @@ -663,7 +700,8 @@ def save_env_value(key: str, value: str): lines[-1] += "\n" lines.append(f"{key}={value}\n") - with open(env_path, 'w', **write_kw) as f: + # Normalize to UTF-8 so future loads are deterministic across platforms. + with open(env_path, 'w', encoding='utf-8', newline='') as f: f.writelines(lines) @@ -739,6 +777,12 @@ def show_config(): print(f" Backend: {terminal.get('backend', 'local')}") print(f" Working dir: {terminal.get('cwd', '.')}") print(f" Timeout: {terminal.get('timeout', 60)}s") + if terminal.get('backend', 'local') == 'local': + try: + from tools.environments.shell_utils import get_local_shell_mode + print(f" Local shell: {get_local_shell_mode()}") + except Exception: + pass if terminal.get('backend') == 'docker': print(f" Docker image: {terminal.get('docker_image', 'python:3.11-slim')}") @@ -835,7 +879,7 @@ def set_config_value(key: str, value: str): user_config = {} if config_path.exists(): try: - with open(config_path) as f: + with open(config_path, encoding="utf-8") as f: user_config = yaml.safe_load(f) or {} except Exception: user_config = {} @@ -863,7 +907,7 @@ def set_config_value(key: str, value: str): # Write only user config back (not the full merged defaults) ensure_hermes_home() - with open(config_path, 'w') as f: + with open(config_path, 'w', encoding='utf-8', newline='') as f: yaml.dump(user_config, f, default_flow_style=False, sort_keys=False) # Keep .env in sync for keys that terminal_tool reads directly from env vars. diff --git a/hermes_cli/doctor.py b/hermes_cli/doctor.py index 031c6eaf8cad..c83cb232c2f5 100644 --- a/hermes_cli/doctor.py +++ b/hermes_cli/doctor.py @@ -11,20 +11,18 @@ from pathlib import Path from hermes_cli.config import get_project_root, get_hermes_home, get_env_path +from agent.env_loader import load_dotenv_with_fallback, read_env_text_with_fallback PROJECT_ROOT = get_project_root() HERMES_HOME = get_hermes_home() +_ENV_LOAD_ERROR = "" -# Load environment variables from ~/.hermes/.env so API key checks work -from dotenv import load_dotenv +# Load environment variables (encoding-safe on Windows) _env_path = get_env_path() if _env_path.exists(): - try: - load_dotenv(_env_path, encoding="utf-8") - except UnicodeDecodeError: - load_dotenv(_env_path, encoding="latin-1") -# Also try project .env as dev fallback -load_dotenv(PROJECT_ROOT / ".env", override=False, encoding="utf-8") + load_dotenv_with_fallback(_env_path) +if (PROJECT_ROOT / ".env").exists(): + load_dotenv_with_fallback(PROJECT_ROOT / ".env", override=False) # Point mini-swe-agent at ~/.hermes/ so it shares our config os.environ.setdefault("MSWEA_GLOBAL_CONFIG_DIR", str(HERMES_HOME)) @@ -129,14 +127,19 @@ def run_doctor(args): env_path = HERMES_HOME / '.env' if env_path.exists(): check_ok("~/.hermes/.env file exists") - - # Check for common issues - content = env_path.read_text() - if "OPENROUTER_API_KEY" in content or "ANTHROPIC_API_KEY" in content: - check_ok("API key configured") + + if _ENV_LOAD_ERROR: + check_fail("Invalid .env encoding/value detected", f"({_ENV_LOAD_ERROR})") + issues.append("Re-save ~/.hermes/.env as UTF-8 and re-enter invalid key values") else: - check_warn("No API key found in ~/.hermes/.env") - issues.append("Run 'hermes setup' to configure API keys") + # Check for common issues + content, _ = read_env_text_with_fallback(env_path) + if "OPENROUTER_API_KEY" in content or "ANTHROPIC_API_KEY" in content: + check_ok("API key configured") + else: + check_warn("No API key found in ~/.hermes/.env") + issues.append("Run 'hermes setup' to configure API keys") + else: # Also check project root as fallback fallback_env = PROJECT_ROOT / '.env' diff --git a/hermes_cli/env_loader.py b/hermes_cli/env_loader.py new file mode 100644 index 000000000000..c10c31ae6662 --- /dev/null +++ b/hermes_cli/env_loader.py @@ -0,0 +1,11 @@ +"""Back-compat shim for env loading helpers. + +Prefer importing from `agent.env_loader`. +""" + +from agent.env_loader import ( # noqa: F401 + DEFAULT_DOTENV_ENCODINGS, + load_dotenv_with_fallback, + read_env_text_with_fallback, + read_text_with_fallback, +) diff --git a/hermes_cli/gateway.py b/hermes_cli/gateway.py index 525950e9ac0f..73bbef59526c 100644 --- a/hermes_cli/gateway.py +++ b/hermes_cli/gateway.py @@ -21,59 +21,93 @@ def find_gateway_pids() -> list: """Find PIDs of running gateway processes.""" pids = [] - patterns = [ - "hermes_cli.main gateway", - "hermes gateway", - "gateway/run.py", - ] - try: if is_windows(): - # Windows: use wmic to search command lines + ps_cmd = ( + "Get-CimInstance Win32_Process | " + "Where-Object { $_.Name -match '^(python|python3|pythonw|hermes|uv)(\\.exe)?$' -and $_.CommandLine -and (" + "$_.CommandLine -like '*hermes.exe* gateway*' -or " + "$_.CommandLine -like '*hermes gateway*' -or " + "$_.CommandLine -like '*hermes_cli.main gateway*' -or " + "$_.CommandLine -like '*gateway/run.py*'" + ") } | " + "Select-Object ProcessId,CommandLine | ConvertTo-Json -Compress" + ) result = subprocess.run( - ["wmic", "process", "get", "ProcessId,CommandLine", "/FORMAT:LIST"], - capture_output=True, text=True + ["powershell", "-NoProfile", "-Command", ps_cmd], + capture_output=True, + text=True, ) - # Parse WMIC LIST output: blocks of "CommandLine=...\nProcessId=...\n" - current_cmd = "" - for line in result.stdout.split('\n'): - line = line.strip() - if line.startswith("CommandLine="): - current_cmd = line[len("CommandLine="):] - elif line.startswith("ProcessId="): - pid_str = line[len("ProcessId="):] - if any(p in current_cmd for p in patterns): + # Fallback for shells where CIM is restricted. + if result.returncode != 0: + ps_cmd = ( + "Get-WmiObject Win32_Process | " + "Where-Object { $_.Name -match '^(python|pythonw|hermes|uv)(\\.exe)?$' -and $_.CommandLine -and (" + "$_.CommandLine -like '*hermes.exe* gateway*' -or " + "$_.CommandLine -like '*hermes gateway*' -or " + "$_.CommandLine -like '*hermes_cli.main gateway*' -or " + "$_.CommandLine -like '*gateway/run.py*'" + ") } | " + "Select-Object ProcessId,CommandLine | ConvertTo-Json -Compress" + ) + result = subprocess.run( + ["powershell", "-NoProfile", "-Command", ps_cmd], + capture_output=True, + text=True, + ) + if result.returncode == 0 and result.stdout.strip(): + import json + data = json.loads(result.stdout.strip()) + if isinstance(data, dict): + data = [data] + excluded = ( + " gateway status", + " gateway stop", + " gateway restart", + " gateway install", + " gateway uninstall", + ) + for item in data: + pid = int(item.get("ProcessId", item.get("Id", 0))) + cmdline = str(item.get("CommandLine", "")).lower() + if any(marker in cmdline for marker in excluded): + continue + if pid and pid != os.getpid() and pid not in pids: + pids.append(pid) + return pids + + # Look for gateway processes with multiple patterns + patterns = [ + "hermes_cli.main gateway", + "hermes gateway", + "gateway/run.py", + ] + + result = subprocess.run( + ["ps", "aux"], + capture_output=True, + text=True + ) + + for line in result.stdout.split('\n'): + # Skip grep and current process + if 'grep' in line or str(os.getpid()) in line: + continue + + for pattern in patterns: + if pattern in line: + parts = line.split() + if len(parts) > 1: try: - pid = int(pid_str) - if pid != os.getpid() and pid not in pids: + pid = int(parts[1]) + if pid not in pids: pids.append(pid) except ValueError: - pass - current_cmd = "" - else: - result = subprocess.run( - ["ps", "aux"], - capture_output=True, - text=True - ) - for line in result.stdout.split('\n'): - # Skip grep and current process - if 'grep' in line or str(os.getpid()) in line: - continue - for pattern in patterns: - if pattern in line: - parts = line.split() - if len(parts) > 1: - try: - pid = int(parts[1]) - if pid not in pids: - pids.append(pid) - except ValueError: - continue - break + continue + break except Exception: pass - + return pids @@ -84,7 +118,15 @@ def kill_gateway_processes(force: bool = False) -> int: for pid in pids: try: - if force and not is_windows(): + if is_windows(): + cmd = ["taskkill", "/PID", str(pid), "/T"] + if force: + cmd.append("/F") + result = subprocess.run(cmd, capture_output=True, text=True) + if result.returncode == 0: + killed += 1 + continue + elif force: os.kill(pid, signal.SIGKILL) else: os.kill(pid, signal.SIGTERM) @@ -122,10 +164,7 @@ def get_launchd_plist_path() -> Path: return Path.home() / "Library" / "LaunchAgents" / "ai.hermes.gateway.plist" def get_python_path() -> str: - if is_windows(): - venv_python = PROJECT_ROOT / "venv" / "Scripts" / "python.exe" - else: - venv_python = PROJECT_ROOT / "venv" / "bin" / "python" + venv_python = PROJECT_ROOT / "venv" / "bin" / "python" if venv_python.exists(): return str(venv_python) return sys.executable @@ -386,7 +425,11 @@ def run_gateway(verbose: bool = False): # Exit with code 1 if gateway fails to connect any platform, # so systemd Restart=on-failure will retry on transient errors - success = asyncio.run(start_gateway()) + try: + success = asyncio.run(start_gateway()) + except KeyboardInterrupt: + print("\nGateway stopped.") + return if not success: sys.exit(1) @@ -432,7 +475,9 @@ def gateway_command(args): elif is_macos(): launchd_start() else: - print("Not supported on this platform.") + print("Gateway service start is not supported on this platform.") + print("Run in foreground instead: hermes gateway") + print("For background execution on Windows, use Task Scheduler.") sys.exit(1) elif subcmd == "stop": @@ -454,7 +499,15 @@ def gateway_command(args): if not service_available: # Kill gateway processes directly - killed = kill_gateway_processes() + killed = kill_gateway_processes(force=is_windows()) + # On some Windows setups, child processes can survive the first pass. + # Do one extra forced sweep before reporting status. + if is_windows(): + import time + time.sleep(0.5) + remaining = find_gateway_pids() + if remaining: + killed += kill_gateway_processes(force=True) if killed: print(f"✓ Stopped {killed} gateway process(es)") else: @@ -479,7 +532,7 @@ def gateway_command(args): if not service_available: # Manual restart: kill existing processes - killed = kill_gateway_processes() + killed = kill_gateway_processes(force=is_windows()) if killed: print(f"✓ Stopped {killed} gateway process(es)") diff --git a/hermes_cli/main.py b/hermes_cli/main.py index 57ab222bf0f2..115dedeb63c5 100644 --- a/hermes_cli/main.py +++ b/hermes_cli/main.py @@ -24,6 +24,7 @@ """ import argparse +import logging import os import sys from pathlib import Path @@ -33,23 +34,33 @@ PROJECT_ROOT = Path(__file__).parent.parent.resolve() sys.path.insert(0, str(PROJECT_ROOT)) -# Load .env from ~/.hermes/.env first, then project root as dev fallback -from dotenv import load_dotenv +def _ensure_utf8_stdio(): + """Best-effort UTF-8 stdout/stderr on Windows to avoid print crashes.""" + if os.name != "nt": + return + for stream in (sys.stdout, sys.stderr): + try: + if hasattr(stream, "reconfigure"): + stream.reconfigure(encoding="utf-8", errors="replace") + except Exception: + pass + +_ensure_utf8_stdio() + +# Load .env from ~/.hermes/.env first, then project root as dev fallback (encoding-safe on Windows) +from agent.env_loader import load_dotenv_with_fallback from hermes_cli.config import get_env_path, get_hermes_home _user_env = get_env_path() if _user_env.exists(): - try: - load_dotenv(dotenv_path=_user_env, encoding="utf-8") - except UnicodeDecodeError: - load_dotenv(dotenv_path=_user_env, encoding="latin-1") -load_dotenv(dotenv_path=PROJECT_ROOT / '.env', override=False) + load_dotenv_with_fallback(_user_env, logger=logging.getLogger(__name__)) +_project_env = PROJECT_ROOT / ".env" +if _project_env.exists(): + load_dotenv_with_fallback(_project_env, override=False, logger=logging.getLogger(__name__)) # Point mini-swe-agent at ~/.hermes/ so it shares our config os.environ.setdefault("MSWEA_GLOBAL_CONFIG_DIR", str(get_hermes_home())) os.environ.setdefault("MSWEA_SILENT_STARTUP", "1") -import logging - from hermes_cli import __version__ from hermes_constants import OPENROUTER_BASE_URL @@ -72,7 +83,8 @@ def _has_any_provider_configured() -> bool: env_file = get_env_path() if env_file.exists(): try: - for line in env_file.read_text().splitlines(): + content, _ = read_env_text_with_fallback(env_file) + for line in content.splitlines(): line = line.strip() if line.startswith("#") or "=" not in line: continue @@ -464,7 +476,7 @@ def _prompt_provider_choice(choices): idx = menu.show() print() return idx - except (ImportError, NotImplementedError): + except ImportError: pass # Fallback: numbered list @@ -1427,12 +1439,12 @@ def cmd_sessions(args): if not data: print(f"Session '{args.session_id}' not found.") return - with open(args.output, "w") as f: + with open(args.output, "w", encoding="utf-8", newline="") as f: f.write(_json.dumps(data, ensure_ascii=False) + "\n") print(f"Exported 1 session to {args.output}") else: sessions = db.export_all(source=args.source) - with open(args.output, "w") as f: + with open(args.output, "w", encoding="utf-8", newline="") as f: for s in sessions: f.write(_json.dumps(s, ensure_ascii=False) + "\n") print(f"Exported {len(sessions)} sessions to {args.output}") diff --git a/hermes_cli/models.py b/hermes_cli/models.py index 8359693f2606..07cb87b1ffaf 100644 --- a/hermes_cli/models.py +++ b/hermes_cli/models.py @@ -7,9 +7,10 @@ # (model_id, display description shown in menus) OPENROUTER_MODELS: list[tuple[str, str]] = [ - ("anthropic/claude-opus-4.6", "recommended"), + ("google/gemini-2.0-flash-001:free", "recommended"), ("anthropic/claude-sonnet-4.5", ""), - ("anthropic/claude-opus-4.5", ""), + ("anthropic/claude-opus-4.6", ""), + ("anthropic/claude-opus-4.5", ""), ("openai/gpt-5.2", ""), ("openai/gpt-5.3-codex", ""), ("google/gemini-3-pro-preview", ""), @@ -26,7 +27,7 @@ def model_ids() -> list[str]: def menu_labels() -> list[str]: - """Return display labels like 'anthropic/claude-opus-4.6 (recommended)'.""" + """Return display labels like 'google/gemini-2.0-flash-001:free (recommended)'.""" labels = [] for mid, desc in OPENROUTER_MODELS: labels.append(f"{mid} ({desc})" if desc else mid) diff --git a/hermes_cli/setup.py b/hermes_cli/setup.py index 0bc9acc0a38e..371123d551ff 100644 --- a/hermes_cli/setup.py +++ b/hermes_cli/setup.py @@ -31,6 +31,33 @@ from hermes_cli.colors import Colors, color +def _has_any_provider_configured() -> bool: + """Check whether at least one inference provider is currently usable.""" + # API-key based providers (treat blank values as not configured) + if ( + get_env_value("OPENROUTER_API_KEY") + or get_env_value("OPENAI_API_KEY") + or get_env_value("ANTHROPIC_API_KEY") + ): + return True + + # OAuth providers (e.g., Nous Portal) + auth_file = get_hermes_home() / "auth.json" + if auth_file.exists(): + try: + import json + + auth = json.loads(auth_file.read_text(encoding="utf-8")) + active = auth.get("active_provider") + if active: + state = auth.get("providers", {}).get(active, {}) + if state.get("access_token") or state.get("refresh_token"): + return True + except Exception: + pass + + return False + def print_header(title: str): """Print a section header.""" print() @@ -100,7 +127,7 @@ def prompt_choice(question: str, choices: list, default: int = 0) -> int: return idx except (ImportError, NotImplementedError): - # Fallback to number-based selection (simple_term_menu doesn't support Windows) + # Fallback to number-based selection for i, choice in enumerate(choices): marker = "●" if i == default else "○" if i == default: @@ -200,7 +227,7 @@ def prompt_checklist(title: str, items: list, pre_selected: list = None) -> list return selected except (ImportError, NotImplementedError): - # Fallback: numbered toggle interface (simple_term_menu doesn't support Windows) + # Fallback: numbered toggle interface selected = set(pre_selected) while True: @@ -390,33 +417,32 @@ def run_setup_wizard(args): config = load_config() hermes_home = get_hermes_home() - # Check if this is an existing installation with a provider configured. - # Just having config.yaml is NOT enough — the installer creates it from - # a template, so it always exists after install. We need an actual - # inference provider to consider it "existing" (otherwise quick mode - # would skip provider selection, leaving hermes non-functional). - from hermes_cli.auth import get_active_provider - active_provider = get_active_provider() + # Check if this is an existing installation with config (any provider or config file) is_existing = ( get_env_value("OPENROUTER_API_KEY") is not None or get_env_value("OPENAI_BASE_URL") is not None - or active_provider is not None + or get_config_path().exists() ) # Import migration helpers from hermes_cli.config import ( get_missing_env_vars, get_missing_config_fields, check_config_version, migrate_config, - REQUIRED_ENV_VARS, OPTIONAL_ENV_VARS ) # Check what's missing - missing_required = [v for v in get_missing_env_vars(required_only=False) if v.get("is_required")] - missing_optional = [v for v in get_missing_env_vars(required_only=False) if not v.get("is_required")] + missing_env = get_missing_env_vars(required_only=False, enabled_only=True, config=config) + missing_required = [v for v in missing_env if v.get("is_required")] + missing_optional = [v for v in missing_env if not v.get("is_required")] missing_config = get_missing_config_fields() current_ver, latest_ver = check_config_version() - has_missing = missing_required or missing_optional or missing_config or current_ver < latest_ver + has_provider = _has_any_provider_configured() + provider_missing = not has_provider + has_missing = ( + bool(missing_required or missing_optional or missing_config or current_ver < latest_ver) + or provider_missing + ) print() print(color("┌─────────────────────────────────────────────────────────┐", Colors.MAGENTA)) @@ -431,7 +457,12 @@ def run_setup_wizard(args): if is_existing and has_missing: print() print_header("Existing Installation Detected") - print_success("You already have Hermes configured!") + if provider_missing: + print_warning("Provider setup is incomplete.") + print_info("No usable inference provider was detected.") + print_info("You'll need to select a provider before Hermes can chat.") + else: + print_success("You already have Hermes configured!") print() if missing_required: @@ -452,22 +483,25 @@ def run_setup_wizard(args): print_info(f" {len(missing_config)} new config option(s) available") print() - - setup_choices = [ - "Quick setup - just configure missing items", - "Full setup - reconfigure everything", - "Skip - exit setup" - ] - - choice = prompt_choice("What would you like to do?", setup_choices, 0) - - if choice == 0: - quick_mode = True - elif choice == 2: - print() - print_info("Exiting. Run 'hermes setup' again when ready.") - return - # choice == 1 continues with full setup + + if provider_missing: + print_info("Continuing to full setup so you can choose a provider.") + else: + setup_choices = [ + "Quick setup - just configure missing items", + "Full setup - reconfigure everything", + "Skip - exit setup" + ] + + choice = prompt_choice("What would you like to do?", setup_choices, 0) + + if choice == 0: + quick_mode = True + elif choice == 2: + print() + print_info("Exiting. Run 'hermes setup' again when ready.") + return + # choice == 1 continues with full setup elif is_existing and not has_missing: print() @@ -824,7 +858,7 @@ def run_setup_wizard(args): if selected_provider != "custom": # Custom already prompted for model name print_header("Default Model") - current_model = config.get('model', 'anthropic/claude-opus-4.6') + current_model = config.get('model', 'google/gemini-2.0-flash-001:free') print_info(f"Current: {current_model}") if selected_provider == "nous" and nous_models: @@ -894,7 +928,7 @@ def run_setup_wizard(args): config['model'] = ids[model_idx] save_env_value("LLM_MODEL", ids[model_idx]) elif model_idx == len(ids): # Custom - custom = prompt("Enter model name (e.g., anthropic/claude-opus-4.6)") + custom = prompt("Enter model name (e.g., anthropic/claude-sonnet-4 or google/gemini-2.0-flash-001:free)") if custom: config['model'] = custom save_env_value("LLM_MODEL", custom) @@ -965,7 +999,10 @@ def run_setup_wizard(args): print_info(" The CLI always uses the directory you run 'hermes' from") print_info(" But messaging bots need a static starting directory") - current_cwd = get_env_value('MESSAGING_CWD') or str(Path.home()) + default_msg_cwd = Path.home() / ".hermes" + if not default_msg_cwd.exists(): + default_msg_cwd = Path.home() + current_cwd = get_env_value('MESSAGING_CWD') or str(default_msg_cwd) print_info(f" Current: {current_cwd}") cwd_input = prompt(" Messaging working directory", current_cwd) diff --git a/hermes_cli/status.py b/hermes_cli/status.py index f1d3a7edf68d..0c72a1c41ae7 100644 --- a/hermes_cli/status.py +++ b/hermes_cli/status.py @@ -154,6 +154,12 @@ def show_status(args): except Exception: terminal_env = "local" print(f" Backend: {terminal_env}") + if terminal_env == "local": + try: + from tools.environments.shell_utils import get_local_shell_mode + print(f" Local shell: {get_local_shell_mode()}") + except Exception: + pass if terminal_env == "ssh": ssh_host = os.getenv("TERMINAL_SSH_HOST", "") @@ -232,7 +238,7 @@ def show_status(args): if jobs_file.exists(): import json try: - with open(jobs_file) as f: + with open(jobs_file, encoding="utf-8") as f: data = json.load(f) jobs = data.get("jobs", []) enabled_jobs = [j for j in jobs if j.get("enabled", True)] @@ -252,7 +258,7 @@ def show_status(args): if sessions_file.exists(): import json try: - with open(sessions_file) as f: + with open(sessions_file, encoding="utf-8") as f: data = json.load(f) print(f" Active: {len(data)} session(s)") except Exception: diff --git a/landingpage/index.html b/landingpage/index.html index 2d1f99972892..bc1aa859e180 100644 --- a/landingpage/index.html +++ b/landingpage/index.html @@ -69,38 +69,14 @@

-
-
-
- - - -
-
- - - -
-
-
- $ - curl -fsSL https://raw.githubusercontent.com/NousResearch/hermes-agent/main/scripts/install.sh | bash - -
-
-

Works on Linux, macOS & WSL · No prerequisites · Installs everything automatically

+
+ curl -fsSL https://raw.githubusercontent.com/NousResearch/hermes-agent/main/scripts/install.sh | bash + +
+

Works on Linux & macOS · No Python prerequisite · Installs everything automatically

@@ -354,16 +330,12 @@

Get started in 60 seconds

Install

-
- - - -
- + bash +
- 
curl -fsSL https://raw.githubusercontent.com/NousResearch/hermes-agent/main/scripts/install.sh | bash
+ 
curl -fsSL https://raw.githubusercontent.com/NousResearch/hermes-agent/main/scripts/install.sh | bash
-

Installs uv, Python 3.11, clones the repo, sets up everything. No sudo needed.

+

Installs uv, Python 3.11, clones the repo, sets up everything. No sudo needed.

@@ -422,7 +394,14 @@

Go multi-platform (optional)

-

🪟 Windows requires Git for Windows — Hermes uses Git Bash internally for shell commands.

+

Windows? Use WSL or PowerShell:

+
+
+ powershell + +
+ 
irm https://raw.githubusercontent.com/NousResearch/hermes-agent/main/scripts/install.ps1 | iex
+
diff --git a/landingpage/script.js b/landingpage/script.js index 422ba3beabd1..6f1c6c105ace 100644 --- a/landingpage/script.js +++ b/landingpage/script.js @@ -2,79 +2,11 @@ // Hermes Agent Landing Page — Interactions // ========================================================================= -// --- Platform install commands --- -const PLATFORMS = { - linux: { - command: 'curl -fsSL https://raw.githubusercontent.com/NousResearch/hermes-agent/main/scripts/install.sh | bash', - prompt: '$', - note: 'Works on Linux, macOS & WSL · No prerequisites · Installs everything automatically', - stepNote: 'Installs uv, Python 3.11, clones the repo, sets up everything. No sudo needed.', - }, - powershell: { - command: 'irm https://raw.githubusercontent.com/NousResearch/hermes-agent/main/scripts/install.ps1 | iex', - prompt: 'PS>', - note: 'Windows PowerShell · Requires Git for Windows · Installs everything automatically', - stepNote: 'Requires Git for Windows. Installs uv, Python 3.11, sets up everything.', - }, - cmd: { - command: 'curl -fsSL https://raw.githubusercontent.com/NousResearch/hermes-agent/main/scripts/install.cmd -o install.cmd && install.cmd && del install.cmd', - prompt: '>', - note: 'Windows CMD · Requires Git for Windows · Installs everything automatically', - stepNote: 'Requires Git for Windows. Downloads and runs the installer, then cleans up.', - }, -}; - -function detectPlatform() { - const ua = navigator.userAgent.toLowerCase(); - if (ua.includes('win')) return 'powershell'; - return 'linux'; -} - -function switchPlatform(platform) { - const cfg = PLATFORMS[platform]; - if (!cfg) return; - - // Update hero install widget - const commandEl = document.getElementById('install-command'); - const promptEl = document.getElementById('install-prompt'); - const noteEl = document.getElementById('install-note'); - - if (commandEl) commandEl.textContent = cfg.command; - if (promptEl) promptEl.textContent = cfg.prompt; - if (noteEl) noteEl.textContent = cfg.note; - - // Update active tab in hero - document.querySelectorAll('.install-tab').forEach(tab => { - tab.classList.toggle('active', tab.dataset.platform === platform); - }); - - // Sync the step section tabs too - switchStepPlatform(platform); -} - -function switchStepPlatform(platform) { - const cfg = PLATFORMS[platform]; - if (!cfg) return; - - const commandEl = document.getElementById('step1-command'); - const copyBtn = document.getElementById('step1-copy'); - const noteEl = document.getElementById('step1-note'); - - if (commandEl) commandEl.textContent = cfg.command; - if (copyBtn) copyBtn.setAttribute('data-text', cfg.command); - if (noteEl) noteEl.textContent = cfg.stepNote; - - // Update active tab in step section - document.querySelectorAll('.code-tab').forEach(tab => { - tab.classList.toggle('active', tab.dataset.platform === platform); - }); -} - // --- Copy to clipboard --- function copyInstall() { const text = document.getElementById('install-command').textContent; navigator.clipboard.writeText(text).then(() => { - const btn = document.querySelector('.install-widget-body .copy-btn'); + const btn = document.querySelector('.hero-install .copy-btn'); const original = btn.querySelector('.copy-text').textContent; btn.querySelector('.copy-text').textContent = 'Copied!'; btn.style.color = 'var(--gold)'; @@ -311,10 +243,6 @@ class TerminalDemo { // --- Initialize --- document.addEventListener('DOMContentLoaded', () => { - // Auto-detect platform and set the right install command - const detectedPlatform = detectPlatform(); - switchPlatform(detectedPlatform); - initScrollAnimations(); // Terminal demo - start when visible diff --git a/landingpage/style.css b/landingpage/style.css index cf05a7a8bc14..f75057d62e48 100644 --- a/landingpage/style.css +++ b/landingpage/style.css @@ -245,132 +245,33 @@ strong { margin-bottom: 32px; } -/* --- Install Widget (hero tabbed installer) --- */ -.install-widget { - max-width: 740px; - margin: 0 auto; +.install-box { + display: flex; + align-items: center; + gap: 0; background: var(--bg-card); border: 1px solid var(--border); border-radius: var(--radius); - overflow: hidden; - transition: border-color 0.3s; -} - -.install-widget:hover { - border-color: var(--border-hover); -} - -.install-widget-header { - display: flex; - align-items: center; - gap: 16px; - padding: 10px 16px; - background: rgba(255, 255, 255, 0.02); - border-bottom: 1px solid var(--border); -} - -.install-dots { - display: flex; - gap: 6px; - flex-shrink: 0; -} - -.install-dots .dot { - width: 10px; - height: 10px; - border-radius: 50%; -} - -.install-tabs { - display: flex; - gap: 4px; - flex-wrap: wrap; -} - -.install-tab { - display: inline-flex; - align-items: center; - gap: 6px; - padding: 5px 14px; - border: none; - border-radius: 6px; - font-family: var(--font-sans); - font-size: 12px; - font-weight: 500; - cursor: pointer; - transition: all 0.2s; - background: transparent; - color: var(--text-muted); -} - -.install-tab:hover { - color: var(--text-dim); - background: rgba(255, 255, 255, 0.04); -} - -.install-tab.active { - background: rgba(255, 215, 0, 0.12); - color: var(--gold); -} - -.install-tab svg { - flex-shrink: 0; -} - -.install-widget-body { - display: flex; - align-items: center; - gap: 10px; padding: 14px 16px; + max-width: 680px; + margin: 0 auto; font-family: var(--font-mono); font-size: 13px; color: var(--text); overflow-x: auto; + transition: border-color 0.3s; } -.install-prompt { - color: var(--gold); - font-weight: 600; - flex-shrink: 0; - opacity: 0.7; +.install-box:hover { + border-color: var(--border-hover); } -.install-widget-body code { +.install-box code { flex: 1; white-space: nowrap; overflow: hidden; text-overflow: ellipsis; text-align: left; - transition: opacity 0.15s; -} - -/* --- Code block tabs (install step section) --- */ -.code-tabs { - display: flex; - gap: 2px; -} - -.code-tab { - padding: 3px 10px; - border: none; - border-radius: 4px; - font-family: var(--font-mono); - font-size: 11px; - font-weight: 500; - cursor: pointer; - transition: all 0.2s; - background: transparent; - color: var(--text-muted); -} - -.code-tab:hover { - color: var(--text-dim); - background: rgba(255, 255, 255, 0.04); -} - -.code-tab.active { - background: rgba(255, 215, 0, 0.1); - color: var(--gold); } .copy-btn { @@ -1047,35 +948,17 @@ strong { margin: 0 auto 28px; } - .install-widget-body { + .install-box { font-size: 10px; padding: 10px 12px; } - .install-widget-body code { + .install-box code { overflow: hidden; text-overflow: ellipsis; display: block; } - .install-widget-header { - padding: 8px 12px; - gap: 10px; - } - - .install-tabs { - gap: 2px; - } - - .install-tab { - padding: 4px 10px; - font-size: 11px; - } - - .install-tab svg { - display: none; - } - .copy-btn { padding: 3px 6px; } diff --git a/mini_swe_runner.py b/mini_swe_runner.py index 6a3871d767a1..43c449d8c220 100644 --- a/mini_swe_runner.py +++ b/mini_swe_runner.py @@ -37,10 +37,13 @@ from typing import List, Dict, Any, Optional, Literal import fire -from dotenv import load_dotenv -# Load environment variables -load_dotenv() +# Load environment variables (encoding-safe on Windows) +from agent.env_loader import load_dotenv_with_fallback +for _p in (Path.home() / ".hermes" / ".env", Path.cwd() / ".env"): + if _p.exists(): + load_dotenv_with_fallback(_p) + break # Add mini-swe-agent to path if not installed mini_swe_path = Path(__file__).parent / "mini-swe-agent" / "src" diff --git a/model_tools.py b/model_tools.py index 8da3d67e8186..d66f2579fd36 100644 --- a/model_tools.py +++ b/model_tools.py @@ -23,6 +23,7 @@ import json import asyncio import os +import sys import logging from typing import Dict, Any, List, Optional, Tuple @@ -106,11 +107,18 @@ def _discover_tools(): _discover_tools() # MCP tool discovery (external MCP servers from config) -try: - from tools.mcp_tool import discover_mcp_tools - discover_mcp_tools() -except Exception as e: - logger.debug("MCP tool discovery failed: %s", e) +_running_under_pytest = "pytest" in sys.modules +_allow_mcp_in_tests = os.getenv("HERMES_TEST_ENABLE_MCP_DISCOVERY", "").strip().lower() in { + "1", "true", "yes", "on", +} +if not _running_under_pytest or _allow_mcp_in_tests: + try: + from tools.mcp_tool import discover_mcp_tools + discover_mcp_tools() + except Exception as e: + logger.debug("MCP tool discovery failed: %s", e) +else: + logger.debug("Skipping MCP discovery during pytest import (set HERMES_TEST_ENABLE_MCP_DISCOVERY=1 to enable).") # ============================================================================= diff --git a/rl_cli.py b/rl_cli.py index 3aa0412d4cce..72f95f9bd61a 100644 --- a/rl_cli.py +++ b/rl_cli.py @@ -72,7 +72,7 @@ from hermes_constants import OPENROUTER_BASE_URL -DEFAULT_MODEL = "anthropic/claude-opus-4.5" +DEFAULT_MODEL = "google/gemini-2.0-flash-001:free" DEFAULT_BASE_URL = OPENROUTER_BASE_URL @@ -92,7 +92,7 @@ def load_hermes_config() -> dict: if config_path.exists(): try: - with open(config_path, "r") as f: + with open(config_path, "r", encoding="utf-8") as f: file_config = yaml.safe_load(f) or {} # Get model from config diff --git a/run_agent.py b/run_agent.py index 4c60b4bd81ea..86a6e9c879ce 100644 --- a/run_agent.py +++ b/run_agent.py @@ -16,7 +16,7 @@ Usage: from run_agent import AIAgent - agent = AIAgent(base_url="http://localhost:30000/v1", model="claude-opus-4-20250514") + agent = AIAgent(base_url="http://localhost:30000/v1", model="google/gemini-2.0-flash-001:free") response = agent.run_conversation("Tell me about the latest Python updates") """ @@ -97,6 +97,49 @@ ) +def _repair_terminal_tool_args(tool_name: str, raw_args: str) -> Optional[str]: + """If the tool is 'terminal' and raw_args is broken JSON, try to salvage the command. + + LLMs often produce invalid JSON for terminal (e.g. unescaped quotes in the command + string, or truncated output). This tries to extract the "command" value and return + valid JSON so we don't have to retry the whole API call. + """ + if tool_name != "terminal" or not raw_args or not raw_args.strip(): + return None + raw = raw_args.strip() + + def try_return(cmd: str) -> Optional[str]: + if not cmd: + return None + try: + return json.dumps({"command": cmd}) + except (TypeError, ValueError): + return None + + # 1) Strict: value with only proper \" escapes (stops at first unescaped ") + m = re.search(r'"command"\s*:\s*"((?:[^"\\]|\\.)*)"', raw) + if m: + return try_return(m.group(1)) + + # 2) Unescaped quote in value (e.g. "command": "powershell -Command "Get-Location..."): + # capture from first " to last " before "}" or ", " + m = re.search(r'"command"\s*:\s*"(.*)"\s*[,}]', raw) + if m: + return try_return(m.group(1)) + + # 3) Truncated: "command": " to end + m2 = re.search(r'"command"\s*:\s*"(.*)', raw, re.DOTALL) + if m2: + cmd = m2.group(1).rstrip() + if cmd.endswith('"}'): + cmd = cmd[:-2] + elif cmd.endswith('"'): + cmd = cmd[:-1] + cmd = re.sub(r'["}\],\s]+$', '', cmd) + return try_return(cmd) if cmd else None + return None + + class AIAgent: """ AI Agent with tool calling capabilities. @@ -329,12 +372,27 @@ def __init__( client_kwargs["base_url"] = OPENROUTER_BASE_URL # Handle API key - OpenRouter is the primary provider - if api_key: - client_kwargs["api_key"] = api_key + if api_key and str(api_key).strip(): + client_kwargs["api_key"] = api_key.strip() else: # Primary: OPENROUTER_API_KEY, fallback to direct provider keys - client_kwargs["api_key"] = os.getenv("OPENROUTER_API_KEY", "") - + client_kwargs["api_key"] = os.getenv("OPENROUTER_API_KEY", "") or os.getenv("OPENAI_API_KEY", "") + # Last-resort: load ~/.hermes/.env if key still empty (e.g. env not loaded before this process) + if not (client_kwargs.get("api_key") and str(client_kwargs["api_key"]).strip()): + try: + from dotenv import load_dotenv + _hermes_home = os.getenv("HERMES_HOME", os.path.expanduser("~/.hermes")) + _env_path = os.path.join(_hermes_home, ".env") + if os.path.isfile(_env_path): + load_dotenv(dotenv_path=_env_path, encoding="utf-8") + client_kwargs["api_key"] = os.getenv("OPENROUTER_API_KEY", "") or os.getenv("OPENAI_API_KEY", "") + except Exception: + pass + if not (client_kwargs.get("api_key") and str(client_kwargs["api_key"]).strip()): + raise ValueError( + "No API key available. Set OPENROUTER_API_KEY (or OPENAI_API_KEY) in ~/.hermes/.env or pass api_key= to AIAgent." + ) + # OpenRouter app attribution — shows hermes-agent in rankings/analytics effective_base = client_kwargs.get("base_url", "") if "openrouter" in effective_base.lower(): @@ -532,7 +590,7 @@ def __init__( model=self.model, threshold_percent=compression_threshold, protect_first_n=3, - protect_last_n=4, + protect_last_n=8, summary_target_tokens=500, summary_model_override=compression_summary_model, quiet_mode=self.quiet_mode, @@ -551,10 +609,10 @@ def __init__( print(f"📊 Context limit: {self.context_compressor.context_length:,} tokens (compress at {int(compression_threshold*100)}% = {self.context_compressor.threshold_tokens:,})") else: print(f"📊 Context limit: {self.context_compressor.context_length:,} tokens (auto-compression disabled)") - + def _max_tokens_param(self, value: int) -> dict: """Return the correct max tokens kwarg for the current provider. - + OpenAI's newer models (gpt-4o, o-series, gpt-5+) require 'max_completion_tokens'. OpenRouter, local models, and older OpenAI models use 'max_tokens'. @@ -570,25 +628,25 @@ def _max_tokens_param(self, value: int) -> dict: def _has_content_after_think_block(self, content: str) -> bool: """ Check if content has actual text after any blocks. - + This detects cases where the model only outputs reasoning but no actual response, which indicates an incomplete generation that should be retried. - + Args: content: The assistant message content to check - + Returns: True if there's meaningful content after think blocks, False otherwise """ if not content: return False - + # Remove all ... blocks (including nested ones, non-greedy) cleaned = re.sub(r'.*?', '', content, flags=re.DOTALL) - + # Check if there's any non-whitespace content remaining return bool(cleaned.strip()) - + def _strip_think_blocks(self, content: str) -> str: """Remove ... blocks from content, returning only visible text.""" if not content: @@ -1359,7 +1417,7 @@ def _build_system_prompt(self, system_message: str = None) -> str: context_files_prompt = build_context_files_prompt() if context_files_prompt: prompt_parts.append(context_files_prompt) - + now = datetime.now() prompt_parts.append( f"Conversation started: {now.strftime('%A, %B %d, %Y %I:%M %p')}" @@ -1371,6 +1429,62 @@ def _build_system_prompt(self, system_message: str = None) -> str: return "\n\n".join(prompt_parts) + def _build_environment_hint(self) -> str: + """ + Build a per-call environment hint describing the local shell/OS. + + This is injected via the ephemeral system prompt so that changes to + HERMES_WINDOWS_SHELL (for example, switching between PowerShell and WSL) + take effect on the very next turn without requiring a new session. + """ + # Only add explicit hints for Windows hosts; POSIX shells are already + # implied by the environment and less likely to conflict. + if os.name != "nt": + return "" + + from tools.environments.shell_utils import get_local_shell_mode + + mode = get_local_shell_mode() + override = os.getenv("HERMES_WINDOWS_SHELL", "auto").strip().lower() or "auto" + + if mode == "powershell": + return ( + "Environment: You are running on a Windows host with a PowerShell local terminal " + f"(HERMES_WINDOWS_SHELL={override}). For commands that run in the *local* terminal, " + "use PowerShell syntax and Windows-style paths. Prefer `Get-ChildItem` instead of " + "`ls`, `Select-String` instead of `grep`, and `New-Item -ItemType Directory -Force` " + "instead of `mkdir -p`. POSIX commands are only appropriate when a tool explicitly " + "targets a remote or containerized Unix environment. " + "To verify the active shell, run: (dir 2>&1 *`|echo CMD);&<# rem #>echo PowerShell — " + "it prints CMD in cmd.exe and PowerShell in PowerShell." + ) + if mode == "wsl": + return ( + "Environment: You are on a Windows host but the local terminal is WSL (Linux) " + f"(HERMES_WINDOWS_SHELL={override}). Commands run in a POSIX shell with tools like " + "`ls`, `grep`, and `mkdir -p` available, and the Windows filesystem is visible under " + "`/mnt//...` (for example, `/mnt/c/Users/...`). Use POSIX paths inside WSL, " + "not raw `C:\\`-style paths." + ) + if mode == "cmd": + return ( + "Environment: You are running on Windows with a cmd.exe local terminal " + f"(HERMES_WINDOWS_SHELL={override}). Run commands *directly in cmd*: use `dir`, `type`, " + "`cd`, `mkdir`, `echo %VAR%` for env vars, and `C:\\`-style paths. Do NOT invoke " + "powershell.exe or use PowerShell syntax (no `powershell -Command ...`). Do NOT use " + "POSIX commands (`pwd`, `uname`, `ls`, `grep`). If a command fails (exit non-zero), " + "do not repeat it; try a different approach or report the error. " + "To verify the active shell, run: (dir 2>&1 *`|echo CMD);&<# rem #>echo PowerShell — " + "it prints CMD in cmd.exe and PowerShell in PowerShell." + ) + + # Fallback for unexpected modes. + return ( + f"Environment: You are running on Windows with local shell mode '{mode}' " + f"(HERMES_WINDOWS_SHELL={override}). Choose command syntax appropriate for this shell " + "and do not assume a Unix/POSIX environment unless explicitly indicated." + ) + def _invalidate_system_prompt(self): """ Invalidate the cached system prompt, forcing a rebuild on the next turn. @@ -1452,6 +1566,8 @@ def _derive_responses_function_call_id( def _chat_messages_to_responses_input(self, messages: List[Dict[str, Any]]) -> List[Dict[str, Any]]: """Convert internal chat-style messages to Responses input items.""" items: List[Dict[str, Any]] = [] + known_function_call_ids: set[str] = set() + deferred_tool_outputs: List[Dict[str, str]] = [] for msg in messages: if not isinstance(msg, dict): @@ -1516,6 +1632,7 @@ def _chat_messages_to_responses_input(self, messages: List[Dict[str, Any]]) -> L "name": fn_name, "arguments": arguments, }) + known_function_call_ids.add(call_id) continue items.append({"role": role, "content": content_text}) @@ -1529,11 +1646,50 @@ def _chat_messages_to_responses_input(self, messages: List[Dict[str, Any]]) -> L call_id = raw_tool_call_id.strip() if not isinstance(call_id, str) or not call_id.strip(): continue - items.append({ + output_text = str(msg.get("content", "") or "") + + # Responses API rejects function_call_output items whose call_id + # is not present as a function_call in the same payload. + # Keep unmatched outputs deferred so we can reconcile call_/fc_ + # variants once all assistant tool_calls are collected. + deferred_tool_outputs.append( + { + "call_id": call_id, + "output": output_text, + } + ) + + def _call_id_variants(raw_call_id: str) -> List[str]: + variants: List[str] = [] + candidate = (raw_call_id or "").strip() + if not candidate: + return variants + variants.append(candidate) + if candidate.startswith("fc_") and len(candidate) > len("fc_"): + variants.append(f"call_{candidate[len('fc_'):]}") + elif candidate.startswith("call_") and len(candidate) > len("call_"): + variants.append(f"fc_{candidate[len('call_'):]}") + return variants + + for entry in deferred_tool_outputs: + raw_call_id = entry.get("call_id", "") + output = entry.get("output", "") + variants = _call_id_variants(raw_call_id) + matched_call_id = next((cid for cid in variants if cid in known_function_call_ids), None) + if not matched_call_id: + logger.debug( + "Skipping orphan function_call_output for unmatched call_id=%s", + raw_call_id, + ) + continue + + items.append( + { "type": "function_call_output", - "call_id": call_id, - "output": str(msg.get("content", "") or ""), - }) + "call_id": matched_call_id, + "output": output, + } + ) return items @@ -1621,6 +1777,52 @@ def _preflight_codex_input_items(self, raw_items: Any) -> List[Dict[str, Any]]: f"Codex Responses input[{idx}] has unsupported item shape (type={item_type!r}, role={role!r})." ) + def _normalize_call_id(raw_call_id: Any) -> str: + if not isinstance(raw_call_id, str): + return "" + cid = raw_call_id.strip() + if not cid: + return "" + if cid.startswith("fc_") and len(cid) > len("fc_"): + return f"call_{cid[len('fc_'):]}" + return cid + + # Final guard: Responses API requires each function_call to have a + # matching function_call_output in the same payload. If history got + # interrupted/truncated and a tool output is missing, synthesize one + # deterministically so the request is still valid. + declared_calls: List[str] = [] + answered_calls: set[str] = set() + for item in normalized: + item_type = item.get("type") + if item_type == "function_call": + cid = item.get("call_id") + if isinstance(cid, str) and cid.strip(): + declared_calls.append(cid.strip()) + elif item_type == "function_call_output": + cid = _normalize_call_id(item.get("call_id")) + if cid: + answered_calls.add(cid) + + for call_id in declared_calls: + if _normalize_call_id(call_id) in answered_calls: + continue + normalized.append( + { + "type": "function_call_output", + "call_id": call_id, + "output": ( + "Error executing tool: Missing tool output recovered automatically " + "during request preflight." + ), + } + ) + answered_calls.add(_normalize_call_id(call_id)) + logger.warning( + "Synthesized missing function_call_output during preflight for call_id=%s", + call_id, + ) + return normalized def _preflight_codex_api_kwargs( @@ -1696,7 +1898,8 @@ def _preflight_codex_api_kwargs( allowed_keys = { "model", "instructions", "input", "tools", "store", - "reasoning", "include", "max_output_tokens", "temperature", + "reasoning", "include", "max_output_tokens", "max_tokens", "temperature", + "extra_body", } normalized: Dict[str, Any] = { "model": model, @@ -1714,10 +1917,16 @@ def _preflight_codex_api_kwargs( if isinstance(include, list): normalized["include"] = include - # Pass through max_output_tokens and temperature - max_output_tokens = api_kwargs.get("max_output_tokens") - if isinstance(max_output_tokens, (int, float)) and max_output_tokens > 0: - normalized["max_output_tokens"] = int(max_output_tokens) + extra_body = api_kwargs.get("extra_body") + normalized_extra_body: Dict[str, Any] = {} + if isinstance(extra_body, dict): + normalized_extra_body = dict(extra_body) + + # Codex Responses currently rejects token-cap parameters in this runtime. + # Accept legacy fields for compatibility but strip them from requests. + normalized_extra_body.pop("max_tokens", None) + if normalized_extra_body: + normalized["extra_body"] = normalized_extra_body temperature = api_kwargs.get("temperature") if isinstance(temperature, (int, float)): normalized["temperature"] = float(temperature) @@ -2089,9 +2298,6 @@ def _build_api_kwargs(self, api_messages: list) -> dict: else: kwargs["include"] = [] - if self.max_tokens is not None: - kwargs["max_output_tokens"] = self.max_tokens - return kwargs provider_preferences = {} @@ -2145,6 +2351,175 @@ def _build_api_kwargs(self, api_messages: list) -> dict: return api_kwargs + def _prepare_api_messages( + self, + messages: list, + active_system_prompt: str, + ) -> tuple[list, int, int]: + """Build API-ready messages and rough size estimates from chat history.""" + api_messages = [] + for msg in messages: + api_msg = msg.copy() + + # Preserve reasoning continuity for providers that support/expect it. + if msg.get("role") == "assistant": + reasoning_text = msg.get("reasoning") + if reasoning_text: + api_msg["reasoning_content"] = reasoning_text + + # Internal-only fields not accepted by strict APIs. + if "reasoning" in api_msg: + api_msg.pop("reasoning") + if "finish_reason" in api_msg: + api_msg.pop("finish_reason") + + api_messages.append(api_msg) + + effective_system = active_system_prompt or "" + extra_ephemeral = [] + if self.ephemeral_system_prompt: + extra_ephemeral.append(self.ephemeral_system_prompt) + env_hint = self._build_environment_hint() + if env_hint: + extra_ephemeral.append(env_hint) + if extra_ephemeral: + effective_system = (effective_system + "\n\n" + "\n\n".join(extra_ephemeral)).strip() + if self._honcho_context: + effective_system = (effective_system + "\n\n" + self._honcho_context).strip() + if effective_system: + api_messages = [{"role": "system", "content": effective_system}] + api_messages + + # Inject ephemeral prefill messages right after system, before history. + if self.prefill_messages: + sys_offset = 1 if effective_system else 0 + for idx, pfm in enumerate(self.prefill_messages): + api_messages.insert(sys_offset + idx, pfm.copy()) + + # Anthropic prompt caching tweaks for Claude on OpenRouter. + if self._use_prompt_caching: + api_messages = apply_anthropic_cache_control(api_messages, cache_ttl=self._cache_ttl) + + total_chars = sum(len(str(msg)) for msg in api_messages) + approx_tokens = total_chars // 4 # Rough estimate: 4 chars/token + return api_messages, total_chars, approx_tokens + + def _hard_trim_context(self, messages: list) -> list: + """Emergency shrink path when summarization-based compression cannot reduce more.""" + if not isinstance(messages, list) or len(messages) <= 24: + return messages + + keep = max(24, min(120, len(messages) // 2)) + trimmed = list(messages[-keep:]) + + # A leading tool role without its assistant tool_call context is invalid. + while trimmed and isinstance(trimmed[0], dict) and trimmed[0].get("role") == "tool": + trimmed.pop(0) + + # Prefer starting on a user/assistant text turn, not an orphan tool-call turn. + while ( + trimmed + and isinstance(trimmed[0], dict) + and trimmed[0].get("role") == "assistant" + and trimmed[0].get("tool_calls") + ): + trimmed.pop(0) + + return trimmed if trimmed else list(messages[-24:]) + + @staticmethod + def _normalize_tool_call_id(raw_id: Any) -> str: + if not isinstance(raw_id, str): + return "" + value = raw_id.strip() + if not value: + return "" + if value.startswith("fc_") and len(value) > len("fc_"): + return f"call_{value[len('fc_'):]}" + return value + + def _has_in_flight_tool_calls(self, messages: list) -> bool: + """True when any assistant tool_call is still missing its tool output.""" + if not isinstance(messages, list) or not messages: + return False + + for idx, msg in enumerate(messages): + if not isinstance(msg, dict): + continue + if msg.get("role") != "assistant" or not msg.get("tool_calls"): + continue + + pending: set[str] = set() + for tc in msg.get("tool_calls", []): + tc_id = "" + if isinstance(tc, dict): + tc_id = self._normalize_tool_call_id(tc.get("id") or tc.get("call_id")) + else: + tc_id = self._normalize_tool_call_id(getattr(tc, "id", None) or getattr(tc, "call_id", None)) + if tc_id: + pending.add(tc_id) + + if not pending: + continue + + for follow in messages[idx + 1:]: + if not isinstance(follow, dict): + continue + if follow.get("role") == "assistant": + break + if follow.get("role") != "tool": + continue + out_id = self._normalize_tool_call_id(follow.get("tool_call_id")) + if out_id in pending: + pending.remove(out_id) + + if pending: + return True + + return False + + def _answers_latest_user_message(self, response_text: str, latest_user_message: str) -> bool: + """Heuristic guard to reduce off-topic final replies. + + This intentionally focuses on catching obvious "acknowledgement" non-answers + (e.g., "I'll check that") while allowing concise legitimate answers. + Overly strict lexical overlap checks can cause false negatives and trigger + unnecessary regeneration loops. + """ + if not isinstance(response_text, str) or not response_text.strip(): + return False + if not isinstance(latest_user_message, str) or not latest_user_message.strip(): + return True + + response_l = response_text.lower() + # Fast-path: common explicit completion/answer markers. + if any( + marker in response_l + for marker in ( + "here's", "here is", "result", "summary", "complete", "completed", + "done", "fixed", "updated", "answer:", "the answer", + ) + ): + return True + + # Reject obvious "not yet an answer" acknowledgements. + ack_prefixes = ( + "sure", "absolutely", "got it", "okay", "ok", "i'll", "i will", + "let me", "i can do that", "i can help with that", + ) + ack_body_hints = ( + "check", "inspect", "look into", "take a look", "dig into", + "work on that", "get started", + ) + stripped = response_l.strip() + is_ack_like = any(stripped.startswith(p) for p in ack_prefixes) and any( + h in response_l for h in ack_body_hints + ) + if is_ack_like: + return False + + # Default permissive behavior for concise, substantive replies. + return True + def _build_assistant_message(self, assistant_message, finish_reason: str) -> dict: """Build a normalized assistant message dict from an API response message. @@ -2301,7 +2676,6 @@ def flush_memories(self, messages: list = None, min_turns: int = None): "messages": api_messages, "tools": [memory_tool_def], "temperature": 0.3, - "max_tokens": 5120, } response = aux_client.chat.completions.create(**api_kwargs, timeout=30.0) elif self.api_mode == "codex_responses": @@ -2309,8 +2683,6 @@ def flush_memories(self, messages: list = None, min_turns: int = None): codex_kwargs = self._build_api_kwargs(api_messages) codex_kwargs["tools"] = self._responses_tools([memory_tool_def]) codex_kwargs["temperature"] = 0.3 - if "max_output_tokens" in codex_kwargs: - codex_kwargs["max_output_tokens"] = 5120 response = self._run_codex_stream(codex_kwargs) else: api_kwargs = { @@ -2370,6 +2742,12 @@ def _compress_context(self, messages: list, system_message: str, *, approx_token Returns: (compressed_messages, new_system_prompt) tuple """ + if self._has_in_flight_tool_calls(messages): + if not self.quiet_mode: + print(f"{self.log_prefix}⏸️ Skipping compression: tool call outputs still in flight.") + current_prompt = self._cached_system_prompt or self._build_system_prompt(system_message) + return messages, current_prompt + # Pre-compression memory flush: let the model save memories before they're lost self.flush_memories(messages, min_turns=0) @@ -2628,6 +3006,143 @@ def _execute_tool_calls(self, assistant_message, messages: list, effective_task_ if self.tool_delay > 0 and i < len(assistant_message.tool_calls): time.sleep(self.tool_delay) + def _fill_missing_tool_outputs(self, messages: list, error_msg: str) -> bool: + """Add synthetic tool outputs for assistant tool calls that were never answered. + + Returns True if any synthetic tool output was appended. + """ + def _extract_call_id(tc: Any) -> Optional[str]: + if isinstance(tc, dict): + tc_id = tc.get("id") + if not tc_id: + tc_id = tc.get("call_id") + else: + tc_id = getattr(tc, "id", None) + if not tc_id: + tc_id = getattr(tc, "call_id", None) + + if not isinstance(tc_id, str): + return None + + tc_id = tc_id.strip() + if not tc_id: + return None + + if tc_id.startswith("fc_"): + tc_id = f"call_{tc_id[len('fc_'):]}" + + return tc_id + + pending_handled = False + for idx in range(len(messages) - 1, -1, -1): + msg = messages[idx] + if not isinstance(msg, dict): + continue + if msg.get("role") != "assistant" or not msg.get("tool_calls"): + continue + + answered_ids = set() + for m in messages[idx + 1:]: + if not isinstance(m, dict) or m.get("role") != "tool": + continue + raw_tool_call_id = m.get("tool_call_id") + if isinstance(raw_tool_call_id, str): + raw_tool_call_id = raw_tool_call_id.strip() + if raw_tool_call_id: + if raw_tool_call_id.startswith("fc_"): + answered_ids.add(f"call_{raw_tool_call_id[len('fc_'):]}") + else: + answered_ids.add(raw_tool_call_id) + + for tc in msg["tool_calls"]: + tc_id = _extract_call_id(tc) + if not tc_id or tc_id in answered_ids: + continue + + messages.append({ + "role": "tool", + "tool_call_id": tc_id, + "content": f"Error executing tool: {error_msg}", + }) + self._log_msg_to_db(messages[-1]) + pending_handled = True + + return pending_handled + + def _tool_call_signature(self, tool_calls: list) -> str: + """Build a deterministic signature for a sequence of tool calls.""" + if not tool_calls: + return "" + + parts = [] + for tc in tool_calls: + if tc is None: + parts.append("null") + continue + + if isinstance(tc, dict): + fn = tc.get("function", {}) + name = fn.get("name", "unknown") + raw_args = fn.get("arguments", "{}") + else: + fn = getattr(tc, "function", None) + name = getattr(fn, "name", "unknown") if fn is not None else "unknown" + raw_args = getattr(fn, "arguments", "{}") if fn is not None else "{}" + + try: + normalized = json.loads(raw_args) if isinstance(raw_args, str) else raw_args + except Exception: + normalized = raw_args + + try: + args_text = json.dumps( + normalized, + sort_keys=True, + separators=(",", ":"), + ensure_ascii=False, + ) + except Exception: + args_text = str(raw_args) + + if len(args_text) > 220: + args_text = args_text[:220] + "..." + + parts.append(f"{name}:{args_text}") + + return "|".join(parts) + + def _build_summary_fallback(self, messages: list) -> str | None: + """Generate a minimal human-readable fallback summary from available messages.""" + if not messages: + return None + + snippets = [] + for msg in messages[-30:]: + if not isinstance(msg, dict): + continue + + role = msg.get("role") + content = msg.get("content") + if not content: + continue + + text = self._strip_think_blocks(content) if isinstance(content, str) else str(content) + text = text.strip() + if not text: + continue + + text = text[:220] + "..." if len(text) > 220 else text + snippets.append(f"{role}: {text}") + + if not snippets: + return None + + lines = ["I reached the iteration limit and could not generate a clean final response."] + lines.append("Recent context:") + for snippet in snippets[-10:]: + lines.append(f"- {snippet}") + return "\n".join(lines) + def _handle_max_iterations(self, messages: list, api_call_count: int) -> str: """Request a summary when max iterations are reached. Returns the final response text.""" print(f"⚠️ Reached maximum iterations ({self.max_iterations}). Requesting summary...") @@ -2637,13 +3152,22 @@ def _handle_max_iterations(self, messages: list, api_call_count: int) -> str: "Please provide a final response summarizing what you've found and accomplished so far, " "without calling any more tools." ) + self._fill_missing_tool_outputs(messages, "Tool execution was skipped while handling iteration limit.") messages.append({"role": "user", "content": summary_request}) try: api_messages = messages.copy() effective_system = self._cached_system_prompt or "" + + # Attach ephemeral system prompt and environment hint at API-call time. + extra_ephemeral = [] if self.ephemeral_system_prompt: - effective_system = (effective_system + "\n\n" + self.ephemeral_system_prompt).strip() + extra_ephemeral.append(self.ephemeral_system_prompt) + env_hint = self._build_environment_hint() + if env_hint: + extra_ephemeral.append(env_hint) + if extra_ephemeral: + effective_system = (effective_system + "\n\n" + "\n\n".join(extra_ephemeral)).strip() if effective_system: api_messages = [{"role": "system", "content": effective_system}] + api_messages if self.prefill_messages: @@ -2668,6 +3192,7 @@ def _handle_max_iterations(self, messages: list, api_call_count: int) -> str: if self.api_mode == "codex_responses": codex_kwargs = self._build_api_kwargs(api_messages) codex_kwargs["tools"] = None + codex_kwargs["tool_choice"] = "none" summary_response = self._run_codex_stream(codex_kwargs) assistant_message, _ = self._normalize_codex_response(summary_response) final_response = (assistant_message.content or "").strip() if assistant_message else "" @@ -2675,6 +3200,7 @@ def _handle_max_iterations(self, messages: list, api_call_count: int) -> str: summary_kwargs = { "model": self.model, "messages": api_messages, + "tools": [], } if self.max_tokens is not None: summary_kwargs.update(self._max_tokens_param(self.max_tokens)) @@ -2697,7 +3223,12 @@ def _handle_max_iterations(self, messages: list, api_call_count: int) -> str: summary_response = self.client.chat.completions.create(**summary_kwargs) - if summary_response.choices and summary_response.choices[0].message.content: + if ( + summary_response is not None + and summary_response.choices + and summary_response.choices[0].message + and summary_response.choices[0].message.content + ): final_response = summary_response.choices[0].message.content else: final_response = "" @@ -2708,12 +3239,15 @@ def _handle_max_iterations(self, messages: list, api_call_count: int) -> str: if final_response: messages.append({"role": "assistant", "content": final_response}) else: - final_response = "I reached the iteration limit and couldn't generate a summary." + final_response = self._build_summary_fallback(messages) or ( + "I reached the iteration limit and couldn't generate a summary." + ) else: # Retry summary generation if self.api_mode == "codex_responses": codex_kwargs = self._build_api_kwargs(api_messages) codex_kwargs["tools"] = None + codex_kwargs["tool_choice"] = "none" retry_response = self._run_codex_stream(codex_kwargs) retry_msg, _ = self._normalize_codex_response(retry_response) final_response = (retry_msg.content or "").strip() if retry_msg else "" @@ -2721,6 +3255,7 @@ def _handle_max_iterations(self, messages: list, api_call_count: int) -> str: summary_kwargs = { "model": self.model, "messages": api_messages, + "tools": [], } if self.max_tokens is not None: summary_kwargs["max_tokens"] = self.max_tokens @@ -2729,7 +3264,12 @@ def _handle_max_iterations(self, messages: list, api_call_count: int) -> str: summary_response = self.client.chat.completions.create(**summary_kwargs) - if summary_response.choices and summary_response.choices[0].message.content: + if ( + summary_response is not None + and summary_response.choices + and summary_response.choices[0].message + and summary_response.choices[0].message.content + ): final_response = summary_response.choices[0].message.content else: final_response = "" @@ -2739,11 +3279,20 @@ def _handle_max_iterations(self, messages: list, api_call_count: int) -> str: final_response = re.sub(r'.*?\s*', '', final_response, flags=re.DOTALL).strip() messages.append({"role": "assistant", "content": final_response}) else: - final_response = "I reached the iteration limit and couldn't generate a summary." + final_response = self._build_summary_fallback(messages) or ( + "I reached the iteration limit and couldn't generate a summary." + ) except Exception as e: logging.warning(f"Failed to get summary response: {e}") + fallback = self._build_summary_fallback(messages) final_response = f"I reached the maximum iterations ({self.max_iterations}) but couldn't summarize. Error: {str(e)}" + if fallback: + final_response = f"{fallback}\n{final_response}" + + if final_response is None: + fallback = self._build_summary_fallback(messages) + final_response = fallback or "I reached the maximum iterations but couldn't generate any response." return final_response @@ -2776,6 +3325,9 @@ def run_conversation( self._last_content_with_tools = None self._turns_since_memory = 0 self._iters_since_skill = 0 + last_tool_signature = None + repeated_tool_signature_count = 0 + tool_signature_repeat_limit = 3 # Initialize conversation (copy to avoid mutating the caller's list) messages = list(conversation_history) if conversation_history else [] @@ -2858,6 +3410,7 @@ def run_conversation( final_response = None interrupted = False codex_ack_continuations = 0 + final_answer_guard_retries = 0 # Clear any stale interrupt state at start self.clear_interrupt() @@ -2872,6 +3425,14 @@ def run_conversation( api_call_count += 1 + # If we ever have pending tool calls from an interrupted or failed + # previous turn, synthesize deterministic error outputs before the + # next API request. This keeps API request history valid. + self._fill_missing_tool_outputs( + messages, + "Recovered tool output(s) before API request after a previous interruption.", + ) + # Fire step_callback for gateway hooks (agent:step event) if self.step_callback is not None: try: @@ -2895,61 +3456,10 @@ def run_conversation( self._iters_since_skill += 1 # Prepare messages for API call - # If we have an ephemeral system prompt, prepend it to the messages - # Note: Reasoning is embedded in content via tags for trajectory storage. - # However, providers like Moonshot AI require a separate 'reasoning_content' field - # on assistant messages with tool_calls. We handle both cases here. - api_messages = [] - for msg in messages: - api_msg = msg.copy() - - # For ALL assistant messages, pass reasoning back to the API - # This ensures multi-turn reasoning context is preserved - if msg.get("role") == "assistant": - reasoning_text = msg.get("reasoning") - if reasoning_text: - # Add reasoning_content for API compatibility (Moonshot AI, Novita, OpenRouter) - api_msg["reasoning_content"] = reasoning_text - - # Remove 'reasoning' field - it's for trajectory storage only - # We've copied it to 'reasoning_content' for the API above - if "reasoning" in api_msg: - api_msg.pop("reasoning") - # Remove finish_reason - not accepted by strict APIs (e.g. Mistral) - if "finish_reason" in api_msg: - api_msg.pop("finish_reason") - # Keep 'reasoning_details' - OpenRouter uses this for multi-turn reasoning context - # The signature field helps maintain reasoning continuity - api_messages.append(api_msg) - - # Build the final system message: cached prompt + ephemeral system prompt. - # The ephemeral part is appended here (not baked into the cached prompt) - # so it stays out of the session DB and logs. - effective_system = active_system_prompt or "" - if self.ephemeral_system_prompt: - effective_system = (effective_system + "\n\n" + self.ephemeral_system_prompt).strip() - if self._honcho_context: - effective_system = (effective_system + "\n\n" + self._honcho_context).strip() - if effective_system: - api_messages = [{"role": "system", "content": effective_system}] + api_messages - - # Inject ephemeral prefill messages right after the system prompt - # but before conversation history. Same API-call-time-only pattern. - if self.prefill_messages: - sys_offset = 1 if effective_system else 0 - for idx, pfm in enumerate(self.prefill_messages): - api_messages.insert(sys_offset + idx, pfm.copy()) - - # Apply Anthropic prompt caching for Claude models via OpenRouter. - # Auto-detected: if model name contains "claude" and base_url is OpenRouter, - # inject cache_control breakpoints (system + last 3 messages) to reduce - # input token costs by ~75% on multi-turn conversations. - if self._use_prompt_caching: - api_messages = apply_anthropic_cache_control(api_messages, cache_ttl=self._cache_ttl) - - # Calculate approximate request size for logging - total_chars = sum(len(str(msg)) for msg in api_messages) - approx_tokens = total_chars // 4 # Rough estimate: 4 chars per token + api_messages, total_chars, approx_tokens = self._prepare_api_messages( + messages, + active_system_prompt, + ) # Thinking spinner for quiet mode (animated during API call) thinking_spinner = None @@ -2976,6 +3486,7 @@ def run_conversation( retry_count = 0 max_retries = 6 # Increased to allow longer backoff periods codex_auth_retry_attempted = False + missing_tool_output_recovery_attempted = False finish_reason = "stop" @@ -3254,6 +3765,34 @@ def run_conversation( or 'payload too large' in error_msg or 'error code: 413' in error_msg ) + is_missing_tool_output_error = ( + "no tool output found for function call" in error_msg + ) + is_context_length_error = any(phrase in error_msg for phrase in [ + 'context length', 'maximum context', 'context window', 'token limit', + 'too many tokens', 'reduce the length', 'exceeds the limit', + 'exceeds the context window', + 'request entity too large', # OpenRouter/Nous 413 safety net + ]) + + if ( + is_missing_tool_output_error + and not missing_tool_output_recovery_attempted + ): + missing_tool_output_recovery_attempted = True + recovered = self._fill_missing_tool_outputs( + messages, + "Recovered missing tool output after provider validation error.", + ) + if recovered: + print( + f"{self.log_prefix}🔧 Recovered missing tool output(s); rebuilding request and retrying..." + ) + api_messages, total_chars, approx_tokens = self._prepare_api_messages( + messages, + active_system_prompt, + ) + continue if is_payload_too_large: print(f"{self.log_prefix}⚠️ Request payload too large (413) - attempting compression...") @@ -3265,8 +3804,23 @@ def run_conversation( if len(messages) < original_len: print(f"{self.log_prefix} 🗜️ Compressed {original_len} → {len(messages)} messages, retrying...") + api_messages, total_chars, approx_tokens = self._prepare_api_messages( + messages, + active_system_prompt, + ) continue # Retry with compressed messages else: + trimmed = self._hard_trim_context(messages) + if len(trimmed) < len(messages): + messages = trimmed + print( + f"{self.log_prefix} ✂️ Hard-trimmed history to {len(messages)} messages after 413; retrying..." + ) + api_messages, total_chars, approx_tokens = self._prepare_api_messages( + messages, + active_system_prompt, + ) + continue print(f"{self.log_prefix}❌ Payload too large and cannot compress further.") logging.error(f"{self.log_prefix}413 payload too large. Cannot compress further.") self._persist_session(messages, conversation_history) @@ -3291,7 +3845,7 @@ def run_conversation( 'unauthorized', 'forbidden', 'not found', ]) - if is_client_error: + if is_client_error and not is_context_length_error: self._dump_api_request_debug( api_kwargs, reason="non_retryable_client_error", error=api_error, ) @@ -3308,13 +3862,6 @@ def run_conversation( "error": str(api_error), } - # Check for non-retryable errors (context length exceeded) - is_context_length_error = any(phrase in error_msg for phrase in [ - 'context length', 'maximum context', 'token limit', - 'too many tokens', 'reduce the length', 'exceeds the limit', - 'request entity too large', # OpenRouter/Nous 413 safety net - ]) - if is_context_length_error: print(f"{self.log_prefix}⚠️ Context length exceeded - attempting compression...") @@ -3325,9 +3872,25 @@ def run_conversation( if len(messages) < original_len: print(f"{self.log_prefix} 🗜️ Compressed {original_len} → {len(messages)} messages, retrying...") + api_messages, total_chars, approx_tokens = self._prepare_api_messages( + messages, + active_system_prompt, + ) continue # Retry with compressed messages else: - # Can't compress further + trimmed = self._hard_trim_context(messages) + if len(trimmed) < len(messages): + messages = trimmed + print( + f"{self.log_prefix} ✂️ Hard-trimmed history to {len(messages)} messages; retrying..." + ) + api_messages, total_chars, approx_tokens = self._prepare_api_messages( + messages, + active_system_prompt, + ) + continue + + # Can't compress or trim further print(f"{self.log_prefix}❌ Context length exceeded and cannot compress further.") print(f"{self.log_prefix} 💡 The conversation has accumulated too much content.") logging.error(f"{self.log_prefix}Context length exceeded: {approx_tokens:,} tokens. Cannot compress further.") @@ -3527,6 +4090,7 @@ def run_conversation( # Validate tool call arguments are valid JSON # Handle empty strings as empty objects (common model quirk) + # For "terminal", try to repair common LLM JSON errors (unescaped quotes, truncation) invalid_json_args = [] for tc in assistant_message.tool_calls: args = tc.function.arguments @@ -3537,7 +4101,11 @@ def run_conversation( try: json.loads(args) except json.JSONDecodeError as e: - invalid_json_args.append((tc.function.name, str(e))) + repaired = _repair_terminal_tool_args(tc.function.name, args) + if repaired is not None: + tc.function.arguments = repaired + else: + invalid_json_args.append((tc.function.name, str(e))) if invalid_json_args: # Track retries for invalid JSON arguments @@ -3577,17 +4145,34 @@ def run_conversation( # answer and calls memory/skill tools as a side-effect in the same # turn. If the follow-up turn after tools is empty, we use this. turn_content = assistant_message.content or "" - if turn_content and self._has_content_after_think_block(turn_content): + if turn_content and turn_content.strip(): self._last_content_with_tools = turn_content # Show intermediate commentary so the user can follow along if self.quiet_mode: clean = self._strip_think_blocks(turn_content).strip() if clean: - print(f" ┊ 💬 {clean}") - + preview = clean[:120] + "..." if len(clean) > 120 else clean + print(f" ┊ 💬 {preview}") + + assistant_signature = self._tool_call_signature(assistant_message.tool_calls) + if assistant_signature == last_tool_signature: + repeated_tool_signature_count += 1 + else: + last_tool_signature = assistant_signature + repeated_tool_signature_count = 1 + messages.append(assistant_msg) self._log_msg_to_db(assistant_msg) - + + if repeated_tool_signature_count >= tool_signature_repeat_limit: + if not self.quiet_mode: + print( + f"{self.log_prefix}⚠️ Repeated tool-call pattern detected. " + "Summarizing now to avoid a loop." + ) + final_response = self._handle_max_iterations(messages, api_call_count) + break + self._execute_tool_calls(assistant_message, messages, effective_task_id) if self.compression_enabled and self.context_compressor.should_compress(): @@ -3607,17 +4192,16 @@ def run_conversation( # No tool calls - this is the final response final_response = assistant_message.content or "" - # Check if response only has think block with no actual content after it - if not self._has_content_after_think_block(final_response): + # Retry only when the model returned truly empty content. + if not final_response.strip(): # Track retries for empty-after-think responses if not hasattr(self, '_empty_content_retries'): self._empty_content_retries = 0 self._empty_content_retries += 1 - # Show the reasoning/thinking content so the user can see - # what the model was thinking even though content is empty + # Show any reasoning/thinking content for debugging context. reasoning_text = self._extract_reasoning(assistant_message) - print(f"{self.log_prefix}⚠️ Response only contains think block with no content after it") + print(f"{self.log_prefix}⚠️ Response content was empty") if reasoning_text: reasoning_preview = reasoning_text[:500] + "..." if len(reasoning_text) > 500 else reasoning_text print(f"{self.log_prefix} Reasoning: {reasoning_preview}") @@ -3710,6 +4294,24 @@ def run_conversation( # Strip blocks from user-facing response (keep raw in messages for trajectory) final_response = self._strip_think_blocks(final_response).strip() + + if ( + final_answer_guard_retries < 1 + and not self._answers_latest_user_message(final_response, original_user_message) + ): + final_answer_guard_retries += 1 + guard_msg = { + "role": "user", + "content": ( + "[System check: Your previous draft did not directly answer the most recent user " + "message. Re-read the latest user request and respond directly, without changing topics.]" + ), + } + messages.append(guard_msg) + self._log_msg_to_db(guard_msg) + continue + + final_answer_guard_retries = 0 final_msg = self._build_assistant_message(assistant_message, finish_reason) @@ -3730,30 +4332,7 @@ def run_conversation( # If an assistant message with tool_calls was already appended, # the API expects a role="tool" result for every tool_call_id. # Fill in error results for any that weren't answered yet. - pending_handled = False - for idx in range(len(messages) - 1, -1, -1): - msg = messages[idx] - if not isinstance(msg, dict): - break - if msg.get("role") == "tool": - continue - if msg.get("role") == "assistant" and msg.get("tool_calls"): - answered_ids = { - m["tool_call_id"] - for m in messages[idx + 1:] - if isinstance(m, dict) and m.get("role") == "tool" - } - for tc in msg["tool_calls"]: - if tc["id"] not in answered_ids: - err_msg = { - "role": "tool", - "tool_call_id": tc["id"], - "content": f"Error executing tool: {error_msg}", - } - messages.append(err_msg) - self._log_msg_to_db(err_msg) - pending_handled = True - break + pending_handled = self._fill_missing_tool_outputs(messages, error_msg) if not pending_handled: # Error happened before tool processing (e.g. response parsing). @@ -3825,7 +4404,7 @@ def chat(self, message: str) -> str: def main( query: str = None, - model: str = "anthropic/claude-opus-4.6", + model: str = "google/gemini-2.0-flash-001:free", api_key: str = None, base_url: str = "https://openrouter.ai/api/v1", max_turns: int = 10, @@ -3842,7 +4421,7 @@ def main( Args: query (str): Natural language query for the agent. Defaults to Python 3.13 example. - model (str): Model name to use (OpenRouter format: provider/model). Defaults to anthropic/claude-sonnet-4-20250514. + model (str): Model name to use (OpenRouter format: provider/model). Defaults to google/gemini-2.0-flash-001:free. api_key (str): API key for authentication. Uses OPENROUTER_API_KEY env var if not provided. base_url (str): Base URL for the model API. Defaults to https://openrouter.ai/api/v1 max_turns (int): Maximum number of API call iterations. Defaults to 10. diff --git a/scripts/install.cmd b/scripts/install.cmd deleted file mode 100644 index 7c4cf7ef698c..000000000000 --- a/scripts/install.cmd +++ /dev/null @@ -1,28 +0,0 @@ -@echo off -REM ============================================================================ -REM Hermes Agent Installer for Windows (CMD wrapper) -REM ============================================================================ -REM This batch file launches the PowerShell installer for users running CMD. -REM -REM Usage: -REM curl -fsSL https://raw.githubusercontent.com/NousResearch/hermes-agent/main/scripts/install.cmd -o install.cmd && install.cmd && del install.cmd -REM -REM Or if you're already in PowerShell, use the direct command instead: -REM irm https://raw.githubusercontent.com/NousResearch/hermes-agent/main/scripts/install.ps1 | iex -REM ============================================================================ - -echo. -echo Hermes Agent Installer -echo Launching PowerShell installer... -echo. - -powershell -ExecutionPolicy ByPass -NoProfile -Command "irm https://raw.githubusercontent.com/NousResearch/hermes-agent/main/scripts/install.ps1 | iex" - -if %ERRORLEVEL% NEQ 0 ( - echo. - echo Installation failed. Please try running PowerShell directly: - echo powershell -ExecutionPolicy ByPass -c "irm https://raw.githubusercontent.com/NousResearch/hermes-agent/main/scripts/install.ps1 | iex" - echo. - pause - exit /b 1 -) diff --git a/scripts/install.ps1 b/scripts/install.ps1 index b4e9758fe13b..be3bfaabf4fb 100644 --- a/scripts/install.ps1 +++ b/scripts/install.ps1 @@ -65,6 +65,31 @@ function Write-Err { Write-Host "✗ $Message" -ForegroundColor Red } +function Invoke-GitCommand { + param([Parameter(Mandatory = $true)][string[]]$Args) + + $oldErrorActionPreference = $ErrorActionPreference + try { + # Clone fallback paths intentionally probe and can fail. + $ErrorActionPreference = "Continue" + $output = & git @Args 2>&1 + $exitCode = $LASTEXITCODE + } finally { + $ErrorActionPreference = $oldErrorActionPreference + } + + return [PSCustomObject]@{ + ExitCode = $exitCode + Output = @($output) + } +} + +function Test-IsHermesOriginSsh { + param([string]$Url) + if (-not $Url) { return $false } + return $Url -match "^(git@github\.com:NousResearch/hermes-agent(\.git)?|ssh://git@github\.com/NousResearch/hermes-agent(\.git)?)$" +} + # ============================================================================ # Dependency checks # ============================================================================ @@ -145,49 +170,17 @@ function Test-Python { # Python not found — use uv to install it (no admin needed!) Write-Info "Python $PythonVersion not found, installing via uv..." try { - $uvOutput = & $UvCmd python install $PythonVersion 2>&1 - if ($LASTEXITCODE -eq 0) { - $pythonPath = & $UvCmd python find $PythonVersion 2>$null - if ($pythonPath) { - $ver = & $pythonPath --version 2>$null - Write-Success "Python installed: $ver" - return $true - } - } else { - Write-Warn "uv python install output:" - Write-Host $uvOutput -ForegroundColor DarkGray - } - } catch { - Write-Warn "uv python install error: $_" - } - - # Fallback: check if ANY Python 3.10+ is already available on the system - Write-Info "Trying to find any existing Python 3.10+..." - foreach ($fallbackVer in @("3.12", "3.13", "3.10")) { - try { - $pythonPath = & $UvCmd python find $fallbackVer 2>$null - if ($pythonPath) { - $ver = & $pythonPath --version 2>$null - Write-Success "Found fallback: $ver" - $script:PythonVersion = $fallbackVer - return $true - } - } catch { } - } - - # Fallback: try system python - if (Get-Command python -ErrorAction SilentlyContinue) { - $sysVer = python --version 2>$null - if ($sysVer -match "3\.(1[0-9]|[1-9][0-9])") { - Write-Success "Using system Python: $sysVer" + & $UvCmd python install $PythonVersion 2>&1 | Out-Null + $pythonPath = & $UvCmd python find $PythonVersion 2>$null + if ($pythonPath) { + $ver = & $pythonPath --version 2>$null + Write-Success "Python installed: $ver" return $true } - } + } catch { } Write-Err "Failed to install Python $PythonVersion" - Write-Info "Install Python 3.11 manually, then re-run this script:" - Write-Info " https://www.python.org/downloads/" - Write-Info " Or: winget install Python.Python.3.11" + Write-Info "Install Python $PythonVersion manually, then re-run this script" return $false } @@ -416,7 +409,22 @@ function Install-Repository { if (Test-Path "$InstallDir\.git") { Write-Info "Existing installation found, updating..." Push-Location $InstallDir - git fetch origin + + $originUrl = (& git remote get-url origin 2>$null) + $fetchResult = Invoke-GitCommand -Args @("fetch", "origin") + if ($fetchResult.ExitCode -ne 0 -and (Test-IsHermesOriginSsh -Url $originUrl)) { + Write-Warn "SSH fetch failed, switching origin remote to HTTPS..." + $setUrlResult = Invoke-GitCommand -Args @("remote", "set-url", "origin", $RepoUrlHttps) + if ($setUrlResult.ExitCode -eq 0) { + $fetchResult = Invoke-GitCommand -Args @("fetch", "origin") + } + } + if ($fetchResult.ExitCode -ne 0) { + Pop-Location + Write-Err "Failed to fetch updates from origin" + exit 1 + } + git checkout $Branch git pull origin $Branch Pop-Location @@ -426,37 +434,28 @@ function Install-Repository { exit 1 } } else { - # Try SSH first (for private repo access), fall back to HTTPS. - # GIT_SSH_COMMAND with BatchMode=yes prevents SSH from hanging - # when no key is configured (fails immediately instead of prompting). - # - # IMPORTANT: Do NOT use 2>&1 on git commands in PowerShell. - # With $ErrorActionPreference = "Stop", PowerShell wraps captured - # stderr lines in ErrorRecord objects, turning git's normal progress - # messages ("Cloning into ...") into terminating NativeCommandErrors. - # Let stderr flow to the console naturally (like OpenClaw does). - Write-Info "Trying SSH clone..." - $env:GIT_SSH_COMMAND = "ssh -o BatchMode=yes -o ConnectTimeout=5" - try { - git clone --branch $Branch --recurse-submodules $RepoUrlSsh $InstallDir - $sshExitCode = $LASTEXITCODE - } catch { - $sshExitCode = 1 - } - $env:GIT_SSH_COMMAND = $null - - if ($sshExitCode -eq 0) { - Write-Success "Cloned via SSH" + # Prefer HTTPS (works without SSH keys), fall back to SSH for private access + Write-Info "Trying HTTPS clone..." + $httpsResult = Invoke-GitCommand -Args @( + "clone", "--branch", $Branch, "--recurse-submodules", $RepoUrlHttps, $InstallDir + ) + + if ($httpsResult.ExitCode -eq 0) { + Write-Success "Cloned via HTTPS" } else { - # Clean up partial SSH clone before retrying - if (Test-Path $InstallDir) { Remove-Item -Recurse -Force $InstallDir -ErrorAction SilentlyContinue } - Write-Info "SSH failed, trying HTTPS..." - git clone --branch $Branch --recurse-submodules $RepoUrlHttps $InstallDir - - if ($LASTEXITCODE -eq 0) { - Write-Success "Cloned via HTTPS" + Write-Info "HTTPS failed, trying SSH..." + $sshResult = Invoke-GitCommand -Args @( + "clone", "--branch", $Branch, "--recurse-submodules", $RepoUrlSsh, $InstallDir + ) + + if ($sshResult.ExitCode -eq 0) { + Write-Success "Cloned via SSH" } else { Write-Err "Failed to clone repository" + Write-Info "If this is a public install, HTTPS should work without SSH keys." + Write-Info "For private repo access, configure GitHub authentication:" + Write-Info " gh auth login" + Write-Info " # or configure SSH and test with: ssh -T git@github.com" exit 1 } } @@ -810,8 +809,8 @@ function Write-Completion { Write-Host "View/edit configuration" Write-Host " hermes config edit " -NoNewline -ForegroundColor Green Write-Host "Open config in editor" - Write-Host " hermes gateway " -NoNewline -ForegroundColor Green - Write-Host "Start messaging gateway (Telegram, Discord, etc.)" + Write-Host " hermes gateway install " -NoNewline -ForegroundColor Green + Write-Host "Install gateway service (messaging + cron)" Write-Host " hermes update " -NoNewline -ForegroundColor Green Write-Host "Update to latest version" Write-Host "" diff --git a/scripts/install.sh b/scripts/install.sh index 0e2cf92a65b0..70f3f8533172 100755 --- a/scripts/install.sh +++ b/scripts/install.sh @@ -537,6 +537,16 @@ show_manual_install_hint() { esac } +is_hermes_origin_ssh() { + local origin_url="$1" + case "$origin_url" in + git@github.com:NousResearch/hermes-agent|git@github.com:NousResearch/hermes-agent.git|ssh://git@github.com/NousResearch/hermes-agent|ssh://git@github.com/NousResearch/hermes-agent.git) + return 0 + ;; + esac + return 1 +} + # ============================================================================ # Installation # ============================================================================ @@ -548,7 +558,17 @@ clone_repo() { if [ -d "$INSTALL_DIR/.git" ]; then log_info "Existing installation found, updating..." cd "$INSTALL_DIR" - git fetch origin + origin_url="$(git remote get-url origin 2>/dev/null || true)" + if ! git fetch origin; then + if is_hermes_origin_ssh "$origin_url"; then + log_warn "SSH fetch failed, switching origin remote to HTTPS..." + git remote set-url origin "$REPO_URL_HTTPS" + git fetch origin + else + log_error "Failed to fetch updates from origin" + exit 1 + fi + fi git checkout "$BRANCH" git pull origin "$BRANCH" else @@ -557,21 +577,21 @@ clone_repo() { exit 1 fi else - # Try SSH first (for private repo access), fall back to HTTPS + # Prefer HTTPS (works without SSH keys), fall back to SSH for private access # Use --recurse-submodules to also clone mini-swe-agent and tinker-atropos - # GIT_SSH_COMMAND disables interactive prompts and sets a short timeout - # so SSH fails fast instead of hanging when no key is configured. - log_info "Trying SSH clone..." - if GIT_SSH_COMMAND="ssh -o BatchMode=yes -o ConnectTimeout=5" \ - git clone --branch "$BRANCH" --recurse-submodules "$REPO_URL_SSH" "$INSTALL_DIR" 2>/dev/null; then - log_success "Cloned via SSH" + log_info "Trying HTTPS clone..." + if git clone --branch "$BRANCH" --recurse-submodules "$REPO_URL_HTTPS" "$INSTALL_DIR"; then + log_success "Cloned via HTTPS" else - rm -rf "$INSTALL_DIR" 2>/dev/null # Clean up partial SSH clone - log_info "SSH failed, trying HTTPS..." - if git clone --branch "$BRANCH" --recurse-submodules "$REPO_URL_HTTPS" "$INSTALL_DIR"; then - log_success "Cloned via HTTPS" + log_info "HTTPS failed, trying SSH..." + if git clone --branch "$BRANCH" --recurse-submodules "$REPO_URL_SSH" "$INSTALL_DIR"; then + log_success "Cloned via SSH" else log_error "Failed to clone repository" + log_info "If this is a public install, HTTPS should work without SSH keys." + log_info "For private repo access, configure GitHub authentication:" + log_info " gh auth login" + log_info " # or configure SSH and test with: ssh -T git@github.com" exit 1 fi fi diff --git a/scripts/sample_and_compress.py b/scripts/sample_and_compress.py index 419111d80fd4..ecaa15343071 100644 --- a/scripts/sample_and_compress.py +++ b/scripts/sample_and_compress.py @@ -22,9 +22,12 @@ from typing import List, Dict, Any, Tuple import fire -# Load environment variables -from dotenv import load_dotenv -load_dotenv() +# Load environment variables (encoding-safe on Windows) +from agent.env_loader import load_dotenv_with_fallback +for _p in (Path.home() / ".hermes" / ".env", Path.cwd() / ".env"): + if _p.exists(): + load_dotenv_with_fallback(_p) + break # Default datasets to sample from diff --git a/skills/media/DESCRIPTION.md b/skills/media/DESCRIPTION.md index 63501dcf297c..d3f2ab334138 100644 --- a/skills/media/DESCRIPTION.md +++ b/skills/media/DESCRIPTION.md @@ -1 +1,3 @@ -Media content extraction and transformation tools — YouTube transcripts, audio, video processing. +--- +description: Skills for fetching, downloading, and processing video and audio from YouTube and other media sources. +--- diff --git a/skills/media/yt-dlp/SKILL.md b/skills/media/yt-dlp/SKILL.md new file mode 100644 index 000000000000..b443c5a7b700 --- /dev/null +++ b/skills/media/yt-dlp/SKILL.md @@ -0,0 +1,105 @@ +--- +name: yt-dlp +description: Use yt-dlp to fetch YouTube metadata, transcripts, and media, plus summarize videos by default when given only a YouTube URL. Use for channel stats, playlist inspection, subtitle extraction, audio or video downloads, and consistent temp-folder workflows with yt-dlp. +version: 1.0.0 +author: community +license: MIT +metadata: + hermes: + tags: [YouTube, yt-dlp, Video, Transcripts, Subtitles, Download, Metadata, Summarize] +--- + +# yt-dlp + +Use a dedicated temp folder for **all yt-dlp work** so you never spray files around the workspace. The default temp root is `TMP/yt-dlp` **inside the workspace root** (for Hermes this means a relative path `TMP/yt-dlp` from the current working directory). Always create it if missing, and keep intermediate JSON, subtitles, and media in that folder unless the user explicitly asks for a different destination. Prefer writing outputs into temp and then moving or copying to a final location if needed. If you create helper scripts, put them under `workspace/scripts/yt-dlp` and delete one-off scripts after use. Before fetching channel stats or user-specific data, check TOOLS.md for usernames and tool variables. + +When constructing commands, **never run yt-dlp in random directories**; always either: + +- `cd` into `TMP/yt-dlp` first, **or** +- pass `--paths TMP/yt-dlp` / `-P TMP/yt-dlp` so all output files land there. + +On Windows / PowerShell (your setup), use **this pattern only**: + +```powershell +New-Item -ItemType Directory -Force -Path "TMP/yt-dlp" | Out-Null +yt-dlp --paths "TMP/yt-dlp" ... +``` + +## Plain YouTube URL (default: summarize) + +When given a **plain YouTube URL with no extra instruction**, treat it as: +“Give me a **thorough written explanation** of this video as if I will *not* watch it.” + +Default flow (run **at most once per URL**; if any yt-dlp step fails, show the error and explain it instead of looping retries): + +1. **Prep temp folder** + - Ensure `TMP/yt-dlp` exists as above. +2. **Metadata JSON** + - Run yt-dlp with `--dump-single-json` into temp, e.g.: + - PowerShell: + ```powershell + yt-dlp --dump-single-json --paths \"TMP/yt-dlp\" -o \"%(title)s [%(id)s].%(ext)s\" > \"TMP/yt-dlp\\meta.json\" + ``` + - POSIX: + ```bash + yt-dlp --dump-single-json --paths TMP/yt-dlp -o '%(title)s [%(id)s].%(ext)s' > TMP/yt-dlp/meta.json + ``` +3. **Subtitles first (preferred)** + - Try captions with **no media download**: + - `--write-auto-subs` or `--write-subs` + - `--sub-langs en` (or the user’s language) + - `--sub-format vtt` + - `--skip-download` + - Always keep output in `TMP/yt-dlp` via `--paths` / `-P`. + - Then use `read_file` to open the `.vtt` from `TMP/yt-dlp` and summarize **from the transcript**. +4. **If no captions exist** + - Extract audio into `TMP/yt-dlp` with `--extract-audio` and an audio format (e.g. `mp3`), then summarize from the audio (or via Whisper if available). +5. **Summary style** + - Write a **detailed, self-contained explanation** as if the user will **never watch the video**: + - Who is speaking / main actors + - Main claims, arguments, and conclusions + - Important evidence, examples, or stories + - Timeline / structure (intro, key sections, ending) + - Any notable quotes or numbers (summarized, not just copied) + - Do **not** just say “here are subtitles”; actually digest and explain. + - Keep the summary in the response text (not as a file) unless the user explicitly asks for a file. + +## Channel or playlist stats + +Use `--dump-single-json` with `--flat-playlist` for speed, then parse fields like `channel`, `channel_id`, `uploader`, `uploader_id`, `channel_follower_count`, and entry counts. If view counts are missing from yt-dlp, say so and offer a browser scrape as a follow up. + +## Media downloads + +Use `-F` for formats, then `-f` to select, or `--extract-audio` with `--audio-format` and `--audio-quality` for audio. Use `-o` to control output names, and `--paths` to set final destinations. Use `--download-archive` to avoid duplicates for batch pulls. + +## Subtitles + +Use `--list-subs` first when accuracy matters, then `--write-subs` or `--write-auto-subs` with `--sub-langs` and `--sub-format`. Use `--convert-subs` if you need srt. + +## Metadata and assets + +Use `--write-info-json`, `--write-thumbnail`, or `--write-all-thumbnails`, and `--embed-metadata` or `--embed-thumbnail` when producing final media. + +## Auth or member-only access + +Use `--cookies-from-browser` or `--cookies`. Keep cookie paths private and never print them. + +## JavaScript runtime warnings + +If yt-dlp warns about missing JavaScript runtimes, note it and continue unless extraction fails. If it fails, suggest adding a JS runtime or switching to browser extraction. + +## Whisper and Windows + +When transcribing with Whisper, prefer CUDA if available by using `--device cuda`, and set UTF-8 output (for example set `PYTHONUTF8=1` or use an output format like srt/json) to avoid Windows UnicodeEncodeError when writing files. + +## General + +Prefer short, direct commands. Avoid listing huge inventories of formats unless explicitly asked. + +If a yt-dlp command fails or you’re unsure about flags, **it is always allowed to run**: + +```bash +yt-dlp --help +``` + +and then adjust the command based on the documented `Usage: yt-dlp [OPTIONS] URL [URL...]` and options. Never keep retrying the same failing command blindly; check `--help` or the error message once, fix the command, and then try again **at most one more time**. diff --git a/tests/integration/test_batch_runner.py b/tests/integration/test_batch_runner.py index 85565ae6e49c..70dd79498935 100644 --- a/tests/integration/test_batch_runner.py +++ b/tests/integration/test_batch_runner.py @@ -25,7 +25,7 @@ def create_test_dataset(): {"prompt": "Explain what Python is in one sentence."}, ] - with open(test_file, 'w') as f: + with open(test_file, "w", encoding="utf-8", newline="") as f: for prompt in prompts: f.write(json.dumps(prompt, ensure_ascii=False) + "\n") @@ -74,7 +74,7 @@ def verify_output(run_name): print(f" - Batch files: {len(batch_files)}") # Load and display statistics - with open(stats_file) as f: + with open(stats_file, encoding="utf-8") as f: stats = json.load(f) print(f"\n📊 Statistics Summary:") diff --git a/tests/integration/test_checkpoint_resumption.py b/tests/integration/test_checkpoint_resumption.py index a5b1a2aa99ff..6cde55b43e78 100644 --- a/tests/integration/test_checkpoint_resumption.py +++ b/tests/integration/test_checkpoint_resumption.py @@ -81,7 +81,7 @@ def monitor_checkpoint_during_run(checkpoint_file: Path, duration: int = 30) -> elapsed = time.time() - start_time try: - with open(checkpoint_file, 'r') as f: + with open(checkpoint_file, "r", encoding="utf-8") as f: checkpoint_data = json.load(f) snapshot = { @@ -238,9 +238,9 @@ def test_interruption_and_resume(): temp_dataset = Path("tests/test_data/checkpoint_test_resume_partial.jsonl") try: # Create a modified dataset with only first 5 prompts for initial run - with open(dataset_file, 'r') as f: + with open(dataset_file, "r", encoding="utf-8") as f: lines = f.readlines()[:5] - with open(temp_dataset, 'w') as f: + with open(temp_dataset, "w", encoding="utf-8", newline="") as f: f.writelines(lines) runner = BatchRunner( @@ -261,7 +261,7 @@ def test_interruption_and_resume(): print("❌ ERROR: Checkpoint file not created after first run") return False - with open(checkpoint_file, 'r') as f: + with open(checkpoint_file, "r", encoding="utf-8") as f: checkpoint_data = json.load(f) initial_completed = len(checkpoint_data.get("completed_prompts", [])) @@ -284,7 +284,7 @@ def test_interruption_and_resume(): runner2.run(resume=True) # Check final checkpoint - with open(checkpoint_file, 'r') as f: + with open(checkpoint_file, "r", encoding="utf-8") as f: final_checkpoint = json.load(f) final_completed = len(final_checkpoint.get("completed_prompts", [])) diff --git a/tests/integration/test_modal_terminal.py b/tests/integration/test_modal_terminal.py index 11943f209405..d3aaf6c7d66f 100644 --- a/tests/integration/test_modal_terminal.py +++ b/tests/integration/test_modal_terminal.py @@ -21,15 +21,17 @@ import json from pathlib import Path -# Try to load .env file if python-dotenv is available +# Try to load .env file (encoding-safe on Windows) try: - from dotenv import load_dotenv - load_dotenv() + from agent.env_loader import load_dotenv_with_fallback + _env_path = Path(__file__).parent.parent.parent / ".env" + if _env_path.exists(): + load_dotenv_with_fallback(_env_path) except ImportError: # Manually load .env if dotenv not available env_file = Path(__file__).parent.parent.parent / ".env" if env_file.exists(): - with open(env_file) as f: + with open(env_file, encoding="utf-8", errors="replace") as f: for line in f: line = line.strip() if line and not line.startswith('#') and '=' in line: diff --git a/tests/integration/test_web_tools.py b/tests/integration/test_web_tools.py index 971d98f2c32c..04d194a79099 100644 --- a/tests/integration/test_web_tools.py +++ b/tests/integration/test_web_tools.py @@ -585,7 +585,7 @@ def save_results(self): } try: - with open(filename, 'w') as f: + with open(filename, "w", encoding="utf-8", newline="") as f: json.dump(results, f, indent=2, ensure_ascii=False) print_info(f"Test results saved to: {filename}") except Exception as e: diff --git a/tests/test_auth_codex_provider.py b/tests/test_auth_codex_provider.py index 4119126e6689..25a838173088 100644 --- a/tests/test_auth_codex_provider.py +++ b/tests/test_auth_codex_provider.py @@ -76,6 +76,7 @@ def test_resolve_codex_runtime_credentials_missing_access_token(tmp_path, monkey hermes_home = tmp_path / "hermes" _setup_hermes_auth(hermes_home, access_token="") monkeypatch.setenv("HERMES_HOME", str(hermes_home)) + monkeypatch.setenv("CODEX_HOME", str(tmp_path / "nonexistent-codex-home")) with pytest.raises(AuthError) as exc: resolve_codex_runtime_credentials() @@ -83,6 +84,48 @@ def test_resolve_codex_runtime_credentials_missing_access_token(tmp_path, monkey assert exc.value.relogin_required is True +def test_resolve_codex_runtime_credentials_recovers_from_invalid_shape_with_cli_tokens(tmp_path, monkeypatch): + hermes_home = tmp_path / "hermes" + hermes_home.mkdir(parents=True, exist_ok=True) + (hermes_home / "auth.json").write_text( + json.dumps( + { + "version": 1, + "providers": { + "openai-codex": { + "auth_file": "/tmp/codex/auth.json", + "source": "codex-auth-json", + } + }, + }, + indent=2, + ) + ) + + codex_home = tmp_path / "codex-cli" + codex_home.mkdir(parents=True, exist_ok=True) + (codex_home / "auth.json").write_text( + json.dumps( + { + "tokens": { + "access_token": "cli-at", + "refresh_token": "cli-rt", + } + } + ) + ) + + monkeypatch.setenv("HERMES_HOME", str(hermes_home)) + monkeypatch.setenv("CODEX_HOME", str(codex_home)) + + resolved = resolve_codex_runtime_credentials(refresh_if_expiring=False) + assert resolved["api_key"] == "cli-at" + + data = _read_codex_tokens() + assert data["tokens"]["access_token"] == "cli-at" + assert data["tokens"]["refresh_token"] == "cli-rt" + + def test_resolve_codex_runtime_credentials_refreshes_expiring_token(tmp_path, monkeypatch): hermes_home = tmp_path / "hermes" expiring_token = _jwt_with_exp(int(time.time()) - 10) diff --git a/tests/test_browser_tool_resolver.py b/tests/test_browser_tool_resolver.py new file mode 100644 index 000000000000..26a20c9ab654 --- /dev/null +++ b/tests/test_browser_tool_resolver.py @@ -0,0 +1,62 @@ +"""Tests for cross-platform agent-browser executable resolution.""" + +from pathlib import Path + +import pytest + +try: + import tools.browser_tool as browser_tool +except ModuleNotFoundError as exc: + pytest.skip(f"Optional dependency missing for browser_tool import: {exc}", allow_module_level=True) + + +def _set_fake_repo_layout(monkeypatch, tmp_path: Path) -> Path: + tools_dir = tmp_path / "tools" + tools_dir.mkdir(parents=True, exist_ok=True) + fake_file = tools_dir / "browser_tool.py" + fake_file.write_text("# test placeholder", encoding="utf-8") + monkeypatch.setattr(browser_tool, "__file__", str(fake_file)) + return tmp_path / "node_modules" / ".bin" + + +def test_find_agent_browser_windows_prefers_local_cmd(monkeypatch, tmp_path: Path): + bin_dir = _set_fake_repo_layout(monkeypatch, tmp_path) + bin_dir.mkdir(parents=True, exist_ok=True) + cmd_path = bin_dir / "agent-browser.cmd" + cmd_path.write_text("@echo off", encoding="utf-8") + (bin_dir / "agent-browser").write_text("#!/bin/sh", encoding="utf-8") + + monkeypatch.setattr(browser_tool.os, "name", "nt", raising=False) + monkeypatch.setattr(browser_tool.shutil, "which", lambda _name: None) + + assert browser_tool._find_agent_browser() == [str(cmd_path)] + + +def test_find_agent_browser_windows_uses_npx_cmd_fallback(monkeypatch, tmp_path: Path): + _set_fake_repo_layout(monkeypatch, tmp_path) + + npx_cmd = r"C:\Program Files\nodejs\npx.cmd" + + def _fake_which(name: str): + if name == "npx.cmd": + return npx_cmd + return None + + monkeypatch.setattr(browser_tool.os, "name", "nt", raising=False) + monkeypatch.setattr(browser_tool.shutil, "which", _fake_which) + + assert browser_tool._find_agent_browser() == [npx_cmd, "agent-browser"] + + +def test_find_agent_browser_posix_prefers_path_binary(monkeypatch, tmp_path: Path): + _set_fake_repo_layout(monkeypatch, tmp_path) + + def _fake_which(name: str): + if name == "agent-browser": + return "/usr/local/bin/agent-browser" + return None + + monkeypatch.setattr(browser_tool.os, "name", "posix", raising=False) + monkeypatch.setattr(browser_tool.shutil, "which", _fake_which) + + assert browser_tool._find_agent_browser() == ["/usr/local/bin/agent-browser"] diff --git a/tests/test_cli_key_resolution.py b/tests/test_cli_key_resolution.py new file mode 100644 index 000000000000..7380a3def886 --- /dev/null +++ b/tests/test_cli_key_resolution.py @@ -0,0 +1,40 @@ +"""Regression tests for CLI API key selection across providers.""" + +from __future__ import annotations + +from cli import HermesCLI + + +def _clear_provider_env(monkeypatch) -> None: + for key in ( + "OPENAI_BASE_URL", + "OPENAI_API_KEY", + "OPENROUTER_BASE_URL", + "OPENROUTER_API_KEY", + "HERMES_INFERENCE_PROVIDER", + ): + monkeypatch.delenv(key, raising=False) + + +def test_cli_prefers_openrouter_key_for_openrouter_base(monkeypatch) -> None: + _clear_provider_env(monkeypatch) + monkeypatch.setenv("OPENROUTER_BASE_URL", "https://openrouter.ai/api/v1") + monkeypatch.setenv("OPENROUTER_API_KEY", "sk-or-test") + monkeypatch.setenv("OPENAI_API_KEY", "sk-proj-test") + + cli = HermesCLI() + + assert cli.base_url == "https://openrouter.ai/api/v1" + assert cli.api_key == "sk-or-test" + + +def test_cli_prefers_openai_key_for_custom_openai_base(monkeypatch) -> None: + _clear_provider_env(monkeypatch) + monkeypatch.setenv("OPENAI_BASE_URL", "https://api.openai.com/v1") + monkeypatch.setenv("OPENAI_API_KEY", "sk-proj-test") + monkeypatch.setenv("OPENROUTER_API_KEY", "sk-or-test") + + cli = HermesCLI() + + assert cli.base_url == "https://api.openai.com/v1" + assert cli.api_key == "sk-proj-test" diff --git a/tests/test_env_loader.py b/tests/test_env_loader.py new file mode 100644 index 000000000000..f1af1a214543 --- /dev/null +++ b/tests/test_env_loader.py @@ -0,0 +1,47 @@ +"""Tests for shared .env loader helpers.""" + +from pathlib import Path + +import pytest + +from agent.env_loader import read_env_text_with_fallback + + +def _write_bytes(path: Path, data: bytes) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(data) + + +def test_read_env_text_with_fallback_accepts_utf8(tmp_path: Path) -> None: + env_file = tmp_path / ".env" + _write_bytes( + env_file, + b"FIRECRAWL_API_KEY=fc_valid_token_1234567890\n", + ) + text, encoding = read_env_text_with_fallback(env_file) + assert "FIRECRAWL_API_KEY=fc_valid_token_1234567890" in text + assert encoding in ("utf-8", "utf-8-sig") + + +def test_read_env_text_with_fallback_rejects_corrupted_sensitive_value(tmp_path: Path) -> None: + env_file = tmp_path / ".env" + # cp1252-encoded non-ASCII byte in a sensitive token value. + _write_bytes(env_file, b"DISCORD_BOT_TOKEN=\xe0bad_token\n") + + with pytest.raises(ValueError) as exc: + read_env_text_with_fallback(env_file) + + assert "non-ASCII bytes" in str(exc.value) + assert "DISCORD_BOT_TOKEN" in str(exc.value) + + +def test_read_env_text_with_fallback_allows_non_ascii_comments(tmp_path: Path) -> None: + env_file = tmp_path / ".env" + # Non-ASCII bytes in comments are tolerated as long as sensitive values are valid. + _write_bytes( + env_file, + b"# comment with em dash \xe2\x80\x94 allowed\nFIRECRAWL_API_KEY=fc_valid_token_1234567890\n", + ) + + text, _ = read_env_text_with_fallback(env_file) + assert "FIRECRAWL_API_KEY=fc_valid_token_1234567890" in text diff --git a/tests/test_file_operations_search_glob.py b/tests/test_file_operations_search_glob.py new file mode 100644 index 000000000000..17137f145dcc --- /dev/null +++ b/tests/test_file_operations_search_glob.py @@ -0,0 +1,34 @@ +from pathlib import Path + +from tools.file_operations import ShellFileOperations + + +class _FakeLocalEnv: + __module__ = "tools.environments.local" + + def __init__(self, cwd: str): + self.cwd = cwd + + def execute(self, command: str, cwd: str = "", **kwargs): + # Search path for Windows local should use native walk implementation. + return {"output": "", "returncode": 0} + + +def test_search_content_with_glob_pattern_routes_to_file_search(tmp_path): + (tmp_path / "a.html").write_text("

A

", encoding="utf-8") + (tmp_path / "b.txt").write_text("hello", encoding="utf-8") + nested = tmp_path / "nested" + nested.mkdir() + (nested / "c.html").write_text("

C

", encoding="utf-8") + + env = _FakeLocalEnv(str(tmp_path)) + ops = ShellFileOperations(env) + + result = ops.search(pattern="*.html", path=str(tmp_path), target="content") + payload = result.to_dict() + + assert "error" not in payload + files = payload.get("files", []) + assert len(files) == 2 + assert any(f.endswith("a.html") for f in files) + assert any(f.endswith("c.html") for f in files) diff --git a/tests/test_file_operations_write_windows.py b/tests/test_file_operations_write_windows.py new file mode 100644 index 000000000000..e0c559dbca18 --- /dev/null +++ b/tests/test_file_operations_write_windows.py @@ -0,0 +1,30 @@ +from pathlib import Path + +from tools.file_operations import ShellFileOperations + + +class _FakeLocalEnv: + __module__ = "tools.environments.local" + + def __init__(self, cwd: str): + self.cwd = cwd + + def execute(self, command: str, cwd: str = "", **kwargs): + # Windows-local write path should not go through shell execution. + raise AssertionError(f"execute() should not be called, got: {command}") + + +def test_write_file_windows_local_uses_native_io(tmp_path): + env = _FakeLocalEnv(str(tmp_path)) + ops = ShellFileOperations(env) + + rel_path = r"agents\memory.md" + content = "EOT check: write via native IO" + result = ops.write_file(rel_path, content) + + assert result.error is None + assert result.bytes_written > 0 + + target = Path(ops._resolve_windows_path(rel_path)) + assert target.exists() + assert target.read_text(encoding="utf-8") == content diff --git a/tests/test_flush_memories_codex.py b/tests/test_flush_memories_codex.py index 22eef5ab034c..0c76519a183c 100644 --- a/tests/test_flush_memories_codex.py +++ b/tests/test_flush_memories_codex.py @@ -211,7 +211,7 @@ def test_codex_mode_no_aux_uses_responses_api(self, monkeypatch): "instructions": "test", "input": [], "tools": [], - "max_output_tokens": 4096, + "max_tokens": 4096, } messages = [ {"role": "user", "content": "Hello"}, diff --git a/tests/test_memory_tool_compaction.py b/tests/test_memory_tool_compaction.py new file mode 100644 index 000000000000..8c5844dd2205 --- /dev/null +++ b/tests/test_memory_tool_compaction.py @@ -0,0 +1,54 @@ +from pathlib import Path + +import tools.memory_tool as memory_tool + + +def test_memory_store_auto_compacts_oversized_legacy_file(monkeypatch, tmp_path): + monkeypatch.setattr(memory_tool, "MEMORY_DIR", tmp_path) + + # Legacy file: huge single entry with no delimiter. + big_entry = "A" * 5000 + (tmp_path / "MEMORY.md").write_text(big_entry, encoding="utf-8") + + store = memory_tool.MemoryStore(memory_char_limit=2200, user_char_limit=1375) + store.load_from_disk() + + assert store._char_count("memory") <= 2200 + disk_content = (tmp_path / "MEMORY.md").read_text(encoding="utf-8") + assert len(disk_content) <= 2200 + + # After compaction, normal writes should work again. + result = store.add("memory", "recent observation") + assert result["success"] is True + + +def test_add_eviction_makes_room_instead_of_error(monkeypatch, tmp_path): + monkeypatch.setattr(memory_tool, "MEMORY_DIR", tmp_path) + store = memory_tool.MemoryStore(memory_char_limit=120, user_char_limit=100) + store.memory_entries = ["old-a", "old-b", "old-c"] + store.save_to_disk("memory") + + # Make the list very full first. + store.memory_entries = [ + "x" * 50, + "y" * 50, + ] + store.save_to_disk("memory") + + res = store.add("memory", "z" * 40) + assert res["success"] is True + assert "Compacted memory" in res.get("message", "") + assert store.memory_entries[-1] == "z" * 40 + + +def test_remove_and_replace_no_match_are_noop_success(monkeypatch, tmp_path): + monkeypatch.setattr(memory_tool, "MEMORY_DIR", tmp_path) + store = memory_tool.MemoryStore(memory_char_limit=2200, user_char_limit=1375) + store.memory_entries = ["alpha note", "beta note"] + store.save_to_disk("memory") + + rep = store.replace("memory", "missing key", "new value") + rem = store.remove("memory", "missing key") + assert rep["success"] is True + assert rem["success"] is True + assert store.memory_entries == ["alpha note", "beta note"] diff --git a/tests/test_response_completion_and_discord_rate_limit.py b/tests/test_response_completion_and_discord_rate_limit.py new file mode 100644 index 000000000000..f7ec80856b7a --- /dev/null +++ b/tests/test_response_completion_and_discord_rate_limit.py @@ -0,0 +1,112 @@ +import asyncio +from types import SimpleNamespace + +from gateway.config import PlatformConfig +from gateway.platforms.discord import DiscordAdapter +from run_agent import AIAgent + + +class _FakeEmbed: + def __init__(self, description: str): + self.description = description + + +class _FakeMessage: + def __init__(self, msg_id: int): + self.id = msg_id + + +class _FakeRetryableError(Exception): + def __init__(self, message: str, retry_after: float = 1.0, status: int = 429, code: int = 0): + super().__init__(message) + self.retry_after = retry_after + self.status = status + self.code = code + self.response = None + + +class _FakeChannel: + def __init__(self, fail_first_send: bool = False): + self.fail_first_send = fail_first_send + self.send_calls = 0 + self.views = [] + + async def send(self, *, embed, view=None, reference=None): + self.send_calls += 1 + self.views.append(view) + if self.fail_first_send and self.send_calls == 1: + raise _FakeRetryableError("rate limited", retry_after=1.25) + return _FakeMessage(self.send_calls) + + +class _FakeClient: + def __init__(self, channel): + self._channel = channel + + def get_channel(self, _chat_id): + return self._channel + + async def fetch_channel(self, _chat_id): + return self._channel + + +def _build_adapter(channel): + adapter = DiscordAdapter(PlatformConfig(enabled=True, token="fake")) + adapter._client = _FakeClient(channel) + return adapter + + +def test_strip_think_blocks_removes_only_standard_tags(): + agent = AIAgent.__new__(AIAgent) + content = ( + "\ninternal\n\n" + "User-facing answer." + ) + assert agent._strip_think_blocks(content).strip() == "User-facing answer." + + +def test_discord_send_retries_with_retry_after(monkeypatch): + import gateway.platforms.discord as discord_module + + channel = _FakeChannel(fail_first_send=True) + adapter = _build_adapter(channel) + + monkeypatch.setattr(discord_module, "discord", SimpleNamespace(Embed=_FakeEmbed)) + monkeypatch.setattr(discord_module, "ListenButtonView", lambda _adapter: object()) + + sleeps = [] + + async def _fake_sleep(delay): + sleeps.append(delay) + + monkeypatch.setattr(discord_module.asyncio, "sleep", _fake_sleep) + + result = asyncio.run(adapter.send(chat_id="123", content="hello world")) + + assert result.success is True + assert channel.send_calls == 2 + assert sleeps + assert sleeps[0] >= 1.25 + + +def test_discord_send_attaches_listen_button_to_each_chunk(monkeypatch): + import gateway.platforms.discord as discord_module + + channel = _FakeChannel(fail_first_send=False) + adapter = _build_adapter(channel) + adapter.MAX_EMBED_DESCRIPTION = 100 + + monkeypatch.setattr(discord_module, "discord", SimpleNamespace(Embed=_FakeEmbed)) + monkeypatch.setattr(discord_module, "ListenButtonView", lambda _adapter: object()) + + async def _fake_sleep(_delay): + return None + + monkeypatch.setattr(discord_module.asyncio, "sleep", _fake_sleep) + + long_text = "x" * 280 + result = asyncio.run(adapter.send(chat_id="123", content=long_text)) + + assert result.success is True + assert channel.send_calls >= 3 + assert all(view is not None for view in channel.views) diff --git a/tests/test_run_agent_codex_responses.py b/tests/test_run_agent_codex_responses.py index a1e5e817e0fe..fc3a5adf8457 100644 --- a/tests/test_run_agent_codex_responses.py +++ b/tests/test_run_agent_codex_responses.py @@ -542,13 +542,15 @@ def test_preflight_codex_api_kwargs_allows_reasoning_and_temperature(monkeypatch kwargs["reasoning"] = {"effort": "high", "summary": "auto"} kwargs["include"] = ["reasoning.encrypted_content"] kwargs["temperature"] = 0.7 - kwargs["max_output_tokens"] = 4096 + kwargs["max_tokens"] = 4096 result = agent._preflight_codex_api_kwargs(kwargs) assert result["reasoning"] == {"effort": "high", "summary": "auto"} assert result["include"] == ["reasoning.encrypted_content"] assert result["temperature"] == 0.7 - assert result["max_output_tokens"] == 4096 + assert "max_tokens" not in result + assert "max_output_tokens" not in result + assert "extra_body" not in result or "max_tokens" not in result.get("extra_body", {}) def test_run_conversation_codex_replay_payload_keeps_call_id(monkeypatch): diff --git a/tests/test_setup_enabled_optional_keys.py b/tests/test_setup_enabled_optional_keys.py new file mode 100644 index 000000000000..522d5db92ac1 --- /dev/null +++ b/tests/test_setup_enabled_optional_keys.py @@ -0,0 +1,91 @@ +"""Tests for setup gating based on enabled optional integrations.""" + +from __future__ import annotations + +import copy +from pathlib import Path + +import yaml + +from hermes_cli.config import DEFAULT_CONFIG, get_missing_env_vars +from hermes_cli.setup import _has_any_provider_configured + + +def _write_config(home: Path, provider: str) -> dict: + """Write a minimal config with the requested TTS provider.""" + home.mkdir(parents=True, exist_ok=True) + cfg = copy.deepcopy(DEFAULT_CONFIG) + cfg.setdefault("tts", {})["provider"] = provider + (home / "config.yaml").write_text( + yaml.safe_dump(cfg, sort_keys=False), + encoding="utf-8", + ) + (home / ".env").write_text("", encoding="utf-8") + return cfg + + +def test_enabled_only_excludes_elevenlabs_when_tts_provider_is_edge(tmp_path: Path, monkeypatch) -> None: + hermes_home = tmp_path / ".hermes" + cfg = _write_config(hermes_home, provider="edge") + monkeypatch.setenv("HERMES_HOME", str(hermes_home)) + monkeypatch.delenv("ELEVENLABS_API_KEY", raising=False) + + missing = get_missing_env_vars(required_only=False, enabled_only=True, config=cfg) + names = {item["name"] for item in missing} + + assert "ELEVENLABS_API_KEY" not in names + + +def test_enabled_only_includes_elevenlabs_when_tts_provider_is_elevenlabs( + tmp_path: Path, monkeypatch +) -> None: + hermes_home = tmp_path / ".hermes" + cfg = _write_config(hermes_home, provider="elevenlabs") + monkeypatch.setenv("HERMES_HOME", str(hermes_home)) + monkeypatch.delenv("ELEVENLABS_API_KEY", raising=False) + + missing = get_missing_env_vars(required_only=False, enabled_only=True, config=cfg) + names = {item["name"] for item in missing} + + assert "ELEVENLABS_API_KEY" in names + + +def test_enabled_only_includes_openai_voice_key_when_tts_provider_is_openai( + tmp_path: Path, monkeypatch +) -> None: + hermes_home = tmp_path / ".hermes" + cfg = _write_config(hermes_home, provider="openai") + monkeypatch.setenv("HERMES_HOME", str(hermes_home)) + monkeypatch.delenv("VOICE_TOOLS_OPENAI_KEY", raising=False) + + missing = get_missing_env_vars(required_only=False, enabled_only=True, config=cfg) + names = {item["name"] for item in missing} + + assert "VOICE_TOOLS_OPENAI_KEY" in names + + +def test_provider_detection_ignores_blank_keys(tmp_path: Path, monkeypatch) -> None: + hermes_home = tmp_path / ".hermes" + hermes_home.mkdir(parents=True, exist_ok=True) + (hermes_home / ".env").write_text( + "OPENROUTER_API_KEY=\nOPENAI_API_KEY=\nANTHROPIC_API_KEY=\n", + encoding="utf-8", + ) + monkeypatch.setenv("HERMES_HOME", str(hermes_home)) + monkeypatch.delenv("OPENROUTER_API_KEY", raising=False) + monkeypatch.delenv("OPENAI_API_KEY", raising=False) + monkeypatch.delenv("ANTHROPIC_API_KEY", raising=False) + + assert _has_any_provider_configured() is False + + +def test_provider_detection_accepts_nonempty_key(tmp_path: Path, monkeypatch) -> None: + hermes_home = tmp_path / ".hermes" + hermes_home.mkdir(parents=True, exist_ok=True) + (hermes_home / ".env").write_text("OPENROUTER_API_KEY=sk-or-test\n", encoding="utf-8") + monkeypatch.setenv("HERMES_HOME", str(hermes_home)) + monkeypatch.delenv("OPENROUTER_API_KEY", raising=False) + monkeypatch.delenv("OPENAI_API_KEY", raising=False) + monkeypatch.delenv("ANTHROPIC_API_KEY", raising=False) + + assert _has_any_provider_configured() is True diff --git a/tests/test_shell_utils_windows_cmd.py b/tests/test_shell_utils_windows_cmd.py new file mode 100644 index 000000000000..a8af36572ef6 --- /dev/null +++ b/tests/test_shell_utils_windows_cmd.py @@ -0,0 +1,59 @@ +from tools.environments import shell_utils + + +def test_cmd_mode_uses_cmd_d_flag_and_no_shell(monkeypatch): + monkeypatch.setattr(shell_utils, "get_local_shell_mode", lambda: "cmd") + monkeypatch.setattr(shell_utils.os, "name", "nt", raising=False) + monkeypatch.setenv("COMSPEC", r"C:\Windows\System32\cmd.exe") + + args, kwargs, mode = shell_utils.build_local_subprocess_invocation( + command="echo hello", + work_dir=r"C:\tmp", + ) + + assert mode == "cmd" + assert kwargs["shell"] is False + assert args[0].lower().endswith("cmd.exe") + lowered = [a.lower() for a in args] + assert "/d" in lowered + assert "/c" in lowered + # Working dir is set via Popen cwd=, not embedded "cd /d" (avoids quote-escaping breakage) + assert kwargs.get("cwd", "").lower().rstrip("\\").endswith("c:\\tmp") + assert any("echo hello" in part.lower() for part in args) + + +def test_cmd_mode_keeps_percent_env_var_syntax(monkeypatch): + monkeypatch.setattr(shell_utils, "get_local_shell_mode", lambda: "cmd") + monkeypatch.setattr(shell_utils.os, "name", "nt", raising=False) + monkeypatch.setenv("COMSPEC", r"C:\Windows\System32\cmd.exe") + + args, _, mode = shell_utils.build_local_subprocess_invocation( + command="echo %COMSPEC%", + work_dir=r"C:\tmp", + ) + + assert mode == "cmd" + payload = " ".join(args) + assert "%COMSPEC%" in payload + + +def test_cmd_mode_strips_trailing_backslash_in_cd(monkeypatch): + monkeypatch.setattr(shell_utils, "get_local_shell_mode", lambda: "cmd") + monkeypatch.setattr(shell_utils.os, "name", "nt", raising=False) + monkeypatch.setenv("COMSPEC", r"C:\Windows\System32\cmd.exe") + + args, kwargs, mode = shell_utils.build_local_subprocess_invocation( + command="echo hello", + work_dir=r"C:\hermes\workspace\\", + ) + + assert mode == "cmd" + # cwd is set via Popen; safe_cwd must not end with single \ (breaks Windows). + cwd = kwargs.get("cwd", "") + assert cwd + assert cwd.endswith(":\\") or not cwd.endswith("\\") + + +def test_windows_path_safe_for_quotes_keeps_drive_root(): + assert shell_utils._windows_path_safe_for_quotes(r"C:\\") == "C:\\" + assert shell_utils._windows_path_safe_for_quotes(r"C:\tmp\\") == r"C:\tmp" diff --git a/tests/test_shell_utils_windows_modes.py b/tests/test_shell_utils_windows_modes.py new file mode 100644 index 000000000000..6bc4a54eee33 --- /dev/null +++ b/tests/test_shell_utils_windows_modes.py @@ -0,0 +1,80 @@ +from tools.environments import shell_utils + + +def _set_windows(monkeypatch): + monkeypatch.setattr(shell_utils.os, "name", "nt", raising=False) + # Reset mode-log state between tests so behavior is deterministic. + shell_utils._last_logged_mode = None + + +def test_get_local_shell_mode_auto_prefers_wsl(monkeypatch): + _set_windows(monkeypatch) + monkeypatch.setenv("HERMES_WINDOWS_SHELL", "auto") + monkeypatch.setattr(shell_utils, "_wsl_available", lambda: True) + monkeypatch.setattr(shell_utils, "_powershell_executable", lambda: r"C:\Windows\System32\WindowsPowerShell\v1.0\powershell.exe") + + assert shell_utils.get_local_shell_mode() == "wsl" + + +def test_get_local_shell_mode_powershell_falls_back_to_cmd(monkeypatch): + _set_windows(monkeypatch) + monkeypatch.setenv("HERMES_WINDOWS_SHELL", "powershell") + monkeypatch.setattr(shell_utils, "_powershell_executable", lambda: "") + + assert shell_utils.get_local_shell_mode() == "cmd" + + +def test_get_local_shell_mode_switching_follows_env(monkeypatch): + """Terminal switching: mode must follow HERMES_WINDOWS_SHELL (no stale cache).""" + _set_windows(monkeypatch) + monkeypatch.setattr(shell_utils, "_wsl_available", lambda: True) + monkeypatch.setattr(shell_utils, "_powershell_executable", lambda: "powershell.exe") + + monkeypatch.setenv("HERMES_WINDOWS_SHELL", "wsl") + shell_utils._last_logged_mode = None + assert shell_utils.get_local_shell_mode() == "wsl" + + monkeypatch.setenv("HERMES_WINDOWS_SHELL", "powershell") + shell_utils._last_logged_mode = None + assert shell_utils.get_local_shell_mode() == "powershell" + + monkeypatch.setenv("HERMES_WINDOWS_SHELL", "wsl") + shell_utils._last_logged_mode = None + assert shell_utils.get_local_shell_mode() == "wsl" + + +def test_build_local_subprocess_invocation_wsl_payload(monkeypatch): + _set_windows(monkeypatch) + monkeypatch.setattr(shell_utils, "get_local_shell_mode", lambda: "wsl") + monkeypatch.setattr(shell_utils, "_wsl_executable", lambda: "wsl.exe") + + args, kwargs, mode = shell_utils.build_local_subprocess_invocation( + command="echo hello", + work_dir=r"C:\tmp", + ) + + assert mode == "wsl" + assert args[:3] == ["wsl.exe", "-e", "bash"] + assert args[3] == "-lc" + assert "cd /mnt/c/tmp && echo hello" == args[4] + assert kwargs["shell"] is False + + +def test_build_local_subprocess_invocation_powershell_payload(monkeypatch): + _set_windows(monkeypatch) + monkeypatch.setattr(shell_utils, "get_local_shell_mode", lambda: "powershell") + monkeypatch.setattr(shell_utils, "_powershell_executable", lambda: "powershell.exe") + monkeypatch.setattr(shell_utils.os.path, "isdir", lambda p: True) + + args, kwargs, mode = shell_utils.build_local_subprocess_invocation( + command="echo hello", + work_dir=r"C:\hermes\workspace\\", + ) + + assert mode == "powershell" + assert args[0].lower().endswith("powershell.exe") + assert "-Command" in args + ps_command = args[-1] + assert "Set-Location -LiteralPath 'C:\\hermes\\workspace'; echo hello" == ps_command + assert kwargs["shell"] is False + diff --git a/tools/browser_tool.py b/tools/browser_tool.py index 208d6e8632c4..ce339f60f7b3 100644 --- a/tools/browser_tool.py +++ b/tools/browser_tool.py @@ -636,34 +636,44 @@ def _get_browserbase_config() -> Dict[str, str]: } -def _find_agent_browser() -> str: +def _find_agent_browser() -> List[str]: """ Find the agent-browser CLI executable. Checks in order: PATH, local node_modules/.bin/, npx fallback. Returns: - Path to agent-browser executable + Command prefix for invoking agent-browser (argv list) Raises: FileNotFoundError: If agent-browser is not installed """ + is_windows = os.name == "nt" # Check if it's in PATH (global install) - which_result = shutil.which("agent-browser") + which_result = shutil.which("agent-browser.cmd" if is_windows else "agent-browser") + if not which_result: + which_result = shutil.which("agent-browser") if which_result: - return which_result + return [which_result] # Check local node_modules/.bin/ (npm install in repo root) repo_root = Path(__file__).parent.parent - local_bin = repo_root / "node_modules" / ".bin" / "agent-browser" + local_bin_dir = repo_root / "node_modules" / ".bin" + if is_windows: + local_cmd = local_bin_dir / "agent-browser.cmd" + if local_cmd.exists(): + return [str(local_cmd)] + local_bin = local_bin_dir / "agent-browser" if local_bin.exists(): - return str(local_bin) + return [str(local_bin)] # Check common npx locations - npx_path = shutil.which("npx") + npx_path = shutil.which("npx.cmd" if is_windows else "npx") + if not npx_path: + npx_path = shutil.which("npx") if npx_path: - return "npx agent-browser" + return [npx_path, "agent-browser"] raise FileNotFoundError( "agent-browser CLI not found. Install it with: npm install -g agent-browser\n" @@ -712,7 +722,7 @@ def _run_browser_command( # IMPORTANT: Do NOT use --session with --cdp. In agent-browser >=0.13, # --session creates a local browser instance and silently ignores --cdp. # Per-task isolation is handled by AGENT_BROWSER_SOCKET_DIR instead. - cmd_parts = browser_cmd.split() + [ + cmd_parts = browser_cmd + [ "--cdp", session_info["cdp_url"], "--json", command @@ -1422,7 +1432,7 @@ def cleanup_browser(task_id: Optional[str] = None) -> None: pid_file = os.path.join(socket_dir, f"{session_name}.pid") if os.path.isfile(pid_file): try: - daemon_pid = int(open(pid_file).read().strip()) + daemon_pid = int(open(pid_file, encoding="utf-8").read().strip()) os.kill(daemon_pid, signal.SIGTERM) logger.debug("Killed daemon pid %s for %s", daemon_pid, session_name) except (ProcessLookupError, ValueError, PermissionError, OSError): diff --git a/tools/code_execution_tool.py b/tools/code_execution_tool.py index 8fb4b443152f..1622eb0e4332 100644 --- a/tools/code_execution_tool.py +++ b/tools/code_execution_tool.py @@ -265,8 +265,8 @@ def _rpc_server_loop( # their status prints don't leak into the CLI spinner. try: _real_stdout, _real_stderr = sys.stdout, sys.stderr - sys.stdout = open(os.devnull, "w") - sys.stderr = open(os.devnull, "w") + sys.stdout = open(os.devnull, "w", encoding="utf-8") + sys.stderr = open(os.devnull, "w", encoding="utf-8") try: result = handle_function_call( tool_name, tool_args, task_id=task_id @@ -361,11 +361,11 @@ def execute_code( tools_src = generate_hermes_tools_module( list(sandbox_tools) if enabled_tools else list(SANDBOX_ALLOWED_TOOLS) ) - with open(os.path.join(tmpdir, "hermes_tools.py"), "w") as f: + with open(os.path.join(tmpdir, "hermes_tools.py"), "w", encoding="utf-8", newline="") as f: f.write(tools_src) # Write the user's script - with open(os.path.join(tmpdir, "script.py"), "w") as f: + with open(os.path.join(tmpdir, "script.py"), "w", encoding="utf-8", newline="") as f: f.write(code) # --- Start UDS server --- diff --git a/tools/environments/local.py b/tools/environments/local.py index 702cca49c1e4..d4b0502129f6 100644 --- a/tools/environments/local.py +++ b/tools/environments/local.py @@ -11,43 +11,10 @@ _IS_WINDOWS = platform.system() == "Windows" from tools.environments.base import BaseEnvironment - - -def _find_shell() -> str: - """Find the best shell for command execution. - - On Unix: uses $SHELL, falls back to bash. - On Windows: uses Git Bash (bundled with Git for Windows). - Raises RuntimeError if no suitable shell is found on Windows. - """ - if not _IS_WINDOWS: - return os.environ.get("SHELL") or shutil.which("bash") or "/bin/bash" - - # Windows: look for Git Bash (installed with Git for Windows). - # Allow override via env var (same pattern as Claude Code). - custom = os.environ.get("HERMES_GIT_BASH_PATH") - if custom and os.path.isfile(custom): - return custom - - # shutil.which finds bash.exe if Git\bin is on PATH - found = shutil.which("bash") - if found: - return found - - # Check common Git for Windows install locations - for candidate in ( - os.path.join(os.environ.get("ProgramFiles", r"C:\Program Files"), "Git", "bin", "bash.exe"), - os.path.join(os.environ.get("ProgramFiles(x86)", r"C:\Program Files (x86)"), "Git", "bin", "bash.exe"), - os.path.join(os.environ.get("LOCALAPPDATA", ""), "Programs", "Git", "bin", "bash.exe"), - ): - if candidate and os.path.isfile(candidate): - return candidate - - raise RuntimeError( - "Git Bash not found. Hermes Agent requires Git for Windows on Windows.\n" - "Install it from: https://git-scm.com/download/win\n" - "Or set HERMES_GIT_BASH_PATH to your bash.exe location." - ) +from tools.environments.shell_utils import ( + build_local_subprocess_invocation, + terminate_process_tree, +) # Noise lines emitted by interactive shells when stdin is not a terminal. # Filtered from output to keep tool results clean. @@ -97,24 +64,19 @@ def execute(self, command: str, cwd: str = "", *, exec_command = self._prepare_command(command) try: - # Use the user's shell as an interactive login shell (-lic) so - # that ALL rc files are sourced — including content after the - # interactive guard in .bashrc (case $- in *i*)..esac) where - # tools like nvm, pyenv, and cargo install their init scripts. - # -l alone isn't enough: .profile sources .bashrc, but the guard - # returns early because the shell isn't interactive. - user_shell = _find_shell() + popen_args, popen_platform_kwargs, _ = build_local_subprocess_invocation( + exec_command, work_dir + ) proc = subprocess.Popen( - [user_shell, "-lic", exec_command], + popen_args, text=True, - cwd=work_dir, env=os.environ | self.env, encoding="utf-8", errors="replace", stdout=subprocess.PIPE, stderr=subprocess.STDOUT, stdin=subprocess.PIPE if stdin_data is not None else subprocess.DEVNULL, - preexec_fn=None if _IS_WINDOWS else os.setsid, + **popen_platform_kwargs, ) if stdin_data is not None: @@ -146,31 +108,22 @@ def _drain_stdout(): while proc.poll() is None: if _interrupt_event.is_set(): + terminate_process_tree(proc, force=False) try: - if _IS_WINDOWS: - proc.terminate() - else: - pgid = os.getpgid(proc.pid) - os.killpg(pgid, signal.SIGTERM) - try: - proc.wait(timeout=1.0) - except subprocess.TimeoutExpired: - os.killpg(pgid, signal.SIGKILL) - except (ProcessLookupError, PermissionError): - proc.kill() + proc.wait(timeout=1.0) + except subprocess.TimeoutExpired: + terminate_process_tree(proc, force=True) reader.join(timeout=2) return { "output": "".join(_output_chunks) + "\n[Command interrupted — user sent a new message]", "returncode": 130, } if time.monotonic() > deadline: + terminate_process_tree(proc, force=False) try: - if _IS_WINDOWS: - proc.terminate() - else: - os.killpg(os.getpgid(proc.pid), signal.SIGTERM) - except (ProcessLookupError, PermissionError): - proc.kill() + proc.wait(timeout=1.0) + except subprocess.TimeoutExpired: + terminate_process_tree(proc, force=True) reader.join(timeout=2) return self._timeout_result(effective_timeout) time.sleep(0.2) diff --git a/tools/environments/shell_utils.py b/tools/environments/shell_utils.py new file mode 100644 index 000000000000..6d15e80c63ab --- /dev/null +++ b/tools/environments/shell_utils.py @@ -0,0 +1,321 @@ +"""Cross-platform helpers for local shell execution and process lifecycle.""" + +from __future__ import annotations + +import logging +import os +import re +import shlex +import shutil +import signal +import subprocess +from functools import lru_cache +from typing import Any, Dict, Tuple, Union + +CommandType = Union[str, list[str]] + +_DRIVE_PATH_RE = re.compile(r"^([A-Za-z]):[\\/]*(.*)$") +_MNT_PATH_RE = re.compile(r"^/mnt/([A-Za-z])(?:/(.*))?$") +logger = logging.getLogger(__name__) +_last_logged_mode: str | None = None + + +def is_windows() -> bool: + return os.name == "nt" + + +@lru_cache(maxsize=1) +def _wsl_executable() -> str: + return shutil.which("wsl.exe") or shutil.which("wsl") or "" + + +@lru_cache(maxsize=1) +def _powershell_executable() -> str: + return ( + shutil.which("pwsh.exe") + or shutil.which("pwsh") + or shutil.which("powershell.exe") + or shutil.which("powershell") + or "" + ) + + +@lru_cache(maxsize=1) +def _wsl_available() -> bool: + if not is_windows(): + return False + exe = _wsl_executable() + if not exe: + return False + try: + probe = subprocess.run( + [exe, "-e", "sh", "-lc", "exit 0"], + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + timeout=4, + check=False, + ) + return probe.returncode == 0 + except Exception: + return False + + +def get_local_shell_mode() -> str: + """Return one of: posix, wsl, powershell, cmd.""" + global _last_logged_mode + + if not is_windows(): + mode = "posix" + else: + override = os.getenv("HERMES_WINDOWS_SHELL", "auto").strip().lower() + if override == "wsl": + mode = "wsl" if _wsl_available() else "powershell" if _powershell_executable() else "cmd" + elif override in {"powershell", "pwsh"}: + mode = "powershell" if _powershell_executable() else "cmd" + elif override in {"cmd", "cmd.exe"}: + mode = "cmd" + else: + if _wsl_available(): + mode = "wsl" + elif _powershell_executable(): + mode = "powershell" + else: + mode = "cmd" + + if _last_logged_mode != mode: + if is_windows(): + override_label = os.getenv("HERMES_WINDOWS_SHELL", "auto") + logger.info( + "Local shell mode selected: %s (HERMES_WINDOWS_SHELL=%s)", + mode, + override_label, + ) + else: + logger.info("Local shell mode selected: %s", mode) + _last_logged_mode = mode + + return mode + + +def to_wsl_path(path: str) -> str: + """Convert C:\\path style to /mnt/c/path when possible.""" + if not path: + return path + normalized = path.replace("\\", "/") + if normalized.startswith("/mnt/"): + return normalized + match = _DRIVE_PATH_RE.match(path) + if not match: + return normalized + drive = match.group(1).lower() + rest = match.group(2).replace("\\", "/").lstrip("/") + if rest: + return f"/mnt/{drive}/{rest}" + return f"/mnt/{drive}" + + +def to_windows_path(path: str) -> str: + """Convert /mnt/c/path style to C:\\path when possible.""" + if not path: + return path + normalized = path.replace("\\", "/") + match = _MNT_PATH_RE.match(normalized) + if not match: + return path + drive = match.group(1).upper() + rest = (match.group(2) or "").replace("/", "\\") + if rest: + return f"{drive}:\\{rest}" + return f"{drive}:\\" + + +def _windows_path_safe_for_quotes(path: str) -> str: + """Return path safe to embed in cmd double-quotes or PowerShell LiteralPath. + + On Windows, \" inside double-quotes escapes the quote, so C:\\path\\ + becomes invalid. Strip trailing backslash (and slash) so we never pass that. + """ + if not path: + return path + # Keep drive roots intact (C:\, C:/, or repeated trailing slashes). + normalized = path.replace("/", "\\") + if re.match(r"^[A-Za-z]:\\*$", normalized): + return normalized[:2] + "\\" + s = path.rstrip("\\/") + return s if s else path + + +def _windows_safe_cwd(work_dir: str | None) -> str: + """Return a directory that exists for use as Popen cwd on Windows. + + If the gateway was started from a dir that no longer exists (e.g. removed drive), + subprocesses inherit that cwd and fail with 'The filename, directory name, or + volume label syntax is incorrect.' So we always pass an explicit cwd that exists. + """ + if work_dir: + try: + resolved = os.path.abspath(os.path.expanduser(work_dir)) + if os.path.isdir(resolved): + return resolved + except (OSError, ValueError): + pass + try: + home = os.path.expanduser("~") + if home and os.path.isdir(home): + return home + except (OSError, ValueError): + pass + for fallback in ("C:\\", "C:\\Users", "."): + try: + if os.path.isdir(fallback): + return os.path.abspath(fallback) + except (OSError, ValueError): + continue + return os.getcwd() + + +def build_local_subprocess_invocation( + command: str, + work_dir: str | None = None, +) -> Tuple[CommandType, Dict[str, Any], str]: + """ + Build a subprocess invocation tuple for local backend execution. + + Returns: + (args, popen_overrides, shell_mode) + where popen_overrides contains platform-specific kwargs like `shell`, + `cwd`, and process-group flags. + """ + mode = get_local_shell_mode() + + if mode == "posix": + return ( + command, + { + "shell": True, + "cwd": work_dir, + "preexec_fn": os.setsid, + }, + mode, + ) + + creationflags = getattr(subprocess, "CREATE_NEW_PROCESS_GROUP", 0) + + if mode == "wsl": + wrapped = command + if work_dir: + wrapped = f"cd {shlex.quote(to_wsl_path(work_dir))} && {command}" + return ( + [_wsl_executable(), "-e", "bash", "-lc", wrapped], + { + "shell": False, + "creationflags": creationflags, + }, + mode, + ) + + # Windows cmd/PowerShell: pass explicit cwd so we never inherit a broken gateway cwd + safe_cwd = _windows_safe_cwd(work_dir) if is_windows() else None + try: + work_dir_exists = bool( + work_dir + and os.path.isdir(os.path.abspath(os.path.expanduser(work_dir))) + ) + except (OSError, ValueError, TypeError): + work_dir_exists = False + + if mode == "powershell": + ps_command = command + if work_dir_exists: + raw = to_windows_path(work_dir) + ps_dir = _windows_path_safe_for_quotes(raw).replace("'", "''") + ps_command = f"Set-Location -LiteralPath '{ps_dir}'; {command}" + logger.info( + "Local invocation: mode=%s cwd=%r ps_command=%s", + mode, + safe_cwd, + ps_command[:300] + ("..." if len(ps_command) > 300 else ""), + ) + return ( + [ + _powershell_executable(), + "-NoLogo", + "-NoProfile", + "-NonInteractive", + "-ExecutionPolicy", + "Bypass", + "-Command", + ps_command, + ], + { + "shell": False, + "creationflags": creationflags, + "cwd": safe_cwd, + }, + mode, + ) + + # Rely on Popen cwd=safe_cwd for working directory. Do not embed "cd /d \"path\"" + # in the /c argument: subprocess escapes the inner quotes when building the + # Windows command line, so cmd sees \"...\" and fails with "The filename, + # directory name, or volume label syntax is incorrect." + cmd_command = command + comspec = os.environ.get("COMSPEC") or shutil.which("cmd.exe") or "cmd.exe" + logger.info( + "Local invocation: mode=%s cwd=%r cmd_command=%s", + mode, + safe_cwd, + cmd_command[:300] + ("..." if len(cmd_command) > 300 else ""), + ) + return ( + [comspec, "/d", "/s", "/c", cmd_command], + { + "shell": False, + "creationflags": creationflags, + "cwd": safe_cwd, + }, + mode, + ) + + +def terminate_process_tree(proc: subprocess.Popen, *, force: bool = False) -> None: + """Terminate a process and its children on both POSIX and Windows.""" + if proc is None or proc.poll() is not None: + return + + if is_windows(): + try: + cmd = ["taskkill", "/PID", str(proc.pid), "/T"] + if force: + cmd.append("/F") + subprocess.run( + cmd, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + timeout=5, + check=False, + ) + except Exception: + pass + + if proc.poll() is None: + try: + if force: + proc.kill() + else: + proc.terminate() + except Exception: + pass + return + + sig = signal.SIGKILL if force else signal.SIGTERM + try: + os.killpg(os.getpgid(proc.pid), sig) + except (ProcessLookupError, PermissionError): + try: + if force: + proc.kill() + else: + proc.terminate() + except Exception: + pass diff --git a/tools/file_operations.py b/tools/file_operations.py index 3649b9ef0490..94f71dc40075 100644 --- a/tools/file_operations.py +++ b/tools/file_operations.py @@ -70,10 +70,12 @@ os.path.join(_HOME, ".gnupg"), os.path.join(_HOME, ".kube"), "/etc/sudoers.d", - "/etc/systemd", + "/etc/systemd", ] ] +READ_DENIED_BASENAMES = {".env"} + def _is_write_denied(path: str) -> bool: """Return True if path is on the write deny list.""" @@ -86,6 +88,14 @@ def _is_write_denied(path: str) -> bool: return False +def _is_env_file_path(path: str) -> bool: + """Return True if path resolves to a file named '.env' (any directory).""" + try: + return os.path.basename(os.path.realpath(os.path.expanduser(path))).lower() == ".env" + except Exception: + return os.path.basename(path).lower() == ".env" + + # ============================================================================= # Result Data Classes # ============================================================================= @@ -311,8 +321,18 @@ def __init__(self, terminal_env, cwd: str = None): # IMPORTANT: do NOT fall back to os.getcwd() -- that's the HOST's local # path which doesn't exist inside container/cloud backends (modal, docker). # If nothing provides a cwd, use "/" as a safe universal default. - self.cwd = cwd or getattr(terminal_env, 'cwd', None) or \ - getattr(getattr(terminal_env, 'config', None), 'cwd', None) or "/" + resolved_cwd = ( + cwd + or getattr(terminal_env, "cwd", None) + or getattr(getattr(terminal_env, "config", None), "cwd", None) + ) + if not resolved_cwd: + resolved_cwd = os.path.expanduser("~") if os.name == "nt" else "/" + # Normalize "~" style working directories on Windows so relative file + # paths like "agents/worldview.md" resolve correctly. + if os.name == "nt": + resolved_cwd = str(Path(str(resolved_cwd)).expanduser()) + self.cwd = resolved_cwd # Cache for command availability checks self._command_cache: Dict[str, bool] = {} @@ -340,8 +360,12 @@ def _exec(self, command: str, cwd: str = None, timeout: int = None, def _has_command(self, cmd: str) -> bool: """Check if a command exists in the environment (cached).""" if cmd not in self._command_cache: - result = self._exec(f"command -v {cmd} >/dev/null 2>&1 && echo 'yes'") - self._command_cache[cmd] = result.stdout.strip() == 'yes' + if os.name == "nt": + # PowerShell/cmd path: "where" is available by default. + result = self._exec(f"where {cmd} >nul 2>nul && echo yes") + else: + result = self._exec(f"command -v {cmd} >/dev/null 2>&1 && echo 'yes'") + self._command_cache[cmd] = result.stdout.strip().lower() == "yes" return self._command_cache[cmd] def _is_likely_binary(self, path: str, content_sample: str = None) -> bool: @@ -389,6 +413,9 @@ def _expand_path(self, path: str) -> str: """ if not path: return path + # Native expansion on Windows local backends. + if os.name == "nt": + return os.path.expandvars(os.path.expanduser(path)) # Handle ~ and ~user if path.startswith('~'): @@ -406,11 +433,105 @@ def _expand_path(self, path: str) -> str: return expand_result.stdout.strip() return path + + @staticmethod + def _normalize_apostrophes(s: str) -> str: + """Normalize Unicode apostrophe variants to ASCII so path matching works across sources.""" + if not s: + return s + for c in ("\u2018", "\u2019", "\u201a", "\u201b", "`"): + s = s.replace(c, "'") + return s + + def _resolve_windows_path_apostrophe_fallback(self, path: str) -> Optional[str]: + """ + When path does not exist, look for a file whose name matches when apostrophe + variants are normalized (e.g. YouTube titles with '). Tries the path's + parent dir, then cwd, workspace, and ~/.hermes/workspace. Returns the first + matching existing path, or None. + """ + expanded = self._expand_path(path) + requested_name = os.path.basename(expanded) or expanded + normalized_requested = self._normalize_apostrophes(requested_name).lower() + parents_to_try = [] + if os.path.isabs(expanded): + parents_to_try.append(os.path.dirname(expanded)) + else: + if self.cwd: + parents_to_try.append(self.cwd) + parents_to_try.append(os.path.join(self.cwd, "workspace")) + parents_to_try.append(os.path.normpath(os.path.expanduser(r"~/.hermes/workspace"))) + for parent in parents_to_try: + if not parent or not os.path.isdir(parent): + continue + try: + for name in os.listdir(parent): + if self._normalize_apostrophes(name).lower() == normalized_requested: + full = os.path.normpath(os.path.join(parent, name)) + if os.path.isfile(full): + return full + except OSError: + continue + return None + + def _resolve_windows_path(self, path: str) -> str: + """ + Resolve a potentially relative Windows path with workspace-aware fallback. + + Resolution order for relative paths: + 1) / + 2) /workspace/ + 3) ~/.hermes/workspace/ + Returns the first existing path, else the first candidate. + If none exist, tries apostrophe-normalized match in the same directory (e.g. YouTube .vtt files). + """ + expanded = self._expand_path(path) + if os.path.isabs(expanded): + resolved = os.path.normpath(expanded) + else: + candidates = [] + if self.cwd: + candidates.append(os.path.normpath(os.path.join(self.cwd, expanded))) + candidates.append(os.path.normpath(os.path.join(self.cwd, "workspace", expanded))) + hermes_ws = os.path.normpath(os.path.expanduser(r"~/.hermes/workspace")) + candidates.append(os.path.normpath(os.path.join(hermes_ws, expanded))) + + resolved = None + for candidate in candidates: + if os.path.exists(candidate): + resolved = candidate + break + if resolved is None: + resolved = candidates[0] if candidates else os.path.normpath(expanded) + + if not os.path.exists(resolved): + fallback = self._resolve_windows_path_apostrophe_fallback(resolved) + if fallback is not None: + return fallback + return resolved + + def _is_windows_local_backend(self) -> bool: + """True when running on Windows host with local backend file access.""" + if os.name != "nt": + return False + module_name = getattr(self.env.__class__, "__module__", "") + return module_name.endswith(".local") def _escape_shell_arg(self, arg: str) -> str: """Escape a string for safe use in shell commands.""" # Use single quotes and escape any single quotes in the string return "'" + arg.replace("'", "'\"'\"'") + "'" + + @staticmethod + def _looks_like_file_glob(pattern: str) -> bool: + """Heuristic: pattern appears to be a filename glob, not content regex.""" + if not pattern: + return False + # Common glob indicators + has_glob = any(ch in pattern for ch in ("*", "?", "[", "]")) + # Typical filename-ish suffixes + has_ext_hint = "." in pattern and "/" not in pattern and "\\" not in pattern + return has_glob and has_ext_hint def _unified_diff(self, old_content: str, new_content: str, filename: str) -> str: """Generate unified diff between old and new content.""" @@ -428,17 +549,15 @@ def _unified_diff(self, old_content: str, new_content: str, filename: str) -> st # ========================================================================= def read_file(self, path: str, offset: int = 1, limit: int = 500) -> ReadResult: - """ - Read a file with pagination, binary detection, and line numbers. - - Args: - path: File path (absolute or relative to cwd) - offset: Line number to start from (1-indexed, default 1) - limit: Maximum lines to return (default 500, max 2000) - - Returns: - ReadResult with content, metadata, or error info - """ + # Block `.env` reads by basename from both POSIX and Windows backends. + if _is_env_file_path(path): + return ReadResult(error="Access to files named '.env' is disabled for security") + + # Windows backend: use native Python/os calls instead of POSIX shell + # tools like stat/head/sed, which aren't available in PowerShell. + if os.name == "nt": + return self._read_file_windows(path, offset, limit) + # POSIX path: use shell tools for compatibility with container/remote backends. # Expand ~ and other shell paths path = self._expand_path(path) @@ -515,6 +634,75 @@ def read_file(self, path: str, offset: int = 1, limit: int = 500) -> ReadResult: truncated=truncated, hint=hint ) + + def _read_file_windows(self, path: str, offset: int = 1, limit: int = 500) -> ReadResult: + """ + Windows implementation of read_file using native os/file APIs. + + Avoids reliance on POSIX tools like stat/head/sed which are not + available in PowerShell-backed environments. + """ + # Resolve to absolute path relative to the current working directory + # of the terminal backend. + path = self._resolve_windows_path(path) + + if _is_env_file_path(path): + return ReadResult(error="Access to files named '.env' is disabled for security") + + if not os.path.exists(path): + return self._suggest_similar_files(path) + + # Directories are not readable as files; return a helpful error. + if os.path.isdir(path): + try: + entries = os.listdir(path) + except OSError: + entries = [] + # Show a small sample of children as a hint + sample = entries[:20] + hint = None + if sample: + hint = "Directory contents:\n" + "\n".join(f"- {name}" for name in sample) + return ReadResult( + error="Path is a directory, not a file.", + hint=hint, + similar_files=[os.path.join(path, name) for name in sample] if sample else [], + ) + + try: + file_size = os.path.getsize(path) + except OSError: + file_size = 0 + + # Clamp limit + limit = min(limit, MAX_LINES) + if offset < 1: + offset = 1 + + # Read all lines once, then slice for pagination + try: + with open(path, "r", encoding="utf-8", errors="replace") as f: + lines = f.readlines() + except Exception as e: + return ReadResult(error=f"Failed to read file: {type(e).__name__}: {e}") + + total_lines = len(lines) + start_idx = offset - 1 + end_idx = start_idx + limit + page_lines = lines[start_idx:end_idx] if start_idx < total_lines else [] + truncated = end_idx < total_lines + hint = None + if truncated: + hint = f"Use offset={end_idx + 1} to continue reading (showing {offset}-{end_idx} of {total_lines} lines)" + + content = "".join(page_lines) + return ReadResult( + content=self._add_line_numbers(content, offset), + total_lines=total_lines, + file_size=file_size, + truncated=truncated, + hint=hint, + ) # Images larger than this are too expensive to inline as base64 in the # conversation context. Return metadata only and suggest vision_analyze. @@ -585,6 +773,8 @@ def _read_image(self, path: str) -> ReadResult: def _suggest_similar_files(self, path: str) -> ReadResult: """Suggest similar files when the requested file is not found.""" + if os.name == "nt": + return self._suggest_similar_files_windows(path) # Get directory and filename dir_path = os.path.dirname(path) or "." filename = os.path.basename(path) @@ -607,6 +797,70 @@ def _suggest_similar_files(self, path: str) -> ReadResult: error=f"File not found: {path}", similar_files=similar[:5] # Limit to 5 suggestions ) + + def _suggest_similar_files_windows(self, path: str) -> ReadResult: + """Windows-native similar-file suggestions (no shell dependencies).""" + dir_path = os.path.dirname(path) or self.cwd or "." + filename = os.path.basename(path) + try: + candidates = os.listdir(dir_path) + except OSError: + candidates = [] + similar = [] + target_lower = filename.lower() + for name in candidates: + common = set(target_lower) & set(name.lower()) + if filename and len(common) >= max(1, int(len(filename) * 0.5)): + similar.append(os.path.join(dir_path, name)) + + # If directory-local suggestions are empty, scan common roots to give + # the model concrete existing paths instead of repeated blind reads. + if not similar: + roots = [] + if self.cwd: + roots.append(self.cwd) + roots.append(os.path.join(self.cwd, "workspace")) + roots.append(os.path.expanduser(r"~/.hermes/workspace")) + roots.append(os.path.expanduser(r"~/.hermes")) + + needle = self._normalize_apostrophes( + (os.path.splitext(filename)[0] or filename).lower() + ) + seen = set() + for root in roots: + if not root or not os.path.isdir(root): + continue + try: + for walk_root, _, files in os.walk(root): + for name in files: + lname = self._normalize_apostrophes(name.lower()) + if needle and needle in lname: + full = os.path.normpath(os.path.join(walk_root, name)) + if full not in seen: + similar.append(full) + seen.add(full) + if len(similar) >= 8: + break + if len(similar) >= 8: + break + except Exception: + continue + if len(similar) >= 8: + break + + hint = None + if similar: + hint_lines = "\n".join(f"- {p}" for p in similar[:5]) + hint = ( + "Closest existing files:\n" + f"{hint_lines}\n" + "Tip: prefer these exact paths or read the containing directory first." + ) + return ReadResult( + error=f"File not found: {path}", + similar_files=similar[:5], + hint=hint, + ) # ========================================================================= # WRITE Implementation @@ -627,6 +881,11 @@ def write_file(self, path: str, content: str) -> WriteResult: Returns: WriteResult with bytes written or error """ + # Windows local backend: avoid shell aliases (cat/Get-Content) and + # write directly with Python file APIs. + if self._is_windows_local_backend(): + return self._write_file_windows(path, content) + # Expand ~ and other shell paths path = self._expand_path(path) @@ -665,6 +924,25 @@ def write_file(self, path: str, content: str) -> WriteResult: bytes_written=bytes_written, dirs_created=dirs_created ) + + def _write_file_windows(self, path: str, content: str) -> WriteResult: + """Windows-native file write path for local backend.""" + try: + resolved = self._resolve_windows_path(path) + parent = os.path.dirname(resolved) + dirs_created = False + if parent and not os.path.exists(parent): + os.makedirs(parent, exist_ok=True) + dirs_created = True + + with open(resolved, "w", encoding="utf-8", newline="") as f: + f.write(content) + f.flush() + + bytes_written = os.path.getsize(resolved) + return WriteResult(bytes_written=bytes_written, dirs_created=dirs_created) + except Exception as e: + return WriteResult(error=f"Failed to write file: {type(e).__name__}: {e}") # ========================================================================= # PATCH Implementation (Replace Mode) @@ -684,21 +962,31 @@ def patch_replace(self, path: str, old_string: str, new_string: str, Returns: PatchResult with diff and lint results """ - # Expand ~ and other shell paths - path = self._expand_path(path) + # Expand path according to platform/backend. + if os.name == "nt": + path = self._resolve_windows_path(path) + else: + path = self._expand_path(path) # Block writes to sensitive paths if _is_write_denied(path): return PatchResult(error=f"Write denied: '{path}' is a protected system/credential file.") # Read current content - read_cmd = f"cat {self._escape_shell_arg(path)} 2>/dev/null" - read_result = self._exec(read_cmd) - - if read_result.exit_code != 0: - return PatchResult(error=f"Failed to read file: {path}") - - content = read_result.stdout + if os.name == "nt": + try: + with open(path, "r", encoding="utf-8", errors="replace") as f: + content = f.read() + except Exception: + return PatchResult(error=f"Failed to read file: {path}") + else: + read_cmd = f"cat {self._escape_shell_arg(path)} 2>/dev/null" + read_result = self._exec(read_cmd) + + if read_result.exit_code != 0: + return PatchResult(error=f"Failed to read file: {path}") + + content = read_result.stdout # Import and use fuzzy matching from tools.fuzzy_match import fuzzy_find_and_replace @@ -818,12 +1106,135 @@ def search(self, pattern: str, path: str = ".", target: str = "content", """ # Expand ~ and other shell paths path = self._expand_path(path) + + # Common model mistake: using grep with a glob pattern (e.g. "*.html"). + # Treat this as file search to avoid regex parse errors and retry loops. + if target == "content" and not file_glob and self._looks_like_file_glob(pattern): + target = "files" + + if os.name == "nt": + return self._search_windows( + pattern=pattern, + path=path, + target=target, + file_glob=file_glob, + limit=limit, + offset=offset, + output_mode=output_mode, + context=context, + ) if target == "files": return self._search_files(pattern, path, limit, offset) else: return self._search_content(pattern, path, file_glob, limit, offset, output_mode, context) + + def _search_windows( + self, + pattern: str, + path: str, + target: str, + file_glob: Optional[str], + limit: int, + offset: int, + output_mode: str, + context: int, + ) -> SearchResult: + """Windows-native search implementation (no grep/find dependency).""" + import fnmatch + import re as _re + + base_path = self._resolve_windows_path(path or ".") + if not os.path.exists(base_path): + return SearchResult(error=f"Path not found: {base_path}") + + if _is_env_file_path(base_path): + return SearchResult(error="Searching '.env' files is disabled for security") + + if target == "files": + search_pattern = pattern + if not any(ch in search_pattern for ch in ["*", "?", "[", "]"]): + search_pattern = f"*{search_pattern}*" + matches = [] + if os.path.isfile(base_path): + matches = [ + base_path + ] if ( + fnmatch.fnmatch(os.path.basename(base_path), search_pattern) + and not _is_env_file_path(base_path) + ) else [] + else: + for root, _, files in os.walk(base_path): + for name in files: + full = os.path.join(root, name) + if _is_env_file_path(full): + continue + if fnmatch.fnmatch(name, search_pattern): + full = os.path.join(root, name) + matches.append((full, os.path.getmtime(full))) + matches.sort(key=lambda x: x[1], reverse=True) + matches = [m[0] for m in matches] + total = len(matches) + page = matches[offset:offset + limit] + return SearchResult(files=page, total_count=total, truncated=total > offset + limit) + + try: + regex = _re.compile(pattern) + except _re.error as e: + return SearchResult(error=f"Invalid regex pattern: {e}") + + if os.path.isfile(base_path): + files_to_scan = [base_path] + else: + files_to_scan = [] + for root, _, files in os.walk(base_path): + for name in files: + if file_glob and not fnmatch.fnmatch(name, file_glob): + continue + if _is_env_file_path(os.path.join(root, name)): + continue + files_to_scan.append(os.path.join(root, name)) + + content_matches: List[SearchMatch] = [] + files_only = set() + counts: Dict[str, int] = {} + + for file_path in files_to_scan: + try: + with open(file_path, "r", encoding="utf-8", errors="ignore") as f: + lines = f.readlines() + except Exception: + continue + hit_count = 0 + for idx, line in enumerate(lines, start=1): + if regex.search(line): + hit_count += 1 + if output_mode == "content": + content_matches.append( + SearchMatch( + path=file_path, + line_number=idx, + content=line.rstrip("\r\n")[:500], + ) + ) + if hit_count: + files_only.add(file_path) + counts[file_path] = hit_count + + if output_mode == "files_only": + all_files = sorted(files_only) + total = len(all_files) + page = all_files[offset:offset + limit] + return SearchResult(files=page, total_count=total, truncated=total > offset + limit) + + if output_mode == "count": + # Keep full counts map and total count for parity with grep mode. + return SearchResult(counts=counts, total_count=sum(counts.values())) + + total = len(content_matches) + page = content_matches[offset:offset + limit] + return SearchResult(matches=page, total_count=total, truncated=total > offset + limit) def _search_files(self, pattern: str, path: str, limit: int, offset: int) -> SearchResult: """Search for files by name pattern (glob-like).""" @@ -834,24 +1245,31 @@ def _search_files(self, pattern: str, path: str, limit: int, offset: int) -> Sea "On Windows, use Git Bash, WSL, or install Unix tools." ) - # Auto-prepend **/ for recursive search if not already present - if not pattern.startswith('**/') and '/' not in pattern: - search_pattern = pattern - else: - search_pattern = pattern.split('/')[-1] + # If the pattern is a plain token (no glob chars), treat it as + # substring match for better UX (e.g., "worldview" -> "*worldview*"). + search_pattern = pattern.split('/')[-1] + if not any(ch in search_pattern for ch in ["*", "?", "[", "]"]): + search_pattern = f"*{search_pattern}*" # Use find with modification time sorting # -printf '%T@ %p\n' outputs: timestamp path # sort -rn sorts by timestamp descending (newest first) - cmd = f"find {self._escape_shell_arg(path)} -type f -name {self._escape_shell_arg(search_pattern)} " \ - f"-printf '%T@ %p\\n' 2>/dev/null | sort -rn | tail -n +{offset + 1} | head -n {limit}" + cmd = ( + f"find {self._escape_shell_arg(path)} -type f -name " + f"{self._escape_shell_arg(search_pattern)} ! -name '.env' " + f"-printf '%T@ %p\\n' 2>/dev/null | sort -rn | " + f"tail -n +{offset + 1} | head -n {limit}" + ) result = self._exec(cmd, timeout=60) if result.exit_code != 0 and not result.stdout.strip(): # Try without -printf (BSD find compatibility) - cmd_simple = f"find {self._escape_shell_arg(path)} -type f -name {self._escape_shell_arg(search_pattern)} " \ - f"2>/dev/null | head -n {limit + offset} | tail -n +{offset + 1}" + cmd_simple = ( + f"find {self._escape_shell_arg(path)} -type f -name " + f"{self._escape_shell_arg(search_pattern)} ! -name '.env' " + f"2>/dev/null | head -n {limit + offset} | tail -n +{offset + 1}" + ) result = self._exec(cmd_simple, timeout=60) files = [] @@ -861,9 +1279,11 @@ def _search_files(self, pattern: str, path: str, limit: int, offset: int) -> Sea # Parse "timestamp path" format parts = line.split(' ', 1) if len(parts) == 2 and parts[0].replace('.', '').isdigit(): - files.append(parts[1]) + if not _is_env_file_path(parts[1]): + files.append(parts[1]) else: - files.append(line) + if not _is_env_file_path(line): + files.append(line) return SearchResult( files=files, @@ -873,6 +1293,9 @@ def _search_files(self, pattern: str, path: str, limit: int, offset: int) -> Sea def _search_content(self, pattern: str, path: str, file_glob: Optional[str], limit: int, offset: int, output_mode: str, context: int) -> SearchResult: """Search for content inside files (grep-like).""" + if _is_env_file_path(path): + return SearchResult(error="Searching '.env' files is disabled for security") + # Try ripgrep first (fast), fallback to grep (slower but works) if self._has_command('rg'): return self._search_with_rg(pattern, path, file_glob, limit, offset, @@ -899,6 +1322,7 @@ def _search_with_rg(self, pattern: str, path: str, file_glob: Optional[str], # Add file glob filter (must be quoted to prevent shell expansion) if file_glob: cmd_parts.extend(["--glob", self._escape_shell_arg(file_glob)]) + cmd_parts.extend(["--glob", self._escape_shell_arg("!.env")]) # Output mode handling if output_mode == "files_only": @@ -921,7 +1345,10 @@ def _search_with_rg(self, pattern: str, path: str, file_glob: Optional[str], # Parse results based on output mode if output_mode == "files_only": - all_files = [f for f in result.stdout.strip().split('\n') if f] + all_files = [ + f for f in result.stdout.strip().split('\n') + if f and not _is_env_file_path(f) + ] total = len(all_files) page = all_files[offset:offset + limit] return SearchResult(files=page, total_count=total) @@ -932,6 +1359,8 @@ def _search_with_rg(self, pattern: str, path: str, file_glob: Optional[str], if ':' in line: parts = line.rsplit(':', 1) if len(parts) == 2: + if _is_env_file_path(parts[0]): + continue try: counts[parts[0]] = int(parts[1]) except ValueError: @@ -952,6 +1381,8 @@ def _search_with_rg(self, pattern: str, path: str, file_glob: Optional[str], parts = line.split(':', 2) if len(parts) >= 3: try: + if _is_env_file_path(parts[0]): + continue matches.append(SearchMatch( path=parts[0], line_number=int(parts[1]), @@ -967,6 +1398,8 @@ def _search_with_rg(self, pattern: str, path: str, file_glob: Optional[str], parts = line.split('-', 2) if len(parts) >= 3: try: + if _is_env_file_path(parts[0]): + continue matches.append(SearchMatch( path=parts[0], line_number=int(parts[1]), @@ -995,6 +1428,7 @@ def _search_with_grep(self, pattern: str, path: str, file_glob: Optional[str], # Add file pattern filter (must be quoted to prevent shell expansion) if file_glob: cmd_parts.extend(["--include", self._escape_shell_arg(file_glob)]) + cmd_parts.extend(["--exclude", self._escape_shell_arg(".env")]) # Output mode handling if output_mode == "files_only": @@ -1014,7 +1448,10 @@ def _search_with_grep(self, pattern: str, path: str, file_glob: Optional[str], result = self._exec(cmd, timeout=60) if output_mode == "files_only": - all_files = [f for f in result.stdout.strip().split('\n') if f] + all_files = [ + f for f in result.stdout.strip().split('\n') + if f and not _is_env_file_path(f) + ] total = len(all_files) page = all_files[offset:offset + limit] return SearchResult(files=page, total_count=total) @@ -1025,6 +1462,8 @@ def _search_with_grep(self, pattern: str, path: str, file_glob: Optional[str], if ':' in line: parts = line.rsplit(':', 1) if len(parts) == 2: + if _is_env_file_path(parts[0]): + continue try: counts[parts[0]] = int(parts[1]) except ValueError: @@ -1043,6 +1482,8 @@ def _search_with_grep(self, pattern: str, path: str, file_glob: Optional[str], parts = line.split(':', 2) if len(parts) >= 3: try: + if _is_env_file_path(parts[0]): + continue matches.append(SearchMatch( path=parts[0], line_number=int(parts[1]), @@ -1056,6 +1497,8 @@ def _search_with_grep(self, pattern: str, path: str, file_glob: Optional[str], parts = line.split('-', 2) if len(parts) >= 3: try: + if _is_env_file_path(parts[0]): + continue matches.append(SearchMatch( path=parts[0], line_number=int(parts[1]), diff --git a/tools/file_tools.py b/tools/file_tools.py index 6182630b03cf..0b3d4a908f14 100644 --- a/tools/file_tools.py +++ b/tools/file_tools.py @@ -208,11 +208,11 @@ def _check_file_reqs(): READ_FILE_SCHEMA = { "name": "read_file", - "description": "Read a text file with line numbers and pagination. Use this instead of cat/head/tail in terminal. Output format: 'LINE_NUM|CONTENT'. Suggests similar filenames if not found. Use offset and limit for large files. NOTE: Cannot read images or binary files — use vision_analyze for images.", + "description": "Read a text file with line numbers and pagination. Use this instead of cat/head/tail in terminal. Output format: 'LINE_NUM|CONTENT'. Suggests similar filenames if not found. Use offset and limit for large files. NOTE: Cannot read images or binary files — use vision_analyze for images. SECURITY: Any file named '.env' is not readable.", "parameters": { "type": "object", "properties": { - "path": {"type": "string", "description": "Path to the file to read (absolute, relative, or ~/path)"}, + "path": {"type": "string", "description": "Path to the file to read (absolute, relative, or ~/path). Files named '.env' are blocked."}, "offset": {"type": "integer", "description": "Line number to start reading from (1-indexed, default: 1)", "default": 1, "minimum": 1}, "limit": {"type": "integer", "description": "Maximum number of lines to read (default: 500, max: 2000)", "default": 500, "maximum": 2000} }, @@ -252,13 +252,13 @@ def _check_file_reqs(): SEARCH_FILES_SCHEMA = { "name": "search_files", - "description": "Search file contents or find files by name. Use this instead of grep/rg/find/ls in terminal. Ripgrep-backed, faster than shell equivalents.\n\nContent search (target='content'): Regex search inside files. Output modes: full matches with line numbers, file paths only, or match counts.\n\nFile search (target='files'): Find files by glob pattern (e.g., '*.py', '*config*'). Also use this instead of ls — results sorted by modification time.", + "description": "Search file contents or find files by name. Use this instead of grep/rg/find/ls in terminal. Ripgrep-backed, faster than shell equivalents.\n\nContent search (target='content'): Regex search inside files. Output modes: full matches with line numbers, file paths only, or match counts.\n\nFile search (target='files'): Find files by glob pattern (e.g., '*.py', '*config*'). Also use this instead of ls — results sorted by modification time.\nSecurity note: '.env' files are excluded from search results and are never read.", "parameters": { "type": "object", "properties": { "pattern": {"type": "string", "description": "Regex pattern for content search, or glob pattern (e.g., '*.py') for file search"}, "target": {"type": "string", "enum": ["content", "files"], "description": "'content' searches inside file contents, 'files' searches for files by name", "default": "content"}, - "path": {"type": "string", "description": "Directory or file to search in (default: current working directory)", "default": "."}, + "path": {"type": "string", "description": "Directory or file to search in (default: current working directory). '.env' files are excluded from results.", "default": "."}, "file_glob": {"type": "string", "description": "Filter files by pattern in grep mode (e.g., '*.py' to only search Python files)"}, "limit": {"type": "integer", "description": "Maximum number of results to return (default: 50)", "default": 50}, "offset": {"type": "integer", "description": "Skip first N results for pagination (default: 0)", "default": 0}, diff --git a/tools/memory_tool.py b/tools/memory_tool.py index 2ce7631240f2..bf1a6f9f0072 100644 --- a/tools/memory_tool.py +++ b/tools/memory_tool.py @@ -114,6 +114,12 @@ def load_from_disk(self): self.memory_entries = list(dict.fromkeys(self.memory_entries)) self.user_entries = list(dict.fromkeys(self.user_entries)) + # Auto-compact oversized legacy files so new writes can succeed. + # This can happen when older installs wrote very large entries or when + # users imported historical notes without entry delimiters. + self._compact_to_limit("memory") + self._compact_to_limit("user") + # Capture frozen snapshot for system prompt injection self._system_prompt_snapshot = { "memory": self._render_block("memory", self.memory_entries), @@ -169,27 +175,27 @@ def add(self, target: str, content: str) -> Dict[str, Any]: if content in entries: return self._success_response(target, "Entry already exists (no duplicate added).") - # Calculate what the new total would be - new_entries = entries + [content] - new_total = len(ENTRY_DELIMITER.join(new_entries)) - - if new_total > limit: - current = self._char_count(target) - return { - "success": False, - "error": ( - f"Memory at {current:,}/{limit:,} chars. " - f"Adding this entry ({len(content)} chars) would exceed the limit. " - f"Replace or remove existing entries first." - ), - "current_entries": entries, - "usage": f"{current:,}/{limit:,}", - } + # Ensure new content itself can fit within the budget. + if len(content) > limit: + content = self._truncate_entry_to_limit(content, limit) + + # If over budget, automatically evict oldest entries to make room. + # This keeps memory useful in long-running sessions and avoids + # repeated model retry loops on deterministic size errors. + dropped = 0 + while len(ENTRY_DELIMITER.join(entries + [content])) > limit and entries: + entries.pop(0) + dropped += 1 + # If still too large (e.g., no entries left), force-truncate content. + if len(ENTRY_DELIMITER.join(entries + [content])) > limit: + content = self._truncate_entry_to_limit(content, limit) entries.append(content) self._set_entries(target, entries) self.save_to_disk(target) + if dropped: + return self._success_response(target, f"Entry added. Compacted memory by removing {dropped} oldest entr{'y' if dropped == 1 else 'ies'}.") return self._success_response(target, "Entry added.") def replace(self, target: str, old_text: str, new_content: str) -> Dict[str, Any]: @@ -210,7 +216,7 @@ def replace(self, target: str, old_text: str, new_content: str) -> Dict[str, Any matches = [(i, e) for i, e in enumerate(entries) if old_text in e] if len(matches) == 0: - return {"success": False, "error": f"No entry matched '{old_text}'."} + return self._success_response(target, f"No matching entry for '{old_text}' (no-op).") if len(matches) > 1: # If all matches are identical (exact duplicates), operate on the first one @@ -257,7 +263,7 @@ def remove(self, target: str, old_text: str) -> Dict[str, Any]: matches = [(i, e) for i, e in enumerate(entries) if old_text in e] if len(matches) == 0: - return {"success": False, "error": f"No entry matched '{old_text}'."} + return self._success_response(target, f"No matching entry for '{old_text}' (no-op).") if len(matches) > 1: # If all matches are identical (exact duplicates), remove the first one @@ -328,6 +334,55 @@ def _render_block(self, target: str, entries: List[str]) -> str: separator = "═" * 46 return f"{separator}\n{header}\n{separator}\n{content}" + def _truncate_entry_to_limit(self, entry: str, limit: int) -> str: + """Ensure a single entry fits within the target char limit.""" + if len(entry) <= limit: + return entry + marker = "[TRUNCATED TO FIT MEMORY LIMIT]\n" + budget = max(limit - len(marker), 0) + if budget <= 0: + return entry[-limit:] + return marker + entry[-budget:] + + def _compact_to_limit(self, target: str) -> None: + """ + Keep the newest entries that fit inside the configured char budget. + + The memory log is append-oriented, so preserving the most recent entries + gives the best chance of retaining relevant context. + """ + entries = [e.strip() for e in self._entries_for(target) if e and e.strip()] + limit = self._char_limit(target) + # Leave some headroom so the next add/replace operation can succeed + # instead of immediately hitting the hard cap again. + reserve = max(int(limit * 0.10), 120) + target_limit = max(limit - reserve, int(limit * 0.5)) + if not entries: + self._set_entries(target, []) + return + + total = len(ENTRY_DELIMITER.join(entries)) + if total <= target_limit: + self._set_entries(target, entries) + return + + kept_rev: List[str] = [] + for raw_entry in reversed(entries): + entry = self._truncate_entry_to_limit(raw_entry, target_limit) + candidate = list(reversed(kept_rev + [entry])) + if len(ENTRY_DELIMITER.join(candidate)) <= target_limit: + kept_rev.append(entry) + elif not kept_rev: + # Ensure at least one newest entry is retained. + kept_rev.append(entry) + break + else: + break + + compacted = list(reversed(kept_rev)) + self._set_entries(target, compacted) + self.save_to_disk(target) + @staticmethod def _read_file(path: Path) -> List[str]: """Read a memory file and split into entries. diff --git a/tools/mixture_of_agents_tool.py b/tools/mixture_of_agents_tool.py index 355419817fd0..694d90c8021a 100644 --- a/tools/mixture_of_agents_tool.py +++ b/tools/mixture_of_agents_tool.py @@ -25,8 +25,8 @@ 3. Multiple layers can be used for iterative refinement (future enhancement) Models Used (via OpenRouter): -- Reference Models: claude-opus-4, gemini-2.5-pro, gpt-4.1, deepseek-r1 -- Aggregator Model: claude-opus-4 (highest capability for synthesis) +- Reference Models: gemini-2.5-pro, claude-sonnet-4, gpt-4.1, deepseek-r1 +- Aggregator Model: google/gemini-2.5-pro (high capability for synthesis) Configuration: To customize the MoA setup, modify the configuration constants at the top of this file: @@ -59,14 +59,14 @@ # Configuration for MoA processing # Reference models - these generate diverse initial responses in parallel (OpenRouter slugs) REFERENCE_MODELS = [ - "anthropic/claude-opus-4.5", - "google/gemini-3-pro-preview", + "anthropic/claude-sonnet-4.5", + "google/gemini-3-pro-preview", "openai/gpt-5.2-pro", "deepseek/deepseek-v3.2" ] # Aggregator model - synthesizes reference responses into final output -AGGREGATOR_MODEL = "anthropic/claude-opus-4.5" # Use highest capability model for aggregation +AGGREGATOR_MODEL = "google/gemini-2.5-pro-preview" # High capability for aggregation # Temperature settings optimized for MoA performance REFERENCE_TEMPERATURE = 0.6 # Balanced creativity for diverse perspectives diff --git a/tools/process_registry.py b/tools/process_registry.py index ecf25c08d2c2..4599e2fe5dc5 100644 --- a/tools/process_registry.py +++ b/tools/process_registry.py @@ -42,11 +42,16 @@ import uuid _IS_WINDOWS = platform.system() == "Windows" -from tools.environments.local import _find_shell from dataclasses import dataclass, field from pathlib import Path from typing import Any, Dict, List, Optional +from tools.environments.shell_utils import ( + build_local_subprocess_invocation, + is_windows, + terminate_process_tree, +) + logger = logging.getLogger(__name__) @@ -145,11 +150,16 @@ def spawn_local( started_at=time.time(), ) + if use_pty: + if is_windows(): + logger.warning("PTY mode is not supported on Windows local backend, falling back to pipe mode") + use_pty = False + if use_pty: # Try PTY mode for interactive CLI tools try: import ptyprocess - user_shell = _find_shell() + user_shell = os.environ.get("SHELL") or shutil.which("bash") or "/bin/bash" pty_env = os.environ | (env_vars or {}) pty_env["PYTHONUNBUFFERED"] = "1" pty_proc = ptyprocess.PtyProcess.spawn( @@ -185,25 +195,24 @@ def spawn_local( logger.warning("PTY spawn failed (%s), falling back to pipe mode", e) # Standard Popen path (non-PTY or PTY fallback) - # Use the user's login shell for consistency with LocalEnvironment -- - # ensures rc files are sourced and user tools are available. - user_shell = _find_shell() + popen_args, popen_platform_kwargs, _ = build_local_subprocess_invocation( + command, session.cwd + ) # Force unbuffered output for Python scripts so progress is visible # during background execution (libraries like tqdm/datasets buffer when # stdout is a pipe, hiding output from process(action="poll")). bg_env = os.environ | (env_vars or {}) bg_env["PYTHONUNBUFFERED"] = "1" proc = subprocess.Popen( - [user_shell, "-lic", command], + popen_args, text=True, - cwd=session.cwd, env=bg_env, encoding="utf-8", errors="replace", stdout=subprocess.PIPE, stderr=subprocess.STDOUT, stdin=subprocess.PIPE, - preexec_fn=None if _IS_WINDOWS else os.setsid, + **popen_platform_kwargs, ) session.process = proc @@ -554,13 +563,11 @@ def kill_process(self, session_id: str) -> dict: os.kill(session.pid, signal.SIGTERM) elif session.process: # Local process -- kill the process group + terminate_process_tree(session.process, force=False) try: - if _IS_WINDOWS: - session.process.terminate() - else: - os.killpg(os.getpgid(session.process.pid), signal.SIGTERM) - except (ProcessLookupError, PermissionError): - session.process.kill() + session.process.wait(timeout=2) + except subprocess.TimeoutExpired: + terminate_process_tree(session.process, force=True) elif session.env_ref and session.pid: # Non-local -- kill inside sandbox session.env_ref.execute(f"kill {session.pid} 2>/dev/null", timeout=5) diff --git a/tools/rl_training_tool.py b/tools/rl_training_tool.py index b98a07d56bd4..679c5703dc46 100644 --- a/tools/rl_training_tool.py +++ b/tools/rl_training_tool.py @@ -163,7 +163,7 @@ def _scan_environments() -> List[EnvironmentInfo]: continue try: - with open(py_file, "r") as f: + with open(py_file, "r", encoding="utf-8", errors="replace") as f: tree = ast.parse(f.read()) for node in ast.walk(tree): @@ -323,7 +323,7 @@ async def _spawn_training_run(run_state: RunState, config_path: Path): # Step 1: Start the Atropos API server (run-api) print(f"[{run_id}] Starting Atropos API server (run-api)...") - api_log_file = open(api_log, "w") + api_log_file = open(api_log, "w", encoding="utf-8", newline="") run_state.api_process = subprocess.Popen( ["run-api"], stdout=api_log_file, @@ -344,7 +344,7 @@ async def _spawn_training_run(run_state: RunState, config_path: Path): # Step 2: Start the Tinker trainer print(f"[{run_id}] Starting Tinker trainer: launch_training.py --config {config_path}") - trainer_log_file = open(trainer_log, "w") + trainer_log_file = open(trainer_log, "w", encoding="utf-8", newline="") run_state.trainer_process = subprocess.Popen( [sys.executable, "launch_training.py", "--config", str(config_path)], stdout=trainer_log_file, @@ -384,7 +384,7 @@ async def _spawn_training_run(run_state: RunState, config_path: Path): print(f"[{run_id}] Starting environment: {env_info.file_path} serve") - env_log_file = open(env_log, "w") + env_log_file = open(env_log, "w", encoding="utf-8", newline="") run_state.env_process = subprocess.Popen( [sys.executable, str(env_info.file_path), "serve", "--config", str(config_path)], stdout=env_log_file, @@ -760,7 +760,7 @@ async def rl_start_training() -> str: if "wandb_name" in _current_config and _current_config["wandb_name"]: run_config["env"]["wandb_name"] = _current_config["wandb_name"] - with open(config_path, "w") as f: + with open(config_path, "w", encoding="utf-8", newline="") as f: yaml.dump(run_config, f, default_flow_style=False) # Create run state @@ -1190,7 +1190,7 @@ async def read_stream(stream, lines_list, prefix=""): stderr_text = "\n".join(stderr_lines) # Write logs to files for inspection outside CLI - with open(log_file, "w") as f: + with open(log_file, "w", encoding="utf-8", newline="") as f: f.write(f"Command: {cmd_display}\n") f.write(f"Working dir: {TINKER_ATROPOS_ROOT}\n") f.write(f"Return code: {process.returncode}\n") @@ -1222,7 +1222,7 @@ async def read_stream(stream, lines_list, prefix=""): # Parse the output JSONL file if output_file.exists(): # Read JSONL file (one JSON object per line = one step) - with open(output_file, "r") as f: + with open(output_file, "r", encoding="utf-8", errors="replace") as f: for line in f: line = line.strip() if not line: diff --git a/tools/send_message_tool.py b/tools/send_message_tool.py index bc8f2d65083b..20a3ee13bdfa 100644 --- a/tools/send_message_tool.py +++ b/tools/send_message_tool.py @@ -130,6 +130,18 @@ def _handle_send(args): f"or set a home channel via: hermes config set {platform_name.upper()}_HOME_CHANNEL " }) + # Guardrail: prevent recursive self-sends from inside a live messaging + # session (most commonly Discord -> send_message(target='discord')). + current_platform = os.getenv("HERMES_SESSION_PLATFORM", "").strip().lower() + current_chat_id = os.getenv("HERMES_SESSION_CHAT_ID", "").strip() + if current_platform == platform_name and current_chat_id and str(chat_id) == current_chat_id: + return json.dumps({ + "error": ( + f"Refusing to send_message to the current active {platform_name} chat " + f"({current_chat_id}) to prevent message loops." + ) + }) + try: from model_tools import _run_async result = _run_async(_send_to_platform(platform, pconfig, chat_id, message)) diff --git a/tools/session_search_tool.py b/tools/session_search_tool.py index b11b79fdae2a..b9b3cc2cacdf 100644 --- a/tools/session_search_tool.py +++ b/tools/session_search_tool.py @@ -165,9 +165,9 @@ async def _summarize_session( {"role": "system", "content": system_prompt}, {"role": "user", "content": user_prompt}, ], - **({} if not _extra else {"extra_body": _extra}), temperature=0.1, **auxiliary_max_tokens_param(MAX_SUMMARY_TOKENS), + **({} if not _extra else {"extra_body": _extra}), ) return response.choices[0].message.content.strip() except Exception as e: diff --git a/tools/skills_hub.py b/tools/skills_hub.py index 1758f678f1e2..6d543769891c 100644 --- a/tools/skills_hub.py +++ b/tools/skills_hub.py @@ -1107,7 +1107,7 @@ def append_audit_log(action: str, skill_name: str, source: str, parts.append(extra) line = " ".join(parts) + "\n" try: - with open(AUDIT_LOG, "a") as f: + with open(AUDIT_LOG, "a", encoding="utf-8", errors="replace", newline="") as f: f.write(line) except OSError as e: logger.debug("Could not write audit log: %s", e) diff --git a/tools/skills_tool.py b/tools/skills_tool.py index f118b2037f73..c33cd3c5f2eb 100644 --- a/tools/skills_tool.py +++ b/tools/skills_tool.py @@ -348,7 +348,14 @@ def skills_list(category: str = None, task_id: str = None) -> str: "categories": [], "message": "No skills found. Skills directory created at ~/.hermes/skills/" }, ensure_ascii=False) - + + # Sync any new bundled skills (e.g. from repo skills/) so they appear without hermes update + try: + from tools.skills_sync import sync_skills + sync_skills(quiet=True) + except Exception: + pass + # Find all skills all_skills = _find_all_skills() diff --git a/tools/terminal_tool.py b/tools/terminal_tool.py index 096ac207fa38..05a7214d38e2 100644 --- a/tools/terminal_tool.py +++ b/tools/terminal_tool.py @@ -38,6 +38,7 @@ import subprocess import tempfile import uuid +import re from pathlib import Path from typing import Optional, Dict, Any @@ -334,10 +335,34 @@ def replace_sudo(match): from tools.environments.ssh import SSHEnvironment as _SSHEnvironment from tools.environments.docker import DockerEnvironment as _DockerEnvironment from tools.environments.modal import ModalEnvironment as _ModalEnvironment +from tools.environments.shell_utils import get_local_shell_mode # Tool description for LLM -TERMINAL_TOOL_DESCRIPTION = """Execute shell commands on a Linux environment. Filesystem persists between calls. +def _build_terminal_tool_description() -> str: + windows_note = "" + if os.name == "nt": + mode = get_local_shell_mode() + if mode == "wsl": + windows_note = ( + "Windows note: with terminal.backend=local, commands run in WSL bash. " + "Use Linux syntax and WSL paths (e.g. /mnt/c/Users/... or /mnt/d/...).\n\n" + ) + elif mode == "powershell": + windows_note = ( + "Windows note: with terminal.backend=local, commands run in PowerShell. " + "Use PowerShell syntax and Windows paths (e.g. C:\\Users\\... or D:\\...). " + "To detect shell: (dir 2>&1 *`|echo CMD);&<# rem #>echo PowerShell (prints CMD or PowerShell).\n\n" + ) + else: + windows_note = ( + "Windows note: with terminal.backend=local, commands run in cmd.exe. " + "Run commands directly in cmd (dir, type, cd, echo %VAR%); do NOT use " + "powershell -Command or POSIX (pwd, uname, ls). Use Windows paths (e.g. C:\\Users\\...). " + "To detect shell: (dir 2>&1 *`|echo CMD);&<# rem #>echo PowerShell (prints CMD or PowerShell).\n\n" + ) + + return windows_note + """Execute shell commands in the configured terminal backend. Filesystem persists between calls. Do NOT use cat/head/tail to read files — use read_file instead. Do NOT use grep/rg/find to search — use search_files instead. @@ -355,6 +380,9 @@ def replace_sudo(match): Do NOT use vim/nano/interactive tools without pty=true — they hang without a pseudo-terminal. Pipe git output to cat if it might page. """ + +TERMINAL_TOOL_DESCRIPTION = _build_terminal_tool_description() + # Global state for environment lifecycle management _active_environments: Dict[str, Any] = {} _last_activity: Dict[str, float] = {} @@ -425,7 +453,8 @@ def _get_env_config() -> Dict[str, Any]: cwd = os.getenv("TERMINAL_CWD", default_cwd) if env_type in ("modal", "docker", "singularity") and cwd: host_prefixes = ("/Users/", "C:\\", "C:/") - if any(cwd.startswith(p) for p in host_prefixes) and cwd != default_cwd: + looks_like_windows_drive = bool(re.match(r"^[A-Za-z]:[\\/]", cwd)) + if (any(cwd.startswith(p) for p in host_prefixes) or looks_like_windows_drive) and cwd != default_cwd: logger.info("Ignoring TERMINAL_CWD=%r for %s backend " "(host path won't exist in sandbox). Using %r instead.", cwd, env_type, default_cwd) @@ -784,6 +813,19 @@ def terminal_tool( # Get configuration config = _get_env_config() env_type = config["env_type"] + if env_type == "local": + logger.info( + "Terminal execution request: backend=%s HERMES_WINDOWS_SHELL=%s TERMINAL_CWD=%s", + env_type, + os.getenv("HERMES_WINDOWS_SHELL", "auto"), + config.get("cwd", ""), + ) + else: + logger.info( + "Terminal execution request: backend=%s cwd=%s", + env_type, + config.get("cwd", ""), + ) # Use task_id for environment isolation effective_task_id = task_id or "default" @@ -1022,7 +1064,13 @@ def terminal_tool( # Extract output output = result.get("output", "") returncode = result.get("returncode", 0) - + if returncode != 0: + logger.info( + "Terminal command failed: exit_code=%s command=%s output=%s", + returncode, + command[:200] + ("..." if len(command) > 200 else ""), + (output[:500] + "..." if len(output) > 500 else output) or "(no output)", + ) # Add helpful message for sudo failures in messaging context output = _handle_sudo_failure(output, env_type) @@ -1064,7 +1112,8 @@ def check_terminal_requirements() -> bool: try: if env_type == "local": - from minisweagent.environments.local import LocalEnvironment + # Local backend uses Hermes' own LocalEnvironment wrapper. + # No external runtime dependency is required. return True elif env_type == "docker": from minisweagent.environments.docker import DockerEnvironment diff --git a/tools/tts_tool.py b/tools/tts_tool.py index 8e8f5e928f7a..af4cbba25303 100644 --- a/tools/tts_tool.py +++ b/tools/tts_tool.py @@ -60,7 +60,8 @@ # Defaults # =========================================================================== DEFAULT_PROVIDER = "edge" -DEFAULT_EDGE_VOICE = "en-US-AriaNeural" +DEFAULT_EDGE_VOICE = "en-US-AvaMultilingualNeural" +DEFAULT_EDGE_RATE = "125%" DEFAULT_ELEVENLABS_VOICE_ID = "pNInz6obpgDQGcFmaJgB" # Adam DEFAULT_ELEVENLABS_MODEL_ID = "eleven_multilingual_v2" DEFAULT_OPENAI_MODEL = "gpt-4o-mini-tts" @@ -144,8 +145,17 @@ async def _generate_edge_tts(text: str, output_path: str, tts_config: Dict[str, """ edge_config = tts_config.get("edge", {}) voice = edge_config.get("voice", DEFAULT_EDGE_VOICE) - - communicate = edge_tts.Communicate(text, voice) + rate = str(edge_config.get("rate", DEFAULT_EDGE_RATE)).strip() + # Edge TTS expects relative rates like "+25%". Accept "125%" in config + # as a user-friendly absolute speed and convert it to relative form. + if rate.endswith("%") and not rate.startswith(("+", "-")): + try: + rate_num = float(rate[:-1]) + rate = f"{rate_num - 100:+g}%" + except ValueError: + rate = "+25%" + + communicate = edge_tts.Communicate(text, voice, rate=rate) await communicate.save(output_path) return output_path @@ -276,6 +286,18 @@ def text_to_speech_tool( # produce Opus natively (no ffmpeg needed). Edge TTS always outputs MP3 # and needs ffmpeg for conversion. platform = os.getenv("HERMES_SESSION_PLATFORM", "").lower() + # Safety default: disable TTS on Discord unless explicitly re-enabled. + # This prevents self-reinforcing audio loops in bot chats. + if platform == "discord": + allow_discord_tts = os.getenv("HERMES_ALLOW_DISCORD_TTS", "false").strip().lower() + if allow_discord_tts not in ("1", "true", "yes", "on"): + return json.dumps({ + "success": False, + "error": ( + "text_to_speech is disabled for Discord by safety guard. " + "Set HERMES_ALLOW_DISCORD_TTS=true to re-enable." + ), + }, ensure_ascii=False) want_opus = (platform == "telegram") # Determine output path diff --git a/tools/web_tools.py b/tools/web_tools.py index 541404e6d22b..0127325e2b7e 100644 --- a/tools/web_tools.py +++ b/tools/web_tools.py @@ -154,8 +154,8 @@ async def process_content_with_llm( return processed_content except Exception as e: - logger.debug("Error processing content with LLM: %s", e) - return f"[Failed to process content: {str(e)[:100]}. Content size: {len(content):,} chars]" + logger.info("LLM processing failed for content (%d chars): %s", len(content), str(e)[:200]) + return None # Caller falls back to raw content async def _call_summarizer_llm( @@ -317,7 +317,7 @@ async def summarize_chunk(chunk_idx: int, chunk_content: str) -> tuple[int, Opti if not summaries: logger.debug("All chunk summarizations failed") - return "[Failed to process large content: all chunk summarizations failed]" + return None # Caller falls back to raw content logger.info("Got %d/%d chunk summaries", len(summaries), len(chunks)) @@ -476,6 +476,9 @@ def web_search_tool(query: str, limit: int = 5) -> str: query=query, limit=limit ) + if response is None: + logger.info("Search returned no response") + return json.dumps({"error": "Search returned no data.", "success": False}) # The response is a SearchData object with web, news, and images attributes # When not scraping, the results are directly in these attributes @@ -532,14 +535,14 @@ def web_search_tool(query: str, limit: int = 5) -> str: return result_json except Exception as e: - error_msg = f"Error searching web: {str(e)}" - logger.debug("%s", error_msg) + error_msg = str(e) or "Search failed (timeout or server error)." + logger.info("Search failed: %s", error_msg[:200]) debug_call_data["error"] = error_msg _debug.log_call("web_search_tool", debug_call_data) _debug.save() - return json.dumps({"error": error_msg}, ensure_ascii=False) + return json.dumps({"error": f"Error searching web: {error_msg}"}, ensure_ascii=False) async def web_extract_tool( @@ -611,76 +614,84 @@ async def web_extract_tool( try: logger.info("Scraping: %s", url) + # Timeout 60s (ms) to avoid long hangs; many sites block or slow-respond scrape_result = _get_firecrawl_client().scrape( url=url, - formats=formats + formats=formats, + timeout=60000, ) - + if scrape_result is None: + results.append({ + "url": url, + "title": "", + "content": "", + "raw_content": "", + "error": "Scrape returned no data (site may block crawlers or require JS).", + }) + continue + # Process the result - properly handle object serialization metadata = {} title = "" content_markdown = None content_html = None - - # Extract data from the scrape result - if hasattr(scrape_result, 'model_dump'): - # Pydantic model - use model_dump to get dict + + if hasattr(scrape_result, "model_dump"): result_dict = scrape_result.model_dump() - content_markdown = result_dict.get('markdown') - content_html = result_dict.get('html') - metadata = result_dict.get('metadata', {}) - elif hasattr(scrape_result, '__dict__'): - # Regular object with attributes - content_markdown = getattr(scrape_result, 'markdown', None) - content_html = getattr(scrape_result, 'html', None) - - # Handle metadata - convert to dict if it's an object - metadata_obj = getattr(scrape_result, 'metadata', {}) - if hasattr(metadata_obj, 'model_dump'): + content_markdown = result_dict.get("markdown") + content_html = result_dict.get("html") + metadata = result_dict.get("metadata") or {} + elif hasattr(scrape_result, "__dict__"): + content_markdown = getattr(scrape_result, "markdown", None) + content_html = getattr(scrape_result, "html", None) + metadata_obj = getattr(scrape_result, "metadata", None) or {} + if hasattr(metadata_obj, "model_dump"): metadata = metadata_obj.model_dump() - elif hasattr(metadata_obj, '__dict__'): - metadata = metadata_obj.__dict__ elif isinstance(metadata_obj, dict): metadata = metadata_obj - else: - metadata = {} + elif hasattr(metadata_obj, "__dict__"): + metadata = getattr(metadata_obj, "__dict__", {}) elif isinstance(scrape_result, dict): - # Already a dictionary - content_markdown = scrape_result.get('markdown') - content_html = scrape_result.get('html') - metadata = scrape_result.get('metadata', {}) - - # Ensure metadata is a dict (not an object) + content_markdown = scrape_result.get("markdown") + content_html = scrape_result.get("html") + metadata = scrape_result.get("metadata") or {} + if not isinstance(metadata, dict): - if hasattr(metadata, 'model_dump'): - metadata = metadata.model_dump() - elif hasattr(metadata, '__dict__'): - metadata = metadata.__dict__ - else: - metadata = {} - - # Get title from metadata - title = metadata.get("title", "") - - # Choose content based on requested format - chosen_content = content_markdown if (format == "markdown" or (format is None and content_markdown)) else content_html or content_markdown or "" - + metadata = metadata.model_dump() if hasattr(metadata, "model_dump") else {} + + title = (metadata or {}).get("title", "") + source_url = (metadata or {}).get("sourceURL") or (metadata or {}).get("source_url") or url + chosen_content = ( + content_markdown + if (format == "markdown" or (format is None and content_markdown)) + else (content_html or content_markdown or "") + ) + if not (chosen_content or "").strip(): + results.append({ + "url": source_url, + "title": title or "", + "content": "", + "raw_content": "", + "error": "Page returned no extractable content (blocked, paywall, or empty).", + }) + continue + results.append({ - "url": metadata.get("sourceURL", url), + "url": source_url, "title": title, "content": chosen_content, "raw_content": chosen_content, - "metadata": metadata # Now guaranteed to be a dict + "metadata": metadata, }) - except Exception as scrape_err: - logger.debug("Scrape failed for %s: %s", url, scrape_err) + err_msg = str(scrape_err) + logger.info("Scrape failed for %s: %s", url, err_msg[:200]) results.append({ "url": url, "title": "", "content": "", "raw_content": "", - "error": str(scrape_err) + "error": err_msg if err_msg else "Scrape failed (timeout, block, or server error).", }) response = {"results": results} @@ -893,8 +904,16 @@ async def web_crawl_tool( **crawl_params ) except Exception as e: - logger.debug("Crawl API call failed: %s", e) - raise + err_msg = str(e) or "Crawl failed (timeout or server error)." + logger.info("Crawl API call failed: %s", err_msg[:200]) + debug_call_data["error"] = err_msg + _debug.log_call("web_crawl_tool", debug_call_data) + _debug.save() + return json.dumps({"error": f"Crawl failed: {err_msg}"}, ensure_ascii=False) + + if crawl_result is None: + logger.info("Crawl returned no result") + return json.dumps({"error": "Crawl returned no data.", "results": []}, ensure_ascii=False) pages: List[Dict[str, Any]] = [] @@ -967,9 +986,10 @@ async def web_crawl_tool( else: metadata = {} - # Extract URL and title from metadata - page_url = metadata.get("sourceURL", metadata.get("url", "Unknown URL")) - title = metadata.get("title", "") + # Extract URL and title from metadata (support sourceURL and source_url) + meta = metadata or {} + page_url = meta.get("sourceURL") or meta.get("source_url") or meta.get("url", "Unknown URL") + title = meta.get("title", "") # Choose content (prefer markdown) content = content_markdown or content_html or "" @@ -979,13 +999,21 @@ async def web_crawl_tool( "title": title, "content": content, "raw_content": content, - "metadata": metadata # Now guaranteed to be a dict + "metadata": metadata }) response = {"results": pages} pages_crawled = len(response.get('results', [])) logger.info("Crawled %d pages", pages_crawled) + if pages_crawled == 0: + debug_call_data["pages_crawled"] = 0 + _debug.log_call("web_crawl_tool", debug_call_data) + _debug.save() + return json.dumps({ + "error": "Crawl returned no pages (site may block crawlers or be unreachable).", + "results": [] + }, ensure_ascii=False) debug_call_data["pages_crawled"] = pages_crawled debug_call_data["original_response_size"] = len(json.dumps(response)) diff --git a/trajectory_compressor.py b/trajectory_compressor.py index dedae1ade0e1..d79d76f4c23d 100644 --- a/trajectory_compressor.py +++ b/trajectory_compressor.py @@ -46,9 +46,12 @@ from rich.console import Console from hermes_constants import OPENROUTER_BASE_URL -# Load environment variables -from dotenv import load_dotenv -load_dotenv() +# Load environment variables (encoding-safe on Windows) +from agent.env_loader import load_dotenv_with_fallback +for _p in (Path.home() / ".hermes" / ".env", Path.cwd() / ".env"): + if _p.exists(): + load_dotenv_with_fallback(_p) + break @dataclass @@ -97,7 +100,7 @@ class CompressionConfig: @classmethod def from_yaml(cls, yaml_path: str) -> "CompressionConfig": """Load configuration from YAML file.""" - with open(yaml_path, 'r') as f: + with open(yaml_path, "r", encoding="utf-8", errors="replace") as f: data = yaml.safe_load(f) config = cls() @@ -1102,7 +1105,7 @@ async def process_single(file_path: Path, entry_idx: int, entry: Dict, # Save metrics if self.config.metrics_enabled: metrics_path = output_dir / self.config.metrics_output_file - with open(metrics_path, 'w') as f: + with open(metrics_path, 'w', encoding='utf-8', newline='') as f: json.dump(self.aggregate_metrics.to_dict(), f, indent=2) console.print(f"\n💾 Metrics saved to {metrics_path}")