diff --git a/agent/agent_runtime_helpers.py b/agent/agent_runtime_helpers.py index 7a9a0961a75ee..5df1ef64c064b 100644 --- a/agent/agent_runtime_helpers.py +++ b/agent/agent_runtime_helpers.py @@ -1513,6 +1513,9 @@ def invoke_tool(agent, function_name: str, function_args: dict, effective_task_i merge=function_args.get("merge", False), store=agent._todo_store, ) + elif function_name == "load_tool_pack": + from tools.lazy_tool_loader import load_tool_pack_for_agent + return load_tool_pack_for_agent(agent, function_args.get("pack", "")) elif function_name == "session_search": session_db = agent._get_session_db_for_recall() if not session_db: diff --git a/agent/tool_executor.py b/agent/tool_executor.py index b161b507e8d67..2ecec01d2630c 100644 --- a/agent/tool_executor.py +++ b/agent/tool_executor.py @@ -605,6 +605,18 @@ def execute_tool_calls_sequential(agent, assistant_message, messages: list, effe tool_duration = time.time() - tool_start_time if agent._should_emit_quiet_tool_messages(): agent._vprint(f" {_get_cute_tool_message_impl('todo', function_args, tool_duration, result=function_result)}") + elif function_name == "load_tool_pack": + function_result = agent._invoke_tool( + function_name, + function_args, + effective_task_id, + tool_call_id=getattr(tool_call, "id", None), + messages=messages, + pre_tool_block_checked=True, + ) + tool_duration = time.time() - tool_start_time + if agent._should_emit_quiet_tool_messages(): + agent._vprint(f" {_get_cute_tool_message_impl('load_tool_pack', function_args, tool_duration, result=function_result)}") elif function_name == "session_search": session_db = agent._get_session_db_for_recall() if not session_db: diff --git a/docs/audits/botparlor-system-prompt-audit-2026-05-20.md b/docs/audits/botparlor-system-prompt-audit-2026-05-20.md new file mode 100644 index 0000000000000..c5ce5e8cec121 --- /dev/null +++ b/docs/audits/botparlor-system-prompt-audit-2026-05-20.md @@ -0,0 +1,631 @@ +# BotParlor / Hermes System Prompt Audit - 2026-05-20 + +Baseline after merging current `upstream/main` into the BotParlor MCP timeout +fork. This audit uses Ria's real BotParlor TUI session +`session_20260520_025717_8915f7.json` as the live prompt sample. + +## Fork State + +- Worktree: `/home/alex/hermes-agent-mcp-timeout` +- Branch: `upstream/mcp-timeout-reconnect` +- Backup ref before merge: `backup/mcp-timeout-before-main-20260520` +- Merge result: `upstream/main` merged cleanly; branch is now one local MCP + timeout commit plus the merge commit ahead of `upstream/main`. +- Important upstream refactor: system prompt assembly moved out of + `run_agent.py` into `agent/system_prompt.py`. + +## Prompt Assembly Path + +Current code builds the system prompt in three tiers: + +- `agent/system_prompt.py::build_system_prompt_parts` + - `stable`: identity, Hermes-help guidance, tool-aware guidance, optional + computer-use guidance, Nous subscription capability block, tool-use + enforcement guidance, skills index, Alibaba model-name workaround, + environment hints, and platform hints. + - `context`: caller `system_message`, then one project context source from + `.hermes.md` / `HERMES.md`, `AGENTS.md`, `CLAUDE.md`, or Cursor rules. + - `volatile`: built-in memory, user profile, external memory provider + context, and the conversation-start/model/provider line. +- `agent/conversation_loop.py::_restore_or_build_system_prompt` + - Restores a previously stored system prompt from the session DB when + possible. + - Builds and persists a new prompt only when no usable stored prompt exists. +- `agent/conversation_loop.py` + - Adds `agent.ephemeral_system_prompt` at API-call time after the cached + system prompt. + - Injects plugin and external-memory recall context into the current user + message, not into the system prompt. + +## Live Ria Baseline + +Ria's live BotParlor TUI session sample: + +- System prompt: `51,096` chars, roughly `12.8k` tokens. +- Tool schemas: `35` tools, `47,041` JSON chars, roughly `11.8k` tokens. +- Combined prompt/tool-schema floor before conversation history: about `98k` + chars, roughly `24.5k` tokens. +- Model: `qwen3.6-27b-huihui-abliterated-awq-lm-104k-3090ab`. +- Platform hint key: `tui`. + +Coarse system-prompt slices: + +| Slice | Chars | Approx tokens | Notes | +| --- | ---: | ---: | --- | +| Bot persona / behavior before skills | 17,899 | 4,474 | Includes SOUL/persona, BotParlor behavior, mood/avatar rules, response format, and other private persona details. | +| Skills index | 14,852 | 3,713 | Largest generic Hermes block; `` alone is about 13,300 chars. | +| Project context | 18,198 | 4,549 | Ria loaded the Hermes repo `AGENTS.md`, including the Hermes development guide, into a BotParlor chat session. | +| Timestamp/model/provider tail | 147 | 36 | Small. | + +Largest tool-schema contributors in the same session: + +| Tool | Description chars | Parameter schema chars | +| --- | ---: | ---: | +| `delegate_task` | 3,015 | 3,307 | +| `terminal` | 1,830 | 2,815 | +| `skill_manage` | 1,787 | 2,205 | +| `execute_code` | 2,032 | 224 | +| `session_search` | 1,588 | 561 | +| `memory` | 1,524 | 546 | +| `mcp_botparlor_create_avatar` | 878 | 1,315 | +| `search_files` | 435 | 1,249 | +| `patch` | 361 | 1,084 | +| `send_message` | 395 | 995 | + +## Generic Code-Path Measurements + +For a minimal Qwen/custom agent with only `memory`, `session_search`, and +`skills` toolsets, skipping project context and memory content: + +- Stable prompt: `16,274` chars, roughly `4.1k` tokens. +- Volatile tail: `80` chars. +- Tools: `5`. + +Major static/dynamic block sizes in that generic path: + +| Block | Chars | Approx tokens | +| --- | ---: | ---: | +| Default identity | 513 | 128 | +| Hermes help guidance | 211 | 52 | +| Memory + session search + skills guidance | 1,999 | 499 | +| Qwen auto tool-use enforcement | 824 | 206 | +| Skills index | 12,345 | 3,086 | +| Environment hints | 126 | 31 | + +## Findings + +1. Project context is currently wrong for BotParlor bot chats. + Ria's BotParlor TUI session loaded the Hermes worktree `AGENTS.md` because + the TUI gateway process runs from the Hermes install/worktree. That adds + about 18k chars of development instructions that are not useful for normal + character chat and can bias tool behavior toward coding-agent concerns. + +2. The skills index is a large always-on block whenever the skills tools are + present. + The current skills prompt tells the model to scan and load skills before + replying, then lists the whole available skill catalog. In Ria's live sample + this costs about 14.9k chars. For BotParlor chats, this is usually not worth + the prompt floor unless the bot is explicitly doing Hermes administration or + a complex external task. + +3. The TUI session has broad core tools in addition to BotParlor MCP tools. + Ria's live session had 35 tools, including `delegate_task`, `terminal`, + `execute_code`, file editing, skills, memory, TTS, vision, and BotParlor MCP. + The bot service already exports `HERMES_TUI_TOOLSETS=hermes-botparlor,botparlor`. + Current upstream TUI code has explicit handling for `HERMES_TUI_TOOLSETS`, + including MCP server names, so deploying the merged fork may reduce this + automatically if the older deployed fork was falling back to configured CLI + toolsets. + +4. Tool schemas are nearly as expensive as the system prompt. + The live tool-schema JSON is about 47k chars. Even if the system prompt is + trimmed, broad tool availability will keep the request floor high and can + encourage unnecessary tool use. + +5. The upstream refactor gives us a cleaner surgery point. + `agent/system_prompt.py` now has a single prompt assembly function with clear + stable/context/volatile tiers. That makes it practical to add BotParlor/TUI + policy switches without editing the full agent loop. + +## First Surgery Candidates + +1. For BotParlor TUI bridge sessions, skip project context files. + Options: + - Set `HERMES_IGNORE_RULES=1` for `tui-ws-bridge.service`; this currently + also skips memory, which may be too broad. + - Add a narrower `HERMES_SKIP_CONTEXT_FILES=1` / TUI config path that passes + `skip_context_files=True` without disabling memory. + - Set `TERMINAL_CWD` to a neutral BotParlor runtime directory with no + `AGENTS.md`; less explicit than a real skip flag. + +2. Make skills prompt optional or narrower for chat sessions. + Candidate policy: if the enabled toolsets are only BotParlor MCP/chat + toolsets, do not include the global skills index even if skills tools are + present, or do not include skills tools at all. + +3. Verify merged `HERMES_TUI_TOOLSETS` behavior on one bot before larger rollout. + After deploying this merged fork to a single bot, create a fresh session and + compare: + - tool count, + - tool names, + - system prompt chars, + - tool-schema JSON chars, + - whether Hermes repo `AGENTS.md` still appears. + +4. Consider a BotParlor-specific profile/toolset floor. + Normal character chat probably needs BotParlor MCP tools and maybe memory, + but not `delegate_task`, file editing, terminal, process management, skills + management, generic TTS, or vision unless explicitly enabled for a task. + +## Verification Run + +- `./scripts/run_tests.sh tests/tools/test_mcp_tool.py tests/tools/test_mcp_tool_session_expired.py` + - `217 passed in 16.87s` +- The shared Hermes venv was missing `pytest-timeout`; installed + `pytest-timeout==2.4.0` so the merged upstream test runner can execute. + +## Ria Skill Prune + +After the initial audit, Ria's active skill install was pruned on +`ria.st-el.com`: + +- Active skills before prune: `131`. +- Active skills after prune: `18`. +- Removed active skill directories: `113`, moved to + `/home/alex/.hermes/skills-uninstalled-20260520-ria`. +- Category `DESCRIPTION.md` files moved out of the active tree: `28`. +- Ria config backup before adding disabled-skill guards: + `/home/alex/.hermes/config.yaml.bak-before-ria-skill-prune-20260520-035101`. +- Hermes re-seeded five bundled MLOps skills after the first restart, so the + removed skill names and directory basenames were also added to + `skills.disabled` in Ria's `~/.hermes/config.yaml`. +- `~/.hermes/.skills_prompt_snapshot.json` was cleared and + `tui-ws-bridge.service` / `hermes-gateway.service` were restarted. + +Kept active skills: + +- `autonomous-ai-agents/hermes-agent` +- `botparlor/botparlor-avatar-maintenance` +- `botparlor/ria-daily-report-template` +- `creative/creative-ideation` (`name: ideation`) +- `creative/humanizer` +- `hermes-context-window-management` +- `mcp/mcporter` +- `mcp/native-mcp` +- `media/youtube-content` +- `note-taking/obsidian` +- `productivity/maps` +- `rig/rig-charlie` +- `rig/rig-delta` +- `rig/rig-hotel` +- `rig/rig-hotel-ops` +- `session-reset` +- `session-reset-workflow` +- `spy-game/ria-spy-game` + +Post-prune measurement in a fresh process with `skills`, `memory`, and +`session_search` available: + +- Skills prompt: `3,823` chars, roughly `955` tokens. +- Displayed skill lines in that measurement: `17`; `maps` remains installed but + is hidden by its conditional metadata unless relevant tool/toolset context is + available. + +## First-Message Size Goal + +Alex's target for Ria is a first-message request floor of about `5k-8k` tokens, +counting both the frozen system prompt and tool schemas before conversation +history. + +Current Ria deployed branch after skill pruning: + +| Toolset config | Tools | System approx tokens | Tool-schema approx tokens | Combined approx tokens | +| --- | ---: | ---: | ---: | ---: | +| `hermes-botparlor,botparlor` | 35 | 5,512 | 11,760 | 17,272 | +| `botparlor,memory,session_search` | 18 | 4,415 | 3,739 | 8,154 | +| `botparlor,memory` | 17 | 4,368 | 3,173 | 7,542 | + +After Alex trimmed Ria's `SOUL.md`, the same measurement improved to: + +| Toolset config | Tools | System approx tokens | Tool-schema approx tokens | Combined approx tokens | +| --- | ---: | ---: | ---: | ---: | +| `hermes-botparlor,botparlor` | 35 | 3,308 | 11,760 | 15,069 | +| `botparlor,memory` | 17 | 2,164 | 3,173 | 5,338 | + +Interpretation: + +- The current `hermes-botparlor` toolset is the main remaining size problem; it + expands to almost all Hermes core tools plus BotParlor MCP. +- `botparlor,memory` meets the initial `5k-8k` target and preserves BotParlor + MCP plus durable memory. +- `session_search` is useful but should be optional or summoned by a narrower + mode because adding it to the default first-message floor pushes Ria just over + the upper target. +- Skills should stay out of Ria's default first-message path unless the user is + explicitly doing Hermes administration or a skill-driven task. + +## Lazy Tool Loading Prototype + +Implemented locally in the Hermes fork after the audit: + +- Added `HERMES_TUI_VISIBLE_TOOLS` / `HERMES_VISIBLE_TOOLS` as an opt-in schema + visibility filter in `model_tools.get_tool_definitions()`. Tools hidden by + this filter remain registered in the process and can still be loaded later. +- Added a small `tool_loader` toolset with `load_tool_pack`. The loader mutates + only the active agent session by appending schemas and updating + `valid_tool_names`; it does not reconnect MCP or alter global registration. +- Initial lazy packs: `avatar`, `reminders`, `media`, `botparlor_resources`, + `recall`, `skills`, and `power`. + +Candidate Ria canary service shape: + +```text +HERMES_TUI_TOOLSETS=botparlor,memory,tool_loader +HERMES_TUI_VISIBLE_TOOLS=mcp_botparlor_set_mood,memory,load_tool_pack +``` + +That should keep first-message schemas to mood + memory + loader while leaving +all BotParlor MCP tools registered for later pack loading. + +Verification: + +- `./scripts/run_tests.sh tests/test_lazy_tool_loader.py tests/test_toolsets.py` + - `30 passed in 1.46s` +- `./scripts/run_tests.sh tests/agent/test_system_prompt_restore.py tests/agent/test_prompt_builder.py tests/tools/test_mcp_tool.py tests/tools/test_mcp_tool_session_expired.py` + - `351 passed in 15.34s` +- `python3 -m py_compile model_tools.py toolsets.py tools/lazy_tool_loader.py agent/agent_runtime_helpers.py agent/tool_executor.py` + +## Ria Canary Deploy + +Ria-only deployment completed on 2026-05-20: + +- Hermes checkout: `ria.st-el.com:/home/alex/.hermes/hermes-agent` +- Branch: `ria/lazy-tool-canary` +- Commit: `8c8b68e19` +- Service drop-in: + - `HERMES_TUI_TOOLSETS=botparlor,memory,tool_loader` + - `HERMES_TUI_VISIBLE_TOOLS=mcp_botparlor_set_mood,memory,load_tool_pack` +- The previous uncommitted Ria `runtime_provider.py` patch was stashed and + backed up; equivalent provider-profile behavior is included in the canary + commit. +- Ria's standalone `tui-ws-bridge.py` needed explicit MCP startup discovery + restored after moving to the newer Hermes branch. The deployed bridge now + calls `discover_mcp_tools()` before starting Uvicorn. + +Ria verification: + +- `python -m py_compile` passed for touched Hermes files. +- `./scripts/run_tests.sh tests/test_lazy_tool_loader.py tests/hermes_cli/test_runtime_provider_resolution.py` + - `119 passed in 22.65s` +- `tui-ws-bridge.service` and `hermes-gateway.service` are active. +- Fresh bridge session visible tools: + - `mcp_botparlor_set_mood` + - `memory` + - `load_tool_pack` +- BotParlor MCP status in that session: connected, 12 registered tools. + +Live canary metrics from BotParlor session `a87df0ca` / Hermes session +`20260520_134210_ae837c`: + +- Visible replies: 7. +- Mood calls: 6 (`6/7` visible replies). +- Avatar flow: `load_tool_pack(pack=avatar)`, then + `mcp_botparlor_get_outfits`, then `mcp_botparlor_create_avatar`. +- First model call input: `8,275` tokens. +- First user-visible reply after mood tool result: `8,361` input tokens. +- Max live context after avatar pack/tool results: `10,955 / 104,448` tokens. +- Session model calls: `15`. +- Cumulative prompt tokens: `136,086`. +- Cumulative generated tokens: `775`. +- Reasoning chars: `0`. +- Compactions: `0`. +- Ria-side summed model-call latency: about `27.3s`, average `1.82s/call`. + +## Chatbot Context-File Trim + +Ria's live canary prompt still loaded the Hermes repo `AGENTS.md` because the +standalone TUI bridge runs with `WorkingDirectory=%h/.hermes/hermes-agent`. +That added `18,196` chars, roughly `4.5k` tokens, of Hermes development +guidance to a normal BotParlor chat session. + +Measured Ria prompt slices from session `20260520_134210_ae837c`: + +| Slice | Chars | Approx tokens | +| --- | ---: | ---: | +| Persona / SOUL | 5,472 | 1,368 | +| Hermes help pointer | 211 | 52 | +| Memory guidance | 1,426 | 356 | +| Tool-use enforcement | 824 | 206 | +| Environment hint | 122 | 30 | +| Hermes `AGENTS.md` project context | 18,196 | 4,549 | +| Memory + user profile | 1,636 | 409 | +| Timestamp/model/provider | 140 | 35 | + +Initial visible tool schemas for Ria's lazy-tool baseline: + +| Tool set | Tools | Schema chars | Approx tokens | +| --- | ---: | ---: | ---: | +| Initial visible tools (`set_mood`, `memory`, `load_tool_pack`) | 3 | 3,656 | 914 | +| After avatar pack load | 8 | 7,702 | 1,925 | + +Implemented `HERMES_TUI_SKIP_CONTEXT_FILES=1` / +`HERMES_SKIP_CONTEXT_FILES=1` for TUI sessions. This skips cwd project context +files while keeping SOUL identity and memory enabled. `HERMES_IGNORE_RULES` +keeps its previous broader behavior: skip context files, SOUL, and memory. + +Projected effect for Ria first-message sessions: + +- Remove about `18.2k` chars / `4.5k` rough tokens from the cached system + prompt. +- Bring the system prompt from `28,041` chars to about `9,845` chars. +- Bring system prompt + initial visible schemas from about `31,697` chars to + about `13,501` chars, before chat-message framing. +- Since the observed first model call was `8,275` input tokens, fresh sessions + with context files skipped should land well inside the original `5k-8k` + target, likely around the low-to-mid `4k` range before conversation history. + +Verification: + +- `./scripts/run_tests.sh tests/tui_gateway/test_make_agent_provider.py` + - `8 passed in 1.72s` +- `python3 -m py_compile tui_gateway/server.py` +- Deployed to Ria at commit `5fa98933e` with + `HERMES_TUI_SKIP_CONTEXT_FILES=1` in + `tui-ws-bridge.service.d/botparlor-lazy-tools.conf`. +- Non-chat prompt-build verification on Ria with the new env: + - `skip_context_files=True` + - `load_soul_identity=True` + - `skip_memory=False` + - `system_chars=9843` + - `has_project_context=False` + - `has_agents=False` + +Fresh live Ria session `20260520_141211_ddaf1b` after the context-file trim: + +- System prompt: `9,843` chars, roughly `2,460` tokens. +- Saved prompt contains no `# Project Context` and no `AGENTS.md`. +- First model call: `3,596` input tokens, down from previous canary's `8,275` + input tokens (`-4,679`, about `56.5%` lower). +- First user-visible reply after mood tool result: `3,685` input tokens, down + from `8,361` (`-4,676`, about `55.9%` lower). +- Normal chat calls before lazy reminder loading stayed around + `3,721`-`4,163` input tokens. +- Reminder flow loaded on demand: + `load_tool_pack(pack=reminders)`, then `mcp_botparlor_create_reminder`. +- Max observed context after reminder pack/tool result: `5,464` input tokens. +- Session through six visible user messages: `12` model calls, + `49,364` cumulative prompt tokens, `396` generated tokens, about `15.1s` + summed model-call latency (`1.26s/call` average). + +## Chatbot Baseline Rollout + +Alex accepted the current Ria configuration as the chatbot baseline: + +```text +HERMES_TUI_TOOLSETS=botparlor,memory,tool_loader +HERMES_TUI_VISIBLE_TOOLS=mcp_botparlor_set_mood,memory,load_tool_pack +HERMES_TUI_SKIP_CONTEXT_FILES=1 +``` + +Baseline properties: + +- Startup visible tools: `load_tool_pack`, `mcp_botparlor_set_mood`, `memory`. +- BotParlor MCP tools remain registered but hidden until loaded by pack. +- Cwd project context files are skipped while SOUL and memory remain enabled. +- Skills stay out of the startup prompt unless the `skills` pack is loaded. + +Pre-rollout group comparison captured from BotParlor group +`PreSlim-GroupChat` (`group_ecc65b24-9796-4ee3-8078-1dff6459358b`), before +rolling the baseline to Katie/Sophia/Lexi/Scarlett: + +| Bot | First reply context | Third reply context | +| --- | ---: | ---: | +| Katie | 22,815 | 24,284 | +| Sophia | 23,331 | 26,094 | +| Lexi | 23,269 | 24,712 | +| Scarlett | 24,909 | 26,441 | +| Ria already slim | 4,237 | 5,009 | + +Group totals for that pre-rollout run: + +- 16 group messages / 15 assistant replies. +- 49 model calls. +- 1,029,255 prompt tokens. +- 5,203 output tokens. +- 10 BotParlor tool calls. +- Span: 2026-05-20 14:34:42-14:37:59 UTC. + +Rollout completed for chatbot hosts: + +| Bot | Host | Branch | Commit | System chars in prompt-build check | +| --- | --- | --- | --- | ---: | +| Ria | `ria.st-el.com` | `ria/lazy-tool-canary` | `246444071` | 9,843 | +| Katie | `katie.st-el.com` | `chatbot/lazy-tool-standard` | `246444071` | 11,316 | +| Sophia | `sophia.st-el.com` | `chatbot/lazy-tool-standard` | `246444071` | 11,300 | +| Lexi | `lexi.st-el.com` | `chatbot/lazy-tool-standard` | `246444071` | 10,998 | +| Scarlett | `scarlett.st-el.com` | `chatbot/lazy-tool-standard` | `246444071` | 17,975 | + +Verification: + +- `python -m py_compile` passed on deployed Hermes files for all four new + rollout hosts. +- `tui-ws-bridge.py` on all four hosts includes MCP startup discovery. +- `tui-ws-bridge.service` and `hermes-gateway.service` are active. +- Bridge `/health` returns OK on all four hosts. +- Production BotParlor reports all chatbot gateways connected. +- Prompt-build checks on all four hosts show no project context / `AGENTS.md`. +- Visible startup tools are exactly `load_tool_pack`, + `mcp_botparlor_set_mood`, and `memory`. + +## Pre/Post Group Comparison + +Post-rollout comparison group: + +- `PostSlim-GroupChat` + (`group_6f1451b6-3c42-4979-9f1f-d983e9d96f44`) +- Span: 2026-05-20 17:53:11-17:55:38 UTC. +- 25 group messages / 24 assistant replies. + +First assistant-turn context after rollout: + +| Bot | Pre-slim first context | Post-slim first context | Change | +| --- | ---: | ---: | ---: | +| Katie | 22,815 | 3,942 | -82.7% | +| Sophia | 23,331 | 3,367 | -85.6% | +| Lexi | 23,269 | 3,105 | -86.7% | +| Scarlett | 24,909 | 4,314 | -82.7% | +| Ria | 4,237 | 4,025 | -5.0% | + +Full-run totals: + +| Metric | PreSlim | PostSlim | Change | +| --- | ---: | ---: | ---: | +| Assistant replies | 15 | 24 | +60.0% | +| Prompt/input tokens | 1,029,255 | 410,097 | -60.2% | +| Output tokens | 5,203 | 3,373 | -35.2% | +| Total tokens | 1,034,458 | 413,470 | -60.0% | +| Model calls | 49 | 95 | +93.9% | +| BotParlor tool calls | 10 | 9 | -10.0% | +| First-context sum | 98,561 | 18,753 | -81.0% | +| Last-context sum | 106,540 | 26,638 | -75.0% | +| Mean context | 20,547 | 4,571 | -77.8% | +| Max context | 26,441 | 5,989 | -77.3% | + +Normalized first three replies per bot: + +| Metric | PreSlim | PostSlim | Change | +| --- | ---: | ---: | ---: | +| Assistant replies | 15 | 15 | 0.0% | +| Prompt/input tokens | 1,029,255 | 166,727 | -83.8% | +| Output tokens | 5,203 | 1,428 | -72.6% | +| Total tokens | 1,034,458 | 168,155 | -83.7% | +| Model calls | 49 | 41 | -16.3% | +| BotParlor tool calls | 10 | 6 | -40.0% | +| First-context sum | 98,561 | 18,753 | -81.0% | +| Last-context sum | 106,540 | 23,234 | -78.2% | +| Mean context | 20,547 | 4,195 | -79.6% | +| Max context | 26,441 | 5,463 | -79.3% | + +Interpretation: + +- The cleanest apples-to-apples result is the normalized first-three-replies + comparison: same reply count, prompt/input tokens down from `1,029,255` to + `166,727`, an `83.8%` reduction. +- Even though the post-slim group ran longer, with `24` replies instead of + `15`, it still used `60.2%` fewer prompt/input tokens overall. +- Post-slim max context stayed under `6k` input tokens across the longer run. +- Behavioral caveats to watch: several post-slim group replies persisted as + very short `...` messages, and one Katie message looked like raw mood-tool + text rather than a parsed tool call. These do not affect the context-size + result but are worth watching in future group-chat behavior checks. + +## Assistant Lazy-Pack Canary + +Implemented assistant-oriented lazy packs in `tools/lazy_tool_loader.py`: + +- `coding`: coding, repo work, debugging, tests, GitHub workflows, and subagents. +- `web_research`: web search/extraction, browser automation, X/Twitter, + YouTube, and PDF research. +- `local_ops`: local machine, network, maps, and Home Assistant operations. +- `hermes_admin`: Hermes internals, MCP work, skill authoring, and TUI + debugging. +- `notes`: Obsidian and local knowledge capture. +- `design`: web-design references and implementation support. +- `persona`: bot personality, dogfood, and security-rule skills. + +Pack loads now return `suggested_skills` so the model can load only relevant +skill docs with `skill_view` after the pack's tool schemas are available. + +Viper canary on `springlab2.st-el.com`: + +- Hermes checkout: branch `assistant/lazy-tool-canary`. +- Base commit: `0f3bc4dcf`. +- Preserved Viper's local `tools/approval.py` fan-out patch as an uncommitted + host-local change. +- Viper active skills were pruned from `110` to Alex's `35`-skill assistant + keep list. Removed skills were moved to: + - `/home/alex/.hermes/skills-uninstalled-20260520-190702-viper-assistant` + - `/home/alex/.hermes/skills-uninstalled-20260520-191134-viper-reseeded` +- Config backups: + - `/home/alex/.hermes/config.yaml.bak-before-viper-assistant-skill-prune-20260520-190702` + - `/home/alex/.hermes/config.yaml.bak-before-viper-reseed-cleanup-20260520-191134` +- Standalone `tui-ws-bridge.py` was refreshed with the MCP startup-discovery + version after the first restart showed BotParlor MCP disconnected. + +Viper assistant floor: + +```text +HERMES_TUI_TOOLSETS=botparlor,memory,session_search,tool_loader +HERMES_TUI_VISIBLE_TOOLS=mcp_botparlor_set_mood,memory,session_search,load_tool_pack +HERMES_TUI_SKIP_CONTEXT_FILES=1 +``` + +Verification: + +- `python3 -m py_compile tools/lazy_tool_loader.py` +- `./scripts/run_tests.sh tests/test_lazy_tool_loader.py tests/test_toolsets.py` + - `31 passed in 1.51s` +- On Viper, `venv/bin/python -m py_compile tools/lazy_tool_loader.py + tools/approval.py tui_gateway/server.py tui-ws-bridge.py`. +- `tui-ws-bridge.service` and `hermes-gateway.service` active. +- Bridge `/health` returns OK. +- BotParlor reports Viper connected. +- Fresh Viper gateway session startup tools: + - `load_tool_pack` + - `mcp_botparlor_set_mood` + - `memory` + - `session_search` +- BotParlor MCP status: connected, `12` tools registered. +- Active visible skills after prune: `35`, exactly the assistant keep list. +- Smoke prompt `Canary smoke test only. Reply exactly: assistant baseline ready` + completed in one model call with `6,750` input tokens and `4` output tokens. + +## Assistant Baseline Rollout + +Rolled the Viper assistant baseline to Hanna, Raven, and Sarah after Alex's +live Viper test looked good. + +Assistant floor on all assistant bots: + +```text +HERMES_TUI_TOOLSETS=botparlor,memory,session_search,tool_loader +HERMES_TUI_VISIBLE_TOOLS=mcp_botparlor_set_mood,memory,session_search,load_tool_pack +HERMES_TUI_SKIP_CONTEXT_FILES=1 +``` + +Rollout notes: + +| Bot | Host / path | Branch | Commit | Host-local changes | +| --- | --- | --- | --- | --- | +| Hanna | `hanna.st-el.com` | `assistant/lazy-tool-standard` | `2cefd860e` | Untracked backup files and standalone `tui-ws-bridge.py`. | +| Raven | `raven.st-el.com` | `assistant/lazy-tool-standard` | `2cefd860e` | Untracked backup file and standalone `tui-ws-bridge.py`. | +| Sarah | local `officedt`, `/home/alex/.hermes/hermes-agent` | `assistant/lazy-tool-standard` | `2cefd860e` | Preserved host-local `tools/approval.py` fan-out patch. | + +Skill pruning: + +| Bot | Active skills after cleanup | Initial removed | Reseeded removed | +| --- | ---: | ---: | ---: | +| Hanna | 27 | 118 | 4 | +| Raven | 25 | 105 | 4 | +| Sarah | 26 | 80 | 8 | + +All three used the same assistant keep-list policy as Viper. Missing keep-list +skills were not newly installed; each host now exposes the intersection of its +installed skills and the assistant keep list. + +Verification: + +| Bot | Startup tools | MCP status | Project context | Smoke input tokens | Calls | +| --- | --- | --- | --- | ---: | ---: | +| Hanna | `load_tool_pack`, `mcp_botparlor_set_mood`, `memory`, `session_search` | connected, 12 tools | skipped | 10,498 | 2 | +| Raven | `load_tool_pack`, `mcp_botparlor_set_mood`, `memory`, `session_search` | connected, 12 tools | skipped | 5,035 | 1 | +| Sarah | `load_tool_pack`, `mcp_botparlor_set_mood`, `memory`, `session_search` | connected, 12 tools | skipped | 5,801 | 1 | + +Smoke prompt for each bot: + +```text +Canary smoke test only. Reply exactly: assistant baseline ready +``` + +All three returned `assistant baseline ready`. Production BotParlor reported +Hanna, Raven, Sarah, and Viper connected after the rollout. diff --git a/hermes_cli/config.py b/hermes_cli/config.py index dd470bdbbf364..fda09dfdd64b6 100644 --- a/hermes_cli/config.py +++ b/hermes_cli/config.py @@ -3008,7 +3008,7 @@ def _normalize_custom_provider_entry( "api_mode", "transport", "model", "default_model", "models", "context_length", "rate_limit_delay", "request_timeout_seconds", "stale_timeout_seconds", - "discover_models", + "discover_models", "provider_profile", "runtime_provider", "provider", } for camel, snake in _CAMEL_ALIASES.items(): if camel in entry and snake not in entry: @@ -3264,6 +3264,7 @@ def check_config_version() -> Tuple[int, int]: _VALID_CUSTOM_PROVIDER_FIELDS = { "name", "base_url", "api_key", "api_mode", "model", "models", "context_length", "rate_limit_delay", + "provider_profile", "runtime_provider", "provider", # key_env is read at runtime by runtime_provider.py and auxiliary_client.py # — include it here so the set accurately describes the supported schema. "key_env", diff --git a/hermes_cli/runtime_provider.py b/hermes_cli/runtime_provider.py index 0765c72cecb4e..0e826a52f50db 100644 --- a/hermes_cli/runtime_provider.py +++ b/hermes_cli/runtime_provider.py @@ -481,6 +481,13 @@ def _get_named_custom_provider(requested_provider: str) -> Optional[Dict[str, An api_mode = _parse_api_mode(entry.get("api_mode") or entry.get("transport")) if api_mode: result["api_mode"] = api_mode + profile = ( + entry.get("provider_profile") + or entry.get("runtime_provider") + or entry.get("provider") + ) + if isinstance(profile, str) and profile.strip(): + result["provider_profile"] = profile.strip() return result # Also check the 'name' field if present display_name = entry.get("name", "") @@ -499,6 +506,13 @@ def _get_named_custom_provider(requested_provider: str) -> Optional[Dict[str, An api_mode = _parse_api_mode(entry.get("api_mode") or entry.get("transport")) if api_mode: result["api_mode"] = api_mode + profile = ( + entry.get("provider_profile") + or entry.get("runtime_provider") + or entry.get("provider") + ) + if isinstance(profile, str) and profile.strip(): + result["provider_profile"] = profile.strip() return result # Fall back to custom_providers: list (legacy format) @@ -539,6 +553,13 @@ def _get_named_custom_provider(requested_provider: str) -> Optional[Dict[str, An result["key_env"] = key_env if provider_key: result["provider_key"] = provider_key + profile = ( + entry.get("provider_profile") + or entry.get("runtime_provider") + or entry.get("provider") + ) + if isinstance(profile, str) and profile.strip(): + result["provider_profile"] = profile.strip() api_mode = _parse_api_mode(entry.get("api_mode")) if api_mode: result["api_mode"] = api_mode @@ -612,7 +633,13 @@ def _resolve_named_custom_runtime( return None # Check if a credential pool exists for this custom endpoint - pool_result = _try_resolve_from_custom_pool(base_url, "custom", custom_provider.get("api_mode"), provider_name=custom_provider.get("name")) + provider_profile = str(custom_provider.get("provider_profile") or "custom").strip() or "custom" + pool_result = _try_resolve_from_custom_pool( + base_url, + provider_profile, + custom_provider.get("api_mode"), + provider_name=custom_provider.get("name"), + ) if pool_result: # Propagate the model name even when using pooled credentials — # the pool doesn't know about the custom_providers model field. @@ -631,7 +658,7 @@ def _resolve_named_custom_runtime( api_key = next((candidate for candidate in api_key_candidates if has_usable_secret(candidate)), "") result = { - "provider": "custom", + "provider": provider_profile, "api_mode": custom_provider.get("api_mode") or _detect_api_mode_for_url(base_url) or "chat_completions", diff --git a/model_tools.py b/model_tools.py index f461afff5ba4b..ce57a0938a1d6 100644 --- a/model_tools.py +++ b/model_tools.py @@ -27,7 +27,7 @@ import logging import threading import time -from typing import Dict, Any, List, Optional, Tuple +from typing import Dict, Any, List, Optional, Set, Tuple from tools.registry import discover_builtin_tools, registry from toolsets import resolve_toolset, validate_toolset @@ -252,6 +252,7 @@ def _run_in_worker(): # inner check_fn TTL cache in registry.py handles environment drift (Docker # daemon start/stop, env var changes, etc.) on a 30 s horizon. _tool_defs_cache: Dict[tuple, List[Dict[str, Any]]] = {} +_VISIBLE_TOOLS_ENV_NAMES = ("HERMES_TUI_VISIBLE_TOOLS", "HERMES_VISIBLE_TOOLS") def _clear_tool_defs_cache() -> None: @@ -261,6 +262,24 @@ def _clear_tool_defs_cache() -> None: _tool_defs_cache.clear() +def _visible_tools_env_fingerprint() -> Tuple[Tuple[str, str], ...]: + return tuple((name, os.environ.get(name, "")) for name in _VISIBLE_TOOLS_ENV_NAMES) + + +def _get_visible_tool_filter() -> Tuple[Optional[Set[str]], Optional[str]]: + for env_name in _VISIBLE_TOOLS_ENV_NAMES: + raw = os.environ.get(env_name) + if not raw: + continue + names = { + part.strip() + for part in re.split(r"[\s,]+", raw) + if part.strip() + } + return names, env_name + return None, None + + def get_tool_definitions( enabled_toolsets: List[str] = None, disabled_toolsets: List[str] = None, @@ -301,6 +320,7 @@ def get_tool_definitions( registry._generation, cfg_fp, bool(os.environ.get("HERMES_KANBAN_TASK")), + _visible_tools_env_fingerprint(), ) cached = _tool_defs_cache.get(cache_key) if cached is not None: @@ -388,6 +408,17 @@ def _compute_tool_definitions( # needed; plugins respect enabled_toolsets / disabled_toolsets like any # other toolset. + visible_tools, visible_source = _get_visible_tool_filter() + if visible_tools is not None: + before_count = len(tools_to_include) + tools_to_include.intersection_update(visible_tools) + if not quiet_mode: + hidden_count = before_count - len(tools_to_include) + print( + f"🔎 Visible tool filter '{visible_source}': " + f"showing {len(tools_to_include)} tools, hiding {hidden_count}" + ) + # Ask the registry for schemas (only returns tools whose check_fn passes) filtered_tools = registry.get_definitions(tools_to_include, quiet=quiet_mode) diff --git a/tests/test_lazy_tool_loader.py b/tests/test_lazy_tool_loader.py new file mode 100644 index 0000000000000..18315c8594873 --- /dev/null +++ b/tests/test_lazy_tool_loader.py @@ -0,0 +1,149 @@ +import json + +import model_tools +from tools.lazy_tool_loader import TOOL_PACKS, ToolPack, load_tool_pack_for_agent +from tools.registry import registry + + +def _dummy_handler(args, **kwargs): + return "{}" + + +def _make_schema(name: str): + return { + "name": name, + "description": f"{name} test tool", + "parameters": {"type": "object", "properties": {}}, + } + + +def test_visible_tools_env_filters_schemas_without_deregistering(monkeypatch): + shown = "test_visible_tools_alpha" + hidden = "test_visible_tools_beta" + try: + registry.register( + name=shown, + toolset="test-visible-tools", + schema=_make_schema(shown), + handler=_dummy_handler, + ) + registry.register( + name=hidden, + toolset="test-visible-tools", + schema=_make_schema(hidden), + handler=_dummy_handler, + ) + model_tools._clear_tool_defs_cache() + monkeypatch.delenv("HERMES_TUI_VISIBLE_TOOLS", raising=False) + monkeypatch.setenv("HERMES_VISIBLE_TOOLS", shown) + + definitions = model_tools.get_tool_definitions( + enabled_toolsets=["test-visible-tools"], + quiet_mode=True, + ) + + assert [definition["function"]["name"] for definition in definitions] == [shown] + assert registry.get_entry(hidden) is not None + finally: + registry.deregister(shown) + registry.deregister(hidden) + model_tools._clear_tool_defs_cache() + + +def test_tui_visible_tools_env_takes_precedence(monkeypatch): + global_tool = "test_visible_tools_global" + tui_tool = "test_visible_tools_tui" + try: + registry.register( + name=global_tool, + toolset="test-visible-tools-precedence", + schema=_make_schema(global_tool), + handler=_dummy_handler, + ) + registry.register( + name=tui_tool, + toolset="test-visible-tools-precedence", + schema=_make_schema(tui_tool), + handler=_dummy_handler, + ) + model_tools._clear_tool_defs_cache() + monkeypatch.setenv("HERMES_VISIBLE_TOOLS", global_tool) + monkeypatch.setenv("HERMES_TUI_VISIBLE_TOOLS", tui_tool) + + definitions = model_tools.get_tool_definitions( + enabled_toolsets=["test-visible-tools-precedence"], + quiet_mode=True, + ) + + assert [definition["function"]["name"] for definition in definitions] == [tui_tool] + finally: + registry.deregister(global_tool) + registry.deregister(tui_tool) + model_tools._clear_tool_defs_cache() + + +def test_load_tool_pack_adds_registered_schemas_to_agent(): + tool_name = "mcp_botparlor_get_avatar_inventory" + + class Agent: + def __init__(self): + self.tools = [] + self.valid_tool_names = {"load_tool_pack"} + + try: + registry.register( + name=tool_name, + toolset="mcp-botparlor", + schema=_make_schema(tool_name), + handler=_dummy_handler, + ) + + agent = Agent() + result = json.loads(load_tool_pack_for_agent(agent, "avatar")) + + assert result["success"] is True + assert tool_name in result["loaded"] + assert tool_name in agent.valid_tool_names + assert any( + definition["function"]["name"] == tool_name + for definition in agent.tools + ) + assert "mcp_botparlor_create_avatar" in result["unavailable"] + assert result["suggested_skills"] == ["botparlor-avatar-maintenance"] + finally: + registry.deregister(tool_name) + + +def test_assistant_pack_returns_skill_hints_and_loads_available_tools(): + tool_name = "test_assistant_pack_tool" + pack_name = "test_assistant_pack" + + class Agent: + def __init__(self): + self.tools = [] + self.valid_tool_names = {"load_tool_pack"} + + try: + TOOL_PACKS[pack_name] = ToolPack( + description="Assistant pack test.", + tools=[tool_name], + suggested_skills=["codex", "test-driven-development"], + ) + registry.register( + name=tool_name, + toolset="test-assistant-pack", + schema=_make_schema(tool_name), + handler=_dummy_handler, + ) + + agent = Agent() + result = json.loads(load_tool_pack_for_agent(agent, pack_name)) + + assert result["success"] is True + assert tool_name in result["loaded"] + assert tool_name in agent.valid_tool_names + assert "codex" in result["suggested_skills"] + assert "test-driven-development" in result["suggested_skills"] + finally: + registry.deregister(tool_name) + TOOL_PACKS.pop(pack_name, None) diff --git a/tests/tools/test_mcp_tool.py b/tests/tools/test_mcp_tool.py index 3212a350c3744..4a099dcfd69d3 100644 --- a/tests/tools/test_mcp_tool.py +++ b/tests/tools/test_mcp_tool.py @@ -572,6 +572,43 @@ def _interrupting_run(coro_or_factory, timeout=30): class TestRunOnMCPLoopInterrupts: + def test_timeout_cancels_waiting_mcp_call(self): + import tools.mcp_tool as mcp_mod + + loop = asyncio.new_event_loop() + thread = threading.Thread(target=loop.run_forever, daemon=True) + thread.start() + + cancelled = threading.Event() + + async def _slow_call(): + try: + await asyncio.sleep(5) + return "done" + except asyncio.CancelledError: + cancelled.set() + raise + + old_loop = mcp_mod._mcp_loop + old_thread = mcp_mod._mcp_thread + mcp_mod._mcp_loop = loop + mcp_mod._mcp_thread = thread + + try: + with pytest.raises(TimeoutError, match="configured timeout: 0.2s"): + mcp_mod._run_on_mcp_loop(_slow_call(), timeout=0.2) + + deadline = time.time() + 2 + while time.time() < deadline and not cancelled.is_set(): + time.sleep(0.05) + assert cancelled.is_set() + finally: + loop.call_soon_threadsafe(loop.stop) + thread.join(timeout=2) + loop.close() + mcp_mod._mcp_loop = old_loop + mcp_mod._mcp_thread = old_thread + def test_interrupt_cancels_waiting_mcp_call(self): import tools.mcp_tool as mcp_mod from tools.interrupt import set_interrupt diff --git a/tests/tools/test_mcp_tool_session_expired.py b/tests/tools/test_mcp_tool_session_expired.py index 59601ba1c3d7c..f73452d532147 100644 --- a/tests/tools/test_mcp_tool_session_expired.py +++ b/tests/tools/test_mcp_tool_session_expired.py @@ -98,6 +98,19 @@ def test_is_session_expired_rejects_empty_message(): assert _is_session_expired_error(Exception()) is False +def test_is_mcp_timeout_error_detects_timeout_variants(): + """Configured tool-call timeouts should be treated as transport + health signals so stale sessions can reconnect.""" + import concurrent.futures + + from tools.mcp_tool import _is_mcp_timeout_error + + assert _is_mcp_timeout_error(TimeoutError("timed out")) is True + assert _is_mcp_timeout_error(concurrent.futures.TimeoutError()) is True + assert _is_mcp_timeout_error(InterruptedError("timed out")) is False + assert _is_mcp_timeout_error(RuntimeError("timed out")) is False + + # --------------------------------------------------------------------------- # Handler integration — verify the recovery plumbing wires end-to-end # --------------------------------------------------------------------------- @@ -189,6 +202,50 @@ async def _call_sequence(*a, **kw): mcp_tool._server_error_counts.pop("wpcom", None) +def test_call_tool_handler_reconnects_on_timeout(monkeypatch, tmp_path): + """If the synchronous MCP call times out, rebuild the transport and + retry once instead of leaving the server behind the circuit breaker.""" + monkeypatch.setenv("HERMES_HOME", str(tmp_path)) + + from tools import mcp_tool + from tools.mcp_tool import _make_tool_handler + + server, reconnect_flag = _install_stub_server("slow") + mcp_tool._servers["slow"] = server + mcp_tool._server_error_counts.pop("slow", None) + + call_count = {"n": 0} + + async def _call_sequence(*a, **kw): + call_count["n"] += 1 + if call_count["n"] == 1: + raise TimeoutError( + "MCP call timed out after 15.0s " + "(configured timeout: 15.0s)" + ) + result = MagicMock() + result.isError = False + result.content = [MagicMock(type="text", text="tool completed")] + result.structuredContent = None + return result + + server.session.call_tool = _call_sequence + + try: + handler = _make_tool_handler("slow", "tool", 10.0) + out = handler({"slug": "hello"}) + parsed = json.loads(out) + assert "error" not in parsed, parsed + assert reconnect_flag.is_set(), ( + "Handler did not trigger transport reconnect on timeout" + ) + assert call_count["n"] == 2 + assert mcp_tool._server_error_counts.get("slow", 0) == 0 + finally: + mcp_tool._servers.pop("slow", None) + mcp_tool._server_error_counts.pop("slow", None) + + def test_call_tool_handler_non_session_expired_error_falls_through( monkeypatch, tmp_path ): @@ -303,6 +360,38 @@ def _retry_raises(): mcp_tool._servers.pop("srv-retry-fail", None) +def test_timeout_handler_returns_none_when_retry_also_fails( + monkeypatch, tmp_path +): + """Timeout recovery retries once and then falls through cleanly if + the post-reconnect call also fails.""" + monkeypatch.setenv("HERMES_HOME", str(tmp_path)) + + from tools import mcp_tool + from tools.mcp_tool import _handle_timeout_and_retry + + server, reconnect_flag = _install_stub_server("srv-timeout-retry-fail") + mcp_tool._servers["srv-timeout-retry-fail"] = server + + def _retry_raises(): + raise RuntimeError("retry blew up too") + + try: + out = _handle_timeout_and_retry( + "srv-timeout-retry-fail", + TimeoutError("MCP call timed out after 15.0s"), + _retry_raises, + "tools/call", + ) + assert out is None + deadline = time.time() + 1 + while time.time() < deadline and not reconnect_flag.is_set(): + time.sleep(0.01) + assert reconnect_flag.is_set() + finally: + mcp_tool._servers.pop("srv-timeout-retry-fail", None) + + # --------------------------------------------------------------------------- # Parallel coverage for resources/list, resources/read, prompts/list, # prompts/get — all four handlers share the same exception path. diff --git a/tests/tui_gateway/test_make_agent_provider.py b/tests/tui_gateway/test_make_agent_provider.py index 896f68a3828a1..19d8696201002 100644 --- a/tests/tui_gateway/test_make_agent_provider.py +++ b/tests/tui_gateway/test_make_agent_provider.py @@ -137,9 +137,48 @@ def test_make_agent_honors_tui_launch_env_flags(): assert kwargs["checkpoints_enabled"] is True assert kwargs["pass_session_id"] is True assert kwargs["skip_context_files"] is True + assert kwargs["load_soul_identity"] is False assert kwargs["skip_memory"] is True +def test_make_agent_can_skip_context_files_without_skipping_soul_or_memory(): + fake_runtime = { + "provider": "openrouter", + "base_url": "https://api.synthetic.new/v1", + "api_key": "sk-test", + "api_mode": "chat_completions", + "command": None, + "args": None, + "credential_pool": None, + } + fake_cfg = {"agent": {"system_prompt": ""}, "model": {"default": "glm-5"}} + + with ( + patch.dict( + os.environ, + { + "HERMES_TUI_SKIP_CONTEXT_FILES": "1", + "HERMES_IGNORE_RULES": "", + }, + ), + patch("tui_gateway.server._load_cfg", return_value=fake_cfg), + patch("tui_gateway.server._get_db", return_value=MagicMock()), + patch( + "hermes_cli.runtime_provider.resolve_runtime_provider", + return_value=fake_runtime, + ), + patch("run_agent.AIAgent") as mock_agent, + ): + from tui_gateway.server import _make_agent + + _make_agent("sid-skip-context-only", "key-skip-context-only") + + kwargs = mock_agent.call_args.kwargs + assert kwargs["skip_context_files"] is True + assert kwargs["load_soul_identity"] is True + assert kwargs["skip_memory"] is False + + def test_probe_config_health_flags_null_sections(): """Bare YAML keys (`agent:` with no value) parse as None and silently drop nested settings; probe must surface them so users can fix.""" diff --git a/tools/lazy_tool_loader.py b/tools/lazy_tool_loader.py new file mode 100644 index 0000000000000..a59b130f486b9 --- /dev/null +++ b/tools/lazy_tool_loader.py @@ -0,0 +1,325 @@ +"""Session-local lazy tool loading. + +Tools remain registered in the global registry, but this helper can add +selected schemas to a running agent after session start. +""" + +import json +from dataclasses import dataclass, field +from typing import Dict, Iterable, List, Set + +from tools.registry import registry + + +TOOL_NAME = "load_tool_pack" + +@dataclass(frozen=True) +class ToolPack: + description: str + tools: List[str] + suggested_skills: List[str] = field(default_factory=list) + + +SKILL_READ_TOOLS = ["skills_list", "skill_view"] +SKILL_EDIT_TOOLS = ["skills_list", "skill_view", "skill_manage"] +BROWSER_TOOLS = [ + "browser_navigate", + "browser_snapshot", + "browser_click", + "browser_type", + "browser_scroll", + "browser_back", + "browser_press", + "browser_get_images", + "browser_vision", + "browser_console", + "browser_cdp", + "browser_dialog", +] +FILE_TOOLS = ["read_file", "write_file", "patch", "search_files"] +TERMINAL_TOOLS = ["terminal", "process"] +HA_TOOLS = ["ha_list_entities", "ha_get_state", "ha_list_services", "ha_call_service"] + + +TOOL_PACKS: Dict[str, ToolPack] = { + "avatar": ToolPack( + description="BotParlor avatar inventory and avatar-generation tools.", + tools=[ + "mcp_botparlor_get_avatar_inventory", + "mcp_botparlor_get_outfits", + "mcp_botparlor_create_avatar", + "mcp_botparlor_get_avatar_job", + "mcp_botparlor_get_moods", + ], + suggested_skills=["botparlor-avatar-maintenance"], + ), + "reminders": ToolPack( + description="BotParlor reminder and scheduled-message tools.", + tools=[ + "mcp_botparlor_create_reminder", + "mcp_botparlor_list_reminders", + "mcp_botparlor_update_reminder", + "mcp_botparlor_cancel_reminder", + ], + ), + "media": ToolPack( + description="BotParlor media display controls.", + tools=[ + "mcp_botparlor_display_media", + "mcp_botparlor_close_media", + ], + ), + "botparlor_resources": ToolPack( + description="BotParlor MCP resources and prompt templates.", + tools=[ + "mcp_botparlor_list_resources", + "mcp_botparlor_read_resource", + "mcp_botparlor_list_prompts", + "mcp_botparlor_get_prompt", + ], + suggested_skills=["botparlor"], + ), + "recall": ToolPack( + description="Past-session search and recall.", + tools=["session_search"], + ), + "skills": ToolPack( + description="Generic skill listing, viewing, and management.", + tools=SKILL_EDIT_TOOLS, + ), + "coding": ToolPack( + description="Coding, repository work, debugging, tests, GitHub workflows, and subagents.", + tools=[ + *SKILL_EDIT_TOOLS, + *TERMINAL_TOOLS, + *FILE_TOOLS, + "todo", + "execute_code", + "delegate_task", + ], + suggested_skills=[ + "claude-code", + "codex", + "github-auth", + "github-code-review", + "github-issues", + "github-pr-workflow", + "github-repo-management", + "plan", + "python-debugpy", + "subagent-driven-development", + "systematic-debugging", + "test-driven-development", + "writing-plans", + ], + ), + "web_research": ToolPack( + description="Web search, extraction, browser automation, social/video/PDF research.", + tools=[ + *SKILL_READ_TOOLS, + "web_search", + "web_extract", + "x_search", + *BROWSER_TOOLS, + "vision_analyze", + ], + suggested_skills=[ + "web-browsing", + "youtube-content", + "x-twitter", + "nano-pdf", + ], + ), + "local_ops": ToolPack( + description="Local machine, network, maps, and Home Assistant operations.", + tools=[ + *SKILL_READ_TOOLS, + *TERMINAL_TOOLS, + *HA_TOOLS, + ], + suggested_skills=[ + "find-nearby", + "maps", + "network-device-discovery", + "homeassistant", + "godmode", + ], + ), + "hermes_admin": ToolPack( + description="Hermes internals, MCP work, skill authoring, and TUI debugging.", + tools=[ + *SKILL_EDIT_TOOLS, + *TERMINAL_TOOLS, + *FILE_TOOLS, + "execute_code", + "delegate_task", + ], + suggested_skills=[ + "hermes-agent", + "hermes-agent-communication", + "debugging-hermes-tui-commands", + "hermes-agent-skill-authoring", + "mcporter", + "native-mcp", + ], + ), + "notes": ToolPack( + description="Note taking, Obsidian, and local knowledge capture.", + tools=[ + *SKILL_READ_TOOLS, + *FILE_TOOLS, + "memory", + ], + suggested_skills=["obsidian"], + ), + "design": ToolPack( + description="Web design references and implementation support.", + tools=[ + *SKILL_READ_TOOLS, + *FILE_TOOLS, + "web_search", + "web_extract", + "image_generate", + ], + suggested_skills=["popular-web-designs"], + ), + "persona": ToolPack( + description="Bot personality, dogfood, and security-rule skills.", + tools=SKILL_READ_TOOLS, + suggested_skills=[ + "dogfood", + "viper-personality", + "7-day-security-rule", + ], + ), + "power": ToolPack( + description="Broad legacy pack for loading most general Hermes tools.", + tools=[ + "web_search", + "web_extract", + *TERMINAL_TOOLS, + *FILE_TOOLS, + *BROWSER_TOOLS, + "vision_analyze", + "image_generate", + "text_to_speech", + "todo", + "execute_code", + "delegate_task", + ], + ), +} + + +def _unique_tool_names(names: Iterable[str]) -> List[str]: + seen: Set[str] = set() + result: List[str] = [] + for name in names: + clean = str(name or "").strip() + if clean and clean not in seen: + seen.add(clean) + result.append(clean) + return result + + +def _current_agent_tool_names(agent) -> Set[str]: + current = set(getattr(agent, "valid_tool_names", set()) or set()) + for tool_def in getattr(agent, "tools", []) or []: + name = tool_def.get("function", {}).get("name") + if name: + current.add(name) + return current + + +def load_tool_pack_for_agent(agent, pack: str) -> str: + """Add schemas from *pack* to a running agent and return a JSON result.""" + pack_name = str(pack or "").strip() + if pack_name not in TOOL_PACKS: + return json.dumps( + { + "success": False, + "error": f"Unknown tool pack: {pack_name}", + "available_packs": sorted(TOOL_PACKS), + } + ) + + tool_pack = TOOL_PACKS[pack_name] + requested = _unique_tool_names(tool_pack.tools) + current_names = _current_agent_tool_names(agent) + definitions = registry.get_definitions(set(requested), quiet=True) + definitions_by_name = { + definition["function"]["name"]: definition + for definition in definitions + if definition.get("function", {}).get("name") + } + + existing_tools = list(getattr(agent, "tools", []) or []) + loaded: List[str] = [] + already_available: List[str] = [] + unavailable: List[str] = [] + + for name in requested: + definition = definitions_by_name.get(name) + if definition is None: + unavailable.append(name) + continue + if name in current_names: + already_available.append(name) + continue + existing_tools.append(definition) + current_names.add(name) + loaded.append(name) + + agent.tools = existing_tools + agent.valid_tool_names = current_names + + return json.dumps( + { + "success": True, + "pack": pack_name, + "description": tool_pack.description, + "loaded": loaded, + "already_available": already_available, + "unavailable": unavailable, + "suggested_skills": tool_pack.suggested_skills, + "message": "Loaded tools are available on the next model call.", + } + ) + + +def _load_tool_pack_handler(args, **kwargs): + return json.dumps( + { + "success": False, + "error": "load_tool_pack requires an active agent session.", + } + ) + + +registry.register( + name=TOOL_NAME, + toolset="tool_loader", + schema={ + "name": TOOL_NAME, + "description": ( + "Load optional tools into this session by pack name. The result may " + "include suggested_skills that can be loaded with skill_view." + ), + "parameters": { + "type": "object", + "properties": { + "pack": { + "type": "string", + "enum": sorted(TOOL_PACKS), + "description": "Tool pack to load.", + } + }, + "required": ["pack"], + }, + }, + handler=_load_tool_pack_handler, + description=( + "Load optional tools into this session by pack name. The result may " + "include suggested_skills that can be loaded with skill_view." + ), +) diff --git a/tools/mcp_tool.py b/tools/mcp_tool.py index e50efc05a0c28..e49f8afdb64b6 100644 --- a/tools/mcp_tool.py +++ b/tools/mcp_tool.py @@ -1980,21 +1980,37 @@ def _is_session_expired_error(exc: BaseException) -> bool: return any(marker in msg for marker in _SESSION_EXPIRED_MARKERS) -def _handle_session_expired_and_retry( +def _is_mcp_timeout_error(exc: BaseException) -> bool: + """Return True if ``exc`` is an MCP call timeout. + + ``_run_on_mcp_loop`` raises ``TimeoutError`` when a synchronous tool + caller waits longer than the server's configured timeout. Treating + that as a transport-health signal lets the long-lived MCP task rebuild + stale HTTP/SSE/stdio sessions instead of leaving the server in a + circuit-breaker state until Hermes is restarted. + """ + if isinstance(exc, InterruptedError): + return False + return isinstance( + exc, + (TimeoutError, concurrent.futures.TimeoutError, asyncio.TimeoutError), + ) + + +def _handle_transport_reconnect_and_retry( server_name: str, exc: BaseException, retry_call, op_description: str, + reason: str, ): - """Trigger a transport reconnect and retry once on session expiry. + """Trigger a transport reconnect and retry an MCP operation once. - Unlike :func:`_handle_auth_error_and_retry`, this does **not** call - the OAuth manager's ``handle_401`` — the access token is still - valid, only the server-side session state is stale. Setting - ``_reconnect_event`` causes the server task's lifecycle loop to - tear down the current ``streamablehttp_client`` + ``ClientSession`` - and rebuild them, reusing the existing OAuth provider instance. - See #13383. + Used for transport-layer failures where OAuth is not involved: + server-side session expiry, closed pipes, and call timeouts. Setting + ``_reconnect_event`` causes the server task's lifecycle loop to tear + down the current transport + ``ClientSession`` and rebuild them, + reusing the existing OAuth provider instance when present. Args: server_name: Name of the MCP server that raised. @@ -2002,16 +2018,14 @@ def _handle_session_expired_and_retry( retry_call: Zero-arg callable that re-runs the operation, returning the same JSON string format as the handler. op_description: Human-readable name of the operation (logs). + reason: Short human-readable failure class for logs. Returns: A JSON string if reconnect + retry was attempted and produced a response, or ``None`` to fall through to the caller's - generic error path (not a session-expired error, no server - record, reconnect didn't ready in time, or retry also failed). + generic error path (no server record, reconnect didn't ready in + time, or retry also failed). """ - if not _is_session_expired_error(exc): - return None - with _lock: srv = _servers.get(server_name) if srv is None or not hasattr(srv, "_reconnect_event"): @@ -2022,9 +2036,9 @@ def _handle_session_expired_and_retry( return None logger.info( - "MCP server '%s': %s failed with session-expired error (%s); " + "MCP server '%s': %s failed with %s (%s); " "signalling transport reconnect and retrying once.", - server_name, op_description, exc, + server_name, op_description, reason, exc, ) # Trigger the same reconnect mechanism the OAuth recovery path @@ -2040,8 +2054,8 @@ def _handle_session_expired_and_retry( if not ready: logger.warning( "MCP server '%s': reconnect did not ready within 15s after " - "session-expired error; falling through to error response.", - server_name, + "%s; falling through to error response.", + server_name, reason, ) return None @@ -2050,10 +2064,10 @@ def _handle_session_expired_and_retry( try: parsed = json.loads(result) if "error" not in parsed: - _server_error_counts[server_name] = 0 + _reset_server_error(server_name) return result except (json.JSONDecodeError, TypeError): - _server_error_counts[server_name] = 0 + _reset_server_error(server_name) return result except Exception as retry_exc: logger.warning( @@ -2063,6 +2077,43 @@ def _handle_session_expired_and_retry( return None +def _handle_session_expired_and_retry( + server_name: str, + exc: BaseException, + retry_call, + op_description: str, +): + """Trigger a transport reconnect and retry once on session expiry. + + Unlike :func:`_handle_auth_error_and_retry`, this does **not** call + the OAuth manager's ``handle_401`` — the access token is still + valid, only the server-side session state is stale. Setting + ``_reconnect_event`` causes the server task's lifecycle loop to + tear down the current ``streamablehttp_client`` + ``ClientSession`` + and rebuild them, reusing the existing OAuth provider instance. + See #13383. + """ + if not _is_session_expired_error(exc): + return None + return _handle_transport_reconnect_and_retry( + server_name, exc, retry_call, op_description, "session-expired error" + ) + + +def _handle_timeout_and_retry( + server_name: str, + exc: BaseException, + retry_call, + op_description: str, +): + """Trigger a transport reconnect and retry once on MCP call timeout.""" + if not _is_mcp_timeout_error(exc): + return None + return _handle_transport_reconnect_and_retry( + server_name, exc, retry_call, op_description, "call timeout" + ) + + # Sanitized server names whose ``supports_parallel_tool_calls`` config is True. # Populated during ``register_mcp_servers()`` and queried by # ``is_mcp_tool_parallel_safe()`` for the parallel-execution check in run_agent. @@ -2414,6 +2465,12 @@ def _call_once(): server_name, exc, _call_once, f"tools/call {tool_name}", ) + if recovered is not None: + return recovered + recovered = _handle_timeout_and_retry( + server_name, exc, _call_once, + f"tools/call {tool_name}", + ) if recovered is not None: return recovered @@ -2475,6 +2532,11 @@ def _call_once(): recovered = _handle_session_expired_and_retry( server_name, exc, _call_once, "resources/list", ) + if recovered is not None: + return recovered + recovered = _handle_timeout_and_retry( + server_name, exc, _call_once, "resources/list", + ) if recovered is not None: return recovered logger.error( @@ -2535,6 +2597,11 @@ def _call_once(): recovered = _handle_session_expired_and_retry( server_name, exc, _call_once, "resources/read", ) + if recovered is not None: + return recovered + recovered = _handle_timeout_and_retry( + server_name, exc, _call_once, "resources/read", + ) if recovered is not None: return recovered logger.error( @@ -2598,6 +2665,11 @@ def _call_once(): recovered = _handle_session_expired_and_retry( server_name, exc, _call_once, "prompts/list", ) + if recovered is not None: + return recovered + recovered = _handle_timeout_and_retry( + server_name, exc, _call_once, "prompts/list", + ) if recovered is not None: return recovered logger.error( @@ -2669,6 +2741,11 @@ def _call_once(): recovered = _handle_session_expired_and_retry( server_name, exc, _call_once, "prompts/get", ) + if recovered is not None: + return recovered + recovered = _handle_timeout_and_retry( + server_name, exc, _call_once, "prompts/get", + ) if recovered is not None: return recovered logger.error( diff --git a/toolsets.py b/toolsets.py index 5de07e4c7a185..5141459c99628 100644 --- a/toolsets.py +++ b/toolsets.py @@ -211,6 +211,12 @@ "tools": ["session_search"], "includes": [] }, + + "tool_loader": { + "description": "Load optional tool packs during a running session", + "tools": ["load_tool_pack"], + "includes": [] + }, "clarify": { "description": "Ask the user clarifying questions (multiple-choice or open-ended)", diff --git a/tui_gateway/server.py b/tui_gateway/server.py index 7b9f286380cbf..fd0ca1b8a661a 100644 --- a/tui_gateway/server.py +++ b/tui_gateway/server.py @@ -1903,6 +1903,10 @@ def _make_agent(sid: str, key: str, session_id: str | None = None): requested=requested_provider, target_model=model or None, ) + ignore_rules = is_truthy_value(os.environ.get("HERMES_IGNORE_RULES")) + skip_context_files = ignore_rules or is_truthy_value( + os.environ.get("HERMES_TUI_SKIP_CONTEXT_FILES") + ) or is_truthy_value(os.environ.get("HERMES_SKIP_CONTEXT_FILES")) return AIAgent( model=model, max_iterations=_cfg_max_turns(cfg, 90), @@ -1924,8 +1928,9 @@ def _make_agent(sid: str, key: str, session_id: str | None = None): ephemeral_system_prompt=system_prompt or None, checkpoints_enabled=is_truthy_value(os.environ.get("HERMES_TUI_CHECKPOINTS")), pass_session_id=is_truthy_value(os.environ.get("HERMES_TUI_PASS_SESSION_ID")), - skip_context_files=is_truthy_value(os.environ.get("HERMES_IGNORE_RULES")), - skip_memory=is_truthy_value(os.environ.get("HERMES_IGNORE_RULES")), + skip_context_files=skip_context_files, + load_soul_identity=skip_context_files and not ignore_rules, + skip_memory=ignore_rules, **_agent_cbs(sid), )