From 41b9b893253fb7af00e1bf7cc0cb579b5a747964 Mon Sep 17 00:00:00 2001 From: binhnt92 Date: Mon, 6 Apr 2026 18:46:50 +0700 Subject: [PATCH 1/2] fix(gateway): release platform locks on Slack/Signal connect failure When Slack's connect() acquires a scoped lock on the app token and then fails (bad token, Socket Mode error), the lock is never released. The next gateway start sees "Slack app token already in use" and refuses to connect until the process dies. Same issue in Signal: if the health check fails after acquiring the phone lock, the lock is held permanently. Discord got this fix in PR #5302. Slack and Signal were missed. Add lock release to both exception/failure paths. Extract Signal's inline release logic into a reusable _release_phone_lock() helper. Also close the leaked httpx client on Signal health check failure. --- gateway/platforms/signal.py | 24 ++++-- gateway/platforms/slack.py | 8 ++ tests/gateway/test_platform_lock_release.py | 84 +++++++++++++++++++++ 3 files changed, 109 insertions(+), 7 deletions(-) create mode 100644 tests/gateway/test_platform_lock_release.py diff --git a/gateway/platforms/signal.py b/gateway/platforms/signal.py index 1629e08631e59..cc2b876736c04 100644 --- a/gateway/platforms/signal.py +++ b/gateway/platforms/signal.py @@ -230,9 +230,15 @@ async def connect(self) -> bool: resp = await self.client.get(f"{self.http_url}/api/v1/check", timeout=10.0) if resp.status_code != 200: logger.error("Signal: health check failed (status %d)", resp.status_code) + await self.client.aclose() + self.client = None + self._release_phone_lock() return False except Exception as e: logger.error("Signal: cannot reach signal-cli at %s: %s", self.http_url, e) + await self.client.aclose() + self.client = None + self._release_phone_lock() return False self._running = True @@ -243,6 +249,16 @@ async def connect(self) -> bool: logger.info("Signal: connected to %s", self.http_url) return True + def _release_phone_lock(self) -> None: + """Release the Signal phone scoped lock if held.""" + if self._phone_lock_identity: + try: + from gateway.status import release_scoped_lock + release_scoped_lock("signal-phone", self._phone_lock_identity) + except Exception as e: + logger.warning("Signal: Error releasing phone lock: %s", e, exc_info=True) + self._phone_lock_identity = None + async def disconnect(self) -> None: """Stop SSE listener and clean up.""" self._running = False @@ -270,13 +286,7 @@ async def disconnect(self) -> None: await self.client.aclose() self.client = None - if self._phone_lock_identity: - try: - from gateway.status import release_scoped_lock - release_scoped_lock("signal-phone", self._phone_lock_identity) - except Exception as e: - logger.warning("Signal: Error releasing phone lock: %s", e, exc_info=True) - self._phone_lock_identity = None + self._release_phone_lock() logger.info("Signal: disconnected") diff --git a/gateway/platforms/slack.py b/gateway/platforms/slack.py index 2e7bbee739ba4..35a46c4fa0519 100644 --- a/gateway/platforms/slack.py +++ b/gateway/platforms/slack.py @@ -189,6 +189,14 @@ async def handle_hermes_command(ack, command): except Exception as e: # pragma: no cover - defensive logging logger.error("[Slack] Connection failed: %s", e, exc_info=True) + # Release the platform lock so the next gateway start isn't blocked. + if self._token_lock_identity: + try: + from gateway.status import release_scoped_lock + release_scoped_lock('slack-app-token', self._token_lock_identity) + except Exception: + pass + self._token_lock_identity = None return False async def disconnect(self) -> None: diff --git a/tests/gateway/test_platform_lock_release.py b/tests/gateway/test_platform_lock_release.py new file mode 100644 index 0000000000000..90f561188aa1d --- /dev/null +++ b/tests/gateway/test_platform_lock_release.py @@ -0,0 +1,84 @@ +"""Tests for platform lock release on connect() failure. + +When a platform adapter acquires a scoped lock during connect() and then +fails (bad token, network error), the lock must be released so the next +gateway start isn't blocked with "already in use" errors. + +Discord got this fix in PR #5302. Slack and Signal were missed. +""" + +import asyncio +from unittest.mock import AsyncMock, MagicMock, patch + +import pytest + + +# --------------------------------------------------------------------------- +# Slack +# --------------------------------------------------------------------------- + +class TestSlackLockReleaseOnConnectFailure: + """gateway/platforms/slack.py — connect() must release lock on exception.""" + + @staticmethod + def _read_source() -> str: + import os + base = os.path.dirname(os.path.dirname(os.path.dirname(__file__))) + with open(os.path.join(base, "gateway", "platforms", "slack.py")) as f: + return f.read() + + def test_except_block_releases_lock(self): + """The except block in connect() should call release_scoped_lock.""" + src = self._read_source() + # Find the except block near the end of connect() + start = src.index("async def connect(") + end = src.index("\n async def disconnect", start) + connect_body = src[start:end] + + assert "release_scoped_lock" in connect_body, ( + "Slack connect() except block does not release the scoped lock — " + "next gateway start will be blocked" + ) + + +# --------------------------------------------------------------------------- +# Signal +# --------------------------------------------------------------------------- + +class TestSignalLockReleaseOnConnectFailure: + """gateway/platforms/signal.py — connect() must release lock on health check failure.""" + + @staticmethod + def _read_source() -> str: + import os + base = os.path.dirname(os.path.dirname(os.path.dirname(__file__))) + with open(os.path.join(base, "gateway", "platforms", "signal.py")) as f: + return f.read() + + def test_health_check_failure_releases_lock(self): + """Both return-False paths after health check must release the phone lock.""" + src = self._read_source() + start = src.index("# Health check") + end = src.index("self._running = True", start) + health_block = src[start:end] + + assert health_block.count("_release_phone_lock") >= 2, ( + "Signal health check failure paths do not release the phone lock — " + "both the non-200 and exception paths need _release_phone_lock()" + ) + + def test_release_phone_lock_method_exists(self): + """_release_phone_lock helper should exist for reuse.""" + src = self._read_source() + assert "def _release_phone_lock(self)" in src + + def test_health_check_failure_closes_client(self): + """Both return-False paths must also close the httpx client.""" + src = self._read_source() + start = src.index("# Health check") + end = src.index("self._running = True", start) + health_block = src[start:end] + + assert health_block.count("await self.client.aclose()") >= 2, ( + "Signal health check failure paths do not close the httpx client" + ) From e29819331d58cc3e69d2361296fb4a0dd1e72268 Mon Sep 17 00:00:00 2001 From: binhnt92 Date: Tue, 7 Apr 2026 20:24:18 +0700 Subject: [PATCH 2/2] docs(guides): add guide for running Hermes locally with Ollama MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Step-by-step guide covering Ollama installation, model selection, Hermes configuration, speed optimization, and optional gateway bot setup — all running on local hardware with zero API cost. Includes hardware requirements, model comparison table with tool-call support status, context window tuning, GPU offloading tips, fallback provider setup, troubleshooting, and cost comparison. --- website/docs/guides/local-ollama-setup.md | 317 ++++++++++++++++++++++ 1 file changed, 317 insertions(+) create mode 100644 website/docs/guides/local-ollama-setup.md diff --git a/website/docs/guides/local-ollama-setup.md b/website/docs/guides/local-ollama-setup.md new file mode 100644 index 0000000000000..ae0cc445a82df --- /dev/null +++ b/website/docs/guides/local-ollama-setup.md @@ -0,0 +1,317 @@ +--- +sidebar_position: 9 +title: "Run Hermes Locally with Ollama — Zero API Cost" +description: "Step-by-step guide to running Hermes Agent entirely on your own machine with Ollama and open-weight models like Gemma 4, no cloud API keys or paid subscriptions needed" +--- + +# Run Hermes Locally with Ollama — Zero API Cost + +## The Problem + +Cloud LLM APIs charge per token. A heavy coding session can cost $5–20. For personal projects, learning, or privacy-sensitive work, that adds up — and you're sending every conversation to a third party. + +## What This Guide Solves + +You'll set up Hermes Agent running entirely on your own hardware, using [Ollama](https://ollama.com) as the model backend. No API keys, no subscriptions, no data leaving your machine. Once configured, Hermes works exactly like it does with OpenRouter or Anthropic — terminal commands, file editing, web browsing, delegation — but the model runs locally. + +By the end, you'll have: + +- Ollama serving one or more open-weight models +- Hermes connected to Ollama as a custom endpoint +- A working local agent that can edit files, run commands, and browse the web +- Optional: a Telegram/Discord bot powered entirely by your own hardware + +## What You Need + +| Component | Minimum | Recommended | +|-----------|---------|-------------| +| **RAM** | 8 GB (for 3B models) | 32+ GB (for 27B+ models) | +| **Storage** | 5 GB free | 30+ GB (for multiple models) | +| **CPU** | 4 cores | 8+ cores (AMD EPYC, Ryzen, Intel Xeon) | +| **GPU** | Not required | NVIDIA GPU with 8+ GB VRAM speeds things up significantly | + +:::tip CPU-only works, but expect slower responses +Ollama runs on CPU-only servers. A 9B model on a modern 8-core CPU gives ~10 tokens/sec. A 31B model on CPU is slower (~2–5 tokens/sec) — each response takes 30–120 seconds, but it works. A GPU dramatically improves this. For CPU-only setups, increase the API timeout in config: + +```yaml +agent: + api_timeout: 1800 # 30 minutes — generous for slow local models +``` +::: + +## Step 1: Install Ollama + +```bash +curl -fsSL https://ollama.com/install.sh | sh +``` + +Verify it's running: + +```bash +ollama --version +curl http://localhost:11434/api/tags # Should return {"models":[]} +``` + +## Step 2: Pull a Model + +Choose based on your hardware: + +| Model | Size on Disk | RAM Needed | Tool Calling | Best For | +|-------|-------------|------------|:------------:|----------| +| `gemma4:31b` | ~20 GB | 24+ GB | Yes | Best quality — strong tool use and reasoning | +| `gemma2:27b` | ~16 GB | 20+ GB | No | Conversational tasks, no tool use | +| `gemma2:9b` | ~5 GB | 8+ GB | No | Fast chat, Q&A — cannot call tools | +| `llama3.2:3b` | ~2 GB | 4+ GB | No | Lightweight quick answers only | + +:::warning Tool calling matters +Hermes is an **agentic** assistant — it edits files, runs commands, and browses the web through tool calls. Models without tool-call support can only chat; they can't take actions. For the full Hermes experience, use a model that supports tools (like `gemma4:31b`). +::: + +Pull your chosen model: + +```bash +ollama pull gemma4:31b +``` + +:::info Multiple models +You can pull several models and switch between them inside Hermes with `/model`. Ollama loads the active model into memory on demand and unloads idle ones automatically. +::: + +Verify the model works: + +```bash +curl http://localhost:11434/v1/chat/completions \ + -H "Content-Type: application/json" \ + -d '{ + "model": "gemma4:31b", + "messages": [{"role": "user", "content": "Say hello"}], + "max_tokens": 50 + }' +``` + +You should see a JSON response with the model's reply. + +## Step 3: Configure Hermes + +Run the Hermes setup wizard: + +```bash +hermes setup +``` + +When prompted for a provider, select **Custom Endpoint** and enter: + +- **Base URL:** `http://localhost:11434/v1` +- **API Key:** Leave empty or type `no-key` (Ollama doesn't need one) +- **Model:** `gemma4:31b` (or whichever model you pulled) + +Alternatively, edit `~/.hermes/config.yaml` directly: + +```yaml +model: + default: "gemma4:31b" + provider: "custom" + base_url: "http://localhost:11434/v1" +``` + +## Step 4: Start Using Hermes + +```bash +hermes +``` + +That's it. You're now running a fully local agent. Try it out: + +``` +You: List all Python files in this directory and count the lines of code in each + +You: Read the README.md and summarize what this project does + +You: Create a Python script that fetches the weather for Ho Chi Minh City +``` + +Hermes will use the terminal tool, file operations, and your local model — no cloud calls. + +## Step 5: Pick the Right Model for Your Task + +Not every task needs the biggest model. Here's a practical guide: + +| Task | Recommended Model | Why | +|------|-------------------|-----| +| File edits, code, terminal commands | `gemma4:31b` | Only model with reliable tool calling | +| Quick Q&A (no tool use needed) | `gemma2:9b` | Fast responses for conversational tasks | +| Lightweight chat | `llama3.2:3b` | Fastest, but very limited capabilities | + +:::note +For full agentic work (editing files, running commands, browsing), `gemma4:31b` is currently the best local option with tool-call support. Check [Ollama's model library](https://ollama.com/library) for newer models — tool-calling support is expanding rapidly. +::: + +Switch models on the fly inside a session: + +``` +/model gemma2:9b +``` + +## Step 6: Optimize for Speed + +### Increase Ollama's Context Window + +By default, Ollama uses a 2048-token context. For agentic work (tool calls, long conversations), you need more: + +```bash +# Create a Modelfile that extends context +cat > /tmp/Modelfile << 'EOF' +FROM gemma4:31b +PARAMETER num_ctx 16384 +EOF + +ollama create gemma4-16k -f /tmp/Modelfile +``` + +Then update your Hermes config to use `gemma4-16k` as the model name. + +### Keep the Model Loaded + +By default, Ollama unloads models after 5 minutes of inactivity. For a persistent gateway bot, keep it loaded: + +```bash +# Set keep-alive to 24 hours +curl http://localhost:11434/api/generate \ + -d '{"model": "gemma4:31b", "keep_alive": "24h"}' +``` + +Or set it globally in Ollama's environment: + +```bash +# /etc/systemd/system/ollama.service.d/override.conf +[Service] +Environment="OLLAMA_KEEP_ALIVE=24h" +``` + +### Use GPU Offloading (If Available) + +If you have an NVIDIA GPU, Ollama automatically offloads layers to it. Check with: + +```bash +ollama ps # Shows which model is loaded and how many GPU layers +``` + +For a 31B model on a 12 GB GPU, you'll get partial offload (~40 layers on GPU, rest on CPU), which still gives a significant speedup. + +## Step 7: Run as a Gateway Bot (Optional) + +Once Hermes works locally in the CLI, you can expose it as a Telegram or Discord bot — still running entirely on your hardware. + +### Telegram + +1. Create a bot via [@BotFather](https://t.me/BotFather) and get the token +2. Add to your `~/.hermes/config.yaml`: + +```yaml +model: + default: "gemma4:31b" + provider: "custom" + base_url: "http://localhost:11434/v1" + +platforms: + telegram: + enabled: true + token: "YOUR_TELEGRAM_BOT_TOKEN" +``` + +3. Start the gateway: + +```bash +hermes gateway +``` + +Now message your bot on Telegram — it responds using your local model. + +### Discord + +1. Create a Discord application at [discord.com/developers](https://discord.com/developers/applications) +2. Add to config: + +```yaml +platforms: + discord: + enabled: true + token: "YOUR_DISCORD_BOT_TOKEN" +``` + +3. Start: `hermes gateway` + +## Step 8: Set Up Fallbacks (Optional) + +Local models can struggle with complex tasks. Set up a cloud fallback that only activates when the local model fails: + +```yaml +model: + default: "gemma4:31b" + provider: "custom" + base_url: "http://localhost:11434/v1" + +fallback_providers: + - provider: openrouter + model: anthropic/claude-sonnet-4 +``` + +This way, 90% of your usage is free (local), and only the hard tasks hit the paid API. + +## Troubleshooting + +### "Connection refused" on startup + +Ollama isn't running. Start it: + +```bash +sudo systemctl start ollama +# or +ollama serve +``` + +### Slow responses + +- **Check model size vs RAM:** If your model needs more RAM than available, it swaps to disk. Use a smaller model or add RAM. +- **Check `ollama ps`:** If no GPU layers are offloaded, responses are CPU-bound. This is normal for CPU-only servers. +- **Reduce context:** Large conversations slow down inference. Use `/compress` regularly, or set a lower compression threshold in config. + +### Model doesn't follow tool calls + +Smaller models (3B, 7B) sometimes ignore tool-call instructions and produce plain text instead of structured function calls. Solutions: + +- **Use a bigger model** — `gemma4:31b` or `gemma2:27b` handle tool calls much better than 3B/7B models. +- **Hermes has auto-repair** — it detects malformed tool calls and attempts to fix them automatically. +- **Set up a fallback** — if the local model fails 3 times, Hermes falls back to a cloud provider. + +### Context window errors + +The default Ollama context (2048 tokens) is too small for agentic work. See [Step 6](#step-6-optimize-for-speed) to increase it. + +## Cost Comparison + +Here's what running locally saves compared to cloud APIs, based on a typical coding session (~100K tokens input, ~20K tokens output): + +| Provider | Cost per Session | Monthly (daily use) | +|----------|-----------------|---------------------| +| Anthropic Claude Sonnet | ~$0.80 | ~$24 | +| OpenRouter (GPT-4o) | ~$0.60 | ~$18 | +| **Ollama (local)** | **$0.00** | **$0.00** | + +Your only cost is electricity — roughly $0.01–0.05 per session depending on hardware. + +## What Works Well Locally + +- **File editing and code generation** — models 9B+ handle this well +- **Terminal commands** — Hermes wraps the command, runs it, reads output regardless of model +- **Web browsing** — the browser tool does the fetching; the model just interprets results +- **Cron jobs and scheduled tasks** — work identically to cloud setups +- **Multi-platform gateway** — Telegram, Discord, Slack all work with local models + +## What's Better with Cloud Models + +- **Very complex multi-step reasoning** — 70B+ or cloud models like Claude Opus are noticeably better +- **Long context windows** — cloud models offer 100K–1M tokens; local models are typically 8K–32K +- **Speed on large responses** — cloud inference is faster than CPU-only local for long generations + +The sweet spot: use local for everyday tasks, set up a cloud fallback for the hard stuff.