STT v3 — route hermes-agent to whisper.service HTTP + cloud OpenAI primary

tools/transcription_tools.py

@@ -26,12 +26,15 @@
         print(result["transcript"])
 """
 
+import json
 import logging
 import os
 import shlex
 import shutil
 import subprocess
 import tempfile
+import urllib.error
+import urllib.request
 from pathlib import Path
 from typing import Optional, Dict, Any
 from urllib.parse import urljoin
@@ -373,8 +376,76 @@ def _load_local_whisper_model(model_name: str):
         return WhisperModel(model_name, device="cpu", compute_type="int8")
 
 
+def _transcribe_via_whisper_service(file_path: str, endpoint: str, language: str = "") -> Dict[str, Any]:
+    """POST audio file to whisper.cpp HTTP server (whisper.service on the VPS).
+
+    The whisper.cpp server (port 8765 by default) exposes /inference accepting
+    OpenAI-compatible multipart form-data. We hit it via stdlib urllib so this
+    module gains no new pip dep and works under SECCOMP without subprocess.
+
+    Returns the same shape as _transcribe_local. On any failure (timeout,
+    non-200, malformed JSON), returns success=False with error — the caller
+    falls back to faster-whisper in-process.
+    """
+    try:
+        with open(file_path, "rb") as f:
+            audio_bytes = f.read()
+        boundary = f"----stt{os.urandom(8).hex()}"
+        filename = Path(file_path).name
+        body = (
+            f"--{boundary}\r\n"
+            f'Content-Disposition: form-data; name="file"; filename="{filename}"\r\n'
+            f"Content-Type: audio/ogg\r\n\r\n"
+        ).encode("utf-8") + audio_bytes + (
+            f"\r\n--{boundary}\r\n"
+            f'Content-Disposition: form-data; name="language"\r\n\r\n{language or "auto"}\r\n'
+            f"--{boundary}\r\n"
+            f'Content-Disposition: form-data; name="response_format"\r\n\r\njson\r\n'
+            f"--{boundary}--\r\n"
+        ).encode("utf-8")
+        req = urllib.request.Request(
+            endpoint,
+            data=body,
+            headers={"Content-Type": f"multipart/form-data; boundary={boundary}"},
+            method="POST",
+        )
+        with urllib.request.urlopen(req, timeout=60) as resp:
+            data = json.loads(resp.read().decode("utf-8", errors="replace"))
+        transcript = (data.get("text") or "").strip()
+        if not transcript:
+            return {"success": False, "transcript": "", "error": "empty transcript from whisper.service"}
+        logger.info(
+            "Transcribed %s via whisper.service HTTP (%s, %d chars)",
+            filename, endpoint, len(transcript),
+        )
+        return {"success": True, "transcript": transcript, "provider": "local_http"}
+    except urllib.error.URLError as e:
+        return {"success": False, "transcript": "", "error": f"whisper.service unreachable: {e.reason}"}
+    except Exception as e:
+        return {"success": False, "transcript": "", "error": f"whisper.service error: {e}"}
+
+
 def _transcribe_local(file_path: str, model_name: str) -> Dict[str, Any]:
-    """Transcribe using faster-whisper (local, free)."""
+    """Transcribe locally: whisper.service HTTP first (large-v3-turbo if running),
+    then fallback to faster-whisper in-process (model from config)."""
+    # Phase 1: try whisper.service HTTP if endpoint is configured.
+    stt_cfg = _load_stt_config()
+    local_cfg = stt_cfg.get("local") or {}
+    endpoint = local_cfg.get("endpoint")
+    if endpoint:
+        result = _transcribe_via_whisper_service(
+            file_path,
+            endpoint,
+            language=local_cfg.get("language") or "",
+        )
+        if result.get("success"):
+            return result
+        logger.info(
+            "whisper.service HTTP failed (%s) — falling back to faster-whisper in-process",
+            result.get("error", "unknown"),
+        )
+
+    # Phase 2: faster-whisper in-process (legacy path, kept as fallback).
     global _local_model, _local_model_name
 
     if not _HAS_FASTER_WHISPER:

tools/voice_io.py

@@ -0,0 +1,166 @@
+#!/usr/bin/env python3
+"""
+voice_io.py — Local TTS/STT loop for hermes-agent via whisper.cpp + Piper services.
+
+Services:
+    whisper.service  127.0.0.1:8765   POST /inference         multipart, returns {"text": ...}
+    tts.service      127.0.0.1:8766   POST /synthesize        JSON in, WAV out
+                                      POST /synthesize-discord JSON in/out (Discord-ready bundle)
+                                      GET  /health             JSON status
+
+Design rule: this module **never** spawns subprocesses. hermes-agent.service runs under
+strict SECCOMP — fork+exec of ffmpeg/ffprobe triggers SIGSYS and kills the process.
+All audio transformation (WAV→Opus, waveform RMS) is offloaded to tts.service via
+the /synthesize-discord endpoint, which runs under a relaxed sandbox.
+
+Format Discord native voice message:
+    flags    = 8192  (IS_VOICE_MESSAGE = 1 << 13)
+    waveform = 256 RMS samples (byte array), base64-encoded
+    audio    = OGG Opus 48 kHz mono
+    https://github.com/discord/discord-api-docs/issues/4406
+"""
+
+from __future__ import annotations
+
+import base64
+import json
+import logging
+import os
+from typing import Tuple
+
+import aiohttp
+
+logger = logging.getLogger(__name__)
+
+# ---------------------------------------------------------------------------
+# Endpoints
+# ---------------------------------------------------------------------------
+
+STT_URL = "http://127.0.0.1:8765/inference"
+TTS_URL = "http://127.0.0.1:8766/synthesize"
+TTS_DISCORD_URL = "http://127.0.0.1:8766/synthesize-discord"
+TTS_HEALTH_URL = "http://127.0.0.1:8766/health"
+
+
+# ---------------------------------------------------------------------------
+# STT — Whisper service
+# ---------------------------------------------------------------------------
+
+async def stt(audio_bytes: bytes, language: str = "fr") -> str:
+    """Transcribe audio via the local whisper.cpp service.
+
+    Args:
+        audio_bytes: raw audio (any format ffmpeg can decode; whisper.cpp uses
+                     --convert and re-encodes to 16 kHz mono internally).
+        language: ISO-639-1 code, or "auto".
+
+    Returns:
+        Transcribed text, stripped.
+
+    Raises:
+        aiohttp.ClientError: on transport / HTTP error.
+    """
+    data = aiohttp.FormData()
+    data.add_field("file", audio_bytes, filename="audio.ogg", content_type="audio/ogg")
+    data.add_field("response_format", "json")
+    data.add_field("language", language)
+
+    async with aiohttp.ClientSession() as session:
+        async with session.post(
+            STT_URL,
+            data=data,
+            timeout=aiohttp.ClientTimeout(total=300),
+        ) as resp:
+            resp.raise_for_status()
+            payload = await resp.json()
+            return payload.get("text", "").strip()
+
+
+# ---------------------------------------------------------------------------
+# TTS — Piper service (raw WAV)
+# ---------------------------------------------------------------------------
+
+async def tts(text: str, voice: str = "fr_FR-upmc-medium") -> bytes:
+    """Synthesize text to WAV bytes (PCM 22050 Hz mono 16-bit).
+
+    Use this when you just need a WAV (e.g. local playback). For Discord
+    native voice messages, prefer ``tts_discord`` which returns OGG Opus
+    + pre-computed waveform without forcing the caller to fork ffmpeg.
+    """
+    async with aiohttp.ClientSession() as session:
+        async with session.post(
+            TTS_URL,
+            json={"text": text, "voice": voice},
+            timeout=aiohttp.ClientTimeout(total=30),
+        ) as resp:
+            resp.raise_for_status()
+            return await resp.read()
+
+
+async def tts_discord(text: str, voice: str = "fr_FR-upmc-medium") -> Tuple[bytes, str, float]:
+    """Synthesize text to a Discord-ready bundle.
+
+    All heavy lifting (WAV→OGG Opus + RMS waveform) happens server-side in
+    tts.service, so hermes-agent does not spawn any subprocess.
+
+    Returns:
+        (ogg_bytes, waveform_base64, duration_seconds)
+
+    Raises:
+        aiohttp.ClientError: on transport / HTTP error.
+    """
+    async with aiohttp.ClientSession() as session:
+        async with session.post(
+            TTS_DISCORD_URL,
+            json={"text": text, "voice": voice},
+            timeout=aiohttp.ClientTimeout(total=60),
+        ) as resp:
+            resp.raise_for_status()
+            payload = await resp.json()
+
+    ogg_bytes = base64.b64decode(payload["ogg_b64"])
+    waveform_b64 = payload["waveform_b64"]
+    duration = float(payload["duration_secs"])
+    return ogg_bytes, waveform_b64, duration
+
+
+async def tts_health_check() -> dict:
+    """Return tts.service /health payload, or a fallback dict on error."""
+    try:
+        async with aiohttp.ClientSession() as session:
+            async with session.get(
+                TTS_HEALTH_URL,
+                timeout=aiohttp.ClientTimeout(total=5),
+            ) as resp:
+                if resp.status == 200:
+                    return await resp.json()
+                return {"status": "unavailable", "code": resp.status}
+    except Exception as e:
+        return {"status": "unreachable", "error": str(e)[:200]}
+
+
+# ---------------------------------------------------------------------------
+# Native voice message detection (Discord input)
+# ---------------------------------------------------------------------------
+
+DISCORD_VOICE_MESSAGE_FLAG = 1 << 13  # IS_VOICE_MESSAGE = 8192
+
+
+def is_native_voice_message(message) -> bool:
+    """True if ``message`` is a Discord native voice message.
+
+    Detection: ``flags & 8192`` AND a single audio attachment.
+    """
+    flags_val = getattr(message, "flags", None)
+    if flags_val is None:
+        return False
+    flags_int = getattr(flags_val, "value", flags_val)
+    if not (flags_int & DISCORD_VOICE_MESSAGE_FLAG):
+        return False
+
+    attachments = getattr(message, "attachments", None) or []
+    if len(attachments) != 1:
+        return False
+
+    content_type = getattr(attachments[0], "content_type", "") or ""
+    return content_type.startswith("audio/")