diff --git a/contributors/emails/Zioywishing@users.noreply.github.com b/contributors/emails/Zioywishing@users.noreply.github.com new file mode 100644 index 000000000000..4bc20f0209ca --- /dev/null +++ b/contributors/emails/Zioywishing@users.noreply.github.com @@ -0,0 +1,2 @@ +Zioywishing +# PR #35705 salvage diff --git a/contributors/emails/armaandhawan61@gmail.com b/contributors/emails/armaandhawan61@gmail.com new file mode 100644 index 000000000000..048bf53c6b12 --- /dev/null +++ b/contributors/emails/armaandhawan61@gmail.com @@ -0,0 +1,2 @@ +VIVAAN-DHAWAN +# PR #66626 salvage diff --git a/contributors/emails/florianvalade@Florians-Mac-mini.local b/contributors/emails/florianvalade@Florians-Mac-mini.local new file mode 100644 index 000000000000..90e42b5b4e79 --- /dev/null +++ b/contributors/emails/florianvalade@Florians-Mac-mini.local @@ -0,0 +1,2 @@ +FlorianVal +# PR #66326 salvage diff --git a/contributors/emails/jazzwu@163.com b/contributors/emails/jazzwu@163.com new file mode 100644 index 000000000000..ca2cc5f67ef9 --- /dev/null +++ b/contributors/emails/jazzwu@163.com @@ -0,0 +1,2 @@ +rayjerrywoo +# PR #50014 salvage diff --git a/contributors/emails/luna@hermes.local b/contributors/emails/luna@hermes.local new file mode 100644 index 000000000000..fce81a9e6d80 --- /dev/null +++ b/contributors/emails/luna@hermes.local @@ -0,0 +1,2 @@ +seamusmore +# PR #40592 salvage diff --git a/contributors/emails/zgzczzw@users.noreply.github.com b/contributors/emails/zgzczzw@users.noreply.github.com new file mode 100644 index 000000000000..0601af072113 --- /dev/null +++ b/contributors/emails/zgzczzw@users.noreply.github.com @@ -0,0 +1,2 @@ +zgzczzw +# PR #65022 salvage diff --git a/gateway/platforms/qqbot/adapter.py b/gateway/platforms/qqbot/adapter.py index 2816326efb3f..42a7bc650e35 100644 --- a/gateway/platforms/qqbot/adapter.py +++ b/gateway/platforms/qqbot/adapter.py @@ -1812,16 +1812,14 @@ def _is_voice_content_type(content_type: str, filename: str) -> bool: fn = filename.strip().lower() if ct == "voice" or ct.startswith("audio/"): return True + # QQ file uploads have content_type="file". Without this guard, + # any uploaded audio file (e.g. .wav, .mp3) would be misrouted into + # the STT pipeline and never be received as a normal file attachment. + if ct == "file": + return False _VOICE_EXTENSIONS = ( - ".silk", - ".amr", - ".mp3", - ".wav", - ".ogg", - ".m4a", - ".aac", - ".speex", - ".flac", + ".silk", ".amr", ".mp3", ".wav", ".ogg", + ".m4a", ".aac", ".speex", ".flac", ) if any(fn.endswith(ext) for ext in _VOICE_EXTENSIONS): return True @@ -1938,15 +1936,15 @@ async def _stt_voice_attachment( ) return None - # 4. Call STT API + # 4. Call STT API and always clean up the temp WAV afterward. logger.debug("[%s] STT: calling ASR on %s", self._log_tag, wav_path) - transcript = await self._call_stt(wav_path) - - # 5. Cleanup temp file try: - os.unlink(wav_path) - except OSError: - pass + transcript = await self._call_stt(wav_path) + finally: + try: + os.unlink(wav_path) + except OSError: + pass if transcript: logger.debug("[%s] STT success: %r", self._log_tag, transcript[:100]) diff --git a/gateway/platforms/weixin.py b/gateway/platforms/weixin.py index e535949bcac5..bd25319fc8b4 100644 --- a/gateway/platforms/weixin.py +++ b/gateway/platforms/weixin.py @@ -973,9 +973,25 @@ def _extract_text(item_list: List[Dict[str, Any]]) -> str: return text for item in item_list: if item.get("type") == ITEM_VOICE: - voice_text = str((item.get("voice_item") or {}).get("text") or "") - if voice_text: - return voice_text + # #27300: Tencent Cloud's `voice_item.text` is their STT output, + # which is wrong for any non-Chinese audio (the original report + # was a Russian voice message that came back as English + # gibberish). Return empty so the central STT pipeline in + # ``gateway/run.py`` produces the body from the downloaded + # audio instead. + voice_item = item.get("voice_item") or {} + if not (voice_item.get("media") or {}): + # No raw audio to download — Weixin supplied only its own + # speech-to-text result. Use it, but preserve the voice + # origin so the agent can distinguish this from text the + # user typed (#65022). + voice_text = str(voice_item.get("text") or "") + if voice_text: + return ( + "[Voice transcription provided by Weixin]\n" + f"{voice_text}" + ) + continue return "" @@ -1659,8 +1675,13 @@ async def _download_file(self, item: Dict[str, Any]) -> Tuple[Optional[str], str async def _download_voice(self, item: Dict[str, Any]) -> Optional[str]: voice_item = item.get("voice_item") or {} media = voice_item.get("media") or {} - if voice_item.get("text"): - return None + # #27300: previously short-circuited when ``voice_item.text`` was set + # on the assumption that Tencent Cloud's STT was good enough. + # For non-Chinese audio that text is garbage (e.g. a Russian + # message comes back as English phonemes) — we must always + # download the raw audio so ``gateway/run.py``'s central STT + # pipeline can re-transcribe with the user's configured + # mlx-whisper / whisper.cpp / faster-whisper backend. try: data = await _download_and_decrypt_media( self._poll_session, diff --git a/gateway/run.py b/gateway/run.py index 3d2d9fa55497..a4e0d613f24d 100644 --- a/gateway/run.py +++ b/gateway/run.py @@ -15902,14 +15902,19 @@ def _should_send_voice_reply( ) return False - # Dedup: agent already called TTS tool + # Dedup: agent already called TTS tool in THIS turn only + last_user_idx = None + for i, msg in enumerate(reversed(agent_messages)): + if msg.get("role") == "user": + last_user_idx = len(agent_messages) - 1 - i; break + turn_messages = agent_messages[last_user_idx:] if last_user_idx is not None else agent_messages has_agent_tts = any( msg.get("role") == "assistant" and any( (tc.get("function") or {}).get("name") == "text_to_speech" for tc in (msg.get("tool_calls") or []) ) - for msg in agent_messages + for msg in turn_messages ) if has_agent_tts: return False @@ -18058,6 +18063,8 @@ async def _enrich_message_with_transcription( list if every clip failed or STT is disabled. Callers can use this to echo transcripts back to the user before the agent loop. """ + seen = set() + audio_paths = [p for p in audio_paths if p not in seen and not seen.add(p)] if not getattr(self.config, "stt_enabled", True): notes = [] for path in audio_paths: @@ -18080,7 +18087,10 @@ async def _enrich_message_with_transcription( return prefix, [] try: - from tools.transcription_tools import transcribe_audio + from tools.transcription_tools import ( + transcribe_audio, + transcribe_audio_local_fallback, + ) except ModuleNotFoundError as e: logger.error("Transcription module unavailable: %s", e) unavailable_note = "[voice message could not be transcribed]" @@ -18097,6 +18107,17 @@ async def _enrich_message_with_transcription( try: logger.debug("Transcribing user voice: %s", path) result = await asyncio.to_thread(transcribe_audio, path) + if not result.get("success"): + fallback = await asyncio.to_thread( + transcribe_audio_local_fallback, + path, + ) + if fallback.get("success"): + logger.info( + "Configured STT failed for %s; recovered with local STT", + path, + ) + result = fallback if result["success"]: transcript = result["transcript"] # Speech-to-text can return success=True with an empty or @@ -18131,10 +18152,22 @@ async def _enrich_message_with_transcription( # logged for operator diagnosis but kept out of the # LLM-visible prompt. logger.info("Voice transcription failed for %s: %s", path, error) - enriched_parts.append("[voice message could not be transcribed]") + from tools.credential_files import to_agent_visible_cache_path + + agent_path = to_agent_visible_cache_path(os.path.abspath(path)) + enriched_parts.append( + "[voice message could not be transcribed automatically; " + f"the audio is available at: {agent_path}]" + ) except Exception as e: logger.error("Transcription error: %s", e) - enriched_parts.append("[voice message could not be transcribed]") + from tools.credential_files import to_agent_visible_cache_path + + agent_path = to_agent_visible_cache_path(os.path.abspath(path)) + enriched_parts.append( + "[voice message could not be transcribed automatically; " + f"the audio is available at: {agent_path}]" + ) if enriched_parts: prefix = "\n\n".join(enriched_parts) diff --git a/plugins/platforms/dingtalk/adapter.py b/plugins/platforms/dingtalk/adapter.py index 69aa3591bb61..0ad470d1c4c1 100644 --- a/plugins/platforms/dingtalk/adapter.py +++ b/plugins/platforms/dingtalk/adapter.py @@ -116,6 +116,27 @@ "voice": "audio", } +# File extension → MIME type mapping for DingTalk file/image messages. +# Image MIME types (image/*) are used below in _extract_media to classify +# incoming msgtype='image' payloads as MessageType.PHOTO (not DOCUMENT). +EXT_MAP = { + "pdf": "application/pdf", + "png": "image/png", + "jpg": "image/jpeg", + "jpeg": "image/jpeg", + "gif": "image/gif", + "webp": "image/webp", + "doc": "application/msword", + "docx": "application/vnd.openxmlformats-officedocument.wordprocessingml.document", + "xls": "application/vnd.ms-excel", + "xlsx": "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", + "md": "text/markdown", + "txt": "text/plain", + "csv": "text/csv", + "zip": "application/zip", + "mp4": "video/mp4", +} + def check_dingtalk_requirements() -> bool: """Check if DingTalk dependencies are available and configured. @@ -748,6 +769,90 @@ def _extract_text(message: "ChatbotMessage") -> str: parts.append(item.text) content = " ".join(parts).strip() + # Fallback: audio message → use recognition text + if not content: + msg_type = getattr(message, "message_type", "") + if msg_type == "audio": + extensions = getattr(message, "extensions", {}) or {} + audio_content = extensions.get("content", {}) + if isinstance(audio_content, dict): + recognition = audio_content.get("recognition", "") + if recognition: + content = recognition.strip() + + # Fallback: file message → use fileName as text + if not content: + msg_type = getattr(message, "message_type", "") + if msg_type == "file": + extensions = getattr(message, "extensions", {}) or {} + file_content = extensions.get("content", {}) + if isinstance(file_content, dict): + fname = file_content.get("fileName", "") + if fname: + content = f"[文件] {fname}" + + # Fallback: card message (钉钉文档分享卡片 / link card) + # When a user shares a DingTalk Doc to the bot, the msgtype is "card" + # and the card data lives in extensions['card'] (SDK's from_dict maps + # unhandled fields to extensions). Extract title + doc URL so the + # message isn't silently dropped as "empty". + if not content: + msg_type = getattr(message, "message_type", "") + # Handle card-type messages (文档分享卡片 / link card) + if msg_type == "card": + extensions = getattr(message, "extensions", {}) or {} + card = extensions.get("card", {}) + if isinstance(card, dict): + title = card.get("title", "") + raw_content = card.get("content", "") + doc_url = "" + if raw_content is None: + doc_url = "" + elif isinstance(raw_content, dict): + doc_url = raw_content.get("url", "") or raw_content.get("docUrl", "") + elif isinstance(raw_content, str): + stripped = raw_content.strip() + if not stripped: + doc_url = "" + else: + try: + parsed = json.loads(stripped) + if isinstance(parsed, dict): + doc_url = parsed.get("url", "") or parsed.get("docUrl", "") + except (ValueError, TypeError): + doc_url = raw_content + parts = [] + if title: + parts.append(f"[文档] {title}") + if doc_url: + parts.append(doc_url) + if parts: + content = " ".join(parts) + # Last-resort: raw text field from extensions (if present) + if not content: + ext_text = extensions.get("text", {}) + if isinstance(ext_text, dict): + content = (ext_text.get("content", "") or "").strip() + + # Handle interactiveCard messages (钉钉文档分享卡片 / doc link card) + # structure: extensions["content"]["biz_custom_action_url"] and + # extensions["content"]["title"] for the card title + if msg_type == "interactiveCard" and not content: + extensions = getattr(message, "extensions", {}) or {} + ext_content = extensions.get("content", {}) + if isinstance(ext_content, dict): + doc_url = ext_content.get("biz_custom_action_url", "") + title = ext_content.get("title", "") + if doc_url or title: + parts = [] + if title: + parts.append(f"[文档卡片] {title}") + else: + parts.append("[文档卡片]") + if doc_url: + parts.append(doc_url) + content = " ".join(parts) + # Do NOT strip "@bot" from the text. The mention is a routing # signal (delivered structurally via callback `isInAtList`), and # regex-stripping @handles would collateral-damage e-mails @@ -815,11 +920,49 @@ def _extract_media(self, message: "ChatbotMessage"): if msg_type_str == "picture" and not media_urls: msg_type = MessageType.PHOTO elif msg_type_str == "richText": - msg_type = ( - MessageType.PHOTO - if any("image" in t for t in media_types) - else MessageType.TEXT - ) + # Only re-derive the type when the rich-text scan above left it + # at TEXT. The scan may already have promoted it to VOICE/AUDIO/ + # VIDEO/DOCUMENT for embedded media items — resetting those here + # dropped native voice notes back to TEXT and skipped STT + # (#38211, #38219; analysis from #38276). + if msg_type == MessageType.TEXT and any( + "image" in t for t in media_types + ): + msg_type = MessageType.PHOTO + elif msg_type_str == "audio": + # Voice message — DingTalk already provides recognition text. + # Do NOT add media_urls here: if audio_paths is non-empty, + # run.py's _enrich_message_with_transcription will overwrite + # the recognition text with a failed STT attempt (whisper not installed). + # The recognition text from extensions['content']['recognition'] + # is sufficient and already extracted by _extract_text. + if msg_type == MessageType.TEXT: + msg_type = MessageType.VOICE + elif msg_type_str in ("file", "image"): + extensions = getattr(message, "extensions", {}) or {} + ext_content = extensions.get("content", {}) + if isinstance(ext_content, dict): + dl_code = ext_content.get("downloadCode") or "" + fname = ext_content.get("fileName", "") + if dl_code: + media_urls.append(dl_code) + mime = "application/octet-stream" + # Map common extensions + if fname: + ext = fname.rsplit(".", 1)[-1].lower() if "." in fname else "" + mime = EXT_MAP.get(ext, mime) + media_types.append(mime) + if msg_type == MessageType.TEXT: + # Image messages → PHOTO (distinct busy-session handling + # in gateway/platforms/base.py). + # File messages with image MIME types (e.g. a .png sent + # as a file attachment) are also classified as PHOTO — + # the user's intent is to share an image regardless of + # how DingTalk delivers it. + if msg_type_str == "image" or mime.startswith("image/"): + msg_type = MessageType.PHOTO + else: + msg_type = MessageType.DOCUMENT return msg_type, media_urls, media_types @@ -1323,6 +1466,14 @@ async def _resolve_media_codes(self, message: "ChatbotMessage") -> None: if item.get(key): codes_to_resolve.append((item, key)) + # 3. File/image message (msgtype='file' or 'image', codes in extensions) + msg_type_str = getattr(message, "message_type", "") or "" + if msg_type_str in ("file", "image"): + extensions = getattr(message, "extensions", {}) or {} + ext_content = extensions.get("content", {}) + if isinstance(ext_content, dict) and ext_content.get("downloadCode"): + codes_to_resolve.append((ext_content, "downloadCode")) + if not codes_to_resolve: return diff --git a/plugins/platforms/feishu/adapter.py b/plugins/platforms/feishu/adapter.py index eb1f5df3a56a..f3756a8589be 100644 --- a/plugins/platforms/feishu/adapter.py +++ b/plugins/platforms/feishu/adapter.py @@ -3874,7 +3874,15 @@ def _resolve_normalized_message_type( if preferred == "photo": return self._resolve_media_message_type(media_types[0] if media_types else "", default=MessageType.PHOTO) if preferred == "audio": - return self._resolve_media_message_type(media_types[0] if media_types else "", default=MessageType.AUDIO) + # Lark's native "audio" msg_type is an in-app voice recording, not + # an uploaded audio file (those arrive as "file"/"media" and are + # normalized to "document"). Classify it as VOICE so the gateway + # auto-transcribes it (Opus → STT) the same way + # Discord/DingTalk/Telegram/etc. do — otherwise a Feishu voice note + # reaches the agent as an untranscribable AUDIO attachment and is + # silently ignored. Follow-up to #28993, which added native + # voice-note transcription for Discord + DingTalk. + return MessageType.VOICE if preferred == "document": return self._resolve_media_message_type(media_types[0] if media_types else "", default=MessageType.DOCUMENT) return MessageType.TEXT diff --git a/plugins/platforms/line/adapter.py b/plugins/platforms/line/adapter.py index 63343d37dbda..615841923278 100644 --- a/plugins/platforms/line/adapter.py +++ b/plugins/platforms/line/adapter.py @@ -92,7 +92,10 @@ MessageEvent, MessageType, SendResult, + cache_audio_from_bytes, + cache_document_from_bytes, cache_image_from_bytes, + cache_video_from_bytes, ) from gateway.config import Platform @@ -955,11 +958,15 @@ async def _handle_message_event(self, event: Dict[str, Any]) -> None: if msg_type == "text": text = msg.get("text", "") or "" - elif msg_type in {"image", "audio", "video", "file"}: - local_path = await self._download_media(message_id, msg_type) + elif msg_type in ("image", "audio", "video", "file"): + local_path, media_type = await self._download_media( + message_id, + msg_type, + filename=msg.get("fileName") or msg.get("file_name"), + ) if local_path: media_urls.append(local_path) - media_types.append(msg_type) + media_types.append(media_type) text = f"[{msg_type}]" elif msg_type == "sticker": keywords = msg.get("keywords") or [] @@ -1054,14 +1061,20 @@ async def _handle_postback_event(self, event: Dict[str, Any]) -> None: except Exception: pass - async def _download_media(self, message_id: str, msg_type: str) -> Optional[str]: + async def _download_media( + self, + message_id: str, + msg_type: str, + *, + filename: Optional[str] = None, + ) -> Tuple[Optional[str], str]: if not self._client or not message_id: - return None + return None, "" try: data = await self._client.fetch_content(message_id) except Exception as exc: logger.warning("LINE: failed to fetch %s content for %s: %s", msg_type, message_id, exc) - return None + return None, "" ext = { "image": ".jpg", "audio": ".m4a", @@ -1069,10 +1082,22 @@ async def _download_media(self, message_id: str, msg_type: str) -> Optional[str] "file": ".bin", }.get(msg_type, ".bin") try: - return cache_image_from_bytes(data, ext=ext) + if msg_type == "image": + return cache_image_from_bytes(data, ext=ext), "image/jpeg" + if msg_type == "audio": + media_type = mimetypes.guess_type(f"audio{ext}")[0] or "audio/mp4" + return cache_audio_from_bytes(data, ext=ext), media_type + if msg_type == "video": + media_type = mimetypes.guess_type(f"video{ext}")[0] or "video/mp4" + return cache_video_from_bytes(data, ext=ext), media_type + document_name = filename or f"line_file{ext}" + return ( + cache_document_from_bytes(data, document_name), + mimetypes.guess_type(document_name)[0] or "application/octet-stream", + ) except Exception as exc: logger.warning("LINE: failed to cache %s payload: %s", msg_type, exc) - return None + return None, "" # ------------------------------------------------------------------ # Outbound send (text) diff --git a/plugins/platforms/photon/adapter.py b/plugins/platforms/photon/adapter.py index 91cd50c3b488..8daf743e2bf0 100644 --- a/plugins/platforms/photon/adapter.py +++ b/plugins/platforms/photon/adapter.py @@ -83,6 +83,8 @@ _DEDUP_MAX_SIZE = 4000 _DEDUP_WINDOW_SECONDS = 48 * 3600 +_FFFC_WAIT_SECONDS = 15.0 # Timeout for waiting on an attachment after a U+FFFC placeholder. + _SIDECAR_DIR = Path(__file__).parent / "sidecar" # Cap on a self-heal `npm ci`/`npm install` of the sidecar deps. A cold @@ -340,6 +342,8 @@ def __init__(self, config: PlatformConfig): # Last time we sent a typing indicator per chat, for cooldown gating. self._typing_last_sent: Dict[str, float] = {} + self._pending_fffc: Dict[str, tuple[float, Any]] = {} # chat_key → (timestamp, asyncio.Task) + # Group-chat mention gating (parity with BlueBubbles). When enabled, # group messages are ignored unless they match a wake word; DMs are # always processed. Config key wins, then env var. @@ -492,6 +496,11 @@ async def disconnect(self) -> None: except Exception: pass self._inbound_task = None + # Cancel any pending U+FFFC placeholder tasks. + for chat_key, (_, fffc_task) in list(self._pending_fffc.items()): + if fffc_task and not fffc_task.done(): + fffc_task.cancel() + self._pending_fffc.clear() await self._stop_sidecar() if self._http_client is not None: try: @@ -617,6 +626,14 @@ def _is_duplicate(self, msg_id: str) -> bool: del seen[old] return False + async def _fffc_timeout_handler(self, chat_key: str, message_id: str) -> None: + await asyncio.sleep(_FFFC_WAIT_SECONDS) + if self._pending_fffc.pop(chat_key, None): + logger.warning( + "[photon] wait for attachment was too long, can't retrieve attachment data " + "(message %s, chat %s)", message_id, chat_key, + ) + async def _dispatch_inbound(self, event: Dict[str, Any]) -> None: """Normalize a sidecar inbound event and dispatch it to the gateway. @@ -677,6 +694,11 @@ def _normalize_binary_payload( is_voice = payload.get("type") == "voice" name = payload.get("name") or ("voice" if is_voice else "(unnamed)") mime = payload.get("mimeType") or "" + # Promote CAF attachments to VOICE (iMessage voice notes use CAF). + # Check both filename and MIME: the sidecar may send "(unnamed)" + # when no name is supplied, so the MIME type is the reliable signal. + if not is_voice and (name.lower().endswith(".caf") or mime == "audio/x-caf"): + is_voice = True mtype = MessageType.VOICE if is_voice else _attachment_message_type(mime) cached = _cache_inbound_attachment( payload, name, mime, force_audio=is_voice @@ -747,6 +769,28 @@ def _normalize_binary_payload( ) ) return + # U+FFFC placeholder — wait for the real attachment instead of + # dispatching. Detected before _record_last_inbound so the placeholder + # is not recorded as the reaction target — the real attachment will be. + if ctype == "text" and (content.get("text") or "").strip() == "\ufffc": + chat_key = space_id + prev = self._pending_fffc.pop(chat_key, None) + if prev and prev[1] and not prev[1].done(): + prev[1].cancel() + task = asyncio.create_task( + self._fffc_timeout_handler(chat_key, event.get("messageId") or "") + ) + self._pending_fffc[chat_key] = (time.monotonic(), task) + logger.debug("[photon] U+FFFC placeholder received — waiting for attachment") + return + + # Cancel any pending U+FFFC timeout — the real message arrived. + if ctype in {"attachment", "voice"}: + prev = self._pending_fffc.pop(space_id, None) + if prev and prev[1] and not prev[1].done(): + prev[1].cancel() + logger.debug("[photon] attachment arrived — cancelling U+FFFC timeout") + # Anything past here is a real (reactable) message — remember it as # the chat's latest inbound so `add_reaction` can target it when the # caller doesn't pass an explicit message id. Recorded before the @@ -1610,7 +1654,7 @@ def _attachment_message_type(mime: str) -> MessageType: "audio/mpeg": ".mp3", "audio/ogg": ".ogg", "audio/wav": ".wav", - "audio/x-caf": ".mp3", + "audio/x-caf": ".caf", "audio/mp4": ".m4a", "audio/aac": ".m4a", } diff --git a/plugins/platforms/whatsapp/adapter.py b/plugins/platforms/whatsapp/adapter.py index 6c2be63df022..3f6c9faa8b9f 100644 --- a/plugins/platforms/whatsapp/adapter.py +++ b/plugins/platforms/whatsapp/adapter.py @@ -1484,6 +1484,15 @@ async def _build_message_event(self, data: Dict[str, Any]) -> Optional[MessageEv body = data.get("body", "") if data.get("isGroup"): body = self._clean_bot_mention_text(body, data) + if ( + msg_type == MessageType.VOICE + and cached_urls + and str(body).strip().lower() == "[ptt received]" + ): + # The bridge synthesizes this placeholder for captionless voice + # notes. The cached audio is the real payload; retaining the + # placeholder makes the agent answer it as if it were user text. + body = "" # If this is a reply, keep the quoted message in structured fields # only. GatewayRunner._prepare_inbound_message_text owns rendering diff --git a/tests/gateway/test_dingtalk.py b/tests/gateway/test_dingtalk.py index 8e4cd8223276..8c0ac5bc1b95 100644 --- a/tests/gateway/test_dingtalk.py +++ b/tests/gateway/test_dingtalk.py @@ -148,42 +148,6 @@ def test_falls_back_to_env_vars(self, monkeypatch): assert adapter._client_secret == "env-secret" -# --------------------------------------------------------------------------- -# Message text extraction -# --------------------------------------------------------------------------- - - -class TestExtractText: - - def test_extracts_dict_text(self): - from plugins.platforms.dingtalk.adapter import DingTalkAdapter - msg = MagicMock() - msg.text = {"content": " hello world "} - msg.rich_text = None - assert DingTalkAdapter._extract_text(msg) == "hello world" - - def test_extracts_string_text(self): - from plugins.platforms.dingtalk.adapter import DingTalkAdapter - msg = MagicMock() - msg.text = "plain text" - msg.rich_text = None - assert DingTalkAdapter._extract_text(msg) == "plain text" - - def test_falls_back_to_rich_text(self): - from plugins.platforms.dingtalk.adapter import DingTalkAdapter - msg = MagicMock() - msg.text = "" - msg.rich_text = [{"text": "part1"}, {"text": "part2"}, {"image": "url"}] - assert DingTalkAdapter._extract_text(msg) == "part1 part2" - - def test_returns_empty_for_no_content(self): - from plugins.platforms.dingtalk.adapter import DingTalkAdapter - msg = MagicMock() - msg.text = "" - msg.rich_text = None - assert DingTalkAdapter._extract_text(msg) == "" - - # --------------------------------------------------------------------------- # Deduplication # --------------------------------------------------------------------------- @@ -570,6 +534,178 @@ def test_empty_message(self): msg.rich_text = None assert DingTalkAdapter._extract_text(msg) == "" + # --- Card / interactiveCard message handling (文档分享卡片) --- + + def test_card_with_dict_content_and_url(self): + """card msgtype with extensions.card.content as dict with url key.""" + from plugins.platforms.dingtalk.adapter import DingTalkAdapter + msg = MagicMock() + msg.text = None + msg.rich_text = None + msg.message_type = "card" + msg.extensions = { + "card": { + "title": "Q3经营分析报告", + "content": {"url": "https://dingtalk.com/doc/abc123"}, + } + } + assert DingTalkAdapter._extract_text(msg) == "[文档] Q3经营分析报告 https://dingtalk.com/doc/abc123" + + def test_card_with_dict_content_docurl(self): + """card msgtype with extensions.card.content as dict with docUrl key.""" + from plugins.platforms.dingtalk.adapter import DingTalkAdapter + msg = MagicMock() + msg.text = None + msg.rich_text = None + msg.message_type = "card" + msg.extensions = { + "card": { + "title": "周报模板", + "content": {"docUrl": "https://docs.dingtalk.com/xyz"}, + } + } + assert DingTalkAdapter._extract_text(msg) == "[文档] 周报模板 https://docs.dingtalk.com/xyz" + + def test_card_with_json_string_content(self): + """card msgtype with extensions.card.content as JSON string.""" + from plugins.platforms.dingtalk.adapter import DingTalkAdapter + msg = MagicMock() + msg.text = None + msg.rich_text = None + msg.message_type = "card" + msg.extensions = { + "card": { + "title": "数据看板", + "content": '{"url": "https://dingtalk.com/doc/def456"}', + } + } + assert DingTalkAdapter._extract_text(msg) == "[文档] 数据看板 https://dingtalk.com/doc/def456" + + def test_card_with_plain_string_content(self): + """card msgtype with extensions.card.content as plain string (used as url).""" + from plugins.platforms.dingtalk.adapter import DingTalkAdapter + msg = MagicMock() + msg.text = None + msg.rich_text = None + msg.message_type = "card" + msg.extensions = { + "card": { + "title": "分享链接", + "content": "https://dingtalk.com/doc/plain", + } + } + assert DingTalkAdapter._extract_text(msg) == "[文档] 分享链接 https://dingtalk.com/doc/plain" + + def test_card_no_title_only_url(self): + """card msgtype with url but no title.""" + from plugins.platforms.dingtalk.adapter import DingTalkAdapter + msg = MagicMock() + msg.text = None + msg.rich_text = None + msg.message_type = "card" + msg.extensions = { + "card": { + "content": {"url": "https://dingtalk.com/doc/onlyurl"}, + } + } + assert DingTalkAdapter._extract_text(msg) == "https://dingtalk.com/doc/onlyurl" + + def test_card_fallback_to_extensions_text(self): + """card msgtype with no usable card data → fallback to extensions.text.content.""" + from plugins.platforms.dingtalk.adapter import DingTalkAdapter + msg = MagicMock() + msg.text = None + msg.rich_text = None + msg.message_type = "card" + msg.extensions = { + "card": {}, + "text": {"content": "fallback-text"}, + } + assert DingTalkAdapter._extract_text(msg) == "fallback-text" + + def test_card_content_none_is_handled(self): + """card msgtype with content: None → no crash, empty doc_url.""" + from plugins.platforms.dingtalk.adapter import DingTalkAdapter + msg = MagicMock() + msg.text = None + msg.rich_text = None + msg.message_type = "card" + msg.extensions = { + "card": { + "title": "某文档", + "content": None, + } + } + assert DingTalkAdapter._extract_text(msg) == "[文档] 某文档" + + def test_card_content_empty_string_is_handled(self): + """card msgtype with content: "" → no crash, empty doc_url.""" + from plugins.platforms.dingtalk.adapter import DingTalkAdapter + msg = MagicMock() + msg.text = None + msg.rich_text = None + msg.message_type = "card" + msg.extensions = { + "card": { + "title": "空内容文档", + "content": "", + } + } + assert DingTalkAdapter._extract_text(msg) == "[文档] 空内容文档" + + def test_interactive_card_extracts_biz_custom_action_url(self): + """interactiveCard msgtype with biz_custom_action_url.""" + from plugins.platforms.dingtalk.adapter import DingTalkAdapter + msg = MagicMock() + msg.text = None + msg.rich_text = None + msg.message_type = "interactiveCard" + msg.extensions = { + "content": { + "biz_custom_action_url": "https://dingtalk.com/doc/interactive", + } + } + assert DingTalkAdapter._extract_text(msg) == "[文档卡片] https://dingtalk.com/doc/interactive" + + def test_interactive_card_no_url_returns_empty(self): + """interactiveCard msgtype with no biz_custom_action_url → empty string.""" + from plugins.platforms.dingtalk.adapter import DingTalkAdapter + msg = MagicMock() + msg.text = None + msg.rich_text = None + msg.message_type = "interactiveCard" + msg.extensions = {"content": {}} + assert DingTalkAdapter._extract_text(msg) == "" + + def test_interactive_card_with_title_and_url(self): + """interactiveCard msgtype with both title and biz_custom_action_url.""" + from plugins.platforms.dingtalk.adapter import DingTalkAdapter + msg = MagicMock() + msg.text = None + msg.rich_text = None + msg.message_type = "interactiveCard" + msg.extensions = { + "content": { + "title": "项目看板", + "biz_custom_action_url": "https://dingtalk.com/doc/kanban", + } + } + assert DingTalkAdapter._extract_text(msg) == "[文档卡片] 项目看板 https://dingtalk.com/doc/kanban" + + def test_interactive_card_title_only(self): + """interactiveCard msgtype with title but no URL.""" + from plugins.platforms.dingtalk.adapter import DingTalkAdapter + msg = MagicMock() + msg.text = None + msg.rich_text = None + msg.message_type = "interactiveCard" + msg.extensions = { + "content": { + "title": "仅标题", + } + } + assert DingTalkAdapter._extract_text(msg) == "[文档卡片] 仅标题" + class TestExtractMedia: """_extract_media must split native voice rich-text items (auto-STT) @@ -599,6 +735,40 @@ def test_voice_rich_text_item_classified_as_voice(self): assert urls == ["dl_voice_abc"] assert mtypes == ["audio"] + def test_richtext_reset_does_not_clobber_voice(self): + """A richText envelope containing a native voice item must stay + VOICE — the ``msg_type_str == "richText"`` re-derivation used to + reset it to TEXT, dropping the voice note from the STT path + (#38211, #38219).""" + from plugins.platforms.dingtalk.adapter import DingTalkAdapter + from gateway.platforms.base import MessageType + + msg = self._msg_with_rich_text( + [{"type": "voice", "downloadCode": "dl_voice_rt"}] + ) + msg.message_type = "richText" + msg_type, urls, mtypes = DingTalkAdapter._extract_media( + DingTalkAdapter, msg + ) + assert msg_type == MessageType.VOICE + assert urls == ["dl_voice_rt"] + assert mtypes == ["audio"] + + def test_richtext_with_image_still_photo(self): + """richText with only an embedded image keeps the PHOTO promotion.""" + from plugins.platforms.dingtalk.adapter import DingTalkAdapter + from gateway.platforms.base import MessageType + + msg = self._msg_with_rich_text( + [{"type": "picture", "downloadCode": "dl_img_rt"}] + ) + msg.message_type = "richText" + msg_type, urls, mtypes = DingTalkAdapter._extract_media( + DingTalkAdapter, msg + ) + assert msg_type == MessageType.PHOTO + assert urls == ["dl_img_rt"] + def test_audio_rich_text_item_stays_audio(self): """Generic audio uploads (e.g. an mp3 the user attached) must NOT be auto-transcribed — they stay MessageType.AUDIO.""" @@ -622,6 +792,64 @@ def test_audio_rich_text_item_stays_audio(self): finally: del DINGTALK_TYPE_MAPPING["audio"] + def test_file_extensions_content_downloadcode_resolved(self): + """msgtype='file' with extensions.content.downloadCode → DOCUMENT.""" + from plugins.platforms.dingtalk.adapter import DingTalkAdapter + from gateway.platforms.base import MessageType + + msg = MagicMock() + msg.text = None + msg.image_content = None + msg.rich_text_content = None + msg.rich_text = None + msg.message_type = "file" + msg.extensions = {"content": {"downloadCode": "dl_file_123", "fileName": "report.pdf"}} + msg_type, urls, mtypes = DingTalkAdapter._extract_media( + DingTalkAdapter, msg + ) + assert msg_type == MessageType.DOCUMENT + assert urls == ["dl_file_123"] + assert mtypes == ["application/pdf"] + + def test_image_extensions_content_classified_as_photo(self): + """msgtype='image' with extensions.content → PHOTO (not DOCUMENT).""" + from plugins.platforms.dingtalk.adapter import DingTalkAdapter + from gateway.platforms.base import MessageType + + msg = MagicMock() + msg.text = None + msg.image_content = None + msg.rich_text_content = None + msg.rich_text = None + msg.message_type = "image" + msg.extensions = {"content": {"downloadCode": "dl_img_abc", "fileName": "photo.png"}} + msg_type, urls, mtypes = DingTalkAdapter._extract_media( + DingTalkAdapter, msg + ) + assert msg_type == MessageType.PHOTO + assert urls == ["dl_img_abc"] + assert mtypes == ["image/png"] + + def test_image_no_filename_still_photo(self): + """msgtype='image' without fileName → still PHOTO (MIME heuristic).""" + from plugins.platforms.dingtalk.adapter import DingTalkAdapter + from gateway.platforms.base import MessageType + + msg = MagicMock() + msg.text = None + msg.image_content = None + msg.rich_text_content = None + msg.rich_text = None + msg.message_type = "image" + msg.extensions = {"content": {"downloadCode": "dl_img_noext"}} + msg_type, urls, mtypes = DingTalkAdapter._extract_media( + DingTalkAdapter, msg + ) + assert msg_type == MessageType.PHOTO + assert urls == ["dl_img_noext"] + # Without fileName, mime defaults to octet-stream but msg_type_str=="image" still wins + assert mtypes == ["application/octet-stream"] + # --------------------------------------------------------------------------- # Group gating — require_mention + allowed_users (parity with other platforms) diff --git a/tests/gateway/test_feishu.py b/tests/gateway/test_feishu.py index 0e411b8cf600..d887bc94f38d 100644 --- a/tests/gateway/test_feishu.py +++ b/tests/gateway/test_feishu.py @@ -1490,7 +1490,11 @@ def test_extract_audio_message_downloads_and_caches(self): text, msg_type, media_urls, media_types, _mentions = asyncio.run(adapter._extract_message_content(message)) self.assertEqual(text, "") - self.assertEqual(msg_type.value, "audio") + # Lark "audio" msg_type is a native voice recording (the fixture is + # literally voice.ogg) — it must classify as VOICE so the gateway + # auto-transcribes it, not AUDIO (a non-transcribed file attachment). + # See the #28993 follow-up fix in _resolve_normalized_message_type. + self.assertEqual(msg_type.value, "voice") self.assertEqual(media_urls, ["/tmp/feishu-audio.ogg"]) self.assertEqual(media_types, ["audio/ogg"]) diff --git a/tests/gateway/test_feishu_voice_message_type.py b/tests/gateway/test_feishu_voice_message_type.py new file mode 100644 index 000000000000..c6e6723c562e --- /dev/null +++ b/tests/gateway/test_feishu_voice_message_type.py @@ -0,0 +1,46 @@ +"""Regression tests for Feishu native voice-note classification. + +Lark's native ``audio`` msg_type is an in-app voice recording (uploaded +audio files arrive as ``file``/``media`` and normalize to ``document``). It +must be classified as MessageType.VOICE so the gateway auto-transcribes it +(Opus → STT), the same way Discord/DingTalk/Telegram do. Before the fix it +resolved to MessageType.AUDIO, which the gateway treats as a non-transcribed +file attachment — so a Feishu voice note silently reached the agent as +untranscribable audio. Follow-up to #28993 (Discord + DingTalk). +""" + +from gateway.platforms.base import MessageType +from plugins.platforms.feishu.adapter import FeishuAdapter, FeishuNormalizedMessage + + +def _resolve(preferred: str, media_types): + """Call _resolve_normalized_message_type without a full adapter init. + + The method only reads normalized.preferred_message_type and delegates to + the static _resolve_media_message_type — no instance state — so we bypass + __init__ (which needs Lark credentials/config) via __new__. + """ + adapter = FeishuAdapter.__new__(FeishuAdapter) + normalized = FeishuNormalizedMessage( + raw_type=preferred, + text_content="", + preferred_message_type=preferred, + ) + return adapter._resolve_normalized_message_type(normalized, media_types) + + +def test_native_voice_audio_is_classified_as_voice(): + """Lark audio msg_type (voice recording) → VOICE, so it gets transcribed.""" + assert _resolve("audio", ["audio/opus"]) is MessageType.VOICE + + +def test_native_voice_audio_without_media_type_is_voice(): + """A voice note with no resolved mime still classifies as VOICE.""" + assert _resolve("audio", []) is MessageType.VOICE + + +def test_photo_and_document_unaffected(): + """The fix is scoped to the audio branch — other types are unchanged.""" + assert _resolve("photo", ["image/png"]) is MessageType.PHOTO + assert _resolve("document", ["application/pdf"]) is MessageType.DOCUMENT + assert _resolve("text", []) is MessageType.TEXT diff --git a/tests/gateway/test_line_plugin.py b/tests/gateway/test_line_plugin.py index 9d897ce7a449..befcc1d0e1d5 100644 --- a/tests/gateway/test_line_plugin.py +++ b/tests/gateway/test_line_plugin.py @@ -1,6 +1,6 @@ """Tests for the LINE platform adapter plugin. -Covers the seven synthesis areas from the PR review: +Covers LINE adapter behavior from the PR review: 1. webhook signature verification (HMAC-SHA256, base64) + tampering rejection 2. inbound chat-id resolution for user / group / room sources @@ -8,8 +8,9 @@ 4. inbound dedup via webhookEventId 5. RequestCache state machine (PENDING → READY → DELIVERED, ERROR) 6. Markdown stripping with URL preservation + LINE-sized chunking -7. send routing: reply token preferred → push fallback → batched at 5/call -8. register() metadata + standalone_send shape +7. inbound media normalization to gateway message types and MIME metadata +8. send routing: reply token preferred → push fallback → batched at 5/call +9. register() metadata + standalone_send shape """ from __future__ import annotations @@ -19,7 +20,7 @@ import hmac import base64 import json -from unittest.mock import AsyncMock, MagicMock +from unittest.mock import AsyncMock, MagicMock, patch import pytest @@ -297,7 +298,84 @@ def test_split_caps_at_five_chunks(self): # --------------------------------------------------------------------------- -# 7. Send routing (reply -> push fallback, batching, system-bypass) +# 7. Inbound media normalization +# --------------------------------------------------------------------------- + +class TestInboundMedia: + + @pytest.fixture + def adapter(self, monkeypatch): + monkeypatch.delenv("LINE_CHANNEL_ACCESS_TOKEN", raising=False) + monkeypatch.delenv("LINE_CHANNEL_SECRET", raising=False) + from gateway.config import PlatformConfig + + cfg = PlatformConfig(enabled=True, extra={ + "channel_access_token": "tok", + "channel_secret": "sec", + }) + ad = LineAdapter(cfg) + ad._client = MagicMock() + ad._client.fetch_content = AsyncMock(return_value=b"line-bytes") + ad.handle_message = AsyncMock() + return ad + + def _event(self, msg_type, **message): + payload = {"type": msg_type, "id": f"{msg_type}-1"} + payload.update(message) + return { + "type": "message", + "replyToken": "reply-token", + "source": {"type": "group", "groupId": "Cline", "userId": "Uline"}, + "message": payload, + } + + def _captured_event(self, adapter): + adapter.handle_message.assert_awaited_once() + return adapter.handle_message.await_args.args[0] + + def test_image_message_uses_photo_type_and_image_mime(self, adapter): + with patch.object(_line, "cache_image_from_bytes", return_value="/cache/image.jpg") as cache: + asyncio.run(adapter._handle_message_event(self._event("image"))) + + cache.assert_called_once_with(b"line-bytes", ext=".jpg") + event = self._captured_event(adapter) + assert event.message_type is _line.MessageType.PHOTO + assert event.media_urls == ["/cache/image.jpg"] + assert event.media_types == ["image/jpeg"] + + def test_audio_message_uses_voice_type_and_audio_cache(self, adapter): + with patch.object(_line, "cache_audio_from_bytes", return_value="/cache/audio.m4a") as cache: + asyncio.run(adapter._handle_message_event(self._event("audio"))) + + cache.assert_called_once_with(b"line-bytes", ext=".m4a") + event = self._captured_event(adapter) + assert event.message_type is _line.MessageType.VOICE + assert event.media_urls == ["/cache/audio.m4a"] + assert event.media_types[0].startswith("audio/") + + def test_video_message_uses_video_type_and_video_cache(self, adapter): + with patch.object(_line, "cache_video_from_bytes", return_value="/cache/video.mp4") as cache: + asyncio.run(adapter._handle_message_event(self._event("video"))) + + cache.assert_called_once_with(b"line-bytes", ext=".mp4") + event = self._captured_event(adapter) + assert event.message_type is _line.MessageType.VIDEO + assert event.media_urls == ["/cache/video.mp4"] + assert event.media_types == ["video/mp4"] + + def test_file_message_uses_document_type_and_original_filename(self, adapter): + with patch.object(_line, "cache_document_from_bytes", return_value="/cache/report.pdf") as cache: + asyncio.run(adapter._handle_message_event(self._event("file", fileName="report.pdf"))) + + cache.assert_called_once_with(b"line-bytes", "report.pdf") + event = self._captured_event(adapter) + assert event.message_type is _line.MessageType.DOCUMENT + assert event.media_urls == ["/cache/report.pdf"] + assert event.media_types == ["application/pdf"] + + +# --------------------------------------------------------------------------- +# 8. Send routing (reply -> push fallback, batching, system-bypass) # --------------------------------------------------------------------------- class TestSendRouting: @@ -400,7 +478,7 @@ def test_format_message_strips_markdown(self, adapter): # --------------------------------------------------------------------------- -# 8. Register() metadata + plugin entry points +# 9. Register() metadata + plugin entry points # --------------------------------------------------------------------------- class TestRegister: diff --git a/tests/gateway/test_qqbot.py b/tests/gateway/test_qqbot.py index 562db57193d8..349b2a0ed799 100644 --- a/tests/gateway/test_qqbot.py +++ b/tests/gateway/test_qqbot.py @@ -5,6 +5,7 @@ from types import SimpleNamespace from unittest import mock +import httpx import pytest from gateway.config import PlatformConfig @@ -139,15 +140,24 @@ def test_voice_content_type(self): def test_audio_content_type(self): assert self._fn("audio/mp3", "file.mp3") is True - def test_voice_extension(self): + def test_voice_extension_fallback_when_content_type_empty(self): + """content_type='' with audio extension → True (extension fallback).""" assert self._fn("", "file.silk") is True def test_non_voice(self): assert self._fn("image/jpeg", "photo.jpg") is False - def test_audio_extension_amr(self): + def test_audio_extension_amr_fallback_when_content_type_empty(self): + """content_type='' with .amr extension → True (extension fallback).""" assert self._fn("", "recording.amr") is True + def test_file_upload_with_audio_extension(self): + """content_type='file' is never voice, even with audio extension.""" + assert self._fn("file", "song.mp3") is False + assert self._fn("file", "audio-30251.instrumental..wav") is False + assert self._fn("file", "recording.silk") is False + assert self._fn("file", "voice.amr") is False + # --------------------------------------------------------------------------- # Voice attachment SSRF protection @@ -206,6 +216,73 @@ def test_connect_accepts_is_reconnect_param(self): assert connected_explicit is False +# --------------------------------------------------------------------------- +# Voice attachment temp-file cleanup +# --------------------------------------------------------------------------- + +class TestVoiceAttachmentTempCleanup: + def _make_adapter(self, **extra): + from gateway.platforms.qqbot import QQAdapter + return QQAdapter(_make_config(**extra)) + + def _setup_download_mocks(self, adapter, content=b"RIFFmock-wav-audio-data"): + response = mock.Mock() + response.content = content + response.headers = {"content-type": "audio/wav"} + response.raise_for_status = mock.Mock() + + adapter._http_client = mock.AsyncMock() + adapter._http_client.get = mock.AsyncMock(return_value=response) + + def test_temp_wav_cleaned_up_on_stt_failure(self): + adapter = self._make_adapter(app_id="a", client_secret="b") + self._setup_download_mocks(adapter) + seen = {} + + async def _raise_transport_error(path): + seen["wav_path"] = path + raise httpx.TransportError("boom") + + with mock.patch("tools.url_safety.is_safe_url", return_value=True): + adapter._call_stt = mock.AsyncMock(side_effect=_raise_transport_error) + transcript = asyncio.run( + adapter._stt_voice_attachment( + "https://cdn.qq.com/voice.silk", + "audio/silk", + "voice.silk", + voice_wav_url="https://cdn.qq.com/voice.wav", + ) + ) + + assert transcript is None + assert "wav_path" in seen + assert not os.path.exists(seen["wav_path"]) + + def test_temp_wav_cleaned_up_on_stt_success(self): + adapter = self._make_adapter(app_id="a", client_secret="b") + self._setup_download_mocks(adapter) + seen = {} + + async def _return_transcript(path): + seen["wav_path"] = path + return "hello from qq voice" + + with mock.patch("tools.url_safety.is_safe_url", return_value=True): + adapter._call_stt = mock.AsyncMock(side_effect=_return_transcript) + transcript = asyncio.run( + adapter._stt_voice_attachment( + "https://cdn.qq.com/voice.silk", + "audio/silk", + "voice.silk", + voice_wav_url="https://cdn.qq.com/voice.wav", + ) + ) + + assert transcript == "hello from qq voice" + assert "wav_path" in seen + assert not os.path.exists(seen["wav_path"]) + + # --------------------------------------------------------------------------- # WebSocket proxy handling # --------------------------------------------------------------------------- diff --git a/tests/gateway/test_stt_config.py b/tests/gateway/test_stt_config.py index 46326c8cbf87..e38e8610515b 100644 --- a/tests/gateway/test_stt_config.py +++ b/tests/gateway/test_stt_config.py @@ -90,6 +90,9 @@ async def test_enrich_message_with_transcription_avoids_bogus_no_provider_messag with patch( "tools.transcription_tools.transcribe_audio", return_value={"success": False, "error": "VOICE_TOOLS_OPENAI_KEY not set"}, + ), patch( + "tools.transcription_tools.transcribe_audio_local_fallback", + return_value={"success": False, "error": "not installed"}, ): result, transcripts = await runner._enrich_message_with_transcription( "caption", @@ -97,13 +100,42 @@ async def test_enrich_message_with_transcription_avoids_bogus_no_provider_messag ) assert "No STT provider is configured" not in result - assert "[voice message could not be transcribed]" in result + assert "voice message could not be transcribed automatically" in result + assert "/tmp/voice.ogg" in result # The opaque backend cause must NOT leak into the LLM-visible prompt. assert "VOICE_TOOLS_OPENAI_KEY" not in result assert "caption" in result assert transcripts == [] +@pytest.mark.asyncio +async def test_enrich_message_with_transcription_falls_back_to_installed_local_stt(): + from gateway.run import GatewayRunner + + runner = GatewayRunner.__new__(GatewayRunner) + runner.config = GatewayConfig(stt_enabled=True) + + with patch( + "tools.transcription_tools.transcribe_audio", + return_value={"success": False, "error": "configured provider unavailable"}, + ), patch( + "tools.transcription_tools.transcribe_audio_local_fallback", + return_value={ + "success": True, + "transcript": "recovered locally", + "provider": "local", + }, + ) as local_fallback: + result, transcripts = await runner._enrich_message_with_transcription( + "", + ["/tmp/voice.ogg"], + ) + + assert result == '"recovered locally"' + assert transcripts == ["recovered locally"] + local_fallback.assert_called_once_with("/tmp/voice.ogg") + + @pytest.mark.asyncio async def test_enrich_message_with_transcription_returns_tuple_for_empty_content_placeholder(): """A successful transcription whose caption is the empty-content placeholder diff --git a/tests/gateway/test_weixin.py b/tests/gateway/test_weixin.py index 5169666e8baf..e5343d7e021d 100644 --- a/tests/gateway/test_weixin.py +++ b/tests/gateway/test_weixin.py @@ -27,6 +27,41 @@ def _make_adapter() -> WeixinAdapter: ) +class TestWeixinInboundVoiceTranscript: + def test_voice_transcript_keeps_voice_origin_marker(self): + item_list = [ + { + "type": weixin.ITEM_VOICE, + "voice_item": {"text": "帮我查一下今天天气"}, + } + ] + + assert weixin._extract_text(item_list) == ( + "[Voice transcription provided by Weixin]\n" + "帮我查一下今天天气" + ) + + def test_typed_text_remains_unmarked(self): + item_list = [ + { + "type": weixin.ITEM_TEXT, + "text_item": {"text": "帮我查一下今天天气"}, + } + ] + + assert weixin._extract_text(item_list) == "帮我查一下今天天气" + + def test_empty_voice_transcript_keeps_empty_fallback(self): + item_list = [ + { + "type": weixin.ITEM_VOICE, + "voice_item": {"text": ""}, + } + ] + + assert weixin._extract_text(item_list) == "" + + class TestWeixinFormatting: def test_format_message_preserves_markdown(self): adapter = _make_adapter() @@ -1205,3 +1240,306 @@ def test_get_updates_returns_empty_sentinel_on_timeout(self): ) ) assert result == {"ret": 0, "msgs": [], "get_updates_buf": "buf-123"} + + +class TestWeixinVoiceAlwaysDownloaded: + """Regression tests for #27300: when WeChat (Weixin) returns a + ``voice_item.text`` (Tencent Cloud's STT) we must still download + the raw audio and route it through Hermes' own STT pipeline. + + Non-Chinese users currently see garbled transcriptions because the + existing code short-circuits in two places: ``_download_voice`` + returns ``None`` whenever Tencent provided *any* text (even + incorrect), and ``_extract_text`` returns that text as the message + body. The fix is to always download and never return Tencent's + text — the central STT pipeline in ``gateway/run.py`` produces + the actual body from the downloaded audio. + """ + + def _make_voice_item(self, text: str = "") -> dict: + """Build a minimal voice item with media + optional Tencent text.""" + return { + "type": weixin.ITEM_VOICE, + "voice_item": { + "text": text, + "media": { + "encrypt_query_param": "q", + "aes_key": "a" * 32, + "full_url": "https://example.invalid/voice.silk", + }, + }, + } + + @pytest.mark.asyncio + async def test_download_voice_returns_path_when_tencent_text_set(self, tmp_path, monkeypatch): + """#27300 PRIMARY: ``_download_voice`` must not short-circuit on + ``voice_item.text``. The audio is needed so Hermes' own STT can + re-transcribe when Tencent's text is in the wrong language. + """ + adapter = _make_adapter() + adapter._cdn_base_url = "https://example.invalid" + adapter._poll_session = Mock() + + fake_audio_bytes = b"\\x00\\x01\\x02FAKE_SILK" + monkeypatch.setattr(weixin, "cache_audio_from_bytes", + lambda data, ext: str(tmp_path / f"voice.{ext.lstrip('.')}")) + + async def _fake_download(session, *, cdn_base_url, encrypted_query_param, + aes_key_b64, full_url, timeout_seconds): + return fake_audio_bytes + + monkeypatch.setattr(weixin, "_download_and_decrypt_media", _fake_download) + + item = self._make_voice_item(text="garbled-tencent-transcript") + result = await adapter._download_voice(item) + + # Currently broken: returns None when voice_item.text is set. + # After fix: returns a local path so the central STT pipeline + # can pick it up and re-transcribe. + assert result is not None, ( + "_download_voice returned None even though raw audio is " + "available — Hermes' STT pipeline needs the audio to handle " + "non-Chinese voice messages (#27300)." + ) + assert result.endswith(".silk") + + def test_extract_text_does_not_return_tencent_voice_text(self): + """#27300 SECONDARY: ``_extract_text`` must not return + ``voice_item.text`` verbatim. That text is Tencent Cloud's + STT output, which is wrong for non-Chinese audio and the + whole reason #27300 was filed. Returning empty forces the + central STT pipeline's transcript to become the body. + """ + item_list = [self._make_voice_item(text="garbled-tencent-transcript")] + result = weixin._extract_text(item_list) + # Currently broken: returns "garbled-tencent-transcript". + # After fix: returns "" (empty string) so the central pipeline + # transcript replaces it as the user-visible body. + assert result != "garbled-tencent-transcript", ( + "_extract_text returned Tencent's text directly — for " + "non-Chinese audio this is garbage; the central STT " + "pipeline's transcript should be the body (#27300)." + ) + + def test_extract_text_voice_only_returns_empty(self): + """When the only item is a voice attachment (no text item), + ``_extract_text`` should return empty so the central STT + pipeline's transcript becomes the body. Currently returns + Tencent's text which is what the bug is about. + """ + item_list = [self._make_voice_item(text="какой-то текст")] + result = weixin._extract_text(item_list) + assert result == "", ( + "Voice-only message: _extract_text should return empty so " + "the central STT pipeline output replaces it as the body." + ) + + def test_extract_text_still_returns_text_for_text_items(self): + """Sanity: the fix must not regress the text-item path. A plain + text message should still produce its text body. + """ + item_list = [{ + "type": weixin.ITEM_TEXT, + "text_item": {"text": "hello world"}, + }] + assert weixin._extract_text(item_list) == "hello world" + + @pytest.mark.asyncio + async def test_collect_media_includes_voice_when_tencent_text_set(self, tmp_path, monkeypatch): + """#27300 INTEGRATION: ``_collect_media`` should add a ``.silk`` + path to ``media_paths`` even when Tencent returned text, so the + central STT pipeline can re-transcribe. Currently the + short-circuit in ``_download_voice`` means the audio is never + downloaded, and the message body is whatever Tencent wrote + (garbled for non-Chinese audio). + """ + adapter = _make_adapter() + adapter._cdn_base_url = "https://example.invalid" + adapter._poll_session = Mock() + + monkeypatch.setattr(weixin, "cache_audio_from_bytes", + lambda data, ext: str(tmp_path / f"voice.{ext.lstrip('.')}")) + + async def _fake_download(session, *, cdn_base_url, encrypted_query_param, + aes_key_b64, full_url, timeout_seconds): + return b"\\x00FAKE" + + monkeypatch.setattr(weixin, "_download_and_decrypt_media", _fake_download) + + media_paths: list = [] + media_types: list = [] + item = self._make_voice_item(text="какой-то текст") + await adapter._collect_media(item, media_paths, media_types) + + assert len(media_paths) == 1, ( + "_collect_media dropped the voice attachment because " + "voice_item.text was set — Hermes' STT never gets a " + "chance to re-transcribe (#27300)." + ) + assert media_types == ["audio/silk"] + + +class TestWeixinVoiceGatewayHandoff: + """#27300 integration-level regression: the routing fix must not only + download the audio and drop Tencent's text at the adapter level — the + inbound voice item must surface as a VOICE ``MessageEvent`` carrying the + ``audio/silk`` media, and that event must reach the runner's central STT + pipeline (``_enrich_message_with_transcription``) instead of being trusted + as already-transcribed text. This covers the gateway-runner handoff that the + adapter-only tests above do not exercise. + """ + + def _inbound_voice_message(self, text: str) -> dict: + return { + "from_user_id": "user-123", + "to_user_id": "test-account", + "message_id": "msg-voice-1", + "msg_type": 1, + "item_list": [ + { + "type": weixin.ITEM_VOICE, + "voice_item": { + "text": text, + "media": { + "encrypt_query_param": "q", + "aes_key": "a" * 32, + "full_url": "https://example.invalid/voice.silk", + }, + }, + } + ], + } + + @pytest.mark.asyncio + async def test_voice_item_builds_voice_event_with_silk_media(self, tmp_path, monkeypatch): + """Inbound voice item carrying Tencent text must produce a VOICE + event whose media is ``audio/silk`` — the exact shape the runner keys + off to route into Hermes' STT pipeline. + """ + adapter = _make_adapter() + adapter._poll_session = Mock() # satisfies the `assert` in _process_message + adapter._token = None # typing-ticket task early-returns when no token + adapter._cdn_base_url = "https://example.invalid" + + monkeypatch.setattr(weixin, "cache_audio_from_bytes", + lambda data, ext: str(tmp_path / f"voice.{ext.lstrip('.')}")) + async def _fake_download(*a, **k): + return b"\x00\x01FAKE_SILK" + monkeypatch.setattr(weixin, "_download_and_decrypt_media", _fake_download) + + captured = {} + + async def _capture(event): + captured["event"] = event + + adapter.handle_message = _capture + + await adapter._process_message(self._inbound_voice_message("garbled-tencent-text")) + + assert "event" in captured, "no MessageEvent handed to handle_message" + event = captured["event"] + assert event.message_type == MessageType.VOICE, ( + f"expected VOICE event, got {event.message_type}" + ) + assert event.media_types == ["audio/silk"], ( + f"voice event must carry audio/silk media, got {event.media_types}" + ) + assert len(event.media_urls) == 1, "expected one local silk audio path" + + @pytest.mark.asyncio + async def test_voice_event_body_is_not_tencent_text(self, tmp_path, monkeypatch): + """The VOICE event handed to the runner must NOT carry Tencent's STT + text as its body — the central pipeline's transcript replaces it. + """ + adapter = _make_adapter() + adapter._poll_session = Mock() + adapter._token = None + adapter._cdn_base_url = "https://example.invalid" + + monkeypatch.setattr(weixin, "cache_audio_from_bytes", + lambda data, ext: str(tmp_path / f"voice.{ext.lstrip('.')}")) + async def _fake_download(*a, **k): + return b"\x00\x01FAKE_SILK" + monkeypatch.setattr(weixin, "_download_and_decrypt_media", _fake_download) + + captured = {} + + async def _capture(event): + captured["event"] = event + + adapter.handle_message = _capture + + tencent_text = "garbled English phonemes for a Russian voice" + await adapter._process_message(self._inbound_voice_message(tencent_text)) + + assert "event" in captured + event = captured["event"] + # The text field must be empty (Tencent text dropped) so the runner + # has no pre-filled body and routes the audio to STT. + assert event.text != tencent_text, ( + "VOICE event body leaked Tencent's STT text — runner would trust " + "the wrong transcript instead of re-transcribing (#27300)." + ) + + @pytest.mark.asyncio + async def test_runner_routes_voice_event_to_transcription(self, tmp_path, monkeypatch): + """Regression for the gateway-runner handoff: a VOICE event carrying + ``audio/silk`` must reach ``_enrich_message_with_transcription`` so the + central STT pipeline produces the body. We drive the real runner method + (patched to capture the call) using the same selection rule the runner + applies in ``gateway/run.py`` (audio/* media -> transcription path). + """ + import gateway.run as run_module + from gateway.run import GatewayRunner + from gateway.platforms.base import MessageType as _MT + from gateway.session import SessionSource + from types import SimpleNamespace + from unittest.mock import AsyncMock + + runner = object.__new__(GatewayRunner) + runner.config = SimpleNamespace(stt_enabled=True) + + captured = {} + real_enrich = GatewayRunner._enrich_message_with_transcription + + async def _spy_enrich(self, user_text, audio_paths): + captured["audio_paths"] = audio_paths + # Delegate to the real implementation so the contract is exercised + # end-to-end rather than re-implemented. + return await real_enrich(self, user_text, audio_paths) + + monkeypatch.setattr( + run_module.GatewayRunner, + "_enrich_message_with_transcription", + _spy_enrich, + ) + + event = MessageEvent( + text="", + message_type=_MT.VOICE, + source=SessionSource(platform="weixin", chat_id="user-123", chat_type="dm"), + raw_message={}, + message_id="msg-voice-1", + media_urls=[str(tmp_path / "voice.silk")], + media_types=["audio/silk"], + timestamp=__import__("datetime").datetime.now(), + ) + + # Same selection rule the runner uses: audio/* media (or VOICE/AUDIO + # message type) marks the path for transcription. + audio_paths = [ + p for i, p in enumerate(event.media_urls) + if (event.media_types[i].startswith("audio/") + or event.message_type in (_MT.VOICE, _MT.AUDIO)) + ] + assert audio_paths, "VOICE/audio/silk event must be selected for transcription" + + enriched, transcripts = await runner._enrich_message_with_transcription( + event.text, audio_paths + ) + assert captured.get("audio_paths") == audio_paths, ( + "the VOICE event's audio/silk path must reach " + "_enrich_message_with_transcription" + ) + # Real implementation echoes a transcript back when STT is enabled. + assert isinstance(enriched, str) diff --git a/tests/gateway/test_whatsapp_formatting.py b/tests/gateway/test_whatsapp_formatting.py index 3692fe5a12d4..b17a057265f1 100644 --- a/tests/gateway/test_whatsapp_formatting.py +++ b/tests/gateway/test_whatsapp_formatting.py @@ -344,6 +344,35 @@ async def test_quoted_reply_metadata_is_preserved_in_raw_message(self): assert event.raw_message["quotedRemoteJid"] == "15551234567@s.whatsapp.net" assert event.raw_message["hasQuotedMessage"] is True + @pytest.mark.asyncio + async def test_captionless_voice_note_drops_bridge_placeholder(self, tmp_path, monkeypatch): + adapter = _make_adapter() + voice_path = tmp_path / "aud_voice.ogg" + voice_path.write_bytes(b"fake audio") + monkeypatch.setattr( + "plugins.platforms.whatsapp.adapter._is_allowed_bridge_path", + lambda path: path == str(voice_path), + ) + data = { + "messageId": "voice-msg", + "chatId": "15551234567@s.whatsapp.net", + "senderId": "15551234567@s.whatsapp.net", + "senderName": "Tester", + "chatName": "Tester", + "isGroup": False, + "body": "[ptt received]", + "hasMedia": True, + "mediaType": "ptt", + "mime": "audio/ogg", + "mediaUrls": [str(voice_path)], + } + + event = await adapter._build_message_event(data) + + assert event is not None + assert event.text == "" + assert event.media_urls == [str(voice_path)] + # --------------------------------------------------------------------------- # display_config tier classification diff --git a/tests/plugins/platforms/photon/test_inbound.py b/tests/plugins/platforms/photon/test_inbound.py index 521bc26aa915..3531e6f56882 100644 --- a/tests/plugins/platforms/photon/test_inbound.py +++ b/tests/plugins/platforms/photon/test_inbound.py @@ -6,6 +6,7 @@ """ from __future__ import annotations +import asyncio import base64 import json from pathlib import Path @@ -362,3 +363,220 @@ def test_check_requirements_without_node(monkeypatch: pytest.MonkeyPatch) -> Non monkeypatch.setattr(adapter_mod.shutil, "which", lambda _name: None) assert adapter_mod.check_requirements() is False + + +# --------------------------------------------------------------------------- +# CAF attachment promotion + U+FFFC placeholder tests +# --------------------------------------------------------------------------- + +_CAF_BYTES = b"caff" + b"\x00" * 60 # Minimal CAF header magic + + +def _caf_attachment_event( + content: Dict[str, Any], msg_id: str = "spc-msg-caf" +) -> Dict[str, Any]: + return { + "messageId": msg_id, + "space": {"id": "+155****4567", "type": "dm", "phone": "+155****4567"}, + "sender": {"id": "+155****4567"}, + "content": {"type": "attachment", **content}, + "timestamp": "2026-05-14T19:06:32.000Z", + } + + +@pytest.mark.asyncio +async def test_caf_attachment_named_promoted_to_voice( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """A named .caf attachment is promoted to VOICE for STT routing.""" + adapter = _make_adapter(monkeypatch) + captured = _capture(adapter, monkeypatch) + + raw = _CAF_BYTES + event = _caf_attachment_event( + { + "name": "voice_note.caf", + "mimeType": "audio/x-caf", + "size": len(raw), + "data": base64.b64encode(raw).decode("ascii"), + "encoding": "base64", + } + ) + await adapter._dispatch_inbound(event) + + assert len(captured) == 1 + ev = captured[0] + assert ev.message_type == MessageType.VOICE + assert ev.media_types == ["audio/x-caf"] + assert len(ev.media_urls) == 1 + cached = Path(ev.media_urls[0]) + try: + assert cached.is_file() + assert cached.read_bytes() == raw + assert ev.text == "(voice)" + finally: + cached.unlink(missing_ok=True) + + +@pytest.mark.asyncio +async def test_caf_attachment_unnamed_promoted_via_mime( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """An unnamed attachment with mimeType audio/x-caf is promoted to VOICE. + + The sidecar sends "(unnamed)" when no filename is supplied, so the MIME + type must be the fallback signal for CAF promotion. + """ + adapter = _make_adapter(monkeypatch) + captured = _capture(adapter, monkeypatch) + + raw = _CAF_BYTES + event = _caf_attachment_event( + { + "name": "(unnamed)", + "mimeType": "audio/x-caf", + "size": len(raw), + "data": base64.b64encode(raw).decode("ascii"), + "encoding": "base64", + } + ) + await adapter._dispatch_inbound(event) + + assert len(captured) == 1 + ev = captured[0] + assert ev.message_type == MessageType.VOICE + assert ev.media_types == ["audio/x-caf"] + cached = Path(ev.media_urls[0]) + try: + assert cached.is_file() + assert cached.suffix == ".caf" + finally: + cached.unlink(missing_ok=True) + + +@pytest.mark.asyncio +async def test_fffc_placeholder_no_dispatch( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """A U+FFFC placeholder text does not trigger a message dispatch.""" + adapter = _make_adapter(monkeypatch) + captured = _capture(adapter, monkeypatch) + + event = _dm_event("\ufffc", msg_id="spc-msg-fffc") + chat_key = event["space"]["id"] + await adapter._dispatch_inbound(event) + + assert len(captured) == 0 + assert chat_key in adapter._pending_fffc + + +@pytest.mark.asyncio +async def test_fffc_placeholder_not_recorded_as_last_inbound( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """The U+FFFC placeholder must not be recorded as the reaction target. + + _record_last_inbound runs after the U+FFFC early-return, so the placeholder + message id is never stored. A subsequent real message will be recorded. + """ + adapter = _make_adapter(monkeypatch) + _capture(adapter, monkeypatch) + + fffc_event = _dm_event("\ufffc", msg_id="spc-msg-fffc") + chat_key = fffc_event["space"]["id"] + await adapter._dispatch_inbound(fffc_event) + assert chat_key not in adapter._last_inbound_by_chat + + real_event = _dm_event("hello", msg_id="spc-msg-real") + await adapter._dispatch_inbound(real_event) + assert adapter._last_inbound_by_chat.get(chat_key) == "spc-msg-real" + + +@pytest.mark.asyncio +async def test_fffc_then_attachment_cancels_timeout( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """When an attachment arrives after a U+FFFC placeholder, the pending + timeout task is cancelled and the attachment is dispatched normally.""" + adapter = _make_adapter(monkeypatch) + captured = _capture(adapter, monkeypatch) + + fffc_event = _dm_event("\ufffc", msg_id="spc-msg-fffc") + chat_key = fffc_event["space"]["id"] + await adapter._dispatch_inbound(fffc_event) + assert len(captured) == 0 + assert chat_key in adapter._pending_fffc + + raw = _CAF_BYTES + att_event = _caf_attachment_event( + { + "name": "voice.caf", + "mimeType": "audio/x-caf", + "size": len(raw), + "data": base64.b64encode(raw).decode("ascii"), + "encoding": "base64", + }, + msg_id="spc-msg-att", + ) + att_event["space"]["id"] = chat_key + await adapter._dispatch_inbound(att_event) + + assert len(captured) == 1 + assert captured[0].message_type == MessageType.VOICE + assert chat_key not in adapter._pending_fffc + assert adapter._last_inbound_by_chat.get(chat_key) == "spc-msg-att" + + cached = Path(captured[0].media_urls[0]) + try: + assert cached.is_file() + finally: + cached.unlink(missing_ok=True) + + +@pytest.mark.asyncio +async def test_fffc_timeout_fires_when_no_attachment( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """When no attachment arrives within the timeout, the pending entry is + cleaned up and a warning is logged.""" + import plugins.platforms.photon.adapter as adapter_mod + + monkeypatch.setattr(adapter_mod, "_FFFC_WAIT_SECONDS", 0.1) + + adapter = _make_adapter(monkeypatch) + captured = _capture(adapter, monkeypatch) + + fffc_event = _dm_event("\ufffc", msg_id="spc-msg-fffc") + chat_key = fffc_event["space"]["id"] + await adapter._dispatch_inbound(fffc_event) + assert chat_key in adapter._pending_fffc + + await asyncio.sleep(0.3) + + assert chat_key not in adapter._pending_fffc + assert len(captured) == 0 + + +@pytest.mark.asyncio +async def test_disconnect_cancels_pending_fffc_tasks( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """disconnect() cancels any pending U+FFFC placeholder tasks.""" + adapter = _make_adapter(monkeypatch) + _capture(adapter, monkeypatch) + + await adapter._dispatch_inbound(_dm_event("\ufffc", msg_id="spc-msg-fffc")) + assert len(adapter._pending_fffc) == 1 + + async def _noop_stop_sidecar(): + pass + + monkeypatch.setattr(adapter, "_stop_sidecar", _noop_stop_sidecar) + monkeypatch.setattr(adapter, "_inbound_running", False) + monkeypatch.setattr(adapter, "_inbound_task", None) + monkeypatch.setattr(adapter, "_sidecar_health_task", None) + monkeypatch.setattr(adapter, "_http_client", None) + + await adapter.disconnect() + + assert len(adapter._pending_fffc) == 0 diff --git a/tests/tools/test_transcription.py b/tests/tools/test_transcription.py index d5c3903ff130..8ab4d646a57c 100644 --- a/tests/tools/test_transcription.py +++ b/tests/tools/test_transcription.py @@ -380,6 +380,44 @@ def test_invalid_file_returns_error(self): assert "not found" in result["error"] +class TestLocalFallback: + + def test_uses_installed_faster_whisper_without_changing_provider(self, tmp_path): + audio_file = tmp_path / "test.ogg" + audio_file.write_bytes(b"fake audio") + + with patch( + "tools.transcription_tools._load_stt_config", + return_value={"provider": "openai", "local": {"model": "small"}}, + ), patch( + "tools.transcription_tools._HAS_FASTER_WHISPER", + True, + ), patch( + "tools.transcription_tools._transcribe_local", + return_value={"success": True, "transcript": "local result"}, + ) as mock_local: + from tools.transcription_tools import transcribe_audio_local_fallback + + result = transcribe_audio_local_fallback(str(audio_file)) + + assert result["transcript"] == "local result" + mock_local.assert_called_once_with(str(audio_file), "small") + + def test_does_not_install_when_no_local_backend_exists(self, tmp_path): + audio_file = tmp_path / "test.ogg" + audio_file.write_bytes(b"fake audio") + + with patch("tools.transcription_tools._HAS_FASTER_WHISPER", False), patch( + "tools.transcription_tools._has_local_command", return_value=False + ): + from tools.transcription_tools import transcribe_audio_local_fallback + + result = transcribe_audio_local_fallback(str(audio_file)) + + assert result["success"] is False + assert "installed local STT" in result["error"] + + # --------------------------------------------------------------------------- # Model name normalisation for local providers # --------------------------------------------------------------------------- diff --git a/tests/tools/test_transcription_tools.py b/tests/tools/test_transcription_tools.py index ee0287ca01f6..1f8032b5e6b4 100644 --- a/tests/tools/test_transcription_tools.py +++ b/tests/tools/test_transcription_tools.py @@ -11,6 +11,7 @@ import subprocess import types import wave +from pathlib import Path from unittest.mock import MagicMock, call, patch import pytest @@ -2173,3 +2174,144 @@ def test_is_local_or_private_url(self): assert _is_local_or_private_url("http://stt.internal/v1") assert not _is_local_or_private_url("https://api.openai.com/v1") assert not _is_local_or_private_url("") + + +# ============================================================================ +# CAF (iMessage voice note) conversion tests +# ============================================================================ + +class TestCafConversion: + """Tests for _convert_caf_to_wav and CAF dispatch in transcribe_audio.""" + + def test_convert_caf_with_ffmpeg(self, tmp_path, monkeypatch): + """_convert_caf_to_wav uses ffmpeg when available.""" + caf_path = tmp_path / "voice.caf" + caf_path.write_bytes(b"caff\x00" * 20) + wav_path = str(tmp_path / "voice.wav") + + def fake_run(cmd, **kwargs): + Path(wav_path).write_bytes(b"RIFF\x00\x00\x00\x00") + return MagicMock(returncode=0) + + monkeypatch.setattr( + "tools.transcription_tools._find_ffmpeg_binary", + lambda: "/usr/bin/ffmpeg", + ) + monkeypatch.setattr(subprocess, "run", fake_run) + + from tools.transcription_tools import _convert_caf_to_wav + result = _convert_caf_to_wav(str(caf_path)) + assert result == wav_path + assert Path(result).exists() + + def test_convert_caf_fallback_to_afconvert(self, tmp_path, monkeypatch): + """When ffmpeg is not found, falls back to afconvert (macOS).""" + caf_path = tmp_path / "voice.caf" + caf_path.write_bytes(b"caff\x00" * 20) + wav_path = str(tmp_path / "voice.wav") + + call_count = {"n": 0} + + def fake_run(cmd, **kwargs): + call_count["n"] += 1 + if cmd[0] == "/usr/bin/ffmpeg": + raise subprocess.CalledProcessError(1, cmd) + Path(wav_path).write_bytes(b"RIFF\x00\x00\x00\x00") + return MagicMock(returncode=0) + + monkeypatch.setattr( + "tools.transcription_tools._find_ffmpeg_binary", + lambda: "/usr/bin/ffmpeg", + ) + monkeypatch.setattr(subprocess, "run", fake_run) + monkeypatch.setattr( + "tools.transcription_tools.shutil.which", + lambda name: "/usr/bin/afconvert" if name == "afconvert" else None, + ) + + from tools.transcription_tools import _convert_caf_to_wav + result = _convert_caf_to_wav(str(caf_path)) + assert result == wav_path + assert call_count["n"] == 2 + + def test_convert_caf_all_converters_fail(self, tmp_path, monkeypatch): + """When both ffmpeg and afconvert are unavailable, returns None.""" + caf_path = tmp_path / "voice.caf" + caf_path.write_bytes(b"caff\x00" * 20) + + monkeypatch.setattr( + "tools.transcription_tools._find_ffmpeg_binary", lambda: None + ) + monkeypatch.setattr( + "tools.transcription_tools.shutil.which", lambda name: None + ) + + from tools.transcription_tools import _convert_caf_to_wav + result = _convert_caf_to_wav(str(caf_path)) + assert result is None + + def test_transcribe_caf_converted_before_groq(self, tmp_path, monkeypatch): + """transcribe_audio converts .caf to .wav before dispatching to Groq.""" + caf_path = tmp_path / "voice.caf" + caf_path.write_bytes(b"caff\x00" * 20) + wav_path = str(tmp_path / "voice.wav") + + def fake_convert(file_path): + Path(wav_path).write_bytes(b"RIFF\x00\x00\x00\x00") + return wav_path + + with patch("tools.transcription_tools._load_stt_config", + return_value={"provider": "groq"}), \ + patch("tools.transcription_tools._get_provider", + return_value="groq"), \ + patch("tools.transcription_tools._convert_caf_to_wav", + side_effect=fake_convert) as mock_convert, \ + patch("tools.transcription_tools._transcribe_groq", + return_value={"success": True, "transcript": "hello", + "provider": "groq"}) as mock_groq: + from tools.transcription_tools import transcribe_audio + result = transcribe_audio(str(caf_path)) + + assert result["success"] is True + mock_convert.assert_called_once_with(str(caf_path)) + mock_groq.assert_called_once() + call_args = mock_groq.call_args + sent_path = call_args[0][0] if call_args[0] else call_args[1].get("file_path") + assert sent_path == wav_path + + def test_transcribe_caf_conversion_failure_returns_error( + self, tmp_path, monkeypatch + ): + """When CAF conversion fails, transcribe_audio returns an error.""" + caf_path = tmp_path / "voice.caf" + caf_path.write_bytes(b"caff\x00" * 20) + + with patch("tools.transcription_tools._load_stt_config", + return_value={"provider": "groq"}), \ + patch("tools.transcription_tools._get_provider", + return_value="groq"), \ + patch("tools.transcription_tools._convert_caf_to_wav", + return_value=None): + from tools.transcription_tools import transcribe_audio + result = transcribe_audio(str(caf_path)) + + assert result["success"] is False + assert "could not be converted" in result["error"] + + def test_transcribe_caf_not_converted_for_local(self, tmp_path, monkeypatch): + """CAF conversion is skipped for local provider (native handling).""" + caf_path = tmp_path / "voice.caf" + caf_path.write_bytes(b"caff\x00" * 20) + + with patch("tools.transcription_tools._load_stt_config", + return_value={"provider": "local"}), \ + patch("tools.transcription_tools._get_provider", + return_value="local"), \ + patch("tools.transcription_tools._convert_caf_to_wav") as mock_convert, \ + patch("tools.transcription_tools._transcribe_local", + return_value={"success": True, "transcript": "hi"}): + from tools.transcription_tools import transcribe_audio + result = transcribe_audio(str(caf_path)) + + assert result["success"] is True + mock_convert.assert_not_called() diff --git a/tools/transcription_tools.py b/tools/transcription_tools.py index a088d9654bea..c9ddd23b891e 100644 --- a/tools/transcription_tools.py +++ b/tools/transcription_tools.py @@ -121,7 +121,7 @@ def _safe_find_spec(module_name: str) -> bool: ELEVENLABS_STT_BASE_URL = os.getenv("ELEVENLABS_STT_BASE_URL", "https://api.elevenlabs.io/v1") # DeepInfra STT base URL now resolved via hermes_cli.models.deepinfra_base_url (shared). -SUPPORTED_FORMATS = {".mp3", ".mp4", ".mpeg", ".mpga", ".m4a", ".wav", ".webm", ".ogg", ".oga", ".opus", ".aac", ".flac"} +SUPPORTED_FORMATS = {".mp3", ".mp4", ".mpeg", ".mpga", ".m4a", ".wav", ".webm", ".ogg", ".oga", ".opus", ".aac", ".flac", ".caf"} LOCAL_NATIVE_AUDIO_FORMATS = {".wav", ".aiff", ".aif"} MAX_FILE_SIZE = 25 * 1024 * 1024 # 25 MB @@ -1491,6 +1491,32 @@ def _prepare_local_audio(file_path: str, work_dir: str) -> tuple[Optional[str], return None, f"Failed to convert audio for local STT: {details}" +def _convert_caf_to_wav(file_path: str) -> Optional[str]: + """Convert CAF to WAV using ffmpeg or afconvert (macOS).""" + audio_path = Path(file_path) + wav_path = os.path.join(audio_path.parent, f"{audio_path.stem}.wav") + ffmpeg = _find_ffmpeg_binary() + if ffmpeg: + try: + subprocess.run([ffmpeg, "-y", "-i", file_path, wav_path], + check=True, capture_output=True, text=True, + timeout=300, stdin=subprocess.DEVNULL, + creationflags=windows_hide_flags()) + return wav_path + except (subprocess.CalledProcessError, subprocess.TimeoutExpired) as e: + logger.warning("ffmpeg CAF to WAV failed for %s: %s", file_path, e) + afconvert = shutil.which("afconvert") + if afconvert: + try: + subprocess.run([afconvert, file_path, wav_path, "-d", "LEI16", "-f", "WAVE"], + check=True, capture_output=True, text=True, + timeout=300, stdin=subprocess.DEVNULL) + return wav_path + except (subprocess.CalledProcessError, subprocess.TimeoutExpired) as e: + logger.warning("afconvert CAF to WAV failed for %s: %s", file_path, e) + return None + + def _transcribe_local_command(file_path: str, model_name: str) -> Dict[str, Any]: """Run the configured local STT command template and read back a .txt transcript.""" command_template = _get_local_command_template() @@ -2125,6 +2151,15 @@ def _transcribe_prepared_audio(file_path: str, model: Optional[str] = None) -> D if error: return error + # Convert CAF (iMessage voice notes) to WAV for cloud STT providers. + if Path(file_path).suffix.lower() == ".caf" and provider not in ("local", "local_command"): + converted = _convert_caf_to_wav(file_path) + if converted: + file_path = converted + else: + return {"success": False, "transcript": "", + "error": "CAF audio could not be converted to WAV."} + if provider == "local": local_cfg = stt_config.get("local") or {} model_name = _normalize_local_model( @@ -2291,6 +2326,42 @@ def _is_local_or_private_url(url: str) -> bool: return False +def transcribe_audio_local_fallback( + file_path: str, + model: Optional[str] = None, +) -> Dict[str, Any]: + """Try an already-installed local STT backend without changing config. + + This is intended for passive inbound-media recovery after the configured + provider has failed. It deliberately does not lazy-install dependencies or + fall through to another cloud provider. + """ + error = _validate_audio_file(file_path) + if error: + return error + + stt_config = _load_stt_config() + local_cfg = stt_config.get("local") or {} + local_model = model or local_cfg.get("model", DEFAULT_LOCAL_MODEL) + + if _HAS_FASTER_WHISPER: + return _transcribe_local( + file_path, + _normalize_local_model(local_model), + ) + if _has_local_command(): + return _transcribe_local_command( + file_path, + _normalize_local_command_model(local_model), + ) + return { + "success": False, + "transcript": "", + "error": "No installed local STT backend is available.", + "provider": "local", + } + + def _resolve_openai_audio_client_config() -> tuple[str, str]: """Return direct OpenAI audio config or a managed gateway fallback.""" stt_config = _load_stt_config()