fix(gateway): inbound voice classification/routing for Feishu, DingTalk, LINE, QQ, Photon, WhatsApp, Weixin

contributors/emails/Zioywishing@users.noreply.github.com

@@ -0,0 +1,2 @@
+Zioywishing
+# PR #35705 salvage

contributors/emails/armaandhawan61@gmail.com

@@ -0,0 +1,2 @@
+VIVAAN-DHAWAN
+# PR #66626 salvage

contributors/emails/florianvalade@Florians-Mac-mini.local

@@ -0,0 +1,2 @@
+FlorianVal
+# PR #66326 salvage

contributors/emails/jazzwu@163.com

@@ -0,0 +1,2 @@
+rayjerrywoo
+# PR #50014 salvage

contributors/emails/luna@hermes.local

@@ -0,0 +1,2 @@
+seamusmore
+# PR #40592 salvage

contributors/emails/zgzczzw@users.noreply.github.com

@@ -0,0 +1,2 @@
+zgzczzw
+# PR #65022 salvage

gateway/platforms/qqbot/adapter.py

@@ -1812,16 +1812,14 @@ def _is_voice_content_type(content_type: str, filename: str) -> bool:
         fn = filename.strip().lower()
         if ct == "voice" or ct.startswith("audio/"):
             return True
+        # QQ file uploads have content_type="file".  Without this guard,
+        # any uploaded audio file (e.g. .wav, .mp3) would be misrouted into
+        # the STT pipeline and never be received as a normal file attachment.
+        if ct == "file":
+            return False
         _VOICE_EXTENSIONS = (
-            ".silk",
-            ".amr",
-            ".mp3",
-            ".wav",
-            ".ogg",
-            ".m4a",
-            ".aac",
-            ".speex",
-            ".flac",
+            ".silk", ".amr", ".mp3", ".wav", ".ogg",
+            ".m4a", ".aac", ".speex", ".flac",
         )
         if any(fn.endswith(ext) for ext in _VOICE_EXTENSIONS):
             return True
@@ -1938,15 +1936,15 @@ async def _stt_voice_attachment(
                     )
                     return None
 
-            # 4. Call STT API
+            # 4. Call STT API and always clean up the temp WAV afterward.
             logger.debug("[%s] STT: calling ASR on %s", self._log_tag, wav_path)
-            transcript = await self._call_stt(wav_path)
-
-            # 5. Cleanup temp file
             try:
-                os.unlink(wav_path)
-            except OSError:
-                pass
+                transcript = await self._call_stt(wav_path)
+            finally:
+                try:
+                    os.unlink(wav_path)
+                except OSError:
+                    pass
 
             if transcript:
                 logger.debug("[%s] STT success: %r", self._log_tag, transcript[:100])

gateway/platforms/weixin.py

@@ -973,9 +973,25 @@ def _extract_text(item_list: List[Dict[str, Any]]) -> str:
             return text
     for item in item_list:
         if item.get("type") == ITEM_VOICE:
-            voice_text = str((item.get("voice_item") or {}).get("text") or "")
-            if voice_text:
-                return voice_text
+            # #27300: Tencent Cloud's `voice_item.text` is their STT output,
+            # which is wrong for any non-Chinese audio (the original report
+            # was a Russian voice message that came back as English
+            # gibberish). Return empty so the central STT pipeline in
+            # ``gateway/run.py`` produces the body from the downloaded
+            # audio instead.
+            voice_item = item.get("voice_item") or {}
+            if not (voice_item.get("media") or {}):
+                # No raw audio to download — Weixin supplied only its own
+                # speech-to-text result. Use it, but preserve the voice
+                # origin so the agent can distinguish this from text the
+                # user typed (#65022).
+                voice_text = str(voice_item.get("text") or "")
+                if voice_text:
+                    return (
+                        "[Voice transcription provided by Weixin]\n"
+                        f"{voice_text}"
+                    )
+            continue
     return ""
 
 
@@ -1659,8 +1675,13 @@ async def _download_file(self, item: Dict[str, Any]) -> Tuple[Optional[str], str
     async def _download_voice(self, item: Dict[str, Any]) -> Optional[str]:
         voice_item = item.get("voice_item") or {}
         media = voice_item.get("media") or {}
-        if voice_item.get("text"):
-            return None
+        # #27300: previously short-circuited when ``voice_item.text`` was set
+        # on the assumption that Tencent Cloud's STT was good enough.
+        # For non-Chinese audio that text is garbage (e.g. a Russian
+        # message comes back as English phonemes) — we must always
+        # download the raw audio so ``gateway/run.py``'s central STT
+        # pipeline can re-transcribe with the user's configured
+        # mlx-whisper / whisper.cpp / faster-whisper backend.
         try:
             data = await _download_and_decrypt_media(
                 self._poll_session,

gateway/run.py

@@ -15902,14 +15902,19 @@ def _should_send_voice_reply(
             )
             return False
 
-        # Dedup: agent already called TTS tool
+        # Dedup: agent already called TTS tool in THIS turn only
+        last_user_idx = None
+        for i, msg in enumerate(reversed(agent_messages)):
+            if msg.get("role") == "user":
+                last_user_idx = len(agent_messages) - 1 - i; break
+        turn_messages = agent_messages[last_user_idx:] if last_user_idx is not None else agent_messages
         has_agent_tts = any(
             msg.get("role") == "assistant"
             and any(
                 (tc.get("function") or {}).get("name") == "text_to_speech"
                 for tc in (msg.get("tool_calls") or [])
             )
-            for msg in agent_messages
+            for msg in turn_messages
         )
         if has_agent_tts:
             return False
@@ -18058,6 +18063,8 @@ async def _enrich_message_with_transcription(
                 list if every clip failed or STT is disabled. Callers can use
                 this to echo transcripts back to the user before the agent loop.
         """
+        seen = set()
+        audio_paths = [p for p in audio_paths if p not in seen and not seen.add(p)]
         if not getattr(self.config, "stt_enabled", True):
             notes = []
             for path in audio_paths:
@@ -18080,7 +18087,10 @@ async def _enrich_message_with_transcription(
             return prefix, []
 
         try:
-            from tools.transcription_tools import transcribe_audio
+            from tools.transcription_tools import (
+                transcribe_audio,
+                transcribe_audio_local_fallback,
+            )
         except ModuleNotFoundError as e:
             logger.error("Transcription module unavailable: %s", e)
             unavailable_note = "[voice message could not be transcribed]"
@@ -18097,6 +18107,17 @@ async def _enrich_message_with_transcription(
             try:
                 logger.debug("Transcribing user voice: %s", path)
                 result = await asyncio.to_thread(transcribe_audio, path)
+                if not result.get("success"):
+                    fallback = await asyncio.to_thread(
+                        transcribe_audio_local_fallback,
+                        path,
+                    )
+                    if fallback.get("success"):
+                        logger.info(
+                            "Configured STT failed for %s; recovered with local STT",
+                            path,
+                        )
+                        result = fallback
                 if result["success"]:
                     transcript = result["transcript"]
                     # Speech-to-text can return success=True with an empty or
@@ -18131,10 +18152,22 @@ async def _enrich_message_with_transcription(
                     # logged for operator diagnosis but kept out of the
                     # LLM-visible prompt.
                     logger.info("Voice transcription failed for %s: %s", path, error)
-                    enriched_parts.append("[voice message could not be transcribed]")
+                    from tools.credential_files import to_agent_visible_cache_path
+
+                    agent_path = to_agent_visible_cache_path(os.path.abspath(path))
+                    enriched_parts.append(
+                        "[voice message could not be transcribed automatically; "
+                        f"the audio is available at: {agent_path}]"
+                    )
             except Exception as e:
                 logger.error("Transcription error: %s", e)
-                enriched_parts.append("[voice message could not be transcribed]")
+                from tools.credential_files import to_agent_visible_cache_path
+
+                agent_path = to_agent_visible_cache_path(os.path.abspath(path))
+                enriched_parts.append(
+                    "[voice message could not be transcribed automatically; "
+                    f"the audio is available at: {agent_path}]"
+                )
 
         if enriched_parts:
             prefix = "\n\n".join(enriched_parts)

plugins/platforms/dingtalk/adapter.py

@@ -116,6 +116,27 @@
     "voice": "audio",
 }
 
+# File extension → MIME type mapping for DingTalk file/image messages.
+# Image MIME types (image/*) are used below in _extract_media to classify
+# incoming msgtype='image' payloads as MessageType.PHOTO (not DOCUMENT).
+EXT_MAP = {
+    "pdf": "application/pdf",
+    "png": "image/png",
+    "jpg": "image/jpeg",
+    "jpeg": "image/jpeg",
+    "gif": "image/gif",
+    "webp": "image/webp",
+    "doc": "application/msword",
+    "docx": "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
+    "xls": "application/vnd.ms-excel",
+    "xlsx": "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
+    "md": "text/markdown",
+    "txt": "text/plain",
+    "csv": "text/csv",
+    "zip": "application/zip",
+    "mp4": "video/mp4",
+}
+
 
 def check_dingtalk_requirements() -> bool:
     """Check if DingTalk dependencies are available and configured.
@@ -748,6 +769,90 @@ def _extract_text(message: "ChatbotMessage") -> str:
                             parts.append(item.text)
                     content = " ".join(parts).strip()
 
+        # Fallback: audio message → use recognition text
+        if not content:
+            msg_type = getattr(message, "message_type", "")
+            if msg_type == "audio":
+                extensions = getattr(message, "extensions", {}) or {}
+                audio_content = extensions.get("content", {})
+                if isinstance(audio_content, dict):
+                    recognition = audio_content.get("recognition", "")
+                    if recognition:
+                        content = recognition.strip()
+
+        # Fallback: file message → use fileName as text
+        if not content:
+            msg_type = getattr(message, "message_type", "")
+            if msg_type == "file":
+                extensions = getattr(message, "extensions", {}) or {}
+                file_content = extensions.get("content", {})
+                if isinstance(file_content, dict):
+                    fname = file_content.get("fileName", "")
+                    if fname:
+                        content = f"[文件] {fname}"
+
+        # Fallback: card message (钉钉文档分享卡片 / link card)
+        # When a user shares a DingTalk Doc to the bot, the msgtype is "card"
+        # and the card data lives in extensions['card'] (SDK's from_dict maps
+        # unhandled fields to extensions).  Extract title + doc URL so the
+        # message isn't silently dropped as "empty".
+        if not content:
+            msg_type = getattr(message, "message_type", "")
+            # Handle card-type messages (文档分享卡片 / link card)
+            if msg_type == "card":
+                extensions = getattr(message, "extensions", {}) or {}
+                card = extensions.get("card", {})
+                if isinstance(card, dict):
+                    title = card.get("title", "")
+                    raw_content = card.get("content", "")
+                    doc_url = ""
+                    if raw_content is None:
+                        doc_url = ""
+                    elif isinstance(raw_content, dict):
+                        doc_url = raw_content.get("url", "") or raw_content.get("docUrl", "")
+                    elif isinstance(raw_content, str):
+                        stripped = raw_content.strip()
+                        if not stripped:
+                            doc_url = ""
+                        else:
+                            try:
+                                parsed = json.loads(stripped)
+                                if isinstance(parsed, dict):
+                                    doc_url = parsed.get("url", "") or parsed.get("docUrl", "")
+                            except (ValueError, TypeError):
+                                doc_url = raw_content
+                    parts = []
+                    if title:
+                        parts.append(f"[文档] {title}")
+                    if doc_url:
+                        parts.append(doc_url)
+                    if parts:
+                        content = " ".join(parts)
+                # Last-resort: raw text field from extensions (if present)
+                if not content:
+                    ext_text = extensions.get("text", {})
+                    if isinstance(ext_text, dict):
+                        content = (ext_text.get("content", "") or "").strip()
+
+            # Handle interactiveCard messages (钉钉文档分享卡片 / doc link card)
+            # structure: extensions["content"]["biz_custom_action_url"] and
+            # extensions["content"]["title"] for the card title
+            if msg_type == "interactiveCard" and not content:
+                extensions = getattr(message, "extensions", {}) or {}
+                ext_content = extensions.get("content", {})
+                if isinstance(ext_content, dict):
+                    doc_url = ext_content.get("biz_custom_action_url", "")
+                    title = ext_content.get("title", "")
+                    if doc_url or title:
+                        parts = []
+                        if title:
+                            parts.append(f"[文档卡片] {title}")
+                        else:
+                            parts.append("[文档卡片]")
+                        if doc_url:
+                            parts.append(doc_url)
+                        content = " ".join(parts)
+
         # Do NOT strip "@bot" from the text.  The mention is a routing
         # signal (delivered structurally via callback `isInAtList`), and
         # regex-stripping @handles would collateral-damage e-mails
@@ -815,11 +920,49 @@ def _extract_media(self, message: "ChatbotMessage"):
         if msg_type_str == "picture" and not media_urls:
             msg_type = MessageType.PHOTO
         elif msg_type_str == "richText":
-            msg_type = (
-                MessageType.PHOTO
-                if any("image" in t for t in media_types)
-                else MessageType.TEXT
-            )
+            # Only re-derive the type when the rich-text scan above left it
+            # at TEXT. The scan may already have promoted it to VOICE/AUDIO/
+            # VIDEO/DOCUMENT for embedded media items — resetting those here
+            # dropped native voice notes back to TEXT and skipped STT
+            # (#38211, #38219; analysis from #38276).
+            if msg_type == MessageType.TEXT and any(
+                "image" in t for t in media_types
+            ):
+                msg_type = MessageType.PHOTO
+        elif msg_type_str == "audio":
+            # Voice message — DingTalk already provides recognition text.
+            # Do NOT add media_urls here: if audio_paths is non-empty,
+            # run.py's _enrich_message_with_transcription will overwrite
+            # the recognition text with a failed STT attempt (whisper not installed).
+            # The recognition text from extensions['content']['recognition']
+            # is sufficient and already extracted by _extract_text.
+            if msg_type == MessageType.TEXT:
+                msg_type = MessageType.VOICE
+        elif msg_type_str in ("file", "image"):
+            extensions = getattr(message, "extensions", {}) or {}
+            ext_content = extensions.get("content", {})
+            if isinstance(ext_content, dict):
+                dl_code = ext_content.get("downloadCode") or ""
+                fname = ext_content.get("fileName", "")
+                if dl_code:
+                    media_urls.append(dl_code)
+                    mime = "application/octet-stream"
+                    # Map common extensions
+                    if fname:
+                        ext = fname.rsplit(".", 1)[-1].lower() if "." in fname else ""
+                        mime = EXT_MAP.get(ext, mime)
+                    media_types.append(mime)
+                    if msg_type == MessageType.TEXT:
+                        # Image messages → PHOTO (distinct busy-session handling
+                        # in gateway/platforms/base.py).
+                        # File messages with image MIME types (e.g. a .png sent
+                        # as a file attachment) are also classified as PHOTO —
+                        # the user's intent is to share an image regardless of
+                        # how DingTalk delivers it.
+                        if msg_type_str == "image" or mime.startswith("image/"):
+                            msg_type = MessageType.PHOTO
+                        else:
+                            msg_type = MessageType.DOCUMENT
 
         return msg_type, media_urls, media_types
 
@@ -1323,6 +1466,14 @@ async def _resolve_media_codes(self, message: "ChatbotMessage") -> None:
                         if item.get(key):
                             codes_to_resolve.append((item, key))
 
+        # 3. File/image message (msgtype='file' or 'image', codes in extensions)
+        msg_type_str = getattr(message, "message_type", "") or ""
+        if msg_type_str in ("file", "image"):
+            extensions = getattr(message, "extensions", {}) or {}
+            ext_content = extensions.get("content", {})
+            if isinstance(ext_content, dict) and ext_content.get("downloadCode"):
+                codes_to_resolve.append((ext_content, "downloadCode"))
+
         if not codes_to_resolve:
             return