From b674df63082bc6c0edc3c83d4baece18c5a37c3c Mon Sep 17 00:00:00 2001 From: arminanton <29869547+arminanton@users.noreply.github.com> Date: Sun, 21 Jun 2026 05:45:41 -0700 Subject: [PATCH] fix(gateway): don't upcast non-image attachments to image on PHOTO events _build_media_placeholder and the image_paths collector classified any attachment on a MessageType.PHOTO event as an image, even when its MIME type was text/* or application/* (a document) or it was part of a mixed album. That mislabels documents as images and routes them into the vision pipeline. Classify by MIME first; only fall back to the PHOTO message_type when MIME is absent/ambiguous, and even then require an image extension or a single attachment with empty/octet-stream MIME before treating it as an image. 6 focused tests cover image MIME, text/application documents under PHOTO, mixed albums, single missing-MIME image-ext, and audio/video. --- gateway/run.py | 52 +++++++++++++-- .../test_media_image_classification.py | 66 +++++++++++++++++++ 2 files changed, 112 insertions(+), 6 deletions(-) create mode 100644 tests/gateway/test_media_image_classification.py diff --git a/gateway/run.py b/gateway/run.py index 2672ab43e95a0..06a348730af9f 100644 --- a/gateway/run.py +++ b/gateway/run.py @@ -1741,13 +1741,34 @@ def _build_media_placeholder(event) -> str: parts = [] media_urls = getattr(event, "media_urls", None) or [] media_types = getattr(event, "media_types", None) or [] + _image_exts = { + ".png", ".jpg", ".jpeg", ".gif", ".webp", ".bmp", + ".tif", ".tiff", ".heic", ".heif", ".avif", + } for i, url in enumerate(media_urls): - mtype = media_types[i] if i < len(media_types) else "" - if mtype.startswith("image/") or getattr(event, "message_type", None) == MessageType.PHOTO: + mtype = (media_types[i] if i < len(media_types) else "") or "" + _mtype = mtype.lower() + _ext = os.path.splitext(url or "")[1].lower() + + # Classify by MIME first; only fall back to the PHOTO message_type when + # MIME is absent/ambiguous, and even then do NOT upcast mixed + # attachments (albums/documents) to images — a PHOTO event can carry a + # non-image document whose MIME is text/*, application/*, etc. + _is_image = False + if _mtype.startswith("image/"): + _is_image = True + elif _mtype.startswith(("text/", "application/", "audio/", "video/")): + _is_image = False + elif getattr(event, "message_type", None) == MessageType.PHOTO: + _is_image = (_ext in _image_exts) or ( + len(media_urls) <= 1 and _mtype in {"", "application/octet-stream"} + ) + + if _is_image: parts.append(f"[User sent an image: {url}]") - elif mtype.startswith("audio/"): + elif _mtype.startswith("audio/"): parts.append(f"[User sent audio: {url}]") - elif mtype.startswith("video/") or getattr(event, "message_type", None) == MessageType.VIDEO: + elif _mtype.startswith("video/") or getattr(event, "message_type", None) == MessageType.VIDEO: parts.append(f"[User sent a video: {url}]") else: parts.append(f"[User sent a file: {url}]") @@ -8436,9 +8457,28 @@ async def _prepare_inbound_message_text( if event.media_urls: image_paths = [] audio_paths = [] + _image_exts = { + ".png", ".jpg", ".jpeg", ".gif", ".webp", ".bmp", + ".tif", ".tiff", ".heic", ".heif", ".avif", + } for i, path in enumerate(event.media_urls): - mtype = event.media_types[i] if i < len(event.media_types) else "" - if mtype.startswith("image/") or event.message_type == MessageType.PHOTO: + mtype = (event.media_types[i] if i < len(event.media_types) else "") or "" + _mtype = mtype.lower() + _ext = os.path.splitext(path or "")[1].lower() + # Classify by MIME first; fall back to the PHOTO message_type only + # when MIME is absent/ambiguous, and don't upcast mixed + # albums/documents (a PHOTO event can carry a non-image document). + _is_image = False + if _mtype.startswith("image/"): + _is_image = True + elif _mtype.startswith(("text/", "application/", "audio/", "video/")): + _is_image = False + elif event.message_type == MessageType.PHOTO: + _is_image = (_ext in _image_exts) or ( + len(event.media_urls) <= 1 + and _mtype in {"", "application/octet-stream"} + ) + if _is_image: image_paths.append(path) # MessageType.AUDIO = audio file attachment (e.g. .mp3, .m4a) — never STT # MessageType.VOICE = voice message (Opus/OGG) — always STT diff --git a/tests/gateway/test_media_image_classification.py b/tests/gateway/test_media_image_classification.py new file mode 100644 index 0000000000000..27e5be805acf9 --- /dev/null +++ b/tests/gateway/test_media_image_classification.py @@ -0,0 +1,66 @@ +"""Tests for gateway media image-classification in _build_media_placeholder. + +Covers the fix that stops upcasting non-image attachments to "image" just +because the event's message_type is PHOTO. A PHOTO event can carry a document +(text/*, application/*) or a mixed album; those must not be labelled as images. +""" +import os +import sys +import types + +import pytest + +# gateway.run imports MessageType from gateway.platforms.base +from gateway.platforms.base import MessageType +from gateway.run import _build_media_placeholder + + +def _event(media_urls, media_types, message_type): + return types.SimpleNamespace( + media_urls=media_urls, + media_types=media_types, + message_type=message_type, + ) + + +def test_image_mime_is_classified_as_image(): + ev = _event(["/tmp/a.png"], ["image/png"], MessageType.PHOTO) + assert "[User sent an image: /tmp/a.png]" in _build_media_placeholder(ev) + + +def test_photo_event_with_text_document_is_not_upcast_to_image(): + # A PHOTO event carrying a text document must NOT be called an image. + ev = _event(["/tmp/notes.txt"], ["text/plain"], MessageType.PHOTO) + out = _build_media_placeholder(ev) + assert "image" not in out + assert "[User sent a file: /tmp/notes.txt]" in out + + +def test_photo_event_with_application_document_is_not_upcast(): + ev = _event(["/tmp/report.pdf"], ["application/pdf"], MessageType.PHOTO) + out = _build_media_placeholder(ev) + assert "image" not in out + + +def test_photo_album_with_missing_mime_is_not_upcast(): + # Multiple attachments + missing MIME → don't blanket-upcast the album. + ev = _event( + ["/tmp/1.bin", "/tmp/2.bin"], + ["", ""], + MessageType.PHOTO, + ) + out = _build_media_placeholder(ev) + assert "image" not in out + + +def test_photo_single_missing_mime_with_image_ext_is_image(): + # Single attachment, missing MIME, but an image extension → image. + ev = _event(["/tmp/pic.jpg"], [""], MessageType.PHOTO) + assert "[User sent an image: /tmp/pic.jpg]" in _build_media_placeholder(ev) + + +def test_audio_and_video_still_classified(): + ev_a = _event(["/tmp/a.mp3"], ["audio/mpeg"], MessageType.AUDIO) + assert "[User sent audio: /tmp/a.mp3]" in _build_media_placeholder(ev_a) + ev_v = _event(["/tmp/v.mp4"], ["video/mp4"], MessageType.VIDEO) + assert "[User sent a video: /tmp/v.mp4]" in _build_media_placeholder(ev_v)