Skip to content

fix(gateway): define missing _MEDIA_EXTS — MEDIA tag and local file extraction #34115

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
proactive-assistant wants to merge 3 commits into from
Closed

fix(gateway): define missing _MEDIA_EXTS — MEDIA tag and local file extraction #34115

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
15976af
fix(gateway): define missing _MEDIA_EXTS and use it in media extracti…
May 28, 2026
4789c69
fix(gateway): consolidate MEDIA extension lists and deliver attachmen…
May 28, 2026
db75de5
fix(gateway): move regex compilations out of loops, fix comment
May 28, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
38 changes: 17 additions & 21 deletions gateway/platforms/base.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1062,6 +1062,19 @@ def validate_media_delivery_path(path: str) -> Optional[str]:
".gif": "image/gif",
}

# Union of all media extensions the gateway can recognise in MEDIA:<path>
# tags and bare local-file-path patterns. Built from the four extension
# dicts in this module (_AUDIO_EXTS, SUPPORTED_VIDEO_TYPES,
# SUPPORTED_DOCUMENT_TYPES, SUPPORTED_IMAGE_DOCUMENT_TYPES) so adding
# a new supported type automatically picks up MEDIA-tag / local-file
# delivery.
_MEDIA_EXTS: frozenset[str] = (
_AUDIO_EXTS
| frozenset(SUPPORTED_VIDEO_TYPES)
| frozenset(SUPPORTED_DOCUMENT_TYPES)
| frozenset(SUPPORTED_IMAGE_DOCUMENT_TYPES)
)


def get_document_cache_dir() -> Path:
"""Return the document cache directory, creating it if it doesn't exist."""
Expand Down Expand Up @@ -2412,8 +2425,9 @@ def extract_media(content: str) -> Tuple[List[Tuple[str, bool]], str]:

# Extract MEDIA:<path> tags, allowing optional whitespace after the colon
# and quoted/backticked paths for LLM-formatted outputs.
_ext_part = '|'.join(e.lstrip('.') for e in _MEDIA_EXTS)
media_pattern = re.compile(
r'''[`"']?MEDIA:\s*(?P<path>`[^`\n]+`|"[^"\n]+"|'[^'\n]+'|(?:~/|/)\S+(?:[^\S\n]+\S+)*?\.(?:png|jpe?g|gif|webp|mp4|mov|avi|mkv|webm|ogg|opus|mp3|wav|m4a|flac|epub|pdf|zip|rar|7z|docx?|xlsx?|pptx?|txt|csv|apk|ipa)(?=[\s`"',;:)\]}]|$))[`"']?'''
r'''[`"']?MEDIA:\s*(?P<path>`[^`\n]+`|"[^"\n]+"|'[^'\n]+'|(?:~/|/)\S+(?:[^\S\n]+\S+)*?\.(?:''' + _ext_part + r''')(?=[\s`"',;:)\]}]|$))[`"']?'''
)
for match in media_pattern.finditer(content):
path = match.group("path").strip()
Expand Down Expand Up @@ -2455,31 +2469,13 @@ def extract_local_files(content: str) -> Tuple[List[str], str]:
Tuple of (list of expanded file paths, cleaned text with the
raw path strings removed).
"""
_LOCAL_MEDIA_EXTS = (
# Images (embed inline)
'.png', '.jpg', '.jpeg', '.gif', '.webp', '.bmp', '.tiff', '.svg',
# Video (embed inline where supported)
'.mp4', '.mov', '.avi', '.mkv', '.webm',
# Audio (delivered as voice/audio where supported)
'.mp3', '.wav', '.ogg', '.m4a', '.flac',
# Documents (uploaded as file attachments)
'.pdf', '.docx', '.doc', '.odt', '.rtf', '.txt', '.md',
# Spreadsheets / data
'.xlsx', '.xls', '.ods', '.csv', '.tsv', '.json', '.xml', '.yaml', '.yml',
# Presentations
'.pptx', '.ppt', '.odp', '.key',
# Archives
'.zip', '.tar', '.gz', '.tgz', '.bz2', '.xz', '.7z', '.rar',
# Web / rendered output
'.html', '.htm',
)
ext_part = '|'.join(e.lstrip('.') for e in _LOCAL_MEDIA_EXTS)
_ext_part = '|'.join(e.lstrip('.') for e in _MEDIA_EXTS)

# (?<![/:\w.]) prevents matching inside URLs (e.g. https://…/img.png)
# and relative paths (./foo.png)
# (?:~/|/) anchors to absolute or home-relative paths
path_re = re.compile(
r'(?<![/:\w.])(?:~/|/)(?:[\w.\-]+/)*[\w.\-]+\.(?:' + ext_part + r')\b',
r'(?<![/:\w.])(?:~/|/)(?:[\w.\-]+/)*[\w.\-]+\.(?:' + _ext_part + r')\b',
re.IGNORECASE,
)

Expand Down
38 changes: 24 additions & 14 deletions gateway/run.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1034,6 +1034,7 @@ def _reload_runtime_env_preserving_config_authority() -> None:
EphemeralReply,
MessageEvent,
MessageType,
_MEDIA_EXTS,
_reply_anchor_for_event,
merge_pending_message_event,
)
Expand Down Expand Up @@ -9165,6 +9166,19 @@ async def _handle_message_with_agent(self, event, source, _quick_key: str, run_g
logger.debug("trailing footer send failed: %s", _e)
return None

# Non-streaming path: extract and deliver MEDIA: files before
# returning the cleaned response text. Streaming handles this
# via _deliver_media_from_response above (which also strips the
# tags from the already-streamed text — a best-effort cleanup).
_ns_adapter = self.adapters.get(source.platform)
if response and _ns_adapter:
await self._deliver_media_from_response(
response, event, _ns_adapter,
)
# Strip MEDIA tags from the text we're about to return so
# the adapter doesn't send them as literal text. _deliver_media
# already extracted and delivered the files.
response, _ = _ns_adapter.extract_media(response)
return response

except Exception as e:
Expand Down Expand Up @@ -16974,17 +16988,15 @@ def _clarify_callback_sync(question: str, choices) -> str:
# from the current turn's extraction. This is compression-safe:
# even if the message list shrinks, we know which paths are old.
_history_media_paths: set = set()
_ext_part = '|'.join(e.lstrip('.') for e in _MEDIA_EXTS)
_TOOL_MEDIA_RE = re.compile(
r'MEDIA:((?:/|~\/)\S+\.(?:' + _ext_part + r'))',
re.IGNORECASE
)
for _hm in agent_history:
if _hm.get("role") in {"tool", "function"}:
_hc = _hm.get("content", "")
if "MEDIA:" in _hc:
_TOOL_MEDIA_RE = re.compile(
r'MEDIA:((?:/|~\/)\S+\.(?:png|jpe?g|gif|webp|'
r'mp4|mov|avi|mkv|webm|ogg|opus|mp3|wav|m4a|'
r'flac|epub|pdf|zip|rar|7z|docx?|xlsx?|pptx?|'
r'txt|csv|apk|ipa))',
re.IGNORECASE
)
for _match in _TOOL_MEDIA_RE.finditer(_hc):
_p = _match.group(1).strip().rstrip('",}')
if _p:
Expand Down Expand Up @@ -17280,17 +17292,15 @@ def _approval_notify_sync(approval_data: dict) -> None:
if "MEDIA:" not in final_response:
media_tags = []
has_voice_directive = False
_ext_part = '|'.join(e.lstrip('.') for e in _MEDIA_EXTS)
_TOOL_MEDIA_RE = re.compile(
r'MEDIA:((?:/|~\/)\S+\.(?:' + _ext_part + r'))',
re.IGNORECASE
)
for msg in result.get("messages", []):
if msg.get("role") in {"tool", "function"}:
content = msg.get("content", "")
if "MEDIA:" in content:
_TOOL_MEDIA_RE = re.compile(
r'MEDIA:((?:/|~\/)\S+\.(?:png|jpe?g|gif|webp|'
r'mp4|mov|avi|mkv|webm|ogg|opus|mp3|wav|m4a|'
r'flac|epub|pdf|zip|rar|7z|docx?|xlsx?|pptx?|'
r'txt|csv|apk|ipa))',
re.IGNORECASE
)
for match in _TOOL_MEDIA_RE.finditer(content):
path = match.group(1).strip().rstrip('",}')
if path and path not in _history_media_paths:
Expand Down