From 15976afb8d3cc816436df08b85ab026a6b778900 Mon Sep 17 00:00:00 2001 From: Proactive Assistant Date: Thu, 28 May 2026 23:32:25 +0200 Subject: [PATCH 1/3] fix(gateway): define missing _MEDIA_EXTS and use it in media extraction regexes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The MEDIA: tag regex in extract_media() was hardcoded with an incomplete extension list — missing .md, .json, .yaml, .html, .svg, and 15 other extensions that extract_local_files() already recognised via its own _LOCAL_MEDIA_EXTS tuple. Consolidate both code paths onto a single module-level _MEDIA_EXTS frozenset built from the union of all four support dicts (_AUDIO_EXTS, SUPPORTED_VIDEO_TYPES, SUPPORTED_DOCUMENT_TYPES, SUPPORTED_IMAGE_DOCUMENT_TYPES). This means: - Adding a new supported document type automatically picks up MEDIA-tag delivery and local-file-path extraction without touching the regex. - extract_media() and extract_local_files() can never drift apart again — they share the same source of truth. The hardcoded extensions already in the regex but missing from the dicts (.epub, .rar, .7z, .apk, .ipa) are intentionally dropped because the gateway has no MIME type / delivery support for them. --- gateway/platforms/base.py | 36 +++++++++++++++--------------------- 1 file changed, 15 insertions(+), 21 deletions(-) diff --git a/gateway/platforms/base.py b/gateway/platforms/base.py index d39601546886..4da9b23e4277 100644 --- a/gateway/platforms/base.py +++ b/gateway/platforms/base.py @@ -1062,6 +1062,17 @@ def validate_media_delivery_path(path: str) -> Optional[str]: ".gif": "image/gif", } +# Union of all media extensions the gateway can recognise in MEDIA: +# tags and bare local-file-path patterns. Kept in sync with the four +# extension dicts directly above so adding a new supported type +# automatically picks up MEDIA-tag / local-file delivery. +_MEDIA_EXTS: frozenset[str] = ( + _AUDIO_EXTS + | frozenset(SUPPORTED_VIDEO_TYPES) + | frozenset(SUPPORTED_DOCUMENT_TYPES) + | frozenset(SUPPORTED_IMAGE_DOCUMENT_TYPES) +) + def get_document_cache_dir() -> Path: """Return the document cache directory, creating it if it doesn't exist.""" @@ -2412,8 +2423,9 @@ def extract_media(content: str) -> Tuple[List[Tuple[str, bool]], str]: # Extract MEDIA: tags, allowing optional whitespace after the colon # and quoted/backticked paths for LLM-formatted outputs. + _ext_part = '|'.join(e.lstrip('.') for e in _MEDIA_EXTS) media_pattern = re.compile( - r'''[`"']?MEDIA:\s*(?P`[^`\n]+`|"[^"\n]+"|'[^'\n]+'|(?:~/|/)\S+(?:[^\S\n]+\S+)*?\.(?:png|jpe?g|gif|webp|mp4|mov|avi|mkv|webm|ogg|opus|mp3|wav|m4a|flac|epub|pdf|zip|rar|7z|docx?|xlsx?|pptx?|txt|csv|apk|ipa)(?=[\s`"',;:)\]}]|$))[`"']?''' + r'''[`"']?MEDIA:\s*(?P`[^`\n]+`|"[^"\n]+"|'[^'\n]+'|(?:~/|/)\S+(?:[^\S\n]+\S+)*?\.(?:''' + _ext_part + r''')(?=[\s`"',;:)\]}]|$))[`"']?''' ) for match in media_pattern.finditer(content): path = match.group("path").strip() @@ -2455,31 +2467,13 @@ def extract_local_files(content: str) -> Tuple[List[str], str]: Tuple of (list of expanded file paths, cleaned text with the raw path strings removed). """ - _LOCAL_MEDIA_EXTS = ( - # Images (embed inline) - '.png', '.jpg', '.jpeg', '.gif', '.webp', '.bmp', '.tiff', '.svg', - # Video (embed inline where supported) - '.mp4', '.mov', '.avi', '.mkv', '.webm', - # Audio (delivered as voice/audio where supported) - '.mp3', '.wav', '.ogg', '.m4a', '.flac', - # Documents (uploaded as file attachments) - '.pdf', '.docx', '.doc', '.odt', '.rtf', '.txt', '.md', - # Spreadsheets / data - '.xlsx', '.xls', '.ods', '.csv', '.tsv', '.json', '.xml', '.yaml', '.yml', - # Presentations - '.pptx', '.ppt', '.odp', '.key', - # Archives - '.zip', '.tar', '.gz', '.tgz', '.bz2', '.xz', '.7z', '.rar', - # Web / rendered output - '.html', '.htm', - ) - ext_part = '|'.join(e.lstrip('.') for e in _LOCAL_MEDIA_EXTS) + _ext_part = '|'.join(e.lstrip('.') for e in _MEDIA_EXTS) # (? Date: Thu, 28 May 2026 23:48:00 +0200 Subject: [PATCH 2/3] fix(gateway): consolidate MEDIA extension lists and deliver attachments in non-streaming responses MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three related fixes in run.py: 1. Replace two hardcoded MEDIA regexes (lines 16982, 17288) — both used the same incomplete list (png|jpe?g|...|apk|ipa) missing .md and 20+ other extensions. Both now build the regex dynamically from _MEDIA_EXTS, imported from gateway.platforms.base. 2. The non-streaming response path returned raw response text with MEDIA: tags still embedded — the adapter sent them as literal text and the files were never extracted or delivered. Only the streaming path had media delivery (via _deliver_media_from_response). Fix: call _deliver_media_from_response + extract_media before returning the response in the non-streaming path. Files are delivered and MEDIA tags are stripped from the text the adapter receives. 3. Added _MEDIA_EXTS to the top-level import from gateway.platforms.base so both regex sites and future consumers can use it. --- gateway/run.py | 26 ++++++++++++++++++-------- 1 file changed, 18 insertions(+), 8 deletions(-) diff --git a/gateway/run.py b/gateway/run.py index 057d15cab915..746910cf6382 100644 --- a/gateway/run.py +++ b/gateway/run.py @@ -1034,6 +1034,7 @@ def _reload_runtime_env_preserving_config_authority() -> None: EphemeralReply, MessageEvent, MessageType, + _MEDIA_EXTS, _reply_anchor_for_event, merge_pending_message_event, ) @@ -9165,6 +9166,19 @@ async def _handle_message_with_agent(self, event, source, _quick_key: str, run_g logger.debug("trailing footer send failed: %s", _e) return None + # Non-streaming path: extract and deliver MEDIA: files before + # returning the cleaned response text. Streaming handles this + # via _deliver_media_from_response above (which also strips the + # tags from the already-streamed text — a best-effort cleanup). + _ns_adapter = self.adapters.get(source.platform) + if response and _ns_adapter: + await self._deliver_media_from_response( + response, event, _ns_adapter, + ) + # Strip MEDIA tags from the text we're about to return so + # the adapter doesn't send them as literal text. _deliver_media + # already extracted and delivered the files. + response, _ = _ns_adapter.extract_media(response) return response except Exception as e: @@ -16978,11 +16992,9 @@ def _clarify_callback_sync(question: str, choices) -> str: if _hm.get("role") in {"tool", "function"}: _hc = _hm.get("content", "") if "MEDIA:" in _hc: + _ext_part = '|'.join(e.lstrip('.') for e in _MEDIA_EXTS) _TOOL_MEDIA_RE = re.compile( - r'MEDIA:((?:/|~\/)\S+\.(?:png|jpe?g|gif|webp|' - r'mp4|mov|avi|mkv|webm|ogg|opus|mp3|wav|m4a|' - r'flac|epub|pdf|zip|rar|7z|docx?|xlsx?|pptx?|' - r'txt|csv|apk|ipa))', + r'MEDIA:((?:/|~\/)\S+\.(?:' + _ext_part + r'))', re.IGNORECASE ) for _match in _TOOL_MEDIA_RE.finditer(_hc): @@ -17284,11 +17296,9 @@ def _approval_notify_sync(approval_data: dict) -> None: if msg.get("role") in {"tool", "function"}: content = msg.get("content", "") if "MEDIA:" in content: + _ext_part = '|'.join(e.lstrip('.') for e in _MEDIA_EXTS) _TOOL_MEDIA_RE = re.compile( - r'MEDIA:((?:/|~\/)\S+\.(?:png|jpe?g|gif|webp|' - r'mp4|mov|avi|mkv|webm|ogg|opus|mp3|wav|m4a|' - r'flac|epub|pdf|zip|rar|7z|docx?|xlsx?|pptx?|' - r'txt|csv|apk|ipa))', + r'MEDIA:((?:/|~\/)\S+\.(?:' + _ext_part + r'))', re.IGNORECASE ) for match in _TOOL_MEDIA_RE.finditer(content): From db75de5e3a6588fcb29ba8f69e1ffebcc9617b9b Mon Sep 17 00:00:00 2001 From: Proactive Assistant Date: Fri, 29 May 2026 00:06:20 +0200 Subject: [PATCH 3/3] fix(gateway): move regex compilations out of loops, fix comment MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Compute _ext_part and _TOOL_MEDIA_RE once before each loop instead of recompiling on every iteration. No behavioral change. - Fix misleading "directly above" comment on _MEDIA_EXTS — the source dicts are spread across the module, not directly above. --- gateway/platforms/base.py | 8 +++++--- gateway/run.py | 20 ++++++++++---------- 2 files changed, 15 insertions(+), 13 deletions(-) diff --git a/gateway/platforms/base.py b/gateway/platforms/base.py index 4da9b23e4277..af0b07c59946 100644 --- a/gateway/platforms/base.py +++ b/gateway/platforms/base.py @@ -1063,9 +1063,11 @@ def validate_media_delivery_path(path: str) -> Optional[str]: } # Union of all media extensions the gateway can recognise in MEDIA: -# tags and bare local-file-path patterns. Kept in sync with the four -# extension dicts directly above so adding a new supported type -# automatically picks up MEDIA-tag / local-file delivery. +# tags and bare local-file-path patterns. Built from the four extension +# dicts in this module (_AUDIO_EXTS, SUPPORTED_VIDEO_TYPES, +# SUPPORTED_DOCUMENT_TYPES, SUPPORTED_IMAGE_DOCUMENT_TYPES) so adding +# a new supported type automatically picks up MEDIA-tag / local-file +# delivery. _MEDIA_EXTS: frozenset[str] = ( _AUDIO_EXTS | frozenset(SUPPORTED_VIDEO_TYPES) diff --git a/gateway/run.py b/gateway/run.py index 746910cf6382..8d5c07064dd4 100644 --- a/gateway/run.py +++ b/gateway/run.py @@ -16988,15 +16988,15 @@ def _clarify_callback_sync(question: str, choices) -> str: # from the current turn's extraction. This is compression-safe: # even if the message list shrinks, we know which paths are old. _history_media_paths: set = set() + _ext_part = '|'.join(e.lstrip('.') for e in _MEDIA_EXTS) + _TOOL_MEDIA_RE = re.compile( + r'MEDIA:((?:/|~\/)\S+\.(?:' + _ext_part + r'))', + re.IGNORECASE + ) for _hm in agent_history: if _hm.get("role") in {"tool", "function"}: _hc = _hm.get("content", "") if "MEDIA:" in _hc: - _ext_part = '|'.join(e.lstrip('.') for e in _MEDIA_EXTS) - _TOOL_MEDIA_RE = re.compile( - r'MEDIA:((?:/|~\/)\S+\.(?:' + _ext_part + r'))', - re.IGNORECASE - ) for _match in _TOOL_MEDIA_RE.finditer(_hc): _p = _match.group(1).strip().rstrip('",}') if _p: @@ -17292,15 +17292,15 @@ def _approval_notify_sync(approval_data: dict) -> None: if "MEDIA:" not in final_response: media_tags = [] has_voice_directive = False + _ext_part = '|'.join(e.lstrip('.') for e in _MEDIA_EXTS) + _TOOL_MEDIA_RE = re.compile( + r'MEDIA:((?:/|~\/)\S+\.(?:' + _ext_part + r'))', + re.IGNORECASE + ) for msg in result.get("messages", []): if msg.get("role") in {"tool", "function"}: content = msg.get("content", "") if "MEDIA:" in content: - _ext_part = '|'.join(e.lstrip('.') for e in _MEDIA_EXTS) - _TOOL_MEDIA_RE = re.compile( - r'MEDIA:((?:/|~\/)\S+\.(?:' + _ext_part + r'))', - re.IGNORECASE - ) for match in _TOOL_MEDIA_RE.finditer(content): path = match.group(1).strip().rstrip('",}') if path and path not in _history_media_paths: