Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
71 changes: 58 additions & 13 deletions gateway/run.py
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,58 @@
_AGENT_CACHE_MAX_SIZE = 128
_AGENT_CACHE_IDLE_TTL_SECS = 3600.0 # evict agents idle for >1h

# Tool results can contain literal MEDIA: examples in docs, logs, or other
# ordinary outputs. Only tools that intentionally create deliverable media
# artifacts should be eligible for automatic append when the model omits them
# from the final gateway reply.
_AUTO_APPEND_MEDIA_TOOL_NAMES = {"text_to_speech", "text_to_speech_tool"}


def _collect_auto_append_media_tags(
messages: List[Dict[str, Any]],
history_offset: int = 0,
history_media_paths: Optional[set] = None,
) -> tuple[List[str], bool]:
"""Collect real media tags from current-turn tool results only.

Avoid scanning arbitrary tool output: documentation, logs, and search
results can contain example strings such as MEDIA:/absolute/path/to/file,
which must never be delivered as attachments.
"""
history_media_paths = history_media_paths or set()
new_messages = messages[history_offset:] if len(messages) > history_offset else []

tool_name_by_call_id: Dict[str, str] = {}
for msg in new_messages:
if msg.get("role") != "assistant":
continue
for call in msg.get("tool_calls") or []:
call_id = call.get("id") or call.get("call_id")
fn = call.get("function") or {}
name = str(fn.get("name") or call.get("name") or "")
if call_id and name:
tool_name_by_call_id[str(call_id)] = name

media_tags: List[str] = []
has_voice_directive = False
for msg in new_messages:
if msg.get("role") not in ("tool", "function"):
continue
call_id = str(msg.get("tool_call_id") or msg.get("call_id") or "")
if tool_name_by_call_id.get(call_id) not in _AUTO_APPEND_MEDIA_TOOL_NAMES:
continue
content = str(msg.get("content") or "")
if "MEDIA:" not in content:
continue
for match in re.finditer(r"MEDIA:(\S+)", content):
path = match.group(1).strip().rstrip('\",}')
if path and path not in history_media_paths:
media_tags.append(f"MEDIA:{path}")
if "[[audio_as_voice]]" in content:
has_voice_directive = True

return media_tags, has_voice_directive

# ---------------------------------------------------------------------------
# SSL certificate auto-detection for NixOS and other non-standard systems.
# Must run BEFORE any HTTP library (discord, aiohttp, etc.) is imported.
Expand Down Expand Up @@ -10562,19 +10614,12 @@ def _approval_notify_sync(approval_data: dict) -> None:
# before run_conversation) instead of index slicing. This is safe even
# when context compression shrinks the message list. (Fixes #160)
if "MEDIA:" not in final_response:
media_tags = []
has_voice_directive = False
for msg in result.get("messages", []):
if msg.get("role") in ("tool", "function"):
content = msg.get("content", "")
if "MEDIA:" in content:
for match in re.finditer(r'MEDIA:(\S+)', content):
path = match.group(1).strip().rstrip('",}')
if path and path not in _history_media_paths:
media_tags.append(f"MEDIA:{path}")
if "[[audio_as_voice]]" in content:
has_voice_directive = True

media_tags, has_voice_directive = _collect_auto_append_media_tags(
result.get("messages", []),
history_offset=len(agent_history),
history_media_paths=_history_media_paths,
)

if media_tags:
seen = set()
unique_tags = []
Expand Down
57 changes: 57 additions & 0 deletions tests/gateway/test_media_extraction.py
Original file line number Diff line number Diff line change
Expand Up @@ -67,6 +67,63 @@ def extract_media_tags_broken(result_messages):

class TestMediaExtraction:
"""Tests for MEDIA tag extraction from tool results."""

def test_gateway_auto_append_ignores_media_examples_in_skill_docs(self):
"""Skill/documentation examples must not be appended as real attachments."""
from gateway.run import _collect_auto_append_media_tags

messages = [
{"role": "user", "content": "How should I format gateway media?"},
{
"role": "assistant",
"tool_calls": [
{"id": "call_skill", "function": {"name": "skill_view"}}
],
},
{
"role": "tool",
"tool_call_id": "call_skill",
"content": """
Recommended pattern:
```text
MEDIA:/absolute/path/to/image.png
```
Second message:
```text
caption
```
""",
},
{"role": "assistant", "content": "Use a standalone media message."},
]

tags, voice = _collect_auto_append_media_tags(messages, history_offset=0)
assert tags == []
assert voice is False

def test_gateway_auto_append_keeps_real_tts_media_tag(self):
"""TTS tool media tags are still auto-appended when the model omits them."""
from gateway.run import _collect_auto_append_media_tags

messages = [
{"role": "user", "content": "Say this as audio"},
{
"role": "assistant",
"tool_calls": [
{"id": "call_tts", "function": {"name": "text_to_speech"}}
],
},
{
"role": "tool",
"tool_call_id": "call_tts",
"content": '{"success": true, "media_tag": "[[audio_as_voice]]\\nMEDIA:/tmp/voice.ogg"}',
},
{"role": "assistant", "content": "Done."},
]

tags, voice = _collect_auto_append_media_tags(messages, history_offset=0)
assert tags == ["MEDIA:/tmp/voice.ogg"]
assert voice is True

def test_media_tags_not_extracted_from_history(self):
"""MEDIA tags from previous turns should NOT be extracted again."""
Expand Down