Fix #521: inline multimodal read_file attachments

tests/test_provider_parity.py

@@ -216,6 +216,69 @@ def test_tool_results_become_function_call_output(self, monkeypatch):
         assert items[0]["call_id"] == "call_abc"
         assert items[0]["output"] == "result here"
 
+    def test_user_multimodal_message_maps_to_codex_input_parts(self, monkeypatch):
+        agent = _make_agent(monkeypatch, "openai-codex", api_mode="codex_responses",
+                            base_url="https://chatgpt.com/backend-api/codex")
+        messages = [{
+            "role": "user",
+            "content": [
+                {"type": "text", "text": "Inspect these"},
+                {"type": "image_url", "image_url": {"url": "data:image/png;base64,AAAA"}},
+                {
+                    "type": "file",
+                    "file": {
+                        "filename": "report.pdf",
+                        "file_data": "data:application/pdf;base64,JVBERi0xLjQK",
+                    },
+                },
+            ],
+        }]
+
+        items = agent._chat_messages_to_responses_input(messages)
+
+        assert items == [{
+            "role": "user",
+            "content": [
+                {"type": "input_text", "text": "Inspect these"},
+                {"type": "input_image", "image_url": "data:image/png;base64,AAAA"},
+                {
+                    "type": "input_file",
+                    "filename": "report.pdf",
+                    "file_data": "data:application/pdf;base64,JVBERi0xLjQK",
+                },
+            ],
+        }]
+
+
+class TestInlineReadFileAttachmentGating:
+    def test_custom_chat_completions_does_not_inject_inline_image(self, monkeypatch):
+        agent = _make_agent(monkeypatch, "custom", api_mode="chat_completions", base_url="http://localhost:1234/v1")
+
+        messages = agent._build_tool_result_messages(
+            function_name="read_file",
+            function_args={"path": "assets/pixel.png"},
+            function_result='{"is_image":true,"is_binary":true,"mime_type":"image/png","base64_content":"AAAA"}',
+            tool_call_id="call_img",
+        )
+
+        assert len(messages) == 1
+        assert messages[0]["role"] == "tool"
+        assert "not supported for the current provider/API mode" in messages[0]["content"]
+
+    def test_codex_does_not_inject_inline_pdf_and_explains_limitation(self, monkeypatch):
+        agent = _make_agent(monkeypatch, "openai-codex", api_mode="codex_responses",
+                            base_url="https://chatgpt.com/backend-api/codex")
+
+        messages = agent._build_tool_result_messages(
+            function_name="read_file",
+            function_args={"path": "docs/report.pdf"},
+            function_result='{"is_pdf":true,"is_binary":true,"mime_type":"application/pdf","base64_content":"JVBERi0xLjQK"}',
+            tool_call_id="call_pdf",
+        )
+
+        assert len(messages) == 1
+        assert "supported OpenRouter chat-completions paths" in messages[0]["content"]
+
     def test_encrypted_reasoning_replayed(self, monkeypatch):
         """Encrypted reasoning items from previous turns must be included in input."""
         agent = _make_agent(monkeypatch, "openai-codex", api_mode="codex_responses",

tests/test_run_agent.py

@@ -217,6 +217,37 @@ def test_extra_newlines_cleaned(self):
         assert "after" in result
 
 
+class TestInlineReadFileAttachments:
+    def test_build_tool_result_messages_injects_image_context(self, agent):
+        tool_result = json.dumps(
+            {
+                "is_image": True,
+                "is_binary": True,
+                "file_size": 68,
+                "mime_type": "image/png",
+                "base64_content": "AAAA",
+            }
+        )
+
+        messages = agent._build_tool_result_messages(
+            function_name="read_file",
+            function_args={"path": "assets/pixel.png"},
+            function_result=tool_result,
+            tool_call_id="call_img",
+        )
+
+        assert messages[0]["role"] == "tool"
+        assert messages[0]["tool_call_id"] == "call_img"
+        assert "base64_content" not in messages[0]["content"]
+        assert messages[1]["role"] == "user"
+        assert messages[1]["content"][0]["type"] == "text"
+        assert "read_file('assets/pixel.png')" in messages[1]["content"][0]["text"]
+        assert messages[1]["content"][1] == {
+            "type": "image_url",
+            "image_url": {"url": "data:image/png;base64,AAAA"},
+        }
+
+
 class TestGetMessagesUpToLastAssistant:
     def test_empty_list(self, agent):
         assert agent._get_messages_up_to_last_assistant([]) == []

tests/tools/test_file_operations.py

@@ -215,12 +215,12 @@ def test_is_likely_binary_by_content(self, file_ops):
         # Normal text -> not binary
         assert file_ops._is_likely_binary("unknown", "Hello world\nLine 2\n") is False
 
-    def test_is_image(self, file_ops):
-        assert file_ops._is_image("photo.png") is True
-        assert file_ops._is_image("pic.jpg") is True
-        assert file_ops._is_image("icon.ico") is True
-        assert file_ops._is_image("data.pdf") is False
-        assert file_ops._is_image("code.py") is False
+    def test_classify_attachment(self, file_ops):
+        assert file_ops._classify_attachment("photo.png") == "image"
+        assert file_ops._classify_attachment("pic.jpg") == "image"
+        assert file_ops._classify_attachment("icon.ico") == "image"
+        assert file_ops._classify_attachment("data.pdf") == "pdf"
+        assert file_ops._classify_attachment("code.py") is None
 
     def test_add_line_numbers(self, file_ops):
         content = "line one\nline two\nline three"

tests/tools/test_file_tools_live.py

@@ -8,6 +8,7 @@
 asserts zero contamination from shell noise via _assert_clean().
 """
 
+import base64
 import json
 import os
 import sys
@@ -59,6 +60,10 @@ def _assert_clean(text: str, context: str = "output"):
 MULTIFILE_A = "def func_alpha():\n    return 42\n"
 MULTIFILE_B = "def func_bravo():\n    return 99\n"
 MULTIFILE_C = "nothing relevant here\n"
+TINY_PNG = base64.b64decode(
+    "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mP8/x8AAusB9Wn9l1cAAAAASUVORK5CYII="
+)
+MINIMAL_PDF = b"%PDF-1.4\n1 0 obj\n<< /Type /Catalog >>\nendobj\ntrailer\n<<>>\n%%EOF\n"
 
 
 @pytest.fixture
@@ -354,6 +359,30 @@ def test_no_noise_in_content(self, ops, tmp_path):
         assert result.error is None
         _assert_clean(result.content)
 
+    def test_small_png_returns_inline_attachment_data(self, ops, tmp_path):
+        f = tmp_path / "pixel.png"
+        f.write_bytes(TINY_PNG)
+
+        result = ops.read_file(str(f))
+
+        assert result.error is None
+        assert result.is_image is True
+        assert result.is_binary is True
+        assert result.mime_type == "image/png"
+        assert base64.b64decode(result.base64_content) == TINY_PNG
+
+    def test_small_pdf_returns_inline_attachment_data(self, ops, tmp_path):
+        f = tmp_path / "report.pdf"
+        f.write_bytes(MINIMAL_PDF)
+
+        result = ops.read_file(str(f))
+
+        assert result.error is None
+        assert result.is_pdf is True
+        assert result.is_binary is True
+        assert result.mime_type == "application/pdf"
+        assert base64.b64decode(result.base64_content) == MINIMAL_PDF
+
 
 # ── write_file ───────────────────────────────────────────────────────────
 

tools/file_operations.py

@@ -100,6 +100,7 @@ class ReadResult:
     hint: Optional[str] = None
     is_binary: bool = False
     is_image: bool = False
+    is_pdf: bool = False
     base64_content: Optional[str] = None
     mime_type: Optional[str] = None
     dimensions: Optional[str] = None  # For images: "WIDTHxHEIGHT"
@@ -274,6 +275,18 @@ def search(self, pattern: str, path: str = ".", target: str = "content",
 # Image extensions (subset of binary that we can return as base64)
 IMAGE_EXTENSIONS = {'.png', '.jpg', '.jpeg', '.gif', '.webp', '.bmp', '.ico'}
 
+# MIME types for inline attachment extensions
+MIME_TYPES = {
+    '.png': 'image/png',
+    '.jpg': 'image/jpeg',
+    '.jpeg': 'image/jpeg',
+    '.gif': 'image/gif',
+    '.webp': 'image/webp',
+    '.bmp': 'image/bmp',
+    '.ico': 'image/x-icon',
+    '.pdf': 'application/pdf',
+}
+
 # Linters by file extension
 LINTERS = {
     '.py': 'python -m py_compile {file} 2>&1',
@@ -364,10 +377,14 @@ def _is_likely_binary(self, path: str, content_sample: str = None) -> bool:
 
         return False
 
-    def _is_image(self, path: str) -> bool:
-        """Check if file is an image we can return as base64."""
+    def _classify_attachment(self, path: str) -> Optional[str]:
+        """Return 'image', 'pdf', or None based on file extension."""
         ext = os.path.splitext(path)[1].lower()
-        return ext in IMAGE_EXTENSIONS
+        if ext in IMAGE_EXTENSIONS:
+            return "image"
+        if ext == ".pdf":
+            return "pdf"
+        return None
 
     def _add_line_numbers(self, content: str, start_line: int = 1) -> str:
         """Add line numbers to content in LINE_NUM|CONTENT format."""
@@ -463,17 +480,9 @@ def read_file(self, path: str, offset: int = 1, limit: int = 500) -> ReadResult:
             # Still try to read, but warn
             pass
 
-        # Images are never inlined — redirect to the vision tool
-        if self._is_image(path):
-            return ReadResult(
-                is_image=True,
-                is_binary=True,
-                file_size=file_size,
-                hint=(
-                    "Image file detected. Automatically redirected to vision_analyze tool. "
-                    "Use vision_analyze with this file path to inspect the image contents."
-                ),
-            )
+        attachment_kind = self._classify_attachment(path)
+        if attachment_kind is not None:
+            return self._read_inline_attachment(path, file_size, attachment_kind)
 
         # Read a sample to check for binary content
         sample_cmd = f"head -c 1000 {self._escape_shell_arg(path)} 2>/dev/null"
@@ -516,71 +525,59 @@ def read_file(self, path: str, offset: int = 1, limit: int = 500) -> ReadResult:
             hint=hint
         )
 
-    # Images larger than this are too expensive to inline as base64 in the
-    # conversation context. Return metadata only and suggest vision_analyze.
-    MAX_IMAGE_BYTES = 512 * 1024  # 512 KB
+    MAX_INLINE_ATTACHMENT_BYTES = 512 * 1024  # 512 KB
 
-    def _read_image(self, path: str) -> ReadResult:
-        """Read an image file, returning base64 content."""
-        # Get file size (wc -c is POSIX, works on Linux + macOS)
-        stat_cmd = f"wc -c < {self._escape_shell_arg(path)} 2>/dev/null"
-        stat_result = self._exec(stat_cmd)
-        try:
-            file_size = int(stat_result.stdout.strip())
-        except ValueError:
-            file_size = 0
-
-        if file_size > self.MAX_IMAGE_BYTES:
+    def _read_inline_attachment(self, path: str, file_size: int, attachment_kind: str) -> ReadResult:
+        """Read an image or PDF file, returning base64 content when small enough."""
+        is_image = attachment_kind == "image"
+        is_pdf = attachment_kind == "pdf"
+
+        if file_size > self.MAX_INLINE_ATTACHMENT_BYTES:
+            noun = "Image" if is_image else "PDF"
+            guidance = (
+                "Use vision_analyze to inspect the image, or reference it by path."
+                if is_image
+                else "Reference it by path or use terminal tools to inspect it."
+            )
             return ReadResult(
-                is_image=True,
+                is_image=is_image,
+                is_pdf=is_pdf,
                 is_binary=True,
                 file_size=file_size,
-                hint=(
-                    f"Image is too large to inline ({file_size:,} bytes). "
-                    "Use vision_analyze to inspect the image, or reference it by path."
-                ),
+                hint=f"{noun} is too large to inline ({file_size:,} bytes). {guidance}",
             )
-
-        # Get base64 content
-        b64_cmd = f"base64 -w 0 {self._escape_shell_arg(path)} 2>/dev/null"
+
+        b64_cmd = f"base64 < {self._escape_shell_arg(path)} | tr -d '\\n'"
         b64_result = self._exec(b64_cmd, timeout=30)
-        
+
         if b64_result.exit_code != 0:
+            noun = "image" if is_image else "PDF"
             return ReadResult(
-                is_image=True,
+                is_image=is_image,
+                is_pdf=is_pdf,
                 is_binary=True,
                 file_size=file_size,
-                error=f"Failed to read image: {b64_result.stdout}"
+                error=f"Failed to read {noun}: {b64_result.stdout}",
             )
-
-        # Try to get dimensions (requires ImageMagick)
+
         dimensions = None
-        if self._has_command('identify'):
+        if is_image and self._has_command('identify'):
             dim_cmd = f"identify -format '%wx%h' {self._escape_shell_arg(path)} 2>/dev/null"
             dim_result = self._exec(dim_cmd)
             if dim_result.exit_code == 0:
                 dimensions = dim_result.stdout.strip()
-
-        # Determine MIME type from extension
+
         ext = os.path.splitext(path)[1].lower()
-        mime_types = {
-            '.png': 'image/png',
-            '.jpg': 'image/jpeg',
-            '.jpeg': 'image/jpeg',
-            '.gif': 'image/gif',
-            '.webp': 'image/webp',
-            '.bmp': 'image/bmp',
-            '.ico': 'image/x-icon',
-        }
-        mime_type = mime_types.get(ext, 'application/octet-stream')
-
+        mime_type = MIME_TYPES.get(ext, 'application/octet-stream')
+
         return ReadResult(
-            is_image=True,
+            is_image=is_image,
+            is_pdf=is_pdf,
             is_binary=True,
             file_size=file_size,
             base64_content=b64_result.stdout,
             mime_type=mime_type,
-            dimensions=dimensions
+            dimensions=dimensions,
         )
 
     def _suggest_similar_files(self, path: str) -> ReadResult:

tools/file_tools.py

@@ -211,7 +211,7 @@ def _check_file_reqs():
 
 READ_FILE_SCHEMA = {
     "name": "read_file",
-    "description": "Read a text file with line numbers and pagination. Use this instead of cat/head/tail in terminal. Output format: 'LINE_NUM|CONTENT'. Suggests similar filenames if not found. Use offset and limit for large files. NOTE: Cannot read images or binary files — use vision_analyze for images.",
+    "description": "Read a text file with line numbers and pagination. Use this instead of cat/head/tail in terminal. Output format: 'LINE_NUM|CONTENT'. Suggests similar filenames if not found. Use offset and limit for large files. Small images may be attached inline for supported multimodal providers. Small PDFs are currently only attached inline on supported OpenRouter chat-completions paths; other providers return metadata only.",
     "parameters": {
         "type": "object",
         "properties": {