From 4d835beed7664e5b28992873208708b3ac61fdf3 Mon Sep 17 00:00:00 2001
From: asakura <0809android@gmail.com>
Date: Thu, 6 Aug 2026 19:21:00 +0900
Subject: [PATCH] fix: stop misclassifying multibyte UTF-8 text as binary
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

read_file samples the first 1000 bytes (head -c 1000) for binary
detection. When the byte boundary cuts through a multibyte UTF-8
character, the terminal backend's incremental decoder (errors=replace)
turns the dangling bytes into U+FFFD at the END of the sample.
_is_likely_binary then treats any U+FFFD as mojibake evidence and
classifies the file as binary — so valid Japanese/Chinese/Korean text
files frequently cannot be read.

Strip a trailing run of <=3 U+FFFD (a 4-byte char yields at most 3
dangling bytes) before judging. Genuine mojibake (U+FFFD throughout
the sample) is still flagged binary, preserving the read-only guard.
---
 tests/tools/test_file_operations.py | 26 ++++++++++++++++++++++++++
 tools/file_operations.py            | 20 +++++++++++++++++---
 2 files changed, 43 insertions(+), 3 deletions(-)

diff --git a/tests/tools/test_file_operations.py b/tests/tools/test_file_operations.py
index 3c53a4e25c992..f04e020e6b6a8 100644
--- a/tests/tools/test_file_operations.py
+++ b/tests/tools/test_file_operations.py
@@ -673,3 +673,29 @@ def test_plain_utf8_text_not_flagged(self, tmp_path):
         ops = ShellFileOperations(make_real_subprocess_env(str(tmp_path)))
         # Proper UTF-8 (including non-ASCII) must still read as text.
         assert ops._is_likely_binary("notes.txt", "café résumé\nsecond\n") is False
+
+    def test_trailing_replacement_from_byte_cut_not_flagged(self, tmp_path):
+        """A U+FFFD at the END of a 1000-byte sample is a `head -c 1000`
+        truncation artifact (multibyte UTF-8 char split at the byte boundary),
+        not evidence the file is binary. Japanese UTF-8 files must not be
+        misclassified as binary."""
+        ops = ShellFileOperations(make_real_subprocess_env(str(tmp_path)))
+        # Japanese text whose 1000-byte sample ends mid-character: the last
+        # byte pair (\xe5\xbe = "自" cut in half) decodes to a trailing U+FFFD.
+        sample = "# サトシ（受付・ワークコーディネーター）運用メモ\n\nこのファイルは、SOUL.md から分離した作業手順の正本。SOULの指示に従い、該当する作業の前に読むこと。" + "テキスト" * 100 + "\uFFFD"
+        assert ops._is_likely_binary("notes.txt", sample) is False
+
+    def test_replacement_in_middle_still_flagged(self, tmp_path):
+        """Genuine mojibake shows U+FFFD throughout the sample (not just at
+        the tail) — such files must still be flagged binary (read-only)."""
+        ops = ShellFileOperations(make_real_subprocess_env(str(tmp_path)))
+        # U+FFFD in the middle, not just at the end.
+        lossy = "caf\ufffd r\ufffdsum\ufffd\nmore text\n"
+        assert ops._is_likely_binary("notes.txt", lossy) is True
+
+    def test_long_trailing_replacement_run_still_flagged(self, tmp_path):
+        """A trailing U+FFFD run of 4+ is beyond any UTF-8 cut artifact
+        (max 3 dangling bytes) and must stay binary (safe direction)."""
+        ops = ShellFileOperations(make_real_subprocess_env(str(tmp_path)))
+        assert ops._is_likely_binary("notes.txt", "plain\n" + "\ufffd" * 5) is True
+        assert ops._is_likely_binary("notes.txt", "plain\n" + "\ufffd" * 3) is False
diff --git a/tools/file_operations.py b/tools/file_operations.py
index 2c1d1a62c9675..b16b9f55f1a50 100644
--- a/tools/file_operations.py
+++ b/tools/file_operations.py
@@ -903,11 +903,25 @@ def _is_likely_binary(self, path: str, content_sample: str = None) -> bool:
             # sample carries the replacement char as binary (read-only) so the
             # agent can't corrupt it. Legitimate UTF-8 text effectively never
             # contains U+FFFD.
-            if "\ufffd" in content_sample[:1000]:
+            sample = content_sample[:1000]
+            # Sampling artifact carve-out: read_file samples with
+            # `head -c 1000`, which cuts at a BYTE boundary. A multibyte
+            # UTF-8 char split across that cut decodes to 1-3 U+FFFD at the
+            # END of the sample (4-byte char → ≤3 dangling bytes), so a
+            # trailing run of ≤3 is not evidence of binary content — Japanese
+            # and other non-ASCII UTF-8 files would otherwise be
+            # misclassified as binary. Genuine mojibake shows U+FFFD
+            # throughout the sample, not just at the tail, so only the
+            # trailing run is stripped before judging. A longer trailing run
+            # (≥4) is kept as binary (safe direction).
+            stripped = sample.rstrip("\ufffd")
+            if len(sample) - len(stripped) <= 3:
+                sample = stripped
+            if "\ufffd" in sample:
                 return True
-            non_printable = sum(1 for c in content_sample[:1000]
+            non_printable = sum(1 for c in sample
                                if ord(c) < 32 and c not in '\n\r\t')
-            return non_printable / min(len(content_sample), 1000) > 0.30
+            return non_printable / min(len(sample), 1000) > 0.30
         
         return False