fix: stop misclassifying multibyte UTF-8 text files as binary in read_file

tests/tools/test_file_operations.py

@@ -673,3 +673,29 @@ def test_plain_utf8_text_not_flagged(self, tmp_path):
         ops = ShellFileOperations(make_real_subprocess_env(str(tmp_path)))
         # Proper UTF-8 (including non-ASCII) must still read as text.
         assert ops._is_likely_binary("notes.txt", "café résumé\nsecond\n") is False
+
+    def test_trailing_replacement_from_byte_cut_not_flagged(self, tmp_path):
+        """A U+FFFD at the END of a 1000-byte sample is a `head -c 1000`
+        truncation artifact (multibyte UTF-8 char split at the byte boundary),
+        not evidence the file is binary. Japanese UTF-8 files must not be
+        misclassified as binary."""
+        ops = ShellFileOperations(make_real_subprocess_env(str(tmp_path)))
+        # Japanese text whose 1000-byte sample ends mid-character: the last
+        # byte pair (\xe5\xbe = "自" cut in half) decodes to a trailing U+FFFD.
+        sample = "# サトシ（受付・ワークコーディネーター）運用メモ\n\nこのファイルは、SOUL.md から分離した作業手順の正本。SOULの指示に従い、該当する作業の前に読むこと。" + "テキスト" * 100 + "\uFFFD"
+        assert ops._is_likely_binary("notes.txt", sample) is False
+
+    def test_replacement_in_middle_still_flagged(self, tmp_path):
+        """Genuine mojibake shows U+FFFD throughout the sample (not just at
+        the tail) — such files must still be flagged binary (read-only)."""
+        ops = ShellFileOperations(make_real_subprocess_env(str(tmp_path)))
+        # U+FFFD in the middle, not just at the end.
+        lossy = "caf\ufffd r\ufffdsum\ufffd\nmore text\n"
+        assert ops._is_likely_binary("notes.txt", lossy) is True
+
+    def test_long_trailing_replacement_run_still_flagged(self, tmp_path):
+        """A trailing U+FFFD run of 4+ is beyond any UTF-8 cut artifact
+        (max 3 dangling bytes) and must stay binary (safe direction)."""
+        ops = ShellFileOperations(make_real_subprocess_env(str(tmp_path)))
+        assert ops._is_likely_binary("notes.txt", "plain\n" + "\ufffd" * 5) is True
+        assert ops._is_likely_binary("notes.txt", "plain\n" + "\ufffd" * 3) is False

tools/file_operations.py

@@ -903,11 +903,25 @@ def _is_likely_binary(self, path: str, content_sample: str = None) -> bool:
             # sample carries the replacement char as binary (read-only) so the
             # agent can't corrupt it. Legitimate UTF-8 text effectively never
             # contains U+FFFD.
-            if "\ufffd" in content_sample[:1000]:
+            sample = content_sample[:1000]
+            # Sampling artifact carve-out: read_file samples with
+            # `head -c 1000`, which cuts at a BYTE boundary. A multibyte
+            # UTF-8 char split across that cut decodes to 1-3 U+FFFD at the
+            # END of the sample (4-byte char → ≤3 dangling bytes), so a
+            # trailing run of ≤3 is not evidence of binary content — Japanese
+            # and other non-ASCII UTF-8 files would otherwise be
+            # misclassified as binary. Genuine mojibake shows U+FFFD
+            # throughout the sample, not just at the tail, so only the
+            # trailing run is stripped before judging. A longer trailing run
+            # (≥4) is kept as binary (safe direction).
+            stripped = sample.rstrip("\ufffd")
+            if len(sample) - len(stripped) <= 3:
+                sample = stripped
+            if "\ufffd" in sample:
                 return True
-            non_printable = sum(1 for c in content_sample[:1000]
+            non_printable = sum(1 for c in sample
                                if ord(c) < 32 and c not in '\n\r\t')
-            return non_printable / min(len(content_sample), 1000) > 0.30
+            return non_printable / min(len(sample), 1000) > 0.30
 
         return False