diff --git a/tests/tools/test_file_operations.py b/tests/tools/test_file_operations.py index 3c53a4e25c99..f04e020e6b6a 100644 --- a/tests/tools/test_file_operations.py +++ b/tests/tools/test_file_operations.py @@ -673,3 +673,29 @@ def test_plain_utf8_text_not_flagged(self, tmp_path): ops = ShellFileOperations(make_real_subprocess_env(str(tmp_path))) # Proper UTF-8 (including non-ASCII) must still read as text. assert ops._is_likely_binary("notes.txt", "café résumé\nsecond\n") is False + + def test_trailing_replacement_from_byte_cut_not_flagged(self, tmp_path): + """A U+FFFD at the END of a 1000-byte sample is a `head -c 1000` + truncation artifact (multibyte UTF-8 char split at the byte boundary), + not evidence the file is binary. Japanese UTF-8 files must not be + misclassified as binary.""" + ops = ShellFileOperations(make_real_subprocess_env(str(tmp_path))) + # Japanese text whose 1000-byte sample ends mid-character: the last + # byte pair (\xe5\xbe = "自" cut in half) decodes to a trailing U+FFFD. + sample = "# サトシ(受付・ワークコーディネーター)運用メモ\n\nこのファイルは、SOUL.md から分離した作業手順の正本。SOULの指示に従い、該当する作業の前に読むこと。" + "テキスト" * 100 + "\uFFFD" + assert ops._is_likely_binary("notes.txt", sample) is False + + def test_replacement_in_middle_still_flagged(self, tmp_path): + """Genuine mojibake shows U+FFFD throughout the sample (not just at + the tail) — such files must still be flagged binary (read-only).""" + ops = ShellFileOperations(make_real_subprocess_env(str(tmp_path))) + # U+FFFD in the middle, not just at the end. + lossy = "caf\ufffd r\ufffdsum\ufffd\nmore text\n" + assert ops._is_likely_binary("notes.txt", lossy) is True + + def test_long_trailing_replacement_run_still_flagged(self, tmp_path): + """A trailing U+FFFD run of 4+ is beyond any UTF-8 cut artifact + (max 3 dangling bytes) and must stay binary (safe direction).""" + ops = ShellFileOperations(make_real_subprocess_env(str(tmp_path))) + assert ops._is_likely_binary("notes.txt", "plain\n" + "\ufffd" * 5) is True + assert ops._is_likely_binary("notes.txt", "plain\n" + "\ufffd" * 3) is False diff --git a/tools/file_operations.py b/tools/file_operations.py index 2c1d1a62c967..b16b9f55f1a5 100644 --- a/tools/file_operations.py +++ b/tools/file_operations.py @@ -903,11 +903,25 @@ def _is_likely_binary(self, path: str, content_sample: str = None) -> bool: # sample carries the replacement char as binary (read-only) so the # agent can't corrupt it. Legitimate UTF-8 text effectively never # contains U+FFFD. - if "\ufffd" in content_sample[:1000]: + sample = content_sample[:1000] + # Sampling artifact carve-out: read_file samples with + # `head -c 1000`, which cuts at a BYTE boundary. A multibyte + # UTF-8 char split across that cut decodes to 1-3 U+FFFD at the + # END of the sample (4-byte char → ≤3 dangling bytes), so a + # trailing run of ≤3 is not evidence of binary content — Japanese + # and other non-ASCII UTF-8 files would otherwise be + # misclassified as binary. Genuine mojibake shows U+FFFD + # throughout the sample, not just at the tail, so only the + # trailing run is stripped before judging. A longer trailing run + # (≥4) is kept as binary (safe direction). + stripped = sample.rstrip("\ufffd") + if len(sample) - len(stripped) <= 3: + sample = stripped + if "\ufffd" in sample: return True - non_printable = sum(1 for c in content_sample[:1000] + non_printable = sum(1 for c in sample if ord(c) < 32 and c not in '\n\r\t') - return non_printable / min(len(content_sample), 1000) > 0.30 + return non_printable / min(len(sample), 1000) > 0.30 return False