diff --git a/tests/tools/test_file_operations_edge_cases.py b/tests/tools/test_file_operations_edge_cases.py index ca5fa188f63c9..93d506c754aaa 100644 --- a/tests/tools/test_file_operations_edge_cases.py +++ b/tests/tools/test_file_operations_edge_cases.py @@ -51,6 +51,31 @@ def test_content_sample_longer_than_1000(self, ops): sample = "\x00" * 200 + "a" * 800 + "\x00" * 1000 assert ops._is_likely_binary("file.xyz", content_sample=sample) is False + def test_single_trailing_replacement_is_sampling_artifact(self, ops): + """A lone trailing U+FFFD is a `head -c 1000` truncation artifact. + + When the 1000-byte sample boundary splits a multi-byte UTF-8 char, + the terminal's errors=replace decode emits exactly one trailing + U+FFFD. That is not binary content and must not block reading a + valid UTF-8 text file (regression for #81480). + """ + sample = "# Instru\u00e7\u00f5es de review \u2014 valida\u00e7\u00e3o\n" * 60 + "\ufffd" + assert sample.endswith("\ufffd") + assert sample.count("\ufffd") == 1 + assert ops._is_likely_binary("notes.md", content_sample=sample) is False + + def test_multiple_scattered_replacements_are_binary(self, ops): + """Genuine binary data produces many scattered U+FFFD chars.""" + # Random non-UTF-8 bytes decoded with errors=replace + sample = "\ufffd" * 40 + "abc" * 100 + "\ufffd" * 40 + assert ops._is_likely_binary("data.bin", content_sample=sample) is True + + def test_trailing_replacement_with_other_garbage_is_binary(self, ops): + """A trailing U+FFFD plus any other replacement char is binary.""" + sample = "text" + "\ufffd" + "more" + "\ufffd" + assert sample.count("\ufffd") == 2 + assert ops._is_likely_binary("f.xyz", content_sample=sample) is True + # ========================================================================= # _check_lint edge cases diff --git a/tools/file_operations.py b/tools/file_operations.py index 2c1d1a62c9675..9c765a513569a 100644 --- a/tools/file_operations.py +++ b/tools/file_operations.py @@ -903,11 +903,22 @@ def _is_likely_binary(self, path: str, content_sample: str = None) -> bool: # sample carries the replacement char as binary (read-only) so the # agent can't corrupt it. Legitimate UTF-8 text effectively never # contains U+FFFD. - if "\ufffd" in content_sample[:1000]: - return True - non_printable = sum(1 for c in content_sample[:1000] + sample = content_sample[:1000] + # One exception to the U+FFFD rule: read_file samples with + # `head -c 1000`, which truncates at a byte boundary. When that + # boundary splits a multi-byte UTF-8 character, the dangling + # continuation bytes decode to exactly one *trailing* U+FFFD — + # a sampling artifact, not evidence of binary content. Genuine + # binary data yields many scattered replacement chars (well above + # the non-printable ratio below too), so a lone trailing U+FFFD + # is treated as text, not binary. (#81480) + if "\ufffd" in sample: + if sample.count("\ufffd") > 1 or not sample.endswith("\ufffd"): + return True + return False + non_printable = sum(1 for c in sample if ord(c) < 32 and c not in '\n\r\t') - return non_printable / min(len(content_sample), 1000) > 0.30 + return non_printable / min(len(sample), 1000) > 0.30 return False