fix: stop misclassifying UTF-8 files cut at the 1000-byte sample boundary as binary

tests/tools/test_file_operations_edge_cases.py

@@ -51,6 +51,37 @@ def test_content_sample_longer_than_1000(self, ops):
         sample = "\x00" * 200 + "a" * 800 + "\x00" * 1000
         assert ops._is_likely_binary("file.xyz", content_sample=sample) is False
 
+    def test_truncation_artifact_trailing_replacement_char(self, ops):
+        """A lone U+FFFD at the sample tail is a head -c 1000 truncation
+        artifact (multi-byte UTF-8 char sliced at the boundary), not binary
+        evidence — legitimate text must not be refused as 'Binary file'."""
+        # Valid UTF-8 file whose 1000th byte lands mid-character: the sample
+        # is 999 ASCII + one replacement char squeezed against the tail.
+        sample = "a" * 999 + "\ufffd"
+        assert ops._is_likely_binary("file.md", content_sample=sample) is False
+        # A few printable chars after the boundary-cut char (e.g. second
+        # sample pass or line-joined output) must also stay text.
+        sample2 = "a" * 998 + "\ufffd" + "b"
+        assert ops._is_likely_binary("file.md", content_sample=sample2) is False
+
+    def test_replacement_char_in_middle_is_binary(self, ops):
+        """U+FFFD away from the tail = real undecodable bytes = binary."""
+        sample = "a" * 100 + "\ufffd" + "b" * 899
+        assert ops._is_likely_binary("file.xyz", content_sample=sample) is True
+
+    def test_multiple_replacement_chars_is_binary(self, ops):
+        """Two U+FFFDs anywhere (even near the tail) = binary: valid UTF-8
+        cut at one boundary produces at most a single replacement char."""
+        sample = "a" * 998 + "\ufffd" + "\ufffd"
+        assert ops._is_likely_binary("file.xyz", content_sample=sample) is True
+
+    def test_trailing_replacement_char_with_binary_ratio(self, ops):
+        """A trailing U+FFFD does NOT mask real binary content: the
+        non-printable ratio check still catches NUL-heavy samples."""
+        # 301 NULs + 697 'a' + trailing artifact char: ratio > 30% → binary
+        sample = "\x00" * 301 + "a" * 697 + "\ufffd"
+        assert ops._is_likely_binary("file.xyz", content_sample=sample) is True
+
 
 # =========================================================================
 # _check_lint edge cases

tools/file_operations.py

@@ -901,13 +901,26 @@ def _is_likely_binary(self, path: str, content_sample: str = None) -> bool:
             # lossy text would let a read→edit→write round-trip silently
             # overwrite the original bytes with mojibake. Treat a file whose
             # sample carries the replacement char as binary (read-only) so the
-            # agent can't corrupt it. Legitimate UTF-8 text effectively never
-            # contains U+FFFD.
-            if "\ufffd" in content_sample[:1000]:
+            # agent can't corrupt it.
+            #
+            # Caveat: `head -c 1000` can slice a multi-byte UTF-8 character
+            # at the sample boundary, and the truncated tail decodes to a
+            # single trailing U+FFFD even for perfectly valid text. That
+            # truncation artifact must NOT be treated as binary evidence —
+            # otherwise legitimate UTF-8 files whose 1000th byte lands mid-
+            # character are refused with "Binary file". Distinguish it from
+            # real undecodable bytes: a true binary sample carries U+FFFD in
+            # the middle of the sample and/or more than once, not a lone
+            # replacement char squeezed against the tail.
+            sample = content_sample[:1000]
+            fffd_count = sample.count("\ufffd")
+            tail_start = max(0, len(sample) - 3)  # UTF-8 seq ≤4 bytes → cut loses ≤3
+            trailing_only = fffd_count == 1 and sample.rfind("\ufffd") >= tail_start
+            if fffd_count and not trailing_only:
                 return True
-            non_printable = sum(1 for c in content_sample[:1000]
+            non_printable = sum(1 for c in sample
                                if ord(c) < 32 and c not in '\n\r\t')
-            return non_printable / min(len(content_sample), 1000) > 0.30
+            return non_printable / min(len(sample), 1000) > 0.30
 
         return False