fix(memory): tolerate non-UTF-8 bytes in USER.md/MEMORY.md reads

tests/tools/test_memory_tool.py

@@ -915,3 +915,68 @@ def test_already_blocked_entry_passes_through(self, tmp_path, monkeypatch):
         # Block marker appears exactly once, not nested
         assert snapshot.count("[BLOCKED:") == 1
         assert "Clean fact" in snapshot
+
+
+class TestNonUtf8Bytes:
+    """Regression for #53833: a stray non-UTF-8 byte in USER.md/MEMORY.md
+    (e.g. a smart-quote/dash saved under a cp1252 mismatch) must not raise
+    UnicodeDecodeError and wedge every future memory save."""
+
+    def test_read_file_replaces_invalid_bytes(self, tmp_path):
+        p = tmp_path / "MEMORY.md"
+        # ENTRY_DELIMITER is "\n§\n" (§ = \xc2\xa7). 0xd1 is an invalid UTF-8
+        # continuation byte embedded in the entry content.
+        p.write_bytes(b"Daniel prefers tables\n\xc2\xa7\nLikes \xd1 punchy prose")
+        entries = MemoryStore._read_file(p)
+        assert any("Daniel prefers tables" in e for e in entries)
+        assert len(entries) == 2
+        # errors="replace" (not ignore/strict): invalid byte becomes U+FFFD
+        second = next(e for e in entries if "Likes" in e)
+        assert "\uFFFD" in second
+        assert second == "Likes \uFFFD punchy prose"
+
+    def test_load_from_disk_survives_invalid_bytes(self, tmp_path, monkeypatch):
+        monkeypatch.setattr("tools.memory_tool.get_memory_dir", lambda: tmp_path)
+        (tmp_path / "MEMORY.md").write_bytes(b"Clean fact\n\xc2\xa7\nBad \xd1 byte fact")
+        (tmp_path / "USER.md").write_bytes(b"User likes \xd1 dashes")
+        s = MemoryStore()
+        s.load_from_disk()  # must not raise UnicodeDecodeError
+        assert any("Clean fact" in e for e in s.memory_entries)
+        assert s.user_entries
+        bad_memory = next(e for e in s.memory_entries if "Bad" in e)
+        assert "\uFFFD" in bad_memory
+        assert "\uFFFD" in s.user_entries[0]
+
+    def test_replace_via_reload_target_survives_invalid_bytes(self, tmp_path, monkeypatch):
+        """replace/remove call _reload_target → _detect_external_drift + _read_file.
+
+        Both disk-read paths must use errors="replace" so a malformed file does
+        not wedge mutation (the path hermes-sweeper asked us to cover).
+        """
+        monkeypatch.setattr("tools.memory_tool.get_memory_dir", lambda: tmp_path)
+        # Two tool-sized entries; second embeds invalid UTF-8 byte 0xd1.
+        (tmp_path / "MEMORY.md").write_bytes(
+            b"Keep this fact\n\xc2\xa7\nLikes \xd1 punchy prose"
+        )
+        (tmp_path / "USER.md").write_text("", encoding="utf-8")
+        s = MemoryStore()
+        result = s.replace("memory", "Likes", "Likes crisp prose")
+        assert result["success"] is True, result
+        assert "Likes crisp prose" in s.memory_entries
+        assert any("Keep this fact" in e for e in s.memory_entries)
+        # On-disk rewrite should be clean UTF-8 after replace
+        raw = (tmp_path / "MEMORY.md").read_text(encoding="utf-8")
+        assert "Likes crisp prose" in raw
+        assert "\uFFFD" not in raw
+
+    def test_remove_via_reload_target_survives_invalid_bytes(self, tmp_path, monkeypatch):
+        monkeypatch.setattr("tools.memory_tool.get_memory_dir", lambda: tmp_path)
+        (tmp_path / "MEMORY.md").write_bytes(
+            b"Keep this fact\n\xc2\xa7\nDrop \xd1 this entry"
+        )
+        (tmp_path / "USER.md").write_text("", encoding="utf-8")
+        s = MemoryStore()
+        result = s.remove("memory", "Drop")
+        assert result["success"] is True, result
+        assert len(s.memory_entries) == 1
+        assert "Keep this fact" in s.memory_entries[0]

tools/memory_tool.py

@@ -689,7 +689,11 @@ def _read_file(path: Path) -> List[str]:
         if not path.exists():
             return []
         try:
-            raw = path.read_text(encoding="utf-8")
+            # errors="replace" so a single stray non-UTF-8 byte in a legacy
+            # USER.md/MEMORY.md (e.g. a smart quote saved under cp1252) does
+            # not raise UnicodeDecodeError and wedge every future memory save
+            # (issue #53833).
+            raw = path.read_text(encoding="utf-8", errors="replace")
         except (OSError, IOError):
             return []
 
@@ -729,7 +733,7 @@ def _detect_external_drift(self, target: str) -> Optional[str]:
         if not path.exists():
             return None
         try:
-            raw = path.read_text(encoding="utf-8")
+            raw = path.read_text(encoding="utf-8", errors="replace")
         except (OSError, IOError):
             return None
         if not raw.strip():