File tree Expand file tree Collapse file tree 1 file changed +15
-4
lines changed
core/src/main/scala/org/apache/spark/network Expand file tree Collapse file tree 1 file changed +15
-4
lines changed Original file line number Diff line number Diff line change @@ -66,16 +66,27 @@ sealed abstract class ManagedBuffer {
6666final class FileSegmentManagedBuffer (val file : File , val offset : Long , val length : Long )
6767 extends ManagedBuffer {
6868
69+ /**
70+ * Memory mapping is expensive and can destabilize the JVM (SPARK-1145, SPARK-3889).
71+ * Avoid unless there's a good reason not to.
72+ */
73+ private val MIN_MEMORY_MAP_BYTES = 2 * 1024 * 1024 ;
74+
6975 override def size : Long = length
7076
7177 override def nioByteBuffer (): ByteBuffer = {
7278 var channel : FileChannel = null
7379 try {
7480 channel = new RandomAccessFile (file, " r" ).getChannel
75- val buf = ByteBuffer .allocate(length.toInt)
76- channel.read(buf, offset)
77- buf.flip()
78- buf
81+ // Just copy the buffer if it's sufficiently small, as memory mapping has a high overhead.
82+ if (length < MIN_MEMORY_MAP_BYTES ) {
83+ val buf = ByteBuffer .allocate(length.toInt)
84+ channel.read(buf, offset)
85+ buf.flip()
86+ buf
87+ } else {
88+ channel.map(MapMode .READ_ONLY , offset, length)
89+ }
7990 } catch {
8091 case e : IOException =>
8192 Try (channel.size).toOption match {
You can’t perform that action at this time.
0 commit comments