Skip to content

Commit a152065

Browse files
committed
Add other pathway back
1 parent 52b6cd2 commit a152065

File tree

1 file changed

+15
-4
lines changed

1 file changed

+15
-4
lines changed

core/src/main/scala/org/apache/spark/network/ManagedBuffer.scala

Lines changed: 15 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -66,16 +66,27 @@ sealed abstract class ManagedBuffer {
6666
final class FileSegmentManagedBuffer(val file: File, val offset: Long, val length: Long)
6767
extends ManagedBuffer {
6868

69+
/**
70+
* Memory mapping is expensive and can destabilize the JVM (SPARK-1145, SPARK-3889).
71+
* Avoid unless there's a good reason not to.
72+
*/
73+
private val MIN_MEMORY_MAP_BYTES = 2 * 1024 * 1024;
74+
6975
override def size: Long = length
7076

7177
override def nioByteBuffer(): ByteBuffer = {
7278
var channel: FileChannel = null
7379
try {
7480
channel = new RandomAccessFile(file, "r").getChannel
75-
val buf = ByteBuffer.allocate(length.toInt)
76-
channel.read(buf, offset)
77-
buf.flip()
78-
buf
81+
// Just copy the buffer if it's sufficiently small, as memory mapping has a high overhead.
82+
if (length < MIN_MEMORY_MAP_BYTES) {
83+
val buf = ByteBuffer.allocate(length.toInt)
84+
channel.read(buf, offset)
85+
buf.flip()
86+
buf
87+
} else {
88+
channel.map(MapMode.READ_ONLY, offset, length)
89+
}
7990
} catch {
8091
case e: IOException =>
8192
Try(channel.size).toOption match {

0 commit comments

Comments
 (0)