File tree Expand file tree Collapse file tree 1 file changed +5
-9
lines changed
core/src/main/scala/org/apache/spark/util/collection Expand file tree Collapse file tree 1 file changed +5
-9
lines changed Original file line number Diff line number Diff line change @@ -20,20 +20,11 @@ package org.apache.spark.util.collection
2020import org .apache .spark .Logging
2121import org .apache .spark .SparkEnv
2222
23- private [spark] object Spillable {
24- // Initial threshold for the size of a collection before we start tracking its memory usage
25- val initialMemoryThreshold : Long =
26- SparkEnv .get.conf.getLong(" spark.shuffle.spill.initialMemoryThreshold" , 5 * 1024 * 1024 )
27- }
28-
2923/**
3024 * Spills contents of an in-memory collection to disk when the memory threshold
3125 * has been exceeded.
3226 */
3327private [spark] trait Spillable [C ] extends Logging {
34-
35- import Spillable ._
36-
3728 /**
3829 * Spills the current in-memory collection to disk, and releases the memory.
3930 *
@@ -51,6 +42,11 @@ private[spark] trait Spillable[C] extends Logging {
5142 // Memory manager that can be used to acquire/release memory
5243 private [this ] val shuffleMemoryManager = SparkEnv .get.shuffleMemoryManager
5344
45+ // Initial threshold for the size of a collection before we start tracking its memory usage
46+ // Exposed for testing
47+ private [this ] val initialMemoryThreshold : Long =
48+ SparkEnv .get.conf.getLong(" spark.shuffle.spill.initialMemoryThreshold" , 5 * 1024 * 1024 )
49+
5450 // Threshold for this collection's size in bytes before we start tracking its memory usage
5551 // To avoid a large number of small spills, initialize this to a value orders of magnitude > 0
5652 private [this ] var myMemoryThreshold = initialMemoryThreshold
You can’t perform that action at this time.
0 commit comments