This repository was archived by the owner on May 9, 2024. It is now read-only.
File tree Expand file tree Collapse file tree 1 file changed +25
-0
lines changed
core/src/test/scala/org/apache/spark Expand file tree Collapse file tree 1 file changed +25
-0
lines changed Original file line number Diff line number Diff line change @@ -141,5 +141,30 @@ class FailureSuite extends SparkFunSuite with LocalSparkContext {
141141 FailureSuiteState .clear()
142142 }
143143
144+ test(" managed memory leak error should not mask other failures (SPARK-9266" ) {
145+ val conf = new SparkConf ().set(" spark.unsafe.exceptionOnMemoryLeak" , " true" )
146+ sc = new SparkContext (" local[1,1]" , " test" , conf)
147+
148+ // If a task leaks memory but fails due to some other cause, then make sure that the original
149+ // cause is preserved
150+ val thrownDueToTaskFailure = intercept[SparkException ] {
151+ sc.parallelize(Seq (0 )).mapPartitions { iter =>
152+ TaskContext .get().taskMemoryManager().allocate(128 )
153+ throw new Exception (" intentional task failure" )
154+ iter
155+ }.count()
156+ }
157+ assert(thrownDueToTaskFailure.getMessage.contains(" intentional task failure" ))
158+
159+ // If the task succeeded but memory was leaked, then the task should fail due to that leak
160+ val thrownDueToMemoryLeak = intercept[SparkException ] {
161+ sc.parallelize(Seq (0 )).mapPartitions { iter =>
162+ TaskContext .get().taskMemoryManager().allocate(128 )
163+ iter
164+ }.count()
165+ }
166+ assert(thrownDueToMemoryLeak.getMessage.contains(" memory leak" ))
167+ }
168+
144169 // TODO: Need to add tests with shuffle fetch failures.
145170}
You can’t perform that action at this time.
0 commit comments