Skip to content

Commit 8959f7e

Browse files
committed
Add test case
1 parent 3792a6d commit 8959f7e

File tree

1 file changed

+26
-0
lines changed

1 file changed

+26
-0
lines changed

repl/src/test/scala/org/apache/spark/repl/SingletonReplSuite.scala

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -380,6 +380,32 @@ class SingletonReplSuite extends SparkFunSuite {
380380
assertDoesNotContain("Exception", output)
381381
}
382382

383+
test("SPARK-31399: should clone+clean line object w/ non-serializable state in ClosureCleaner") {
384+
// Can't use :paste mode because PipedOutputStream/PipedInputStream doesn't work well with the
385+
// EOT control character (i.e. Ctrl+D).
386+
// Just write things on a single line to emulate :paste mode.
387+
388+
// NOTE: in order for this test case to trigger the intended scenario, the following three
389+
// variables need to be in the same "input", which will make the REPL pack them into the
390+
// same REPL line object:
391+
// - ns: a non-serializable state, not accessed by the closure;
392+
// - topLevelValue: a serializable state, accessed by the closure;
393+
// - closure: the starting closure, captures the enclosing REPL line object.
394+
val output = runInterpreter(
395+
"""
396+
|class NotSerializableClass(val x: Int)
397+
|val ns = new NotSerializableClass(42); val topLevelValue = "someValue"; val closure =
398+
|(j: Int) => {
399+
| (1 to j).flatMap { x =>
400+
| (1 to x).map { y => y + topLevelValue }
401+
| }
402+
|}
403+
|val r = sc.parallelize(0 to 2).map(closure).collect
404+
""".stripMargin)
405+
assertContains("r: Array[scala.collection.immutable.IndexedSeq[String]] = Array(Vector", output)
406+
assertDoesNotContain("Exception", output)
407+
}
408+
383409
test("newProductSeqEncoder with REPL defined class") {
384410
val output = runInterpreter(
385411
"""

0 commit comments

Comments
 (0)