diff --git a/core/src/main/scala/org/apache/spark/rdd/RDD.scala b/core/src/main/scala/org/apache/spark/rdd/RDD.scala index fe55a5124f3b6..f9a826643446b 100644 --- a/core/src/main/scala/org/apache/spark/rdd/RDD.scala +++ b/core/src/main/scala/org/apache/spark/rdd/RDD.scala @@ -1162,7 +1162,7 @@ abstract class RDD[T: ClassTag]( } else { // the left side of max is >=1 whenever partsScanned >= 2 numPartsToTry = Math.max((1.5 * num * partsScanned / buf.size).toInt - partsScanned, 1) - numPartsToTry = Math.min(numPartsToTry, partsScanned * 4) + numPartsToTry = Math.min(numPartsToTry, partsScanned * 4) } } @@ -1253,9 +1253,9 @@ abstract class RDD[T: ClassTag]( /** * @return true if and only if the RDD contains no elements at all. Note that an RDD - * may be empty even when it has at least 1 partition. + * may be empty even when it has 1 or more partitions. */ - def isEmpty(): Boolean = partitions.length == 0 || take(1).length == 0 + def isEmpty(): Boolean = partitions.length == 0 || mapPartitions(it => Iterator(!it.hasNext)).reduce(_&&_) /** * Save this RDD as a text file, using string representations of elements. diff --git a/core/src/test/scala/org/apache/spark/rdd/RDDSuite.scala b/core/src/test/scala/org/apache/spark/rdd/RDDSuite.scala index bede1ffb3e2d0..0dc108d8bd9db 100644 --- a/core/src/test/scala/org/apache/spark/rdd/RDDSuite.scala +++ b/core/src/test/scala/org/apache/spark/rdd/RDDSuite.scala @@ -570,6 +570,8 @@ class RDDSuite extends FunSuite with SharedSparkContext { assert(!sc.parallelize(Seq(1)).isEmpty()) assert(sc.parallelize(Seq(1,2,3), 3).filter(_ < 0).isEmpty()) assert(!sc.parallelize(Seq(1,2,3), 3).filter(_ > 1).isEmpty()) + // Test empty partition (SPARK-5744) + assert(sc.parallelize(Seq(), 1).isEmpty()) } test("sample preserves partitioner") {