Skip to content

Commit 3348619

Browse files
author
liguoqiang
committed
Optimize performance for partitions check
1 parent 61e5a87 commit 3348619

File tree

1 file changed

+4
-4
lines changed

1 file changed

+4
-4
lines changed

core/src/main/scala/org/apache/spark/SparkContext.scala

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -847,8 +847,8 @@ class SparkContext(
847847
partitions: Seq[Int],
848848
allowLocal: Boolean,
849849
resultHandler: (Int, U) => Unit) {
850-
val rddPartitions = rdd.partitions.map(_.index)
851-
require(partitions.forall(rddPartitions.contains(_)), "partition index out of range")
850+
val partitionRange = (0 until rdd.partitions.size)
851+
require(partitions.forall(partitionRange.contains(_)), "partition index out of range")
852852
val callSite = getCallSite
853853
val cleanedFunc = clean(func)
854854
logInfo("Starting job: " + callSite)
@@ -952,8 +952,8 @@ class SparkContext(
952952
resultHandler: (Int, U) => Unit,
953953
resultFunc: => R): SimpleFutureAction[R] =
954954
{
955-
val rddPartitions = rdd.partitions.map(_.index)
956-
require(partitions.forall(rddPartitions.contains(_)), "partition index out of range")
955+
val partitionRange = (0 until rdd.partitions.size)
956+
require(partitions.forall(partitionRange.contains(_)), "partition index out of range")
957957
val cleanF = clean(processPartition)
958958
val callSite = getCallSite
959959
val waiter = dagScheduler.submitJob(

0 commit comments

Comments
 (0)