RDD take method: overestimate too much

yingjieMiao · yingjieMiao · commit a2aa36b6838f · 2014-10-03T15:26:01.000-07:00
diff --git a/core/src/main/scala/org/apache/spark/rdd/RDD.scala b/core/src/main/scala/org/apache/spark/rdd/RDD.scala
@@ -1084,10 +1084,9 @@ abstract class RDD[T: ClassTag](
         if (buf.size == 0) {
           numPartsToTry = partsScanned * 4
         } else {
-          numPartsToTry = (1.5 * num * partsScanned / buf.size).toInt
+          numPartsToTry = (1.5 * num * partsScanned / buf.size).toInt - partsScanned
         }
       }
-      numPartsToTry = math.max(0, numPartsToTry)  // guard against negative num of partitions
 
       val left = num - buf.size
       val p = partsScanned until math.min(partsScanned + numPartsToTry, totalParts)

Original file line number	Diff line number	Diff line change
`@@ -1084,10 +1084,9 @@ abstract class RDD[T: ClassTag](`
`1084`	`1084`	`if (buf.size == 0) {`
`1085`	`1085`	`numPartsToTry = partsScanned * 4`
`1086`	`1086`	`} else {`
`1087`		`- numPartsToTry = (1.5 * num * partsScanned / buf.size).toInt`
	`1087`	`+ numPartsToTry = (1.5 * num * partsScanned / buf.size).toInt - partsScanned`
`1088`	`1088`	`}`
`1089`	`1089`	`}`
`1090`		`- numPartsToTry = math.max(0, numPartsToTry) // guard against negative num of partitions`
`1091`	`1090`
`1092`	`1091`	`val left = num - buf.size`
`1093`	`1092`	`val p = partsScanned until math.min(partsScanned + numPartsToTry, totalParts)`