Skip to content

Commit 70de2b4

Browse files
author
Vinod K C
committed
Handled review comments
1 parent 4e6ce62 commit 70de2b4

File tree

1 file changed

+21
-16
lines changed
  • core/src/main/scala/org/apache/spark/rdd

1 file changed

+21
-16
lines changed

core/src/main/scala/org/apache/spark/rdd/RDD.scala

Lines changed: 21 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -480,27 +480,32 @@ abstract class RDD[T: ClassTag](
480480
"Cannot support a sample size > Int.MaxValue - " +
481481
s"$numStDev * math.sqrt(Int.MaxValue)")
482482

483-
if (num == 0 || this.count() == 0) {
483+
if (num == 0) {
484484
new Array[T](0)
485485
} else {
486486
val initialCount = this.count()
487-
val rand = new Random(seed)
488-
if (!withReplacement && num >= initialCount) {
489-
Utils.randomizeInPlace(this.collect(), rand)
487+
if(initialCount ==0)
488+
{
489+
new Array[T](0)
490490
} else {
491-
val fraction = SamplingUtils.computeFractionForSampleSize(num, initialCount,
492-
withReplacement)
493-
var samples = this.sample(withReplacement, fraction, rand.nextInt()).collect()
494-
495-
// If the first sample didn't turn out large enough, keep trying to take samples;
496-
// this shouldn't happen often because we use a big multiplier for the initial size
497-
var numIters = 0
498-
while (samples.length < num) {
499-
logWarning(s"Needed to re-sample due to insufficient sample size. Repeat #$numIters")
500-
samples = this.sample(withReplacement, fraction, rand.nextInt()).collect()
501-
numIters += 1
491+
val rand = new Random(seed)
492+
if (!withReplacement && num >= initialCount) {
493+
Utils.randomizeInPlace(this.collect(), rand)
494+
} else {
495+
val fraction = SamplingUtils.computeFractionForSampleSize(num, initialCount,
496+
withReplacement)
497+
var samples = this.sample(withReplacement, fraction, rand.nextInt()).collect()
498+
499+
// If the first sample didn't turn out large enough, keep trying to take samples;
500+
// this shouldn't happen often because we use a big multiplier for the initial size
501+
var numIters = 0
502+
while (samples.length < num) {
503+
logWarning(s"Needed to re-sample due to insufficient sample size. Repeat #$numIters")
504+
samples = this.sample(withReplacement, fraction, rand.nextInt()).collect()
505+
numIters += 1
506+
}
507+
Utils.randomizeInPlace(samples, rand).take(num)
502508
}
503-
Utils.randomizeInPlace(samples, rand).take(num)
504509
}
505510
}
506511
}

0 commit comments

Comments
 (0)