File tree Expand file tree Collapse file tree 2 files changed +6
-6
lines changed
main/scala/org/apache/spark/util/random
test/scala/org/apache/spark/util/random Expand file tree Collapse file tree 2 files changed +6
-6
lines changed Original file line number Diff line number Diff line change @@ -22,13 +22,13 @@ import scala.reflect.ClassTag
2222private [spark] object SamplingUtils {
2323
2424 /**
25- * Reservoir Sampling implementation.
25+ * Reservoir sampling implementation that also returns the input size .
2626 *
2727 * @param input input size
2828 * @param k reservoir size
2929 * @return (samples, input size)
3030 */
31- def reservoirSample [T : ClassTag ](input : Iterator [T ], k : Int ): (Array [T ], Int ) = {
31+ def reservoirSampleAndCount [T : ClassTag ](input : Iterator [T ], k : Int ): (Array [T ], Int ) = {
3232 val reservoir = new Array [T ](k)
3333 // Put the first k elements in the reservoir.
3434 var i = 0
Original file line number Diff line number Diff line change @@ -24,21 +24,21 @@ import org.scalatest.FunSuite
2424
2525class SamplingUtilsSuite extends FunSuite {
2626
27- test(" reservoirSample " ) {
27+ test(" reservoirSampleAndCount " ) {
2828 val input = Seq .fill(100 )(Random .nextInt())
2929
3030 // input size < k
31- val (sample1, count1) = SamplingUtils .reservoirSample (input.iterator, 150 )
31+ val (sample1, count1) = SamplingUtils .reservoirSampleAndCount (input.iterator, 150 )
3232 assert(count1 === 100 )
3333 assert(input === sample1.toSeq)
3434
3535 // input size == k
36- val (sample2, count2) = SamplingUtils .reservoirSample (input.iterator, 100 )
36+ val (sample2, count2) = SamplingUtils .reservoirSampleAndCount (input.iterator, 100 )
3737 assert(count2 === 100 )
3838 assert(input === sample2.toSeq)
3939
4040 // input size > k
41- val (sample3, count3) = SamplingUtils .reservoirSample (input.iterator, 10 )
41+ val (sample3, count3) = SamplingUtils .reservoirSampleAndCount (input.iterator, 10 )
4242 assert(count3 === 100 )
4343 assert(sample3.length === 10 )
4444 }
You can’t perform that action at this time.
0 commit comments