Initial pass

Author: pwendell

core/src/main/scala/org/apache/spark/Aggregator.scala

@@ -20,6 +20,8 @@ package org.apache.spark
 import org.apache.spark.util.collection.{AppendOnlyMap, ExternalAppendOnlyMap}
 
 /**
+ * <span class="badge badge-red" style="float: right;">SEMI-PRIVATE</span>
+ *
  * A set of functions used to aggregate data.
  *
  * @param createCombiner function to create the initial value of the aggregation.

core/src/main/scala/org/apache/spark/Dependency.scala

@@ -21,12 +21,16 @@ import org.apache.spark.rdd.RDD
 import org.apache.spark.serializer.Serializer
 
 /**
+ * <span class="badge badge-red" style="float: right;">SEMI-PRIVATE</span>
+ *
  * Base class for dependencies.
  */
 abstract class Dependency[T](val rdd: RDD[T]) extends Serializable
 
 
 /**
+ * <span class="badge badge-red" style="float: right;">SEMI-PRIVATE</span>
+ *
  * Base class for dependencies where each partition of the parent RDD is used by at most one
  * partition of the child RDD.  Narrow dependencies allow for pipelined execution.
  */
@@ -41,6 +45,8 @@ abstract class NarrowDependency[T](rdd: RDD[T]) extends Dependency(rdd) {
 
 
 /**
+ * <span class="badge badge-red" style="float: right;">SEMI-PRIVATE</span>
+ *
  * Represents a dependency on the output of a shuffle stage.
  * @param rdd the parent RDD
  * @param partitioner partitioner used to partition the shuffle output
@@ -59,6 +65,8 @@ class ShuffleDependency[K, V](
 
 
 /**
+ * <span class="badge badge-red" style="float: right;">SEMI-PRIVATE</span>
+ *
  * Represents a one-to-one dependency between partitions of the parent and child RDDs.
  */
 class OneToOneDependency[T](rdd: RDD[T]) extends NarrowDependency[T](rdd) {
@@ -67,6 +75,8 @@ class OneToOneDependency[T](rdd: RDD[T]) extends NarrowDependency[T](rdd) {
 
 
 /**
+ * <span class="badge badge-red" style="float: right;">SEMI-PRIVATE</span>
+ *
  * Represents a one-to-one dependency between ranges of partitions in the parent and child RDDs.
  * @param rdd the parent RDD
  * @param inStart the start of the range in the parent RDD

core/src/main/scala/org/apache/spark/FutureAction.scala

@@ -25,6 +25,8 @@ import org.apache.spark.rdd.RDD
 import org.apache.spark.scheduler.{JobFailed, JobSucceeded, JobWaiter}
 
 /**
+ * <span class="badge badge-red" style="float: right;">EXPERIMENTAL</span>
+ *
  * A future for the result of an action to support cancellation. This is an extension of the
  * Scala Future interface to support cancellation.
  */
@@ -148,6 +150,8 @@ class SimpleFutureAction[T] private[spark](jobWaiter: JobWaiter[_], resultFunc:
 
 
 /**
+ * <span class="badge badge-red" style="float: right;">EXPERIMENTAL</span>
+ *
  * A [[FutureAction]] for actions that could trigger multiple Spark jobs. Examples include take,
  * takeSample. Cancellation works by setting the cancelled flag to true and interrupting the
  * action thread if it is being blocked by a job.

core/src/main/scala/org/apache/spark/SerializableWritable.scala

@@ -23,7 +23,7 @@ import org.apache.hadoop.conf.Configuration
 import org.apache.hadoop.io.ObjectWritable
 import org.apache.hadoop.io.Writable
 
-class SerializableWritable[T <: Writable](@transient var t: T) extends Serializable {
+private[spark] class SerializableWritable[T <: Writable](@transient var t: T) extends Serializable {
   def value = t
   override def toString = t.toString
 

core/src/main/scala/org/apache/spark/SparkEnv.scala

@@ -41,7 +41,7 @@ import org.apache.spark.util.{AkkaUtils, Utils}
  * objects needs to have the right SparkEnv set. You can get the current environment with
  * SparkEnv.get (e.g. after creating a SparkContext) and set it with SparkEnv.set.
  */
-class SparkEnv private[spark] (
+private[spark] class SparkEnv private[spark] (
     val executorId: String,
     val actorSystem: ActorSystem,
     val serializer: Serializer,

core/src/main/scala/org/apache/spark/TaskContext.scala

@@ -21,6 +21,11 @@ import scala.collection.mutable.ArrayBuffer
 
 import org.apache.spark.executor.TaskMetrics
 
+/**
+ * <span class="badge badge-red" style="float: right;">SEMI-PRIVATE</span>
+ *
+ * Contextual information about a task which can be read or mutated during execution.
+ */
 class TaskContext(
   val stageId: Int,
   val partitionId: Int,

core/src/main/scala/org/apache/spark/api/java/JavaUtils.scala

@@ -19,7 +19,7 @@ package org.apache.spark.api.java
 
 import com.google.common.base.Optional
 
-object JavaUtils {
+private[spark] object JavaUtils {
   def optionToOptional[T](option: Option[T]): Optional[T] =
     option match {
       case Some(value) => Optional.of(value)

core/src/main/scala/org/apache/spark/executor/ExecutorSource.scala

@@ -24,7 +24,7 @@ import org.apache.hadoop.fs.FileSystem
 
 import org.apache.spark.metrics.source.Source
 
-class ExecutorSource(val executor: Executor, executorId: String) extends Source {
+private[spark] class ExecutorSource(val executor: Executor, executorId: String) extends Source {
   private def fileStats(scheme: String) : Option[FileSystem.Statistics] =
     FileSystem.getAllStatistics().filter(s => s.getScheme.equals(scheme)).headOption
 

core/src/main/scala/org/apache/spark/executor/TaskMetrics.scala

@@ -19,6 +19,11 @@ package org.apache.spark.executor
 
 import org.apache.spark.storage.{BlockId, BlockStatus}
 
+/**
+ * <span class="badge badge-red" style="float: right;">SEMI-PRIVATE</span>
+ *
+ * Metrics tracked during the execution of a task.
+ */
 class TaskMetrics extends Serializable {
   /**
    * Host's name the task runs on
@@ -82,6 +87,11 @@ object TaskMetrics {
 }
 
 
+/**
+ * <span class="badge badge-red" style="float: right;">SEMI-PRIVATE</span>
+ *
+ * Metrics pertaining to shuffle data read in a given task.
+ */
 class ShuffleReadMetrics extends Serializable {
   /**
    * Absolute time when this task finished reading shuffle data
@@ -116,6 +126,11 @@ class ShuffleReadMetrics extends Serializable {
   var remoteBytesRead: Long = _
 }
 
+/**
+ * <span class="badge badge-red" style="float: right;">SEMI-PRIVATE</span>
+ *
+ * Metrics pertaining to shuffle data written in a given task.
+ */
 class ShuffleWriteMetrics extends Serializable {
   /**
    * Number of bytes written for the shuffle by this task

core/src/main/scala/org/apache/spark/io/CompressionCodec.scala

@@ -25,8 +25,14 @@ import org.xerial.snappy.{SnappyInputStream, SnappyOutputStream}
 import org.apache.spark.SparkConf
 
 /**
+ * <span class="badge badge-red" style="float: right;">SEMI-PRIVATE</span>
+ *
  * CompressionCodec allows the customization of choosing different compression implementations
  * to be used in block storage.
+ *
+ * Note: The wire protocol for a codec is not guaranteed compatible across versions of Spark.
+ *       This is intended for use as an internal compression utility within a single
+ *       Spark application.
  */
 trait CompressionCodec {
 
@@ -52,7 +58,13 @@ private[spark] object CompressionCodec {
 
 
 /**
+ * <span class="badge badge-red" style="float: right;">SEMI-PRIVATE</span>
+ *
  * LZF implementation of [[org.apache.spark.io.CompressionCodec]].
+ *
+ * Note: The wire protocol for this codec is not guaranteed to be compatible across versions
+ *       of Spark. This is intended for use as an internal compression utility within a single Spark
+ *       application.
  */
 class LZFCompressionCodec(conf: SparkConf) extends CompressionCodec {
 
@@ -65,8 +77,14 @@ class LZFCompressionCodec(conf: SparkConf) extends CompressionCodec {
 
 
 /**
+ * <span class="badge badge-red" style="float: right;">SEMI-PRIVATE</span>
+ *
  * Snappy implementation of [[org.apache.spark.io.CompressionCodec]].
  * Block size can be configured by spark.io.compression.snappy.block.size.
+ *
+ * Note: The wire protocol for this codec is not guaranteed to be compatible across versions
+ *       of Spark. This is intended for use as an internal compression utility within a single Spark
+ *       application.
  */
 class SnappyCompressionCodec(conf: SparkConf) extends CompressionCodec {