docs

Author: squito

core/src/main/java/org/apache/spark/status/api/v1/TaskSorting.java

@@ -23,6 +23,7 @@
 import java.util.HashSet;
 import java.util.Set;
 
+// exposed for jersey
 @DeveloperApi
 public enum TaskSorting {
   ID,

core/src/main/scala/org/apache/spark/status/api/v1/JacksonMessageWriter.scala

@@ -28,6 +28,15 @@ import javax.ws.rs.ext.{MessageBodyWriter, Provider}
 import com.fasterxml.jackson.annotation.JsonInclude
 import com.fasterxml.jackson.databind.{ObjectMapper, SerializationFeature}
 
+/**
+ * This class converts the POJO metric responses into json, using jackson.
+ *
+ * This doesn't follow the standard jersey-jackson plugin options, because we want to stick
+ * with an old version of jersey (since we have it from yarn anyway) and don't want to pull in lots
+ * of dependencies from a new plugin.
+ *
+ * Note that jersey automatically discovers this class based on its package and its annotations.
+ */
 @Provider
 @Produces(Array(MediaType.APPLICATION_JSON))
 private[v1] class JacksonMessageWriter extends MessageBodyWriter[Object]{

core/src/main/scala/org/apache/spark/status/api/v1/JsonRootResource.scala

@@ -28,6 +28,16 @@ import org.eclipse.jetty.servlet.{ServletContextHandler, ServletHolder}
 import org.apache.spark.SecurityManager
 import org.apache.spark.ui.SparkUI
 
+/**
+ * Main entry point for serving spark application metrics as json, using JAX-RS.
+ *
+ * Each resource should have endpoints that return **public** classes defined in api.scala.  Mima
+ * binary compatibility checks ensure that we don't inadvertently make changes that break the api.
+ * The returned objects are automatically converted to json by jackson with JacksonMessageWriter.
+ * In addition, there are a number of tests in HistoryServerSuite that compare the json to "golden
+ * files".  Any changes and additions should be reflected there as well -- see the notes in
+ * HistoryServerSuite.
+ */
 @Path("/v1")
 private[v1] class JsonRootResource extends UIRootFromServletContext {
 
@@ -156,4 +166,8 @@ private[v1] class BadParameterException(msg: String) extends WebApplicationExcep
   }
 }
 
+/**
+ * Signal to JacksonMessageWriter to not convert the message into json (which would result in an
+ * extra set of quotes).
+ */
 private[v1] case class ErrorWrapper(s: String)

core/src/test/scala/org/apache/spark/deploy/history/HistoryServerSuite.scala

@@ -28,6 +28,17 @@ import org.scalatest.mock.MockitoSugar
 import org.apache.spark.{JsonTestUtils, SecurityManager, SparkConf}
 import org.apache.spark.ui.SparkUI
 
+/**
+ * A collection of tests against the historyserver, including comparing responses from the json
+ * metrics api to a set of known "golden files".  If new endpoints / parameters are added,
+ * cases should be added to this test suite.  The expected outcomes can be genered by running
+ * the HistoryServerSuite.main.  Note that this will blindly generate new expectation files matching
+ * the current behavior -- the developer must verify that behavior is correct.
+ *
+ * Similarly, if the behavior is changed, HistoryServerSuite.main can be run to update the
+ * expectations.  However, in general this should be done with extreme caution, as the metrics
+ * are considered part of Spark's public api.
+ */
 class HistoryServerSuite extends FunSuite with BeforeAndAfter with Matchers with MockitoSugar
   with JsonTestUtils {