switch to using java.util.Dates for times

Author: squito

core/src/main/scala/org/apache/spark/status/api/v1/AllJobsResource.scala

@@ -16,6 +16,7 @@
  */
 package org.apache.spark.status.api.v1
 
+import java.util.Date
 import javax.ws.rs._
 import javax.ws.rs.core.MediaType
 
@@ -75,8 +76,8 @@ object AllJobsResource {
         jobId = job.jobId,
         name = lastStageName,
         description = lastStageDescription,
-        submissionTime = job.submissionTime,
-        completionTime = job.completionTime,
+        submissionTime = job.submissionTime.map{new Date(_)},
+        completionTime = job.completionTime.map{new Date(_)},
         stageIds = job.stageIds,
         jobGroup = job.jobGroup,
         status = job.status,

core/src/main/scala/org/apache/spark/status/api/v1/AllStagesResource.scala

@@ -16,6 +16,7 @@
  */
 package org.apache.spark.status.api.v1
 
+import java.util.Date
 import javax.ws.rs.{QueryParam, PathParam, GET, Produces}
 import javax.ws.rs.core.MediaType
 
@@ -135,7 +136,7 @@ object AllStagesResource {
       taskId = uiData.taskInfo.taskId,
       index = uiData.taskInfo.index,
       attempt = uiData.taskInfo.attempt,
-      launchTime = uiData.taskInfo.launchTime,
+      launchTime = new Date(uiData.taskInfo.launchTime),
       executorId = uiData.taskInfo.executorId,
       host = uiData.taskInfo.host,
       taskLocality = uiData.taskInfo.taskLocality.toString(),

core/src/main/scala/org/apache/spark/status/api/v1/ApplicationInfo.scala

@@ -17,10 +17,12 @@
 
 package org.apache.spark.status.api.v1
 
+import java.util.Date
+
 case class ApplicationInfo(
   id: String,
   name: String,
-  startTime: Long,
-  endTime: Long,
+  startTime: Date,
+  endTime: Date,
   sparkUser: String,
   completed: Boolean = false)

core/src/main/scala/org/apache/spark/status/api/v1/ApplicationListResource.scala

@@ -16,6 +16,7 @@
  */
 package org.apache.spark.status.api.v1
 
+import java.util.Date
 import javax.ws.rs.core.MediaType
 import javax.ws.rs.{DefaultValue, QueryParam, Produces, GET}
 
@@ -45,7 +46,7 @@ class ApplicationListResource(uiRoot: UIRoot) {
     allApps.filter{app =>
       val statusOk = (app.completed && includeCompleted) ||
         (!app.completed && includeRunning)
-      val dateOk = app.startTime >= minDate.timestamp && app.startTime <= maxDate.timestamp
+      val dateOk = app.startTime.getTime >= minDate.timestamp && app.startTime.getTime <= maxDate.timestamp
       statusOk && dateOk
     }
   }
@@ -56,8 +57,8 @@ object ApplicationsListResource {
     ApplicationInfo(
       id = app.id,
       name = app.name,
-      startTime = app.startTime,
-      endTime = app.endTime,
+      startTime = new Date(app.startTime),
+      endTime = new Date(app.endTime),
       sparkUser = app.sparkUser,
       completed = app.completed
     )
@@ -67,8 +68,8 @@ object ApplicationsListResource {
     ApplicationInfo(
       id = internal.id,
       name = internal.desc.name,
-      startTime = internal.startTime,
-      endTime = internal.endTime,
+      startTime = new Date(internal.startTime),
+      endTime = new Date(internal.endTime),
       sparkUser = internal.desc.user,
       completed = completed
     )

core/src/main/scala/org/apache/spark/status/api/v1/CustomObjectMapper.scala

@@ -16,6 +16,8 @@
  */
 package org.apache.spark.status.api.v1
 
+import java.text.SimpleDateFormat
+import java.util.{SimpleTimeZone, Calendar}
 import javax.ws.rs.Produces
 import javax.ws.rs.core.MediaType
 import javax.ws.rs.ext.{ContextResolver, Provider}
@@ -34,9 +36,20 @@ class CustomObjectMapper extends ContextResolver[ObjectMapper]{
   mapper.registerModule(com.fasterxml.jackson.module.scala.DefaultScalaModule)
   mapper.enable(SerializationFeature.INDENT_OUTPUT)
   mapper.setSerializationInclusion(JsonInclude.Include.NON_NULL)
+  mapper.setDateFormat(CustomObjectMapper.makeISODateFormat)
 
   override def getContext(tpe: Class[_]): ObjectMapper = {
     mapper
   }
 
 }
+
+object CustomObjectMapper {
+  def makeISODateFormat: SimpleDateFormat = {
+    val iso8601 = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS'GMT'");
+    val cal = Calendar.getInstance(new SimpleTimeZone(0, "GMT"));
+    iso8601.setCalendar(cal);
+    iso8601;
+  }
+
+}

core/src/main/scala/org/apache/spark/status/api/v1/JobData.scala

@@ -16,14 +16,16 @@
  */
 package org.apache.spark.status.api.v1
 
+import java.util.Date
+
 import org.apache.spark.JobExecutionStatus
 
 case class JobData(
   jobId: Int,
   name: String,
   description: Option[String],
-  submissionTime: Option[Long],
-  completionTime: Option[Long],
+  submissionTime: Option[Date],
+  completionTime: Option[Date],
   stageIds: Seq[Int],
   jobGroup: Option[String],
   status: JobExecutionStatus,

core/src/main/scala/org/apache/spark/status/api/v1/SimpleDateParam.scala

@@ -41,7 +41,7 @@ private[api] class SimpleDateParam(val originalValue: String) {
 
 private[api] object SimpleDateParam {
   val formats = Seq(
-    "yyyy-MM-dd'T'HH:mm:ssz",
+    "yyyy-MM-dd'T'HH:mm:ss.SSSz",
     "yyyy-MM-dd"
   ).map{new SimpleDateFormat(_)}
 }

core/src/main/scala/org/apache/spark/status/api/v1/TaskData.scala

@@ -17,11 +17,13 @@
 
 package org.apache.spark.status.api.v1
 
+import java.util.Date
+
 case class TaskData(
   taskId: Long,
   index: Int,
   attempt: Int,
-  launchTime: Long,
+  launchTime: Date,
   executorId: String,
   host: String,
   taskLocality: String,

core/src/main/scala/org/apache/spark/ui/SparkUI.scala

@@ -17,6 +17,8 @@
 
 package org.apache.spark.ui
 
+import java.util.Date
+
 import org.apache.spark.status.api.v1.{ApplicationInfo, JsonRootResource}
 import org.apache.spark.status.api.v1.UIRoot
 import org.apache.spark.{Logging, SecurityManager, SparkConf, SparkContext}
@@ -96,8 +98,8 @@ private[spark] class SparkUI private (
       id = appName,
       name = appName,
       //TODO
-      startTime = -1,
-      endTime = -1,
+      startTime = new Date(-1),
+      endTime = new Date(-1),
       sparkUser = "",
       completed = false
     ))

core/src/test/resources/HistoryServerExpectations/applications/json_expectation

@@ -1,22 +1,22 @@
 [ {
   "id" : "local-1425081759269",
   "name" : "Spark shell",
-  "startTime" : 1425081758277,
-  "endTime" : 1425081766912,
+  "startTime" : "2015-02-28T00:02:38.277GMT",
+  "endTime" : "2015-02-28T00:02:46.912GMT",
   "sparkUser" : "irashid",
   "completed" : true
 }, {
   "id" : "local-1422981780767",
   "name" : "Spark shell",
-  "startTime" : 1422981779720,
-  "endTime" : 1422981788731,
+  "startTime" : "2015-02-03T16:42:59.720GMT",
+  "endTime" : "2015-02-03T16:43:08.731GMT",
   "sparkUser" : "irashid",
   "completed" : true
 }, {
   "id" : "local-1422981759269",
   "name" : "Spark shell",
-  "startTime" : 1422981758277,
-  "endTime" : 1422981766912,
+  "startTime" : "2015-02-03T16:42:38.277GMT",
+  "endTime" : "2015-02-03T16:42:46.912GMT",
   "sparkUser" : "irashid",
   "completed" : true
 } ]