diff --git a/core/src/main/scala/org/apache/spark/status/AppStatusListener.scala b/core/src/main/scala/org/apache/spark/status/AppStatusListener.scala index 7da0a9d2285b..a5850fc2ac4b 100644 --- a/core/src/main/scala/org/apache/spark/status/AppStatusListener.scala +++ b/core/src/main/scala/org/apache/spark/status/AppStatusListener.scala @@ -355,6 +355,8 @@ private[spark] class AppStatusListener( val lastStageInfo = event.stageInfos.sortBy(_.stageId).lastOption val jobName = lastStageInfo.map(_.name).getOrElse("") + val description = Option(event.properties) + .flatMap { p => Option(p.getProperty(SparkContext.SPARK_JOB_DESCRIPTION)) } val jobGroup = Option(event.properties) .flatMap { p => Option(p.getProperty(SparkContext.SPARK_JOB_GROUP_ID)) } val sqlExecutionId = Option(event.properties) @@ -363,6 +365,7 @@ private[spark] class AppStatusListener( val job = new LiveJob( event.jobId, jobName, + description, if (event.time > 0) Some(new Date(event.time)) else None, event.stageIds, jobGroup, diff --git a/core/src/main/scala/org/apache/spark/status/LiveEntity.scala b/core/src/main/scala/org/apache/spark/status/LiveEntity.scala index 00c991b49920..a0ef8da0a4b6 100644 --- a/core/src/main/scala/org/apache/spark/status/LiveEntity.scala +++ b/core/src/main/scala/org/apache/spark/status/LiveEntity.scala @@ -62,6 +62,7 @@ private[spark] abstract class LiveEntity { private class LiveJob( val jobId: Int, name: String, + description: Option[String], val submissionTime: Option[Date], val stageIds: Seq[Int], jobGroup: Option[String], @@ -92,7 +93,7 @@ private class LiveJob( val info = new v1.JobData( jobId, name, - None, // description is always None? + description, submissionTime, completionTime, stageIds, diff --git a/core/src/test/scala/org/apache/spark/status/AppStatusListenerSuite.scala b/core/src/test/scala/org/apache/spark/status/AppStatusListenerSuite.scala index 6bf163506e0c..a289dddbdc9e 100644 --- a/core/src/test/scala/org/apache/spark/status/AppStatusListenerSuite.scala +++ b/core/src/test/scala/org/apache/spark/status/AppStatusListenerSuite.scala @@ -155,6 +155,7 @@ class AppStatusListenerSuite extends SparkFunSuite with BeforeAndAfter { new StageInfo(2, 0, "stage2", 4, Nil, Seq(1), "details2")) val jobProps = new Properties() + jobProps.setProperty(SparkContext.SPARK_JOB_DESCRIPTION, "jobDescription") jobProps.setProperty(SparkContext.SPARK_JOB_GROUP_ID, "jobGroup") jobProps.setProperty(SparkContext.SPARK_SCHEDULER_POOL, "schedPool") @@ -163,7 +164,7 @@ class AppStatusListenerSuite extends SparkFunSuite with BeforeAndAfter { check[JobDataWrapper](1) { job => assert(job.info.jobId === 1) assert(job.info.name === stages.last.name) - assert(job.info.description === None) + assert(job.info.description === Some("jobDescription")) assert(job.info.status === JobExecutionStatus.RUNNING) assert(job.info.submissionTime === Some(new Date(time))) assert(job.info.jobGroup === Some("jobGroup"))