diff --git a/core/src/main/scala/org/apache/spark/ui/jobs/AllJobsPage.scala b/core/src/main/scala/org/apache/spark/ui/jobs/AllJobsPage.scala index 18be0870746e..7065f6a4df1a 100644 --- a/core/src/main/scala/org/apache/spark/ui/jobs/AllJobsPage.scala +++ b/core/src/main/scala/org/apache/spark/ui/jobs/AllJobsPage.scala @@ -629,7 +629,8 @@ private[ui] class JobPagedTable( {if (job.numSkippedStages > 0) s"(${job.numSkippedStages} skipped)"} - {UIUtils.makeProgressBar(started = job.numActiveTasks, completed = job.numCompletedTasks, + {UIUtils.makeProgressBar(started = job.numActiveTasks, + completed = job.completedIndices.size, failed = job.numFailedTasks, skipped = job.numSkippedTasks, reasonToNumKilled = job.reasonToNumKilled, total = job.numTasks - job.numSkippedTasks)} diff --git a/core/src/main/scala/org/apache/spark/ui/jobs/JobProgressListener.scala b/core/src/main/scala/org/apache/spark/ui/jobs/JobProgressListener.scala index 1cf03e1541d1..46c50d656f5d 100644 --- a/core/src/main/scala/org/apache/spark/ui/jobs/JobProgressListener.scala +++ b/core/src/main/scala/org/apache/spark/ui/jobs/JobProgressListener.scala @@ -423,6 +423,7 @@ class JobProgressListener(conf: SparkConf) extends SparkListener with Logging { jobData.numActiveTasks -= 1 taskEnd.reason match { case Success => + jobData.completedIndices.add((taskEnd.stageId, info.index)) jobData.numCompletedTasks += 1 case kill: TaskKilled => jobData.reasonToNumKilled = jobData.reasonToNumKilled.updated( diff --git a/core/src/main/scala/org/apache/spark/ui/jobs/UIData.scala b/core/src/main/scala/org/apache/spark/ui/jobs/UIData.scala index ac1a74ad8029..fde21bc51f68 100644 --- a/core/src/main/scala/org/apache/spark/ui/jobs/UIData.scala +++ b/core/src/main/scala/org/apache/spark/ui/jobs/UIData.scala @@ -62,6 +62,7 @@ private[spark] object UIData { var numTasks: Int = 0, var numActiveTasks: Int = 0, var numCompletedTasks: Int = 0, + var completedIndices: OpenHashSet[(Int, Int)] = new OpenHashSet[(Int, Int)](), var numSkippedTasks: Int = 0, var numFailedTasks: Int = 0, var reasonToNumKilled: Map[String, Int] = Map.empty, diff --git a/core/src/test/scala/org/apache/spark/ui/UISeleniumSuite.scala b/core/src/test/scala/org/apache/spark/ui/UISeleniumSuite.scala index 422837303642..d75b0fa86956 100644 --- a/core/src/test/scala/org/apache/spark/ui/UISeleniumSuite.scala +++ b/core/src/test/scala/org/apache/spark/ui/UISeleniumSuite.scala @@ -285,12 +285,7 @@ class UISeleniumSuite extends SparkFunSuite with WebBrowser with Matchers with B eventually(timeout(5 seconds), interval(50 milliseconds)) { goToUi(sc, "/jobs") find(cssSelector(".stage-progress-cell")).get.text should be ("2/2 (1 failed)") - // Ideally, the following test would pass, but currently we overcount completed tasks - // if task recomputations occur: - // find(cssSelector(".progress-cell .progress")).get.text should be ("2/2 (1 failed)") - // Instead, we guarantee that the total number of tasks is always correct, while the number - // of completed tasks may be higher: - find(cssSelector(".progress-cell .progress")).get.text should be ("3/2 (1 failed)") + find(cssSelector(".progress-cell .progress")).get.text should be ("2/2 (1 failed)") } val jobJson = getJson(sc.ui.get, "jobs") (jobJson \ "numTasks").extract[Int]should be (2)