Skip to content

[SPARK-15590] [WEBUI] Paginate Job Table in Jobs tab #13620

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
nblintao wants to merge 4 commits into from
Closed

[SPARK-15590] [WEBUI] Paginate Job Table in Jobs tab #13620

Changes from 1 commit
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
4db0e09
paginate job tables
nblintao Jun 11, 2016
6682e86
fix default order & fix for job groups & rm unneccessary def
nblintao Jun 14, 2016
41bb189
add notions of override
nblintao Jun 15, 2016
649bb19
fix link problems
nblintao Jul 7, 2016
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
357 changes: 296 additions & 61 deletions core/src/main/scala/org/apache/spark/ui/jobs/AllJobsPage.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@

package org.apache.spark.ui.jobs

import java.net.URLEncoder
import java.util.Date
import javax.servlet.http.HttpServletRequest

Expand All @@ -26,8 +27,10 @@ import scala.xml._
import org.apache.commons.lang3.StringEscapeUtils

import org.apache.spark.JobExecutionStatus
import org.apache.spark.ui.{ToolTips, UIUtils, WebUIPage}
import org.apache.spark.ui.jobs.UIData.{ExecutorUIData, JobUIData}
import org.apache.spark.scheduler.StageInfo
import org.apache.spark.ui._
import org.apache.spark.ui.jobs.UIData.{ExecutorUIData, JobUIData, StageUIData}
import org.apache.spark.util.Utils

/** Page showing list of all ongoing and recently finished jobs */
private[ui] class AllJobsPage(parent: JobsTab) extends WebUIPage("") {
Expand Down Expand Up @@ -210,64 +213,56 @@ private[ui] class AllJobsPage(parent: JobsTab) extends WebUIPage("") {
</script>
}

private def jobsTable(jobs: Seq[JobUIData]): Seq[Node] = {
val someJobHasJobGroup = jobs.exists(_.jobGroup.isDefined)
private def jobsTable(
request: HttpServletRequest,
jobTag: String,
jobs: Seq[JobUIData]): Seq[Node] = {
val parameterJobPage = request.getParameter(jobTag + ".page")
val parameterJobSortColumn = request.getParameter(jobTag + ".sort")
val parameterJobSortDesc = request.getParameter(jobTag + ".desc")
val parameterJobPageSize = request.getParameter(jobTag + ".pageSize")
val parameterJobPrevPageSize = request.getParameter(jobTag + ".prevPageSize")

val columns: Seq[Node] = {
<th>{if (someJobHasJobGroup) "Job Id (Job Group)" else "Job Id"}</th>
<th>Description</th>
<th>Submitted</th>
<th>Duration</th>
<th class="sorttable_nosort">Stages: Succeeded/Total</th>
<th class="sorttable_nosort">Tasks (for all stages): Succeeded/Total</th>
}
val jobPage = Option(parameterJobPage).map(_.toInt).getOrElse(1)
val jobSortColumn = Option(parameterJobSortColumn).map { sortColumn =>
UIUtils.decodeURLParameter(sortColumn)
}.getOrElse("Job Id")
val jobSortDesc = Option(parameterJobSortDesc).map(_.toBoolean).getOrElse(false)
val jobPageSize = Option(parameterJobPageSize).map(_.toInt).getOrElse(100)
val jobPrevPageSize = Option(parameterJobPrevPageSize).map(_.toInt).getOrElse(jobPageSize)

def makeRow(job: JobUIData): Seq[Node] = {
val (lastStageName, lastStageDescription) = getLastStageNameAndDescription(job)
val duration: Option[Long] = {
job.submissionTime.map { start =>
val end = job.completionTime.getOrElse(System.currentTimeMillis())
end - start
}
val page: Int = {
// If the user has changed to a larger page size, then go to page 1 in order to avoid
// IndexOutOfBoundsException.
if (jobPageSize <= jobPrevPageSize) {
jobPage
} else {
1
}
val formattedDuration = duration.map(d => UIUtils.formatDuration(d)).getOrElse("Unknown")
val formattedSubmissionTime = job.submissionTime.map(UIUtils.formatDate).getOrElse("Unknown")
val basePathUri = UIUtils.prependBaseUri(parent.basePath)
val jobDescription =
UIUtils.makeDescription(lastStageDescription, basePathUri, plainText = false)

val detailUrl = "%s/jobs/job?id=%s".format(basePathUri, job.jobId)
<tr id={"job-" + job.jobId}>
<td sorttable_customkey={job.jobId.toString}>
{job.jobId} {job.jobGroup.map(id => s"($id)").getOrElse("")}
</td>
<td>
{jobDescription}
<a href={detailUrl} class="name-link">{lastStageName}</a>
</td>
<td sorttable_customkey={job.submissionTime.getOrElse(-1).toString}>
{formattedSubmissionTime}
</td>
<td sorttable_customkey={duration.getOrElse(-1).toString}>{formattedDuration}</td>
<td class="stage-progress-cell">
{job.completedStageIndices.size}/{job.stageIds.size - job.numSkippedStages}
{if (job.numFailedStages > 0) s"(${job.numFailedStages} failed)"}
{if (job.numSkippedStages > 0) s"(${job.numSkippedStages} skipped)"}
</td>
<td class="progress-cell">
{UIUtils.makeProgressBar(started = job.numActiveTasks, completed = job.numCompletedTasks,
failed = job.numFailedTasks, skipped = job.numSkippedTasks, killed = job.numKilledTasks,
total = job.numTasks - job.numSkippedTasks)}
</td>
</tr>
}
val currentTime = System.currentTimeMillis()

<table class="table table-bordered table-striped table-condensed sortable">
<thead>{columns}</thead>
<tbody>
{jobs.map(makeRow)}
</tbody>
</table>
try {
new JobPagedTable(
jobs,
jobTag,
UIUtils.prependBaseUri(parent.basePath),
parent.jobProgresslistener.stageIdToInfo,
parent.jobProgresslistener.stageIdToData,
currentTime,
pageSize = jobPageSize,
sortColumn = jobSortColumn,
desc = jobSortDesc
).table(page)
} catch {
case e @ (_ : IllegalArgumentException | _ : IndexOutOfBoundsException) =>

@yhuai yhuai Jul 24, 2016

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

When will this these two exceptions get triggered?

@nblintao nblintao Jul 25, 2016

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It was following @zsxwing's code for TaskPagedTable (4f7f1ee#diff-fa4cfb2cce1b925f55f41f2dfa8c8501R265). I haven't found concrete positions where they will be triggered yet.
But I think catching exceptions here is necessary, because I'm not sure my code for JobPagedTable is robust enough without exceptions. Since this exception information is shown in the UI, the user could easily notice which table cannot be rendered.

@zsxwing zsxwing Jul 25, 2016

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

When will this these two exceptions get triggered?

As the parameters are from the user input (the url), the user may type some illegal paramters in the url, which causes these exceptions.

<div class="alert alert-error">
<p>Error while rendering job table:</p>
<pre>
{Utils.exceptionString(e)}
</pre>
</div>
}
}

def render(request: HttpServletRequest): Seq[Node] = {
Expand All @@ -279,12 +274,9 @@ private[ui] class AllJobsPage(parent: JobsTab) extends WebUIPage("") {
val completedJobs = listener.completedJobs.reverse.toSeq
val failedJobs = listener.failedJobs.reverse.toSeq

val activeJobsTable =
jobsTable(activeJobs.sortBy(_.submissionTime.getOrElse(-1L)).reverse)
val completedJobsTable =
jobsTable(completedJobs.sortBy(_.completionTime.getOrElse(-1L)).reverse)
val failedJobsTable =
jobsTable(failedJobs.sortBy(_.completionTime.getOrElse(-1L)).reverse)
val activeJobsTable = jobsTable(request, "activeJob", activeJobs)
val completedJobsTable = jobsTable(request, "completedJob", completedJobs)
val failedJobsTable = jobsTable(request, "failedJob", failedJobs)

val shouldShowActiveJobs = activeJobs.nonEmpty
val shouldShowCompletedJobs = completedJobs.nonEmpty
Expand Down Expand Up @@ -369,3 +361,246 @@ private[ui] class AllJobsPage(parent: JobsTab) extends WebUIPage("") {
}
}
}

private[ui] class JobTableRowData(
val jobData: JobUIData,
val lastStageName: String,
val lastStageDescription: String,
val duration: Long,
val formattedDuration: String,
val submissionTime: Long,
val formattedSubmissionTime: String,
val jobDescription: NodeSeq,
val detailUrl: String)

private[ui] class JobDataSource(
jobs: Seq[JobUIData],
stageIdToInfo: HashMap[Int, StageInfo],
stageIdToData: HashMap[(Int, Int), StageUIData],
basePath: String,
currentTime: Long,
pageSize: Int,
sortColumn: String,
desc: Boolean) extends PagedDataSource[JobTableRowData](pageSize) {

// Convert JobUIData to JobTableRowData which contains the final contents to show in the table
// so that we can avoid creating duplicate contents during sorting the data
private val data = jobs.map(jobRow).sorted(ordering(sortColumn, desc))

private var _slicedJobIds: Set[Int] = null

override def dataSize: Int = data.size

override def sliceData(from: Int, to: Int): Seq[JobTableRowData] = {
val r = data.slice(from, to)
_slicedJobIds = r.map(_.jobData.jobId).toSet
r
}

def slicedJobIds: Set[Int] = _slicedJobIds

@andrewor14 andrewor14 Jun 13, 2016

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

used anywhere?

@nblintao nblintao Jun 14, 2016

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Sorry, it's deprecated. I will remove it. Thanks.


private def getLastStageNameAndDescription(job: JobUIData): (String, String) = {
val lastStageInfo = Option(job.stageIds)
.filter(_.nonEmpty)
.flatMap { ids => stageIdToInfo.get(ids.max)}
val lastStageData = lastStageInfo.flatMap { s =>
stageIdToData.get((s.stageId, s.attemptId))
}
val name = lastStageInfo.map(_.name).getOrElse("(Unknown Stage Name)")
val description = lastStageData.flatMap(_.description).getOrElse("")
(name, description)
}

private def jobRow(jobData: JobUIData): JobTableRowData = {
val (lastStageName, lastStageDescription) = getLastStageNameAndDescription(jobData)
val duration: Option[Long] = {
jobData.submissionTime.map { start =>
val end = jobData.completionTime.getOrElse(System.currentTimeMillis())
end - start
}
}
val formattedDuration = duration.map(d => UIUtils.formatDuration(d)).getOrElse("Unknown")
val submissionTime = jobData.submissionTime
val formattedSubmissionTime = submissionTime.map(UIUtils.formatDate).getOrElse("Unknown")
val jobDescription = UIUtils.makeDescription(lastStageDescription, basePath, plainText = false)

val detailUrl = "%s/jobs/job?id=%s".format(basePath, jobData.jobId)

new JobTableRowData (
jobData,
lastStageName,
lastStageDescription,
duration.getOrElse(-1),
formattedDuration,
submissionTime.getOrElse(-1),
formattedSubmissionTime,
jobDescription,
detailUrl
)
}

/**
* Return Ordering according to sortColumn and desc
*/
private def ordering(sortColumn: String, desc: Boolean): Ordering[JobTableRowData] = {
val ordering = sortColumn match {
case "Job Id" | "Job Id (Job Group)" => new Ordering[JobTableRowData] {
override def compare(x: JobTableRowData, y: JobTableRowData): Int =
Ordering.Int.compare(x.jobData.jobId, y.jobData.jobId)
}
case "Description" => new Ordering[JobTableRowData] {
override def compare(x: JobTableRowData, y: JobTableRowData): Int =
Ordering.String.compare(x.lastStageDescription, y.lastStageDescription)
}
case "Submitted" => new Ordering[JobTableRowData] {
override def compare(x: JobTableRowData, y: JobTableRowData): Int =
Ordering.Long.compare(x.submissionTime, y.submissionTime)
}
case "Duration" => new Ordering[JobTableRowData] {
override def compare(x: JobTableRowData, y: JobTableRowData): Int =
Ordering.Long.compare(x.duration, y.duration)
}
case "Stages: Succeeded/Total" | "Tasks (for all stages): Succeeded/Total" =>
throw new IllegalArgumentException(s"Unsortable column: $sortColumn")
case unknownColumn => throw new IllegalArgumentException(s"Unknown column: $unknownColumn")
}
if (desc) {
ordering.reverse
} else {
ordering
}
}

}
private[ui] class JobPagedTable(
data: Seq[JobUIData],
jobTag: String,
basePath: String,
// TODO: Should use StageID instead of Int below, but cannot resolve. How?

@andrewor14 andrewor14 Jun 13, 2016

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

what do you mean by this?

@nblintao nblintao Jun 14, 2016

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

in the class JobProgressListener, the stageIdToData is defined as HashMap[(StageId, StageAttemptId), StageUIData] and the stageIdToInfo is defined as HashMap[StageId, StageInfo].
But they cannot be resolved when the types like StageId, StageAttemptId are used. (They are defined in JobProgressListener, which I think could be used by others.) So I used Int here temporary.
I think this should be fixed before the patch is merged, but I don't know how for now.
Any suggestions?

@nblintao nblintao Jun 15, 2016

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The todo is removed.

stageIdToInfo: HashMap[Int, StageInfo],
stageIdToData: HashMap[(Int, Int), StageUIData],
currentTime: Long,
pageSize: Int,
sortColumn: String,
desc: Boolean
) extends PagedTable[JobTableRowData] {

override def tableId: String = jobTag + "-table"

override def tableCssClass: String =
"table table-bordered table-condensed table-striped table-head-clickable"

override def pageSizeFormField: String = jobTag + ".pageSize"

override def prevPageSizeFormField: String = jobTag + ".prevPageSize"

override def pageNumberFormField: String = jobTag + ".page"

override val dataSource = new JobDataSource(
data,
stageIdToInfo,
stageIdToData,
basePath,
currentTime,
pageSize,
sortColumn,
desc)

override def pageLink(page: Int): String = {
val encodedSortColumn = URLEncoder.encode(sortColumn, "UTF-8")
basePath +
s"?$pageNumberFormField=$page" +
s"&$jobTag.sort=$encodedSortColumn" +
s"&$jobTag.desc=$desc" +
s"&$pageSizeFormField=$pageSize"
}

override def goButtonFormPath: String = {
val encodedSortColumn = URLEncoder.encode(sortColumn, "UTF-8")
s"$basePath?$jobTag.sort=$encodedSortColumn&$jobTag.desc=$desc"
}

def headers: Seq[Node] = {

@andrewor14 andrewor14 Jun 13, 2016

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

is this used anywhere? I can't find it

@nblintao nblintao Jun 14, 2016

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It implements the headers in the trait PagedTable. Shall I add a notion of override here to make it clear?

@andrewor14 andrewor14 Jun 14, 2016

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

yes

@nblintao nblintao Jun 15, 2016

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

added

val someJobHasJobGroup = data.exists(_.jobGroup.isDefined)
val jobHeadersAndCssClasses: Seq[(String, String, Boolean)] =
Seq(
( {
if (someJobHasJobGroup) "Job Id (Job Group)" else "Job Id"
}, "", true),
("Description", "", true), ("Submitted", "", true), ("Duration", "", true),
("Stages: Succeeded/Total", "", false),
("Tasks (for all stages): Succeeded/Total", "", false)
)

if (!jobHeadersAndCssClasses.filter(_._3).map(_._1).contains(sortColumn)) {
throw new IllegalArgumentException(s"Unknown column: $sortColumn")
}

val headerRow: Seq[Node] = {
jobHeadersAndCssClasses.map { case (header, cssClass, sortable) =>
if (header == sortColumn) {
val headerLink = Unparsed(
basePath +
s"?$jobTag.sort=${URLEncoder.encode(header, "UTF-8")}" +
s"&$jobTag.desc=${!desc}" +
s"&$jobTag.pageSize=$pageSize")
val arrow = if (desc) "&#x25BE;" else "&#x25B4;" // UP or DOWN

<th class={cssClass}>
<a href={headerLink}>
{header}<span>
&nbsp;{Unparsed(arrow)}
</span>
</a>
</th>
} else {
if (sortable) {
val headerLink = Unparsed(
basePath +
s"?$jobTag.sort=${URLEncoder.encode(header, "UTF-8")}" +
s"&$jobTag.pageSize=$pageSize")

<th class={cssClass}>
<a href={headerLink}>
{header}
</a>
</th>
} else {
<th class={cssClass}>
{header}
</th>
}
}
}
}
<thead>{headerRow}</thead>
}

def row(jobTableRow: JobTableRowData): Seq[Node] = {

@andrewor14 andrewor14 Jun 13, 2016

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

same here, is this used?

@nblintao nblintao Jun 14, 2016

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Same here, it implements the row in the trait PagedTable.

val job = jobTableRow.jobData

<tr id={"job-" + job.jobId}>

@yhuai yhuai Jul 24, 2016

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Copied from https://github.com/apache/spark/pull/13620/files#diff-9a486b2f4e30d5375c3541571aec616cL240?

@nblintao nblintao Jul 25, 2016

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yes.

<td>
{job.jobId} {job.jobGroup.map(id => s"($id)").getOrElse("")}
</td>
<td>
{jobTableRow.jobDescription}
<a href={jobTableRow.detailUrl} class="name-link">{jobTableRow.lastStageName}</a>
</td>
<td>
{jobTableRow.formattedSubmissionTime}
</td>
<td>{jobTableRow.formattedDuration}</td>
<td class="stage-progress-cell">
{job.completedStageIndices.size}/{job.stageIds.size - job.numSkippedStages}
{if (job.numFailedStages > 0) s"(${job.numFailedStages} failed)"}
{if (job.numSkippedStages > 0) s"(${job.numSkippedStages} skipped)"}
</td>
<td class="progress-cell">
{UIUtils.makeProgressBar(started = job.numActiveTasks, completed = job.numCompletedTasks,
failed = job.numFailedTasks, skipped = job.numSkippedTasks, killed = job.numKilledTasks,
total = job.numTasks - job.numSkippedTasks)}
</td>
</tr>
}
}