Skip to content

[SPARK-29599][WEBUI] Support pagination for session table in JDBC/ODBC Tab #26253

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
AngersZhuuuu wants to merge 8 commits into from
Closed

[SPARK-29599][WEBUI] Support pagination for session table in JDBC/ODBC Tab #26253

Changes from 4 commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
dfa225b
Update ThriftServerPage.scala
AngersZhuuuu Oct 25, 2019
f209cb4
Update ThriftServerPage.scala
AngersZhuuuu Oct 25, 2019
43a1621
remove unused code
AngersZhuuuu Oct 25, 2019
c7c9479
Update ThriftServerPage.scala
AngersZhuuuu Oct 25, 2019
ce588e5
Update ThriftServerPage.scala
AngersZhuuuu Oct 25, 2019
59697bc
Merge branch 'master' into SPARK-29599
AngersZhuuuu Oct 27, 2019
691eed1
follow comment
AngersZhuuuu Oct 27, 2019
5315c58
fix bug
AngersZhuuuu Oct 27, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
223 changes: 185 additions & 38 deletions ...iftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/ui/ThriftServerPage.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,6 @@ private[ui] class ThriftServerPage(parent: ThriftServerTab) extends WebUIPage(""

private val listener = parent.listener
private val startTime = Calendar.getInstance().getTime()
private val emptyCell = "-"

/** Render the page */
def render(request: HttpServletRequest): Seq[Node] = {
Expand Down Expand Up @@ -139,33 +138,60 @@ private[ui] class ThriftServerPage(parent: ThriftServerTab) extends WebUIPage(""

/** Generate stats of batch sessions of the thrift server program */
private def generateSessionStatsTable(request: HttpServletRequest): Seq[Node] = {
val sessionList = listener.getSessionList
val numBatches = sessionList.size
val table = if (numBatches > 0) {
val dataRows = sessionList.sortBy(_.startTimestamp).reverse
val headerRow = Seq("User", "IP", "Session ID", "Start Time", "Finish Time", "Duration",
"Total Execute")
def generateDataRow(session: SessionInfo): Seq[Node] = {
val sessionLink = "%s/%s/session/?id=%s".format(
UIUtils.prependBaseUri(request, parent.basePath), parent.prefix, session.sessionId)
<tr>
<td> {session.userName} </td>
<td> {session.ip} </td>
<td> <a href={sessionLink}> {session.sessionId} </a> </td>
<td> {formatDate(session.startTimestamp)} </td>
<td> {if (session.finishTimestamp > 0) formatDate(session.finishTimestamp)} </td>
<td sorttable_customkey={session.totalTime.toString}>
{formatDurationOption(Some(session.totalTime))} </td>
<td> {session.totalExecution.toString} </td>
</tr>
val numSessions = listener.getSessionList.size
val table = if (numSessions > 0) {

val sessionTableTag = "sessionstat"

val parameterOtherTable = request.getParameterMap().asScala
.filterNot(_._1.startsWith(sessionTableTag))
.map { case (name, vals) =>
name + "=" + vals(0)
}

val parameterSessionTablePage = request.getParameter(s"$sessionTableTag.page")
val parameterSessionTableSortColumn = request.getParameter(s"$sessionTableTag.sort")
val parameterSessionTableSortDesc = request.getParameter(s"$sessionTableTag.desc")
val parameterSessionPageSize = request.getParameter(s"$sessionTableTag.pageSize")

val sessionTablePage = Option(parameterSessionTablePage).map(_.toInt).getOrElse(1)
val sessionTableSortColumn = Option(parameterSessionTableSortColumn).map { sortColumn =>
UIUtils.decodeURLParameter(sortColumn)
}.getOrElse("Start Time")
val sessionTableSortDesc = Option(parameterSessionTableSortDesc).map(_.toBoolean).getOrElse(
// Old session should be shown above new session by default.
!(sessionTableSortColumn == "Start Time")
Copy link
Contributor

@shahidki31 shahidki31 Oct 25, 2019

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@AngersZhuuuu By default new session should come above right? (Screenshot from master branch)
Screenshot 2019-10-25 at 11 04 15 PM

Copy link
Contributor Author

@AngersZhuuuu AngersZhuuuu Oct 25, 2019

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@AngersZhuuuu By default new session should come above right? (Screenshot from master branch)

Updated, I could have been misled by the modified thriftserver I was using

)
val sessionTablePageSize = Option(parameterSessionPageSize).map(_.toInt).getOrElse(100)

try {
Some(new SessionStatsPagedTable(
request,
parent,
listener.getSessionList,
"sqlserver",
UIUtils.prependBaseUri(request, parent.basePath),
parameterOtherTable,
sessionTableTag,
pageSize = sessionTablePageSize,
sortColumn = sessionTableSortColumn,
desc = sessionTableSortDesc
).table(sessionTablePage))
} catch {
case e@(_: IllegalArgumentException | _: IndexOutOfBoundsException) =>
Some(<div class="alert alert-error">
<p>Error while rendering job table:</p>
<pre>
{Utils.exceptionString(e)}
</pre>
</div>)
}
Some(UIUtils.listingTable(headerRow, generateDataRow, dataRows, true, None, Seq(null), false))
Copy link
Contributor

@shahidki31 shahidki31 Oct 27, 2019

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

nit: Here for the sessions table fixedWidth is true. right? Are we making the paged table also fixed width?

Copy link
Contributor Author

@AngersZhuuuu AngersZhuuuu Oct 27, 2019 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

fixedWidth

Changed. Not so familiar with UI HTML part, thanks a lot for your suggestion

} else {
None
}

val content =
<h5 id="sessionstat">Session Statistics ({numBatches})</h5> ++
<h5 id="sessionstat">Session Statistics ({numSessions})</h5> ++
<div>
<ul class="unstyled">
{table.getOrElse("No statistics have been generated yet.")}
Expand All @@ -174,21 +200,6 @@ private[ui] class ThriftServerPage(parent: ThriftServerTab) extends WebUIPage(""

content
}

/**
* Returns a human-readable string representing a duration such as "5 second 35 ms"
*/
private def formatDurationOption(msOption: Option[Long]): String = {
msOption.map(formatDurationVerbose).getOrElse(emptyCell)
}

/** Generate HTML table from string data */
private def listingTable(headers: Seq[String], data: Seq[Seq[String]]) = {
def generateDataRow(data: Seq[String]): Seq[Node] = {
<tr> {data.map(d => <td>{d}</td>)} </tr>
}
UIUtils.listingTable(headers, generateDataRow, data, fixedWidth = true)
}
}

private[ui] class SqlStatsPagedTable(
Expand Down Expand Up @@ -377,6 +388,102 @@ private[ui] class SqlStatsPagedTable(
"%s/jobs/job/?id=%s".format(UIUtils.prependBaseUri(request, parent.basePath), jobId)
}

private[ui] class SessionStatsPagedTable(
request: HttpServletRequest,
parent: ThriftServerTab,
data: Seq[SessionInfo],
subPath: String,
basePath: String,
parameterOtherTable: Iterable[String],
sessionStatsTableTag: String,
pageSize: Int,
sortColumn: String,
desc: Boolean) extends PagedTable[SessionInfo] {

override val dataSource = new SessionStatsTableDataSource(data, pageSize, sortColumn, desc)

private val parameterPath = s"$basePath/$subPath/?${parameterOtherTable.mkString("&")}"

override def tableId: String = sessionStatsTableTag

override def tableCssClass: String =
"table table-bordered table-condensed table-striped " +
"table-head-clickable table-cell-width-limited"

override def pageLink(page: Int): String = {
val encodedSortColumn = URLEncoder.encode(sortColumn, UTF_8.name())
parameterPath +
s"&$pageNumberFormField=$page" +
s"&$sessionStatsTableTag.sort=$encodedSortColumn" +
s"&$sessionStatsTableTag.desc=$desc" +
s"&$pageSizeFormField=$pageSize"
}

override def pageSizeFormField: String = s"$sessionStatsTableTag.pageSize"

override def pageNumberFormField: String = s"$sessionStatsTableTag.page"

override def goButtonFormPath: String = {
val encodedSortColumn = URLEncoder.encode(sortColumn, UTF_8.name())
s"$parameterPath&$sessionStatsTableTag.sort=$encodedSortColumn&$sessionStatsTableTag.desc=$desc"
}

override def headers: Seq[Node] = {
val sessionTableHeaders =
Seq("User", "IP", "Session ID", "Start Time", "Finish Time", "Duration", "Total Execute")

val headerRow: Seq[Node] = {
sessionTableHeaders.map { case header =>
if (header == sortColumn) {
val headerLink = Unparsed(
parameterPath +
s"&$sessionStatsTableTag.sort=${URLEncoder.encode(header, UTF_8.name())}" +
s"&$sessionStatsTableTag.desc=${!desc}" +
s"&$sessionStatsTableTag.pageSize=$pageSize" +
s"#$sessionStatsTableTag")
val arrow = if (desc) "&#x25BE;" else "&#x25B4;" // UP or DOWN

<th>
<a href={headerLink}>
{header}&nbsp;{Unparsed(arrow)}
</a>
</th>
} else {
val headerLink = Unparsed(
parameterPath +
s"&$sessionStatsTableTag.sort=${URLEncoder.encode(header, UTF_8.name())}" +
s"&$sessionStatsTableTag.pageSize=$pageSize" +
s"#$sessionStatsTableTag")

<th>
<a href={headerLink}>
{header}
</a>
</th>
}
}
}
<thead>
{headerRow}
</thead>
}

override def row(session: SessionInfo): Seq[Node] = {
val sessionLink = "%s/%s/session/?id=%s".format(
UIUtils.prependBaseUri(request, parent.basePath), parent.prefix, session.sessionId)
<tr>
<td> {session.userName} </td>
<td> {session.ip} </td>
<td> <a href={sessionLink}> {session.sessionId} </a> </td>
<td> {formatDate(session.startTimestamp)} </td>
<td> {if (session.finishTimestamp > 0) formatDate(session.finishTimestamp)} </td>
<td sorttable_customkey={session.totalTime.toString}>
Copy link
Contributor

@shahidki31 shahidki31 Oct 27, 2019 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

nit: Is it necessary to add sorttable_customkey as we are already sorting using raw data in ordering method?

Copy link
Contributor Author

@AngersZhuuuu AngersZhuuuu Oct 27, 2019

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

nit: Is it necessary to add sorttable_customkey as we are already sorting using raw data in ordering method?

This seems to be the previous pr, change to use raw data for sort column, old version didn't have this and trouble me a lot. Don't need it.

Copy link
Contributor

@shahidki31 shahidki31 Oct 27, 2019

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Also, please confirm if the sorting is working properly after changing this. Thanks

Copy link
Contributor Author

@AngersZhuuuu AngersZhuuuu Oct 27, 2019

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Also, please confirm if the sorting is working properly after changing this. Thanks

Checked ok. Thanks

{formatDurationVerbose(session.totalTime)} </td>
<td> {session.totalExecution.toString} </td>
</tr>
}
}

private[ui] class SqlStatsTableRow(
val jobId: Seq[String],
val duration: Long,
Expand All @@ -400,7 +507,6 @@ private[ui] class SqlStatsPagedTable(

override def sliceData(from: Int, to: Int): Seq[SqlStatsTableRow] = {
val r = data.slice(from, to)
r.map(x => x)
_slicedStartTime = r.map(_.executionInfo.startTimestamp).toSet
r
}
Expand Down Expand Up @@ -442,3 +548,44 @@ private[ui] class SqlStatsPagedTable(
}

}

private[ui] class SessionStatsTableDataSource(
info: Seq[SessionInfo],
pageSize: Int,
sortColumn: String,
desc: Boolean) extends PagedDataSource[SessionInfo](pageSize) {

// Sorting SessionInfo data
private val data = info.sorted(ordering(sortColumn, desc))

private var _slicedStartTime: Set[Long] = null

override def dataSize: Int = data.size

override def sliceData(from: Int, to: Int): Seq[SessionInfo] = {
val r = data.slice(from, to)
_slicedStartTime = r.map(_.startTimestamp).toSet
r
}

/**
* Return Ordering according to sortColumn and desc.
*/
private def ordering(sortColumn: String, desc: Boolean): Ordering[SessionInfo] = {
val ordering: Ordering[SessionInfo] = sortColumn match {
case "User" => Ordering.by(_.userName)
case "IP" => Ordering.by(_.ip)
case "Session ID" => Ordering.by(_.sessionId)
case "Start Time" => Ordering by (_.startTimestamp)
case "Finish Time" => Ordering.by(_.finishTimestamp)
case "Duration" => Ordering.by(_.totalTime)
case "Total Execute" => Ordering.by(_.totalExecution)
case unknownColumn => throw new IllegalArgumentException(s"Unknown column: $unknownColumn")
}
if (desc) {
ordering.reverse
} else {
ordering
}
}
}