Skip to content

[SPARK-25564][CORE] Add output bytes metrics for each Executor #22578

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
LantaoJin wants to merge 2 commits into from
Closed

[SPARK-25564][CORE] Add output bytes metrics for each Executor #22578

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
64101a2
[SPARK-25564][CORE] Add output bytes metrics for each Executor
LantaoJin Sep 28, 2018
1797802
some ui change
LantaoJin Sep 29, 2018
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions core/src/main/resources/org/apache/spark/ui/static/executorspage-template.html
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,7 @@ <h4 style="clear: left; display: inline-block;">Summary</h4>
title="Shaded red when garbage collection (GC) time is over 10% of task time">
Task Time (GC Time)</span></th>
<th><span data-toggle="tooltip"
title="Bytes and records read from Hadoop or from Spark storage.">Input</span></th>
title="Bytes read from Hadoop or from Spark storage / Bytes written to Hadoop">Iuput/Output</span></th>
Copy link
Contributor

@vanzin vanzin Jan 25, 2019

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Typo. Also in the other change.

<th><span data-toggle="tooltip"
title="Total shuffle bytes and records read (includes both data read locally and data read from remote executors).">
Shuffle Read</span></th>
Expand Down Expand Up @@ -102,7 +102,7 @@ <h4 style="clear: left; display: inline-block;">Executors</h4>
</scan>
</th>
<th><span data-toggle="tooltip" data-placement="top"
title="Bytes and records read from Hadoop or from Spark storage.">Input</span></th>
title="Bytes read from Hadoop or from Spark storage / Bytes written to Hadoop">Iuput/Output</span></th>
<th>
<span data-toggle="tooltip" data-placement="top"
title="Total shuffle bytes and records read (includes both data read locally and data read from remote executors).">
Expand Down
23 changes: 21 additions & 2 deletions core/src/main/resources/org/apache/spark/ui/static/executorspage.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -209,6 +209,7 @@ $(document).ready(function () {
var allTotalDuration = 0;
var allTotalGCTime = 0;
var allTotalInputBytes = 0;
var allTotalOutputBytes = 0;
var allTotalShuffleRead = 0;
var allTotalShuffleWrite = 0;
var allTotalBlacklisted = 0;
Expand All @@ -231,6 +232,7 @@ $(document).ready(function () {
var activeTotalDuration = 0;
var activeTotalGCTime = 0;
var activeTotalInputBytes = 0;
var activeTotalOutputBytes = 0;
var activeTotalShuffleRead = 0;
var activeTotalShuffleWrite = 0;
var activeTotalBlacklisted = 0;
Expand All @@ -253,6 +255,7 @@ $(document).ready(function () {
var deadTotalDuration = 0;
var deadTotalGCTime = 0;
var deadTotalInputBytes = 0;
var deadTotalOutputBytes = 0;
var deadTotalShuffleRead = 0;
var deadTotalShuffleWrite = 0;
var deadTotalBlacklisted = 0;
Expand Down Expand Up @@ -287,6 +290,7 @@ $(document).ready(function () {
allTotalDuration += exec.totalDuration;
allTotalGCTime += exec.totalGCTime;
allTotalInputBytes += exec.totalInputBytes;
allTotalOutputBytes += exec.totalOutputBytes;
allTotalShuffleRead += exec.totalShuffleRead;
allTotalShuffleWrite += exec.totalShuffleWrite;
allTotalBlacklisted += exec.isBlacklisted ? 1 : 0;
Expand All @@ -309,6 +313,7 @@ $(document).ready(function () {
activeTotalDuration += exec.totalDuration;
activeTotalGCTime += exec.totalGCTime;
activeTotalInputBytes += exec.totalInputBytes;
activeTotalOutputBytes += exec.totalOutputBytes;
activeTotalShuffleRead += exec.totalShuffleRead;
activeTotalShuffleWrite += exec.totalShuffleWrite;
activeTotalBlacklisted += exec.isBlacklisted ? 1 : 0;
Expand All @@ -331,6 +336,7 @@ $(document).ready(function () {
deadTotalDuration += exec.totalDuration;
deadTotalGCTime += exec.totalGCTime;
deadTotalInputBytes += exec.totalInputBytes;
deadTotalOutputBytes += exec.totalOutputBytes;
deadTotalShuffleRead += exec.totalShuffleRead;
deadTotalShuffleWrite += exec.totalShuffleWrite;
deadTotalBlacklisted += exec.isBlacklisted ? 1 : 0;
Expand All @@ -356,6 +362,7 @@ $(document).ready(function () {
"allTotalDuration": allTotalDuration,
"allTotalGCTime": allTotalGCTime,
"allTotalInputBytes": allTotalInputBytes,
"allTotalOutputBytes": allTotalOutputBytes,
"allTotalShuffleRead": allTotalShuffleRead,
"allTotalShuffleWrite": allTotalShuffleWrite,
"allTotalBlacklisted": allTotalBlacklisted
Expand All @@ -379,6 +386,7 @@ $(document).ready(function () {
"allTotalDuration": activeTotalDuration,
"allTotalGCTime": activeTotalGCTime,
"allTotalInputBytes": activeTotalInputBytes,
"allTotalOutputBytes": activeTotalOutputBytes,
"allTotalShuffleRead": activeTotalShuffleRead,
"allTotalShuffleWrite": activeTotalShuffleWrite,
"allTotalBlacklisted": activeTotalBlacklisted
Expand All @@ -402,6 +410,7 @@ $(document).ready(function () {
"allTotalDuration": deadTotalDuration,
"allTotalGCTime": deadTotalGCTime,
"allTotalInputBytes": deadTotalInputBytes,
"allTotalOutputBytes": deadTotalOutputBytes,
"allTotalShuffleRead": deadTotalShuffleRead,
"allTotalShuffleWrite": deadTotalShuffleWrite,
"allTotalBlacklisted": deadTotalBlacklisted
Expand Down Expand Up @@ -494,7 +503,12 @@ $(document).ready(function () {
}
}
},
{data: 'totalInputBytes', render: formatBytes},
{
data: function (row, type) {
return (formatBytes(row.totalInputBytes, type) + ' / ' +
formatBytes(row.totalOutputBytes, type));
}
},
{data: 'totalShuffleRead', render: formatBytes},
{data: 'totalShuffleWrite', render: formatBytes},
{name: 'executorLogsCol', data: 'executorLogs', render: formatLogsCells},
Expand Down Expand Up @@ -590,7 +604,12 @@ $(document).ready(function () {
}
}
},
{data: 'allTotalInputBytes', render: formatBytes},
{
data: function (row, type) {
return (formatBytes(row.allTotalInputBytes, type) + ' / ' +
formatBytes(row.allTotalOutputBytes, type));
}
},
{data: 'allTotalShuffleRead', render: formatBytes},
{data: 'allTotalShuffleWrite', render: formatBytes},
{data: 'allTotalBlacklisted'}
Expand Down
1 change: 1 addition & 0 deletions core/src/main/scala/org/apache/spark/status/AppStatusListener.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -592,6 +592,7 @@ private[spark] class AppStatusListener(
val readMetrics = event.taskMetrics.shuffleReadMetrics
exec.totalGcTime += event.taskMetrics.jvmGCTime
exec.totalInputBytes += event.taskMetrics.inputMetrics.bytesRead
exec.totalOutputBytes += event.taskMetrics.outputMetrics.bytesWritten
exec.totalShuffleRead += readMetrics.localBytesRead + readMetrics.remoteBytesRead
exec.totalShuffleWrite += event.taskMetrics.shuffleWriteMetrics.bytesWritten
}
Expand Down
2 changes: 2 additions & 0 deletions core/src/main/scala/org/apache/spark/status/LiveEntity.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -251,6 +251,7 @@ private class LiveExecutor(val executorId: String, _addTime: Long) extends LiveE
var totalDuration = 0L
var totalGcTime = 0L
var totalInputBytes = 0L
var totalOutputBytes = 0L
var totalShuffleRead = 0L
var totalShuffleWrite = 0L
var isBlacklisted = false
Expand Down Expand Up @@ -295,6 +296,7 @@ private class LiveExecutor(val executorId: String, _addTime: Long) extends LiveE
totalDuration,
totalGcTime,
totalInputBytes,
totalOutputBytes,
totalShuffleRead,
totalShuffleWrite,
isBlacklisted,
Expand Down
1 change: 1 addition & 0 deletions core/src/main/scala/org/apache/spark/status/api/v1/api.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -94,6 +94,7 @@ class ExecutorSummary private[spark](
val totalDuration: Long,
val totalGCTime: Long,
val totalInputBytes: Long,
val totalOutputBytes: Long,
val totalShuffleRead: Long,
val totalShuffleWrite: Long,
val isBlacklisted: Boolean,
Expand Down
1 change: 1 addition & 0 deletions core/src/test/resources/HistoryServerExpectations/executor_list_json_expectation.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@
"totalDuration" : 8820,
"totalGCTime" : 352,
"totalInputBytes" : 28000288,
"totalOutputBytes" : 0,
Copy link
Contributor

@attilapiros attilapiros Feb 28, 2019

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It would be nice to see at least one example where the totalOutputByte is nonzero. For example you could modify an existing spark events in core/src/test/resources/spark-events as a new test test could be overkill for this.

"totalShuffleRead" : 0,
"totalShuffleWrite" : 13180,
"isBlacklisted" : false,
Expand Down
8 changes: 8 additions & 0 deletions ...urces/HistoryServerExpectations/executor_list_with_executor_metrics_json_expectation.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@
"totalDuration" : 0,
"totalGCTime" : 0,
"totalInputBytes" : 0,
"totalOutputBytes" : 0,
"totalShuffleRead" : 0,
"totalShuffleWrite" : 0,
"isBlacklisted" : false,
Expand Down Expand Up @@ -55,6 +56,7 @@
"totalDuration" : 0,
"totalGCTime" : 0,
"totalInputBytes" : 0,
"totalOutputBytes" : 0,
"totalShuffleRead" : 0,
"totalShuffleWrite" : 0,
"isBlacklisted" : false,
Expand Down Expand Up @@ -87,6 +89,7 @@
"totalDuration" : 0,
"totalGCTime" : 0,
"totalInputBytes" : 0,
"totalOutputBytes" : 0,
"totalShuffleRead" : 0,
"totalShuffleWrite" : 0,
"isBlacklisted" : false,
Expand Down Expand Up @@ -119,6 +122,7 @@
"totalDuration" : 9252,
"totalGCTime" : 920,
"totalInputBytes" : 36838295,
"totalOutputBytes" : 0,
"totalShuffleRead" : 0,
"totalShuffleWrite" : 355051,
"isBlacklisted" : false,
Expand Down Expand Up @@ -151,6 +155,7 @@
"totalDuration" : 15645,
"totalGCTime" : 405,
"totalInputBytes" : 87272855,
"totalOutputBytes" : 0,
"totalShuffleRead" : 438675,
"totalShuffleWrite" : 26773039,
"isBlacklisted" : false,
Expand Down Expand Up @@ -195,6 +200,7 @@
"totalDuration" : 14491,
"totalGCTime" : 342,
"totalInputBytes" : 50409514,
"totalOutputBytes" : 0,
"totalShuffleRead" : 0,
"totalShuffleWrite" : 31362123,
"isBlacklisted" : false,
Expand Down Expand Up @@ -239,6 +245,7 @@
"totalDuration" : 14113,
"totalGCTime" : 326,
"totalInputBytes" : 50423423,
"totalOutputBytes" : 0,
"totalShuffleRead" : 0,
"totalShuffleWrite" : 22950296,
"isBlacklisted" : false,
Expand Down Expand Up @@ -283,6 +290,7 @@
"totalDuration" : 15665,
"totalGCTime" : 471,
"totalInputBytes" : 98905018,
"totalOutputBytes" : 0,
"totalShuffleRead" : 0,
"totalShuffleWrite" : 20594744,
"isBlacklisted" : false,
Expand Down
5 changes: 5 additions & 0 deletions core/src/test/resources/HistoryServerExpectations/executor_memory_usage_expectation.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@
"totalDuration" : 0,
"totalGCTime" : 0,
"totalInputBytes" : 0,
"totalOutputBytes" : 0,
"totalShuffleRead" : 0,
"totalShuffleWrite" : 0,
"isBlacklisted" : true,
Expand Down Expand Up @@ -43,6 +44,7 @@
"totalDuration" : 2453,
"totalGCTime" : 72,
"totalInputBytes" : 0,
"totalOutputBytes" : 0,
"totalShuffleRead" : 0,
"totalShuffleWrite" : 0,
"isBlacklisted" : true,
Expand Down Expand Up @@ -75,6 +77,7 @@
"totalDuration" : 2537,
"totalGCTime" : 88,
"totalInputBytes" : 0,
"totalOutputBytes" : 0,
"totalShuffleRead" : 0,
"totalShuffleWrite" : 0,
"isBlacklisted" : true,
Expand Down Expand Up @@ -107,6 +110,7 @@
"totalDuration" : 3152,
"totalGCTime" : 68,
"totalInputBytes" : 0,
"totalOutputBytes" : 0,
"totalShuffleRead" : 0,
"totalShuffleWrite" : 0,
"isBlacklisted" : true,
Expand Down Expand Up @@ -139,6 +143,7 @@
"totalDuration" : 2551,
"totalGCTime" : 116,
"totalInputBytes" : 0,
"totalOutputBytes" : 0,
"totalShuffleRead" : 0,
"totalShuffleWrite" : 0,
"isBlacklisted" : true,
Expand Down
5 changes: 5 additions & 0 deletions .../src/test/resources/HistoryServerExpectations/executor_node_blacklisting_expectation.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@
"totalDuration" : 0,
"totalGCTime" : 0,
"totalInputBytes" : 0,
"totalOutputBytes" : 0,
"totalShuffleRead" : 0,
"totalShuffleWrite" : 0,
"isBlacklisted" : true,
Expand Down Expand Up @@ -43,6 +44,7 @@
"totalDuration" : 2453,
"totalGCTime" : 72,
"totalInputBytes" : 0,
"totalOutputBytes" : 0,
"totalShuffleRead" : 0,
"totalShuffleWrite" : 0,
"isBlacklisted" : true,
Expand Down Expand Up @@ -75,6 +77,7 @@
"totalDuration" : 2537,
"totalGCTime" : 88,
"totalInputBytes" : 0,
"totalOutputBytes" : 0,
"totalShuffleRead" : 0,
"totalShuffleWrite" : 0,
"isBlacklisted" : true,
Expand Down Expand Up @@ -107,6 +110,7 @@
"totalDuration" : 3152,
"totalGCTime" : 68,
"totalInputBytes" : 0,
"totalOutputBytes" : 0,
"totalShuffleRead" : 0,
"totalShuffleWrite" : 0,
"isBlacklisted" : true,
Expand Down Expand Up @@ -139,6 +143,7 @@
"totalDuration" : 2551,
"totalGCTime" : 116,
"totalInputBytes" : 0,
"totalOutputBytes" : 0,
"totalShuffleRead" : 0,
"totalShuffleWrite" : 0,
"isBlacklisted" : true,
Expand Down
5 changes: 5 additions & 0 deletions ...rces/HistoryServerExpectations/executor_node_blacklisting_unblacklisting_expectation.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@
"totalDuration" : 0,
"totalGCTime" : 0,
"totalInputBytes" : 0,
"totalOutputBytes" : 0,
"totalShuffleRead" : 0,
"totalShuffleWrite" : 0,
"isBlacklisted" : false,
Expand All @@ -37,6 +38,7 @@
"totalDuration" : 3457,
"totalGCTime" : 72,
"totalInputBytes" : 0,
"totalOutputBytes" : 0,
"totalShuffleRead" : 0,
"totalShuffleWrite" : 0,
"isBlacklisted" : false,
Expand All @@ -63,6 +65,7 @@
"totalDuration" : 2792,
"totalGCTime" : 128,
"totalInputBytes" : 0,
"totalOutputBytes" : 0,
"totalShuffleRead" : 0,
"totalShuffleWrite" : 0,
"isBlacklisted" : false,
Expand All @@ -89,6 +92,7 @@
"totalDuration" : 2613,
"totalGCTime" : 84,
"totalInputBytes" : 0,
"totalOutputBytes" : 0,
"totalShuffleRead" : 0,
"totalShuffleWrite" : 0,
"isBlacklisted" : false,
Expand All @@ -115,6 +119,7 @@
"totalDuration" : 2741,
"totalGCTime" : 120,
"totalInputBytes" : 0,
"totalOutputBytes" : 0,
"totalShuffleRead" : 0,
"totalShuffleWrite" : 0,
"isBlacklisted" : false,
Expand Down