diff --git a/core/src/main/resources/org/apache/spark/ui/static/executorspage-template.html b/core/src/main/resources/org/apache/spark/ui/static/executorspage-template.html
index 5c91304e49fd..e9a8e4454518 100644
--- a/core/src/main/resources/org/apache/spark/ui/static/executorspage-template.html
+++ b/core/src/main/resources/org/apache/spark/ui/static/executorspage-template.html
@@ -44,7 +44,7 @@
Summary
title="Shaded red when garbage collection (GC) time is over 10% of task time">
Task Time (GC Time)
Input |
+ title="Bytes read from Hadoop or from Spark storage / Bytes written to Hadoop">Iuput/Output
Shuffle Read |
@@ -102,7 +102,7 @@ Executors
Input |
+ title="Bytes read from Hadoop or from Spark storage / Bytes written to Hadoop">Iuput/Output
diff --git a/core/src/main/resources/org/apache/spark/ui/static/executorspage.js b/core/src/main/resources/org/apache/spark/ui/static/executorspage.js
index 6717af3ac4da..5a7fb660e0b9 100644
--- a/core/src/main/resources/org/apache/spark/ui/static/executorspage.js
+++ b/core/src/main/resources/org/apache/spark/ui/static/executorspage.js
@@ -209,6 +209,7 @@ $(document).ready(function () {
var allTotalDuration = 0;
var allTotalGCTime = 0;
var allTotalInputBytes = 0;
+ var allTotalOutputBytes = 0;
var allTotalShuffleRead = 0;
var allTotalShuffleWrite = 0;
var allTotalBlacklisted = 0;
@@ -231,6 +232,7 @@ $(document).ready(function () {
var activeTotalDuration = 0;
var activeTotalGCTime = 0;
var activeTotalInputBytes = 0;
+ var activeTotalOutputBytes = 0;
var activeTotalShuffleRead = 0;
var activeTotalShuffleWrite = 0;
var activeTotalBlacklisted = 0;
@@ -253,6 +255,7 @@ $(document).ready(function () {
var deadTotalDuration = 0;
var deadTotalGCTime = 0;
var deadTotalInputBytes = 0;
+ var deadTotalOutputBytes = 0;
var deadTotalShuffleRead = 0;
var deadTotalShuffleWrite = 0;
var deadTotalBlacklisted = 0;
@@ -287,6 +290,7 @@ $(document).ready(function () {
allTotalDuration += exec.totalDuration;
allTotalGCTime += exec.totalGCTime;
allTotalInputBytes += exec.totalInputBytes;
+ allTotalOutputBytes += exec.totalOutputBytes;
allTotalShuffleRead += exec.totalShuffleRead;
allTotalShuffleWrite += exec.totalShuffleWrite;
allTotalBlacklisted += exec.isBlacklisted ? 1 : 0;
@@ -309,6 +313,7 @@ $(document).ready(function () {
activeTotalDuration += exec.totalDuration;
activeTotalGCTime += exec.totalGCTime;
activeTotalInputBytes += exec.totalInputBytes;
+ activeTotalOutputBytes += exec.totalOutputBytes;
activeTotalShuffleRead += exec.totalShuffleRead;
activeTotalShuffleWrite += exec.totalShuffleWrite;
activeTotalBlacklisted += exec.isBlacklisted ? 1 : 0;
@@ -331,6 +336,7 @@ $(document).ready(function () {
deadTotalDuration += exec.totalDuration;
deadTotalGCTime += exec.totalGCTime;
deadTotalInputBytes += exec.totalInputBytes;
+ deadTotalOutputBytes += exec.totalOutputBytes;
deadTotalShuffleRead += exec.totalShuffleRead;
deadTotalShuffleWrite += exec.totalShuffleWrite;
deadTotalBlacklisted += exec.isBlacklisted ? 1 : 0;
@@ -356,6 +362,7 @@ $(document).ready(function () {
"allTotalDuration": allTotalDuration,
"allTotalGCTime": allTotalGCTime,
"allTotalInputBytes": allTotalInputBytes,
+ "allTotalOutputBytes": allTotalOutputBytes,
"allTotalShuffleRead": allTotalShuffleRead,
"allTotalShuffleWrite": allTotalShuffleWrite,
"allTotalBlacklisted": allTotalBlacklisted
@@ -379,6 +386,7 @@ $(document).ready(function () {
"allTotalDuration": activeTotalDuration,
"allTotalGCTime": activeTotalGCTime,
"allTotalInputBytes": activeTotalInputBytes,
+ "allTotalOutputBytes": activeTotalOutputBytes,
"allTotalShuffleRead": activeTotalShuffleRead,
"allTotalShuffleWrite": activeTotalShuffleWrite,
"allTotalBlacklisted": activeTotalBlacklisted
@@ -402,6 +410,7 @@ $(document).ready(function () {
"allTotalDuration": deadTotalDuration,
"allTotalGCTime": deadTotalGCTime,
"allTotalInputBytes": deadTotalInputBytes,
+ "allTotalOutputBytes": deadTotalOutputBytes,
"allTotalShuffleRead": deadTotalShuffleRead,
"allTotalShuffleWrite": deadTotalShuffleWrite,
"allTotalBlacklisted": deadTotalBlacklisted
@@ -494,7 +503,12 @@ $(document).ready(function () {
}
}
},
- {data: 'totalInputBytes', render: formatBytes},
+ {
+ data: function (row, type) {
+ return (formatBytes(row.totalInputBytes, type) + ' / ' +
+ formatBytes(row.totalOutputBytes, type));
+ }
+ },
{data: 'totalShuffleRead', render: formatBytes},
{data: 'totalShuffleWrite', render: formatBytes},
{name: 'executorLogsCol', data: 'executorLogs', render: formatLogsCells},
@@ -590,7 +604,12 @@ $(document).ready(function () {
}
}
},
- {data: 'allTotalInputBytes', render: formatBytes},
+ {
+ data: function (row, type) {
+ return (formatBytes(row.allTotalInputBytes, type) + ' / ' +
+ formatBytes(row.allTotalOutputBytes, type));
+ }
+ },
{data: 'allTotalShuffleRead', render: formatBytes},
{data: 'allTotalShuffleWrite', render: formatBytes},
{data: 'allTotalBlacklisted'}
diff --git a/core/src/main/scala/org/apache/spark/status/AppStatusListener.scala b/core/src/main/scala/org/apache/spark/status/AppStatusListener.scala
index f21eee196576..0ce4fda5cf17 100644
--- a/core/src/main/scala/org/apache/spark/status/AppStatusListener.scala
+++ b/core/src/main/scala/org/apache/spark/status/AppStatusListener.scala
@@ -592,6 +592,7 @@ private[spark] class AppStatusListener(
val readMetrics = event.taskMetrics.shuffleReadMetrics
exec.totalGcTime += event.taskMetrics.jvmGCTime
exec.totalInputBytes += event.taskMetrics.inputMetrics.bytesRead
+ exec.totalOutputBytes += event.taskMetrics.outputMetrics.bytesWritten
exec.totalShuffleRead += readMetrics.localBytesRead + readMetrics.remoteBytesRead
exec.totalShuffleWrite += event.taskMetrics.shuffleWriteMetrics.bytesWritten
}
diff --git a/core/src/main/scala/org/apache/spark/status/LiveEntity.scala b/core/src/main/scala/org/apache/spark/status/LiveEntity.scala
index 8708e64db3c1..1821849e3512 100644
--- a/core/src/main/scala/org/apache/spark/status/LiveEntity.scala
+++ b/core/src/main/scala/org/apache/spark/status/LiveEntity.scala
@@ -251,6 +251,7 @@ private class LiveExecutor(val executorId: String, _addTime: Long) extends LiveE
var totalDuration = 0L
var totalGcTime = 0L
var totalInputBytes = 0L
+ var totalOutputBytes = 0L
var totalShuffleRead = 0L
var totalShuffleWrite = 0L
var isBlacklisted = false
@@ -295,6 +296,7 @@ private class LiveExecutor(val executorId: String, _addTime: Long) extends LiveE
totalDuration,
totalGcTime,
totalInputBytes,
+ totalOutputBytes,
totalShuffleRead,
totalShuffleWrite,
isBlacklisted,
diff --git a/core/src/main/scala/org/apache/spark/status/api/v1/api.scala b/core/src/main/scala/org/apache/spark/status/api/v1/api.scala
index 77466b62ff6e..2e679dd26a44 100644
--- a/core/src/main/scala/org/apache/spark/status/api/v1/api.scala
+++ b/core/src/main/scala/org/apache/spark/status/api/v1/api.scala
@@ -94,6 +94,7 @@ class ExecutorSummary private[spark](
val totalDuration: Long,
val totalGCTime: Long,
val totalInputBytes: Long,
+ val totalOutputBytes: Long,
val totalShuffleRead: Long,
val totalShuffleWrite: Long,
val isBlacklisted: Boolean,
diff --git a/core/src/test/resources/HistoryServerExpectations/executor_list_json_expectation.json b/core/src/test/resources/HistoryServerExpectations/executor_list_json_expectation.json
index 7bb8fe8fd8f9..9ecfaf0e117f 100644
--- a/core/src/test/resources/HistoryServerExpectations/executor_list_json_expectation.json
+++ b/core/src/test/resources/HistoryServerExpectations/executor_list_json_expectation.json
@@ -14,6 +14,7 @@
"totalDuration" : 8820,
"totalGCTime" : 352,
"totalInputBytes" : 28000288,
+ "totalOutputBytes" : 0,
"totalShuffleRead" : 0,
"totalShuffleWrite" : 13180,
"isBlacklisted" : false,
diff --git a/core/src/test/resources/HistoryServerExpectations/executor_list_with_executor_metrics_json_expectation.json b/core/src/test/resources/HistoryServerExpectations/executor_list_with_executor_metrics_json_expectation.json
index 9bf2086cc8e7..da8e7a91bff5 100644
--- a/core/src/test/resources/HistoryServerExpectations/executor_list_with_executor_metrics_json_expectation.json
+++ b/core/src/test/resources/HistoryServerExpectations/executor_list_with_executor_metrics_json_expectation.json
@@ -14,6 +14,7 @@
"totalDuration" : 0,
"totalGCTime" : 0,
"totalInputBytes" : 0,
+ "totalOutputBytes" : 0,
"totalShuffleRead" : 0,
"totalShuffleWrite" : 0,
"isBlacklisted" : false,
@@ -55,6 +56,7 @@
"totalDuration" : 0,
"totalGCTime" : 0,
"totalInputBytes" : 0,
+ "totalOutputBytes" : 0,
"totalShuffleRead" : 0,
"totalShuffleWrite" : 0,
"isBlacklisted" : false,
@@ -87,6 +89,7 @@
"totalDuration" : 0,
"totalGCTime" : 0,
"totalInputBytes" : 0,
+ "totalOutputBytes" : 0,
"totalShuffleRead" : 0,
"totalShuffleWrite" : 0,
"isBlacklisted" : false,
@@ -119,6 +122,7 @@
"totalDuration" : 9252,
"totalGCTime" : 920,
"totalInputBytes" : 36838295,
+ "totalOutputBytes" : 0,
"totalShuffleRead" : 0,
"totalShuffleWrite" : 355051,
"isBlacklisted" : false,
@@ -151,6 +155,7 @@
"totalDuration" : 15645,
"totalGCTime" : 405,
"totalInputBytes" : 87272855,
+ "totalOutputBytes" : 0,
"totalShuffleRead" : 438675,
"totalShuffleWrite" : 26773039,
"isBlacklisted" : false,
@@ -195,6 +200,7 @@
"totalDuration" : 14491,
"totalGCTime" : 342,
"totalInputBytes" : 50409514,
+ "totalOutputBytes" : 0,
"totalShuffleRead" : 0,
"totalShuffleWrite" : 31362123,
"isBlacklisted" : false,
@@ -239,6 +245,7 @@
"totalDuration" : 14113,
"totalGCTime" : 326,
"totalInputBytes" : 50423423,
+ "totalOutputBytes" : 0,
"totalShuffleRead" : 0,
"totalShuffleWrite" : 22950296,
"isBlacklisted" : false,
@@ -283,6 +290,7 @@
"totalDuration" : 15665,
"totalGCTime" : 471,
"totalInputBytes" : 98905018,
+ "totalOutputBytes" : 0,
"totalShuffleRead" : 0,
"totalShuffleWrite" : 20594744,
"isBlacklisted" : false,
diff --git a/core/src/test/resources/HistoryServerExpectations/executor_memory_usage_expectation.json b/core/src/test/resources/HistoryServerExpectations/executor_memory_usage_expectation.json
index dd5b1dcb7372..11dd30d1dc99 100644
--- a/core/src/test/resources/HistoryServerExpectations/executor_memory_usage_expectation.json
+++ b/core/src/test/resources/HistoryServerExpectations/executor_memory_usage_expectation.json
@@ -14,6 +14,7 @@
"totalDuration" : 0,
"totalGCTime" : 0,
"totalInputBytes" : 0,
+ "totalOutputBytes" : 0,
"totalShuffleRead" : 0,
"totalShuffleWrite" : 0,
"isBlacklisted" : true,
@@ -43,6 +44,7 @@
"totalDuration" : 2453,
"totalGCTime" : 72,
"totalInputBytes" : 0,
+ "totalOutputBytes" : 0,
"totalShuffleRead" : 0,
"totalShuffleWrite" : 0,
"isBlacklisted" : true,
@@ -75,6 +77,7 @@
"totalDuration" : 2537,
"totalGCTime" : 88,
"totalInputBytes" : 0,
+ "totalOutputBytes" : 0,
"totalShuffleRead" : 0,
"totalShuffleWrite" : 0,
"isBlacklisted" : true,
@@ -107,6 +110,7 @@
"totalDuration" : 3152,
"totalGCTime" : 68,
"totalInputBytes" : 0,
+ "totalOutputBytes" : 0,
"totalShuffleRead" : 0,
"totalShuffleWrite" : 0,
"isBlacklisted" : true,
@@ -139,6 +143,7 @@
"totalDuration" : 2551,
"totalGCTime" : 116,
"totalInputBytes" : 0,
+ "totalOutputBytes" : 0,
"totalShuffleRead" : 0,
"totalShuffleWrite" : 0,
"isBlacklisted" : true,
diff --git a/core/src/test/resources/HistoryServerExpectations/executor_node_blacklisting_expectation.json b/core/src/test/resources/HistoryServerExpectations/executor_node_blacklisting_expectation.json
index 3e55d3d9d7eb..a8217a992a3e 100644
--- a/core/src/test/resources/HistoryServerExpectations/executor_node_blacklisting_expectation.json
+++ b/core/src/test/resources/HistoryServerExpectations/executor_node_blacklisting_expectation.json
@@ -14,6 +14,7 @@
"totalDuration" : 0,
"totalGCTime" : 0,
"totalInputBytes" : 0,
+ "totalOutputBytes" : 0,
"totalShuffleRead" : 0,
"totalShuffleWrite" : 0,
"isBlacklisted" : true,
@@ -43,6 +44,7 @@
"totalDuration" : 2453,
"totalGCTime" : 72,
"totalInputBytes" : 0,
+ "totalOutputBytes" : 0,
"totalShuffleRead" : 0,
"totalShuffleWrite" : 0,
"isBlacklisted" : true,
@@ -75,6 +77,7 @@
"totalDuration" : 2537,
"totalGCTime" : 88,
"totalInputBytes" : 0,
+ "totalOutputBytes" : 0,
"totalShuffleRead" : 0,
"totalShuffleWrite" : 0,
"isBlacklisted" : true,
@@ -107,6 +110,7 @@
"totalDuration" : 3152,
"totalGCTime" : 68,
"totalInputBytes" : 0,
+ "totalOutputBytes" : 0,
"totalShuffleRead" : 0,
"totalShuffleWrite" : 0,
"isBlacklisted" : true,
@@ -139,6 +143,7 @@
"totalDuration" : 2551,
"totalGCTime" : 116,
"totalInputBytes" : 0,
+ "totalOutputBytes" : 0,
"totalShuffleRead" : 0,
"totalShuffleWrite" : 0,
"isBlacklisted" : true,
diff --git a/core/src/test/resources/HistoryServerExpectations/executor_node_blacklisting_unblacklisting_expectation.json b/core/src/test/resources/HistoryServerExpectations/executor_node_blacklisting_unblacklisting_expectation.json
index e87f3e78f2dc..5a72506b3c35 100644
--- a/core/src/test/resources/HistoryServerExpectations/executor_node_blacklisting_unblacklisting_expectation.json
+++ b/core/src/test/resources/HistoryServerExpectations/executor_node_blacklisting_unblacklisting_expectation.json
@@ -14,6 +14,7 @@
"totalDuration" : 0,
"totalGCTime" : 0,
"totalInputBytes" : 0,
+ "totalOutputBytes" : 0,
"totalShuffleRead" : 0,
"totalShuffleWrite" : 0,
"isBlacklisted" : false,
@@ -37,6 +38,7 @@
"totalDuration" : 3457,
"totalGCTime" : 72,
"totalInputBytes" : 0,
+ "totalOutputBytes" : 0,
"totalShuffleRead" : 0,
"totalShuffleWrite" : 0,
"isBlacklisted" : false,
@@ -63,6 +65,7 @@
"totalDuration" : 2792,
"totalGCTime" : 128,
"totalInputBytes" : 0,
+ "totalOutputBytes" : 0,
"totalShuffleRead" : 0,
"totalShuffleWrite" : 0,
"isBlacklisted" : false,
@@ -89,6 +92,7 @@
"totalDuration" : 2613,
"totalGCTime" : 84,
"totalInputBytes" : 0,
+ "totalOutputBytes" : 0,
"totalShuffleRead" : 0,
"totalShuffleWrite" : 0,
"isBlacklisted" : false,
@@ -115,6 +119,7 @@
"totalDuration" : 2741,
"totalGCTime" : 120,
"totalInputBytes" : 0,
+ "totalOutputBytes" : 0,
"totalShuffleRead" : 0,
"totalShuffleWrite" : 0,
"isBlacklisted" : false,
|