From 64101a2c0a8aae2e7fc7ff35e3502b5aa5ded867 Mon Sep 17 00:00:00 2001
From: LantaoJin <jinlantao@gmail.com>
Date: Fri, 28 Sep 2018 15:10:17 +0800
Subject: [PATCH 1/2] [SPARK-25564][CORE] Add output bytes metrics for each
 Executor

---
 .../ui/static/executorspage-template.html     |  4 ++--
 .../apache/spark/ui/static/executorspage.js   | 23 +++++++++++++++++--
 .../spark/status/AppStatusListener.scala      |  1 +
 .../org/apache/spark/status/LiveEntity.scala  |  2 ++
 .../org/apache/spark/status/api/v1/api.scala  |  1 +
 .../executor_list_json_expectation.json       |  1 +
 ...ith_executor_metrics_json_expectation.json |  8 +++++++
 .../executor_memory_usage_expectation.json    |  5 ++++
 ...xecutor_node_blacklisting_expectation.json |  5 ++++
 ...acklisting_unblacklisting_expectation.json |  5 ++++
 10 files changed, 51 insertions(+), 4 deletions(-)

diff --git a/core/src/main/resources/org/apache/spark/ui/static/executorspage-template.html b/core/src/main/resources/org/apache/spark/ui/static/executorspage-template.html
index 5c91304e49fd..0faed286068d 100644
--- a/core/src/main/resources/org/apache/spark/ui/static/executorspage-template.html
+++ b/core/src/main/resources/org/apache/spark/ui/static/executorspage-template.html
@@ -44,7 +44,7 @@ <h4 style="clear: left; display: inline-block;">Summary</h4>
                   title="Shaded red when garbage collection (GC) time is over 10% of task time">
           Task Time (GC Time)</span></th>
         <th><span data-toggle="tooltip"
-                  title="Bytes and records read from Hadoop or from Spark storage.">Input</span></th>
+                  title="Bytes read from Hadoop or from Spark storage / Bytes written to Hadoop">I/O</span></th>
         <th><span data-toggle="tooltip"
                   title="Total shuffle bytes and records read (includes both data read locally and data read from remote executors).">
           Shuffle Read</span></th>
@@ -102,7 +102,7 @@ <h4 style="clear: left; display: inline-block;">Executors</h4>
             </scan>
           </th>
           <th><span data-toggle="tooltip" data-placement="top"
-                    title="Bytes and records read from Hadoop or from Spark storage.">Input</span></th>
+                    title="Bytes read from Hadoop or from Spark storage / Bytes written to Hadoop">I/O</span></th>
           <th>
             <span data-toggle="tooltip" data-placement="top"
                   title="Total shuffle bytes and records read (includes both data read locally and data read from remote executors).">
diff --git a/core/src/main/resources/org/apache/spark/ui/static/executorspage.js b/core/src/main/resources/org/apache/spark/ui/static/executorspage.js
index 6717af3ac4da..5a7fb660e0b9 100644
--- a/core/src/main/resources/org/apache/spark/ui/static/executorspage.js
+++ b/core/src/main/resources/org/apache/spark/ui/static/executorspage.js
@@ -209,6 +209,7 @@ $(document).ready(function () {
             var allTotalDuration = 0;
             var allTotalGCTime = 0;
             var allTotalInputBytes = 0;
+            var allTotalOutputBytes = 0;
             var allTotalShuffleRead = 0;
             var allTotalShuffleWrite = 0;
             var allTotalBlacklisted = 0;
@@ -231,6 +232,7 @@ $(document).ready(function () {
             var activeTotalDuration = 0;
             var activeTotalGCTime = 0;
             var activeTotalInputBytes = 0;
+            var activeTotalOutputBytes = 0;
             var activeTotalShuffleRead = 0;
             var activeTotalShuffleWrite = 0;
             var activeTotalBlacklisted = 0;
@@ -253,6 +255,7 @@ $(document).ready(function () {
             var deadTotalDuration = 0;
             var deadTotalGCTime = 0;
             var deadTotalInputBytes = 0;
+            var deadTotalOutputBytes = 0;
             var deadTotalShuffleRead = 0;
             var deadTotalShuffleWrite = 0;
             var deadTotalBlacklisted = 0;
@@ -287,6 +290,7 @@ $(document).ready(function () {
                 allTotalDuration += exec.totalDuration;
                 allTotalGCTime += exec.totalGCTime;
                 allTotalInputBytes += exec.totalInputBytes;
+                allTotalOutputBytes += exec.totalOutputBytes;
                 allTotalShuffleRead += exec.totalShuffleRead;
                 allTotalShuffleWrite += exec.totalShuffleWrite;
                 allTotalBlacklisted += exec.isBlacklisted ? 1 : 0;
@@ -309,6 +313,7 @@ $(document).ready(function () {
                     activeTotalDuration += exec.totalDuration;
                     activeTotalGCTime += exec.totalGCTime;
                     activeTotalInputBytes += exec.totalInputBytes;
+                    activeTotalOutputBytes += exec.totalOutputBytes;
                     activeTotalShuffleRead += exec.totalShuffleRead;
                     activeTotalShuffleWrite += exec.totalShuffleWrite;
                     activeTotalBlacklisted += exec.isBlacklisted ? 1 : 0;
@@ -331,6 +336,7 @@ $(document).ready(function () {
                     deadTotalDuration += exec.totalDuration;
                     deadTotalGCTime += exec.totalGCTime;
                     deadTotalInputBytes += exec.totalInputBytes;
+                    deadTotalOutputBytes += exec.totalOutputBytes;
                     deadTotalShuffleRead += exec.totalShuffleRead;
                     deadTotalShuffleWrite += exec.totalShuffleWrite;
                     deadTotalBlacklisted += exec.isBlacklisted ? 1 : 0;
@@ -356,6 +362,7 @@ $(document).ready(function () {
                 "allTotalDuration": allTotalDuration,
                 "allTotalGCTime": allTotalGCTime,
                 "allTotalInputBytes": allTotalInputBytes,
+                "allTotalOutputBytes": allTotalOutputBytes,
                 "allTotalShuffleRead": allTotalShuffleRead,
                 "allTotalShuffleWrite": allTotalShuffleWrite,
                 "allTotalBlacklisted": allTotalBlacklisted
@@ -379,6 +386,7 @@ $(document).ready(function () {
                 "allTotalDuration": activeTotalDuration,
                 "allTotalGCTime": activeTotalGCTime,
                 "allTotalInputBytes": activeTotalInputBytes,
+                "allTotalOutputBytes": activeTotalOutputBytes,
                 "allTotalShuffleRead": activeTotalShuffleRead,
                 "allTotalShuffleWrite": activeTotalShuffleWrite,
                 "allTotalBlacklisted": activeTotalBlacklisted
@@ -402,6 +410,7 @@ $(document).ready(function () {
                 "allTotalDuration": deadTotalDuration,
                 "allTotalGCTime": deadTotalGCTime,
                 "allTotalInputBytes": deadTotalInputBytes,
+                "allTotalOutputBytes": deadTotalOutputBytes,
                 "allTotalShuffleRead": deadTotalShuffleRead,
                 "allTotalShuffleWrite": deadTotalShuffleWrite,
                 "allTotalBlacklisted": deadTotalBlacklisted
@@ -494,7 +503,12 @@ $(document).ready(function () {
                                 }
                             }
                         },
-                        {data: 'totalInputBytes', render: formatBytes},
+                        {
+                            data: function (row, type) {
+                                return (formatBytes(row.totalInputBytes, type) + ' / ' +
+                                    formatBytes(row.totalOutputBytes, type));
+                            }
+                        },
                         {data: 'totalShuffleRead', render: formatBytes},
                         {data: 'totalShuffleWrite', render: formatBytes},
                         {name: 'executorLogsCol', data: 'executorLogs', render: formatLogsCells},
@@ -590,7 +604,12 @@ $(document).ready(function () {
                                 }
                             }
                         },
-                        {data: 'allTotalInputBytes', render: formatBytes},
+                        {
+                            data: function (row, type) {
+                                return (formatBytes(row.allTotalInputBytes, type) + ' / ' +
+                                    formatBytes(row.allTotalOutputBytes, type));
+                            }
+                        },
                         {data: 'allTotalShuffleRead', render: formatBytes},
                         {data: 'allTotalShuffleWrite', render: formatBytes},
                         {data: 'allTotalBlacklisted'}
diff --git a/core/src/main/scala/org/apache/spark/status/AppStatusListener.scala b/core/src/main/scala/org/apache/spark/status/AppStatusListener.scala
index f21eee196576..0ce4fda5cf17 100644
--- a/core/src/main/scala/org/apache/spark/status/AppStatusListener.scala
+++ b/core/src/main/scala/org/apache/spark/status/AppStatusListener.scala
@@ -592,6 +592,7 @@ private[spark] class AppStatusListener(
           val readMetrics = event.taskMetrics.shuffleReadMetrics
           exec.totalGcTime += event.taskMetrics.jvmGCTime
           exec.totalInputBytes += event.taskMetrics.inputMetrics.bytesRead
+          exec.totalOutputBytes += event.taskMetrics.outputMetrics.bytesWritten
           exec.totalShuffleRead += readMetrics.localBytesRead + readMetrics.remoteBytesRead
           exec.totalShuffleWrite += event.taskMetrics.shuffleWriteMetrics.bytesWritten
         }
diff --git a/core/src/main/scala/org/apache/spark/status/LiveEntity.scala b/core/src/main/scala/org/apache/spark/status/LiveEntity.scala
index 8708e64db3c1..1821849e3512 100644
--- a/core/src/main/scala/org/apache/spark/status/LiveEntity.scala
+++ b/core/src/main/scala/org/apache/spark/status/LiveEntity.scala
@@ -251,6 +251,7 @@ private class LiveExecutor(val executorId: String, _addTime: Long) extends LiveE
   var totalDuration = 0L
   var totalGcTime = 0L
   var totalInputBytes = 0L
+  var totalOutputBytes = 0L
   var totalShuffleRead = 0L
   var totalShuffleWrite = 0L
   var isBlacklisted = false
@@ -295,6 +296,7 @@ private class LiveExecutor(val executorId: String, _addTime: Long) extends LiveE
       totalDuration,
       totalGcTime,
       totalInputBytes,
+      totalOutputBytes,
       totalShuffleRead,
       totalShuffleWrite,
       isBlacklisted,
diff --git a/core/src/main/scala/org/apache/spark/status/api/v1/api.scala b/core/src/main/scala/org/apache/spark/status/api/v1/api.scala
index 77466b62ff6e..2e679dd26a44 100644
--- a/core/src/main/scala/org/apache/spark/status/api/v1/api.scala
+++ b/core/src/main/scala/org/apache/spark/status/api/v1/api.scala
@@ -94,6 +94,7 @@ class ExecutorSummary private[spark](
     val totalDuration: Long,
     val totalGCTime: Long,
     val totalInputBytes: Long,
+    val totalOutputBytes: Long,
     val totalShuffleRead: Long,
     val totalShuffleWrite: Long,
     val isBlacklisted: Boolean,
diff --git a/core/src/test/resources/HistoryServerExpectations/executor_list_json_expectation.json b/core/src/test/resources/HistoryServerExpectations/executor_list_json_expectation.json
index 7bb8fe8fd8f9..9ecfaf0e117f 100644
--- a/core/src/test/resources/HistoryServerExpectations/executor_list_json_expectation.json
+++ b/core/src/test/resources/HistoryServerExpectations/executor_list_json_expectation.json
@@ -14,6 +14,7 @@
   "totalDuration" : 8820,
   "totalGCTime" : 352,
   "totalInputBytes" : 28000288,
+  "totalOutputBytes" : 0,
   "totalShuffleRead" : 0,
   "totalShuffleWrite" : 13180,
   "isBlacklisted" : false,
diff --git a/core/src/test/resources/HistoryServerExpectations/executor_list_with_executor_metrics_json_expectation.json b/core/src/test/resources/HistoryServerExpectations/executor_list_with_executor_metrics_json_expectation.json
index 9bf2086cc8e7..da8e7a91bff5 100644
--- a/core/src/test/resources/HistoryServerExpectations/executor_list_with_executor_metrics_json_expectation.json
+++ b/core/src/test/resources/HistoryServerExpectations/executor_list_with_executor_metrics_json_expectation.json
@@ -14,6 +14,7 @@
   "totalDuration" : 0,
   "totalGCTime" : 0,
   "totalInputBytes" : 0,
+  "totalOutputBytes" : 0,
   "totalShuffleRead" : 0,
   "totalShuffleWrite" : 0,
   "isBlacklisted" : false,
@@ -55,6 +56,7 @@
   "totalDuration" : 0,
   "totalGCTime" : 0,
   "totalInputBytes" : 0,
+  "totalOutputBytes" : 0,
   "totalShuffleRead" : 0,
   "totalShuffleWrite" : 0,
   "isBlacklisted" : false,
@@ -87,6 +89,7 @@
   "totalDuration" : 0,
   "totalGCTime" : 0,
   "totalInputBytes" : 0,
+  "totalOutputBytes" : 0,
   "totalShuffleRead" : 0,
   "totalShuffleWrite" : 0,
   "isBlacklisted" : false,
@@ -119,6 +122,7 @@
   "totalDuration" : 9252,
   "totalGCTime" : 920,
   "totalInputBytes" : 36838295,
+  "totalOutputBytes" : 0,
   "totalShuffleRead" : 0,
   "totalShuffleWrite" : 355051,
   "isBlacklisted" : false,
@@ -151,6 +155,7 @@
   "totalDuration" : 15645,
   "totalGCTime" : 405,
   "totalInputBytes" : 87272855,
+  "totalOutputBytes" : 0,
   "totalShuffleRead" : 438675,
   "totalShuffleWrite" : 26773039,
   "isBlacklisted" : false,
@@ -195,6 +200,7 @@
   "totalDuration" : 14491,
   "totalGCTime" : 342,
   "totalInputBytes" : 50409514,
+  "totalOutputBytes" : 0,
   "totalShuffleRead" : 0,
   "totalShuffleWrite" : 31362123,
   "isBlacklisted" : false,
@@ -239,6 +245,7 @@
   "totalDuration" : 14113,
   "totalGCTime" : 326,
   "totalInputBytes" : 50423423,
+  "totalOutputBytes" : 0,
   "totalShuffleRead" : 0,
   "totalShuffleWrite" : 22950296,
   "isBlacklisted" : false,
@@ -283,6 +290,7 @@
   "totalDuration" : 15665,
   "totalGCTime" : 471,
   "totalInputBytes" : 98905018,
+  "totalOutputBytes" : 0,
   "totalShuffleRead" : 0,
   "totalShuffleWrite" : 20594744,
   "isBlacklisted" : false,
diff --git a/core/src/test/resources/HistoryServerExpectations/executor_memory_usage_expectation.json b/core/src/test/resources/HistoryServerExpectations/executor_memory_usage_expectation.json
index dd5b1dcb7372..11dd30d1dc99 100644
--- a/core/src/test/resources/HistoryServerExpectations/executor_memory_usage_expectation.json
+++ b/core/src/test/resources/HistoryServerExpectations/executor_memory_usage_expectation.json
@@ -14,6 +14,7 @@
   "totalDuration" : 0,
   "totalGCTime" : 0,
   "totalInputBytes" : 0,
+  "totalOutputBytes" : 0,
   "totalShuffleRead" : 0,
   "totalShuffleWrite" : 0,
   "isBlacklisted" : true,
@@ -43,6 +44,7 @@
   "totalDuration" : 2453,
   "totalGCTime" : 72,
   "totalInputBytes" : 0,
+  "totalOutputBytes" : 0,
   "totalShuffleRead" : 0,
   "totalShuffleWrite" : 0,
   "isBlacklisted" : true,
@@ -75,6 +77,7 @@
   "totalDuration" : 2537,
   "totalGCTime" : 88,
   "totalInputBytes" : 0,
+  "totalOutputBytes" : 0,
   "totalShuffleRead" : 0,
   "totalShuffleWrite" : 0,
   "isBlacklisted" : true,
@@ -107,6 +110,7 @@
   "totalDuration" : 3152,
   "totalGCTime" : 68,
   "totalInputBytes" : 0,
+  "totalOutputBytes" : 0,
   "totalShuffleRead" : 0,
   "totalShuffleWrite" : 0,
   "isBlacklisted" : true,
@@ -139,6 +143,7 @@
   "totalDuration" : 2551,
   "totalGCTime" : 116,
   "totalInputBytes" : 0,
+  "totalOutputBytes" : 0,
   "totalShuffleRead" : 0,
   "totalShuffleWrite" : 0,
   "isBlacklisted" : true,
diff --git a/core/src/test/resources/HistoryServerExpectations/executor_node_blacklisting_expectation.json b/core/src/test/resources/HistoryServerExpectations/executor_node_blacklisting_expectation.json
index 3e55d3d9d7eb..a8217a992a3e 100644
--- a/core/src/test/resources/HistoryServerExpectations/executor_node_blacklisting_expectation.json
+++ b/core/src/test/resources/HistoryServerExpectations/executor_node_blacklisting_expectation.json
@@ -14,6 +14,7 @@
   "totalDuration" : 0,
   "totalGCTime" : 0,
   "totalInputBytes" : 0,
+  "totalOutputBytes" : 0,
   "totalShuffleRead" : 0,
   "totalShuffleWrite" : 0,
   "isBlacklisted" : true,
@@ -43,6 +44,7 @@
   "totalDuration" : 2453,
   "totalGCTime" : 72,
   "totalInputBytes" : 0,
+  "totalOutputBytes" : 0,
   "totalShuffleRead" : 0,
   "totalShuffleWrite" : 0,
   "isBlacklisted" : true,
@@ -75,6 +77,7 @@
   "totalDuration" : 2537,
   "totalGCTime" : 88,
   "totalInputBytes" : 0,
+  "totalOutputBytes" : 0,
   "totalShuffleRead" : 0,
   "totalShuffleWrite" : 0,
   "isBlacklisted" : true,
@@ -107,6 +110,7 @@
   "totalDuration" : 3152,
   "totalGCTime" : 68,
   "totalInputBytes" : 0,
+  "totalOutputBytes" : 0,
   "totalShuffleRead" : 0,
   "totalShuffleWrite" : 0,
   "isBlacklisted" : true,
@@ -139,6 +143,7 @@
   "totalDuration" : 2551,
   "totalGCTime" : 116,
   "totalInputBytes" : 0,
+  "totalOutputBytes" : 0,
   "totalShuffleRead" : 0,
   "totalShuffleWrite" : 0,
   "isBlacklisted" : true,
diff --git a/core/src/test/resources/HistoryServerExpectations/executor_node_blacklisting_unblacklisting_expectation.json b/core/src/test/resources/HistoryServerExpectations/executor_node_blacklisting_unblacklisting_expectation.json
index e87f3e78f2dc..5a72506b3c35 100644
--- a/core/src/test/resources/HistoryServerExpectations/executor_node_blacklisting_unblacklisting_expectation.json
+++ b/core/src/test/resources/HistoryServerExpectations/executor_node_blacklisting_unblacklisting_expectation.json
@@ -14,6 +14,7 @@
   "totalDuration" : 0,
   "totalGCTime" : 0,
   "totalInputBytes" : 0,
+  "totalOutputBytes" : 0,
   "totalShuffleRead" : 0,
   "totalShuffleWrite" : 0,
   "isBlacklisted" : false,
@@ -37,6 +38,7 @@
   "totalDuration" : 3457,
   "totalGCTime" : 72,
   "totalInputBytes" : 0,
+  "totalOutputBytes" : 0,
   "totalShuffleRead" : 0,
   "totalShuffleWrite" : 0,
   "isBlacklisted" : false,
@@ -63,6 +65,7 @@
   "totalDuration" : 2792,
   "totalGCTime" : 128,
   "totalInputBytes" : 0,
+  "totalOutputBytes" : 0,
   "totalShuffleRead" : 0,
   "totalShuffleWrite" : 0,
   "isBlacklisted" : false,
@@ -89,6 +92,7 @@
   "totalDuration" : 2613,
   "totalGCTime" : 84,
   "totalInputBytes" : 0,
+  "totalOutputBytes" : 0,
   "totalShuffleRead" : 0,
   "totalShuffleWrite" : 0,
   "isBlacklisted" : false,
@@ -115,6 +119,7 @@
   "totalDuration" : 2741,
   "totalGCTime" : 120,
   "totalInputBytes" : 0,
+  "totalOutputBytes" : 0,
   "totalShuffleRead" : 0,
   "totalShuffleWrite" : 0,
   "isBlacklisted" : false,

From 17978026900e82290c7cccd8247bed131ff49493 Mon Sep 17 00:00:00 2001
From: LantaoJin <jinlantao@gmail.com>
Date: Sat, 29 Sep 2018 16:43:59 +0800
Subject: [PATCH 2/2] some ui change

---
 .../org/apache/spark/ui/static/executorspage-template.html    | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/core/src/main/resources/org/apache/spark/ui/static/executorspage-template.html b/core/src/main/resources/org/apache/spark/ui/static/executorspage-template.html
index 0faed286068d..e9a8e4454518 100644
--- a/core/src/main/resources/org/apache/spark/ui/static/executorspage-template.html
+++ b/core/src/main/resources/org/apache/spark/ui/static/executorspage-template.html
@@ -44,7 +44,7 @@ <h4 style="clear: left; display: inline-block;">Summary</h4>
                   title="Shaded red when garbage collection (GC) time is over 10% of task time">
           Task Time (GC Time)</span></th>
         <th><span data-toggle="tooltip"
-                  title="Bytes read from Hadoop or from Spark storage / Bytes written to Hadoop">I/O</span></th>
+                  title="Bytes read from Hadoop or from Spark storage / Bytes written to Hadoop">Iuput/Output</span></th>
         <th><span data-toggle="tooltip"
                   title="Total shuffle bytes and records read (includes both data read locally and data read from remote executors).">
           Shuffle Read</span></th>
@@ -102,7 +102,7 @@ <h4 style="clear: left; display: inline-block;">Executors</h4>
             </scan>
           </th>
           <th><span data-toggle="tooltip" data-placement="top"
-                    title="Bytes read from Hadoop or from Spark storage / Bytes written to Hadoop">I/O</span></th>
+                    title="Bytes read from Hadoop or from Spark storage / Bytes written to Hadoop">Iuput/Output</span></th>
           <th>
             <span data-toggle="tooltip" data-placement="top"
                   title="Total shuffle bytes and records read (includes both data read locally and data read from remote executors).">