From b7e56eef4d66af977bd05af58a81e14faf33c211 Mon Sep 17 00:00:00 2001 From: Dmitry Parfenchik Date: Mon, 5 Jun 2017 22:42:41 +0200 Subject: [PATCH 1/5] [SPARK-21254] [WebUI] History UI: Improving performance by detaching table DOM before processing Currently all the DOM manipulations are handled in a loop after Mustache template is parsed. This causes severe performance issues especially within loops iteration over thousands of (attempt/application) records and causing all kinds of unnecessary browser work: reflow, repaint, etc. This could be easily fixed by preparing a DOM node beforehand and doing all manipulations within the loops on detached node, reattaching it to the document only after the work is done. The most costly operation in this case was setting innerHTML for `duration` column within a loop, which is extremely unperformant: https://jsperf.com/jquery-append-vs-html-list-performance/24 While duration parsing could be done before mustache-template processing without any additional DOM alteratoins. --- .../spark/ui/static/historypage-template.html | 2 +- .../org/apache/spark/ui/static/historypage.js | 18 +++++++----------- 2 files changed, 8 insertions(+), 12 deletions(-) diff --git a/core/src/main/resources/org/apache/spark/ui/static/historypage-template.html b/core/src/main/resources/org/apache/spark/ui/static/historypage-template.html index bfe31aae555b..879d7048fae9 100644 --- a/core/src/main/resources/org/apache/spark/ui/static/historypage-template.html +++ b/core/src/main/resources/org/apache/spark/ui/static/historypage-template.html @@ -74,7 +74,7 @@ {{attemptId}} {{startTime}} {{endTime}} - {{duration}} + {{duration}} {{sparkUser}} {{lastUpdated}} Download diff --git a/core/src/main/resources/org/apache/spark/ui/static/historypage.js b/core/src/main/resources/org/apache/spark/ui/static/historypage.js index 5ec1ce15a212..0e1093dd6fd6 100644 --- a/core/src/main/resources/org/apache/spark/ui/static/historypage.js +++ b/core/src/main/resources/org/apache/spark/ui/static/historypage.js @@ -122,7 +122,8 @@ $(document).ready(function() { attempt["lastUpdated"] = formatDate(attempt["lastUpdated"]); attempt["log"] = uiRoot + "/api/v1/applications/" + id + "/" + (attempt.hasOwnProperty("attemptId") ? attempt["attemptId"] + "/" : "") + "logs"; - + attempt["durationMillisec"] = attempt["duration"]; + attempt["duration"] = formatDuration(attempt["duration"]); var app_clone = {"id" : id, "name" : name, "num" : num, "attempts" : [attempt]}; array.push(app_clone); } @@ -134,7 +135,7 @@ $(document).ready(function() { } $.get("static/historypage-template.html", function(template) { - historySummary.append(Mustache.render($(template).filter("#history-summary-template").html(),data)); + var apps = $(Mustache.render($(template).filter("#history-summary-template").html(),data)); var selector = "#history-summary-table"; var conf = { "columns": [ @@ -164,31 +165,26 @@ $(document).ready(function() { if (hasMultipleAttempts) { jQuery.extend(conf, rowGroupConf); - var rowGroupCells = document.getElementsByClassName("rowGroupColumn"); + var rowGroupCells = apps.find(".rowGroupColumn"); for (i = 0; i < rowGroupCells.length; i++) { rowGroupCells[i].style='background-color: #ffffff'; } } if (!hasMultipleAttempts) { - var attemptIDCells = document.getElementsByClassName("attemptIDSpan"); + var attemptIDCells = apps.find(".attemptIDSpan"); for (i = 0; i < attemptIDCells.length; i++) { attemptIDCells[i].style.display='none'; } } if (requestedIncomplete) { - var completedCells = document.getElementsByClassName("completedColumn"); + var completedCells = apps.find(".completedColumn"); for (i = 0; i < completedCells.length; i++) { completedCells[i].style.display='none'; } } - - var durationCells = document.getElementsByClassName("durationClass"); - for (i = 0; i < durationCells.length; i++) { - var timeInMilliseconds = parseInt(durationCells[i].title); - durationCells[i].innerHTML = formatDuration(timeInMilliseconds); - } + historySummary.append(apps); if ($(selector.concat(" tr")).length < 20) { $.extend(conf, {paging: false}); From aeeeeb520d156a7293a707aa6bc053a2f83b9ac2 Mon Sep 17 00:00:00 2001 From: Dmitry Parfenchik Date: Sun, 30 Jul 2017 16:52:24 +0200 Subject: [PATCH 2/5] [SPARK-21254] [WebUI] Performance optimization for pagination check Check whether to display pagination or not on large data sets (10-20k rows) was taking up to 50ms because it was iterating over all rows. This could be easily done by testing length of array before passing it to mustache. --- .../resources/org/apache/spark/ui/static/historypage.js | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/core/src/main/resources/org/apache/spark/ui/static/historypage.js b/core/src/main/resources/org/apache/spark/ui/static/historypage.js index 0e1093dd6fd6..003cefef9d64 100644 --- a/core/src/main/resources/org/apache/spark/ui/static/historypage.js +++ b/core/src/main/resources/org/apache/spark/ui/static/historypage.js @@ -128,6 +128,9 @@ $(document).ready(function() { array.push(app_clone); } } + if(array.length < 20) { + $.fn.dataTable.defaults.paging = false; + } var data = { "uiroot": uiRoot, @@ -185,11 +188,6 @@ $(document).ready(function() { } } historySummary.append(apps); - - if ($(selector.concat(" tr")).length < 20) { - $.extend(conf, {paging: false}); - } - $(selector).DataTable(conf); $('#history-summary [data-toggle="tooltip"]').tooltip(); }); From e25be9a66b018ba0cc53884f242469b515cb2bf4 Mon Sep 17 00:00:00 2001 From: Dmitry Parfenchik Date: Sun, 30 Jul 2017 17:23:37 +0200 Subject: [PATCH 3/5] [SPARK-21254] [WebUI] Performance improvement: removing unnecessary DOM processing Logic related to `hasMultipleAttempts` flag: - Hiding attmptId column (if `hasMultipleAttempts = false`) - Seting white background color for first 2 columns (if `hasMultipleAttempts = true`) was updating DOM after mustache template processing, which was causing 2 unnecessary iterations over full data set (first through jquery selector, than through for-loop). Refactoring it inside mustache template helps saving 80-90ms on large data sets (10k+ rows) --- .../spark/ui/static/historypage-template.html | 12 ++- .../org/apache/spark/ui/static/historypage.js | 77 ++++++++++--------- 2 files changed, 48 insertions(+), 41 deletions(-) diff --git a/core/src/main/resources/org/apache/spark/ui/static/historypage-template.html b/core/src/main/resources/org/apache/spark/ui/static/historypage-template.html index 879d7048fae9..d0f9c05c357b 100644 --- a/core/src/main/resources/org/apache/spark/ui/static/historypage-template.html +++ b/core/src/main/resources/org/apache/spark/ui/static/historypage-template.html @@ -29,11 +29,13 @@ App Name - + {{#hasMultipleAttempts}} + Attempt ID + {{/hasMultipleAttempts}} Started @@ -68,10 +70,12 @@ {{#applications}} - {{id}} - {{name}} + {{id}} + {{name}} {{#attempts}} - {{attemptId}} + {{#hasMultipleAttempts}} + {{attemptId}} + {{/hasMultipleAttempts}} {{startTime}} {{endTime}} {{duration}} diff --git a/core/src/main/resources/org/apache/spark/ui/static/historypage.js b/core/src/main/resources/org/apache/spark/ui/static/historypage.js index 003cefef9d64..59b85a95051a 100644 --- a/core/src/main/resources/org/apache/spark/ui/static/historypage.js +++ b/core/src/main/resources/org/apache/spark/ui/static/historypage.js @@ -48,6 +48,18 @@ function getParameterByName(name, searchString) { return results === null ? "" : decodeURIComponent(results[1].replace(/\+/g, " ")); } +function removeColumnByName(columns, columnName) { + return columns.filter(function(col) {return col.name != columnName}) +} + +function getColumnIndex(columns, columnName) { + for(var i = 0; i < columns.length; i++) { + if (columns[i].name == columnName) + return i; + } + return -1; +} + jQuery.extend( jQuery.fn.dataTableExt.oSort, { "title-numeric-pre": function ( a ) { var x = a.match(/title="*(-?[0-9\.]+)/)[1]; @@ -134,51 +146,38 @@ $(document).ready(function() { var data = { "uiroot": uiRoot, - "applications": array - } + "applications": array, + "hasMultipleAttempts": hasMultipleAttempts, + } $.get("static/historypage-template.html", function(template) { var apps = $(Mustache.render($(template).filter("#history-summary-template").html(),data)); var selector = "#history-summary-table"; + var attemptIdColumnName = 'attemptId'; + var defaultSortColumn = completedColumnName = 'completed'; + var durationColumnName = 'duration'; var conf = { - "columns": [ - {name: 'first', type: "appid-numeric"}, - {name: 'second'}, - {name: 'third'}, - {name: 'fourth'}, - {name: 'fifth'}, - {name: 'sixth', type: "title-numeric"}, - {name: 'seventh'}, - {name: 'eighth'}, - {name: 'ninth'}, - ], - "columnDefs": [ - {"searchable": false, "targets": [5]} - ], - "autoWidth": false, - "order": [[ 4, "desc" ]] - }; - - var rowGroupConf = { - "rowsGroup": [ - 'first:name', - 'second:name' - ], + "columns": [ + {name: 'appId', type: "appid-numeric"}, + {name: 'appName'}, + {name: attemptIdColumnName}, + {name: 'started'}, + {name: completedColumnName}, + {name: durationColumnName, type: "title-numeric"}, + {name: 'user'}, + {name: 'lastUpdated'}, + {name: 'eventLog'}, + ], + "autoWidth": false, }; if (hasMultipleAttempts) { - jQuery.extend(conf, rowGroupConf); - var rowGroupCells = apps.find(".rowGroupColumn"); - for (i = 0; i < rowGroupCells.length; i++) { - rowGroupCells[i].style='background-color: #ffffff'; - } - } - - if (!hasMultipleAttempts) { - var attemptIDCells = apps.find(".attemptIDSpan"); - for (i = 0; i < attemptIDCells.length; i++) { - attemptIDCells[i].style.display='none'; - } + conf.rowsGroup = [ + 'appId:name', + 'appName:name' + ]; + } else { + conf.columns = removeColumnByName(conf.columns, attemptIdColumnName); } if (requestedIncomplete) { @@ -187,6 +186,10 @@ $(document).ready(function() { completedCells[i].style.display='none'; } } + conf.order = [[ getColumnIndex(conf.columns, defaultSortColumn), "desc" ]]; + conf.columnDefs = [ + {"searchable": false, "targets": [getColumnIndex(conf.columns, durationColumnName)]} + ]; historySummary.append(apps); $(selector).DataTable(conf); $('#history-summary [data-toggle="tooltip"]').tooltip(); From 91697079a29138b7581e64f2aa79247fa1a4e4af Mon Sep 17 00:00:00 2001 From: Dmitry Parfenchik Date: Sun, 30 Jul 2017 22:06:32 +0200 Subject: [PATCH 4/5] [SPARK-21254] [WebUI] Performance improvement: further reducing DOM manipulations Refactoring incomplete requests filter behavior due to inefficency in DOM manipulations. We were traversing DOM 2 more times just to hide columns that we could have avoided rendering in mustache. Factoring this logic in mustache template (`showCompletedColumn`) saves 70-80ms on 10k+ rows. --- .../apache/spark/ui/static/historypage-template.html | 8 ++++++-- .../org/apache/spark/ui/static/historypage.js | 11 ++++++----- 2 files changed, 12 insertions(+), 7 deletions(-) diff --git a/core/src/main/resources/org/apache/spark/ui/static/historypage-template.html b/core/src/main/resources/org/apache/spark/ui/static/historypage-template.html index d0f9c05c357b..20cd7bfdb223 100644 --- a/core/src/main/resources/org/apache/spark/ui/static/historypage-template.html +++ b/core/src/main/resources/org/apache/spark/ui/static/historypage-template.html @@ -41,11 +41,13 @@ Started - + {{#showCompletedColumn}} + Completed + {{/showCompletedColumn}} Duration @@ -77,7 +79,9 @@ {{attemptId}} {{/hasMultipleAttempts}} {{startTime}} - {{endTime}} + {{#showCompletedColumn}} + {{endTime}} + {{/showCompletedColumn}} {{duration}} {{sparkUser}} {{lastUpdated}} diff --git a/core/src/main/resources/org/apache/spark/ui/static/historypage.js b/core/src/main/resources/org/apache/spark/ui/static/historypage.js index 59b85a95051a..1b0bcc0575af 100644 --- a/core/src/main/resources/org/apache/spark/ui/static/historypage.js +++ b/core/src/main/resources/org/apache/spark/ui/static/historypage.js @@ -148,12 +148,14 @@ $(document).ready(function() { "uiroot": uiRoot, "applications": array, "hasMultipleAttempts": hasMultipleAttempts, + "showCompletedColumn": !requestedIncomplete, } $.get("static/historypage-template.html", function(template) { var apps = $(Mustache.render($(template).filter("#history-summary-template").html(),data)); var selector = "#history-summary-table"; var attemptIdColumnName = 'attemptId'; + var startedColumnName = 'started'; var defaultSortColumn = completedColumnName = 'completed'; var durationColumnName = 'duration'; var conf = { @@ -161,7 +163,7 @@ $(document).ready(function() { {name: 'appId', type: "appid-numeric"}, {name: 'appName'}, {name: attemptIdColumnName}, - {name: 'started'}, + {name: startedColumnName}, {name: completedColumnName}, {name: durationColumnName, type: "title-numeric"}, {name: 'user'}, @@ -180,11 +182,10 @@ $(document).ready(function() { conf.columns = removeColumnByName(conf.columns, attemptIdColumnName); } + var defaultSortColumn = completedColumnName; if (requestedIncomplete) { - var completedCells = apps.find(".completedColumn"); - for (i = 0; i < completedCells.length; i++) { - completedCells[i].style.display='none'; - } + defaultSortColumn = startedColumnName; + conf.columns = removeColumnByName(conf.columns, completedColumnName); } conf.order = [[ getColumnIndex(conf.columns, defaultSortColumn), "desc" ]]; conf.columnDefs = [ From 3630ca212baa94d60c5fe7e4109cf6da26288cec Mon Sep 17 00:00:00 2001 From: Dmitry Parfenchik Date: Sun, 30 Jul 2017 22:26:59 +0200 Subject: [PATCH 5/5] [SPARK-21254] [WebUI] Performance improvements: detaching DOM before DataTables plugin processing Detaching history table wrapper from document before parsing it with DataTables plugin and reattaching back right after plugin has processed nested DOM. This allows to avoid huge amount of browser repaints and reflows, reducing initial page load time in Chrome from 15s to 4s for 20k+ rows --- .../resources/org/apache/spark/ui/static/historypage.js | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/core/src/main/resources/org/apache/spark/ui/static/historypage.js b/core/src/main/resources/org/apache/spark/ui/static/historypage.js index 1b0bcc0575af..3e2bba8a8941 100644 --- a/core/src/main/resources/org/apache/spark/ui/static/historypage.js +++ b/core/src/main/resources/org/apache/spark/ui/static/historypage.js @@ -152,8 +152,9 @@ $(document).ready(function() { } $.get("static/historypage-template.html", function(template) { + var sibling = historySummary.prev(); + historySummary.detach(); var apps = $(Mustache.render($(template).filter("#history-summary-template").html(),data)); - var selector = "#history-summary-table"; var attemptIdColumnName = 'attemptId'; var startedColumnName = 'started'; var defaultSortColumn = completedColumnName = 'completed'; @@ -192,7 +193,8 @@ $(document).ready(function() { {"searchable": false, "targets": [getColumnIndex(conf.columns, durationColumnName)]} ]; historySummary.append(apps); - $(selector).DataTable(conf); + apps.DataTable(conf); + sibling.after(historySummary); $('#history-summary [data-toggle="tooltip"]').tooltip(); }); });