From 312986f1fbe997d6b96727eafb397c7f9151f615 Mon Sep 17 00:00:00 2001 From: Kent Yao Date: Tue, 7 Mar 2023 18:14:46 +0800 Subject: [PATCH 1/2] [SPARK-42697][WEBUI] Fix /api/v1/applications to return total uptime instead of 0 for duration --- core/src/main/scala/org/apache/spark/ui/SparkUI.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/src/main/scala/org/apache/spark/ui/SparkUI.scala b/core/src/main/scala/org/apache/spark/ui/SparkUI.scala index db1f8bc1a2ff..ac154b793856 100644 --- a/core/src/main/scala/org/apache/spark/ui/SparkUI.scala +++ b/core/src/main/scala/org/apache/spark/ui/SparkUI.scala @@ -167,7 +167,7 @@ private[spark] class SparkUI private ( attemptId = None, startTime = new Date(startTime), endTime = new Date(-1), - duration = 0, + duration = System.currentTimeMillis() - startTime, lastUpdated = new Date(startTime), sparkUser = getSparkUser, completed = false, From c85fd4d88474647fc245fc1d07881ece7a000e2f Mon Sep 17 00:00:00 2001 From: Kent Yao Date: Wed, 8 Mar 2023 10:58:25 +0800 Subject: [PATCH 2/2] fix tests --- .../test/scala/org/apache/spark/ui/UISeleniumSuite.scala | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/core/src/test/scala/org/apache/spark/ui/UISeleniumSuite.scala b/core/src/test/scala/org/apache/spark/ui/UISeleniumSuite.scala index 45348b2e9a7b..79496bba6674 100644 --- a/core/src/test/scala/org/apache/spark/ui/UISeleniumSuite.scala +++ b/core/src/test/scala/org/apache/spark/ui/UISeleniumSuite.scala @@ -700,7 +700,14 @@ class UISeleniumSuite extends SparkFunSuite with WebBrowser with Matchers { parseDate(attempts(0) \ "startTime") should be (sc.startTime) parseDate(attempts(0) \ "endTime") should be (-1) val oneAppJsonAst = getJson(sc.ui.get, "") - oneAppJsonAst should be (appListJsonAst.children(0)) + val duration = attempts(0) \ "duration" + oneAppJsonAst \\ "duration" should not be duration + // SPARK-42697: duration will increase as the app is running + // Replace the duration before we compare the full JObjects + val durationAdjusted = oneAppJsonAst.transformField { + case ("duration", _) => ("duration", duration) + } + durationAdjusted should be (appListJsonAst.children(0)) } }