-
Notifications
You must be signed in to change notification settings - Fork 29k
[SPARK-37383][SQL][WEBUI]Show the parsing time for each phase of a SQL on spark ui #35744
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Changes from 6 commits
4e7d02b
74e6d35
92781bc
09fc0d5
d741418
bdb389e
fb2095b
0d6b33a
d9dbce9
c7bb365
ea79c1e
f395490
943d410
c7af808
2d42b10
32f4efc
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -19,9 +19,9 @@ package org.apache.spark.sql.catalyst | |
|
|
||
| import scala.collection.JavaConverters._ | ||
|
|
||
| import org.apache.spark.internal.Logging | ||
| import org.apache.spark.util.BoundedPriorityQueue | ||
|
|
||
|
|
||
| /** | ||
| * A simple utility for tracking runtime and associated stats in query planning. | ||
| * | ||
|
|
@@ -90,7 +90,7 @@ object QueryPlanningTracker { | |
| } | ||
|
|
||
|
|
||
| class QueryPlanningTracker { | ||
| class QueryPlanningTracker extends Logging { | ||
|
||
|
|
||
| import QueryPlanningTracker._ | ||
|
|
||
|
|
@@ -120,6 +120,24 @@ class QueryPlanningTracker { | |
| ret | ||
| } | ||
|
|
||
| /** | ||
| * print out the timeSpent for each phase of a SQL | ||
| */ | ||
| def logTimeSpent(): Unit = { | ||
| var totalTimeSpent = 0L | ||
| val timeSpentSummary: StringBuffer = new StringBuffer() | ||
|
||
| Seq(QueryPlanningTracker.PARSING, QueryPlanningTracker.ANALYSIS, | ||
| QueryPlanningTracker.OPTIMIZATION, QueryPlanningTracker.PLANNING).foreach { phase => | ||
| val duration = phasesMap.getOrDefault(phase, new PhaseSummary(-1, -1)).durationMs | ||
| timeSpentSummary.append(s"phase: $phase, timeSpent: $duration ms\n") | ||
| totalTimeSpent += duration | ||
| } | ||
| logInfo( | ||
| s"""Query planning time spent:\n ${timeSpentSummary.toString} | ||
| |Total time spent: $totalTimeSpent ms. | ||
| """.stripMargin) | ||
| } | ||
|
|
||
| /** | ||
| * Record a specific invocation of a rule. | ||
| * | ||
|
|
||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
no need to remove this line
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
thanks, updated