1515 * limitations under the License.
1616 */
1717
18- package org .apache .spark .ui .viz
18+ package org .apache .spark .ui .scope
1919
2020import scala .collection .mutable
2121import scala .collection .mutable .ListBuffer
2222
2323import org .apache .spark .Logging
24- import org .apache .spark .rdd .OperatorScope
2524import org .apache .spark .scheduler .StageInfo
2625
2726/**
28- * A representation of a generic cluster graph used for storing visualization information.
27+ * A representation of a generic cluster graph used for storing information on RDD operations .
2928 *
3029 * Each graph is defined with a set of edges and a root cluster, which may contain children
3130 * nodes and children clusters. Additionally, a graph may also have edges that enter or exit
3231 * the graph from nodes that belong to adjacent graphs.
3332 */
34- private [ui] case class VizGraph (
35- edges : Seq [VizEdge ],
36- outgoingEdges : Seq [VizEdge ],
37- incomingEdges : Seq [VizEdge ],
38- rootCluster : VizCluster )
33+ private [ui] case class RDDOperationGraph (
34+ edges : Seq [RDDOperationEdge ],
35+ outgoingEdges : Seq [RDDOperationEdge ],
36+ incomingEdges : Seq [RDDOperationEdge ],
37+ rootCluster : RDDOperationCluster )
3938
40- /** A node in a VizGraph . This represents an RDD. */
41- private [ui] case class VizNode (id : Int , name : String )
39+ /** A node in an RDDOperationGraph . This represents an RDD. */
40+ private [ui] case class RDDOperationNode (id : Int , name : String )
4241
43- /** A directed edge connecting two nodes in a VizGraph. This represents an RDD dependency. */
44- private [ui] case class VizEdge (fromId : Int , toId : Int )
42+ /**
43+ * A directed edge connecting two nodes in an RDDOperationGraph.
44+ * This represents an RDD dependency.
45+ */
46+ private [ui] case class RDDOperationEdge (fromId : Int , toId : Int )
4547
4648/**
47- * A cluster that groups nodes together in a VizGraph .
49+ * A cluster that groups nodes together in an RDDOperationGraph .
4850 *
4951 * This represents any grouping of RDDs, including operator scopes (e.g. textFile, flatMap),
5052 * stages, jobs, or any higher level construct. A cluster may be nested inside of other clusters.
5153 */
52- private [ui] class VizCluster (val id : String , val name : String ) {
53- private val _childrenNodes = new ListBuffer [VizNode ]
54- private val _childrenClusters = new ListBuffer [VizCluster ]
55-
56- def childrenNodes : Seq [VizNode ] = _childrenNodes.iterator.toSeq
57- def childrenClusters : Seq [VizCluster ] = _childrenClusters.iterator.toSeq
58- def attachChildNode (childNode : VizNode ): Unit = { _childrenNodes += childNode }
59- def attachChildCluster (childCluster : VizCluster ): Unit = { _childrenClusters += childCluster }
54+ private [ui] class RDDOperationCluster (val id : String , val name : String ) {
55+ private val _childrenNodes = new ListBuffer [RDDOperationNode ]
56+ private val _childrenClusters = new ListBuffer [RDDOperationCluster ]
57+
58+ def childrenNodes : Seq [RDDOperationNode ] = _childrenNodes.iterator.toSeq
59+ def childrenClusters : Seq [RDDOperationCluster ] = _childrenClusters.iterator.toSeq
60+ def attachChildNode (childNode : RDDOperationNode ): Unit = { _childrenNodes += childNode }
61+ def attachChildCluster (childCluster : RDDOperationCluster ): Unit = {
62+ _childrenClusters += childCluster
63+ }
6064}
6165
62- private [ui] object VizGraph extends Logging {
66+ private [ui] object RDDOperationGraph extends Logging {
6367
6468 /**
65- * Construct a VizGraph for a given stage.
69+ * Construct a RDDOperationGraph for a given stage.
6670 *
6771 * The root cluster represents the stage, and all children clusters represent RDD operations.
6872 * Each node represents an RDD, and each edge represents a dependency between two RDDs pointing
6973 * from the parent to the child.
7074 *
71- * This does not currently merge common scopes across stages. This may be worth supporting in
72- * the future when we decide to group certain stages within the same job under a common scope
73- * (e.g. part of a SQL query).
75+ * This does not currently merge common operator scopes across stages. This may be worth
76+ * supporting in the future if we decide to group certain stages within the same job under
77+ * a common scope (e.g. part of a SQL query).
7478 */
75- def makeVizGraph (stage : StageInfo ): VizGraph = {
76- val edges = new ListBuffer [VizEdge ]
77- val nodes = new mutable.HashMap [Int , VizNode ]
78- val clusters = new mutable.HashMap [String , VizCluster ] // cluster ID -> VizCluster
79+ def makeOperationGraph (stage : StageInfo ): RDDOperationGraph = {
80+ val edges = new ListBuffer [RDDOperationEdge ]
81+ val nodes = new mutable.HashMap [Int , RDDOperationNode ]
82+ val clusters = new mutable.HashMap [String , RDDOperationCluster ] // indexed by cluster ID
7983
8084 // Root cluster is the stage cluster
8185 val stageClusterId = s " stage_ ${stage.stageId}"
8286 val stageClusterName = s " Stage ${stage.stageId}" +
8387 { if (stage.attemptId == 0 ) " " else s " (attempt ${stage.attemptId}) " }
84- val rootCluster = new VizCluster (stageClusterId, stageClusterName)
88+ val rootCluster = new RDDOperationCluster (stageClusterId, stageClusterName)
8589
8690 // Find nodes, edges, and operator scopes that belong to this stage
8791 stage.rddInfos.foreach { rdd =>
88- edges ++= rdd.parentIds.map { parentId => VizEdge (parentId, rdd.id) }
89- val node = nodes.getOrElseUpdate(rdd.id, VizNode (rdd.id, rdd.name))
92+ edges ++= rdd.parentIds.map { parentId => RDDOperationEdge (parentId, rdd.id) }
93+ val node = nodes.getOrElseUpdate(rdd.id, RDDOperationNode (rdd.id, rdd.name))
9094
9195 if (rdd.scope == null ) {
9296 // This RDD has no encompassing scope, so we put it directly in the root cluster
@@ -99,7 +103,7 @@ private[ui] object VizGraph extends Logging {
99103 val rddClusters = rddScopes.map { scope =>
100104 val clusterId = scope.name + " _" + scope.id
101105 val clusterName = scope.name
102- clusters.getOrElseUpdate(clusterId, new VizCluster (clusterId, clusterName))
106+ clusters.getOrElseUpdate(clusterId, new RDDOperationCluster (clusterId, clusterName))
103107 }
104108 // Build the cluster hierarchy for this RDD
105109 rddClusters.sliding(2 ).foreach { pc =>
@@ -117,10 +121,10 @@ private[ui] object VizGraph extends Logging {
117121
118122 // Classify each edge as internal, outgoing or incoming
119123 // This information is needed to reason about how stages relate to each other
120- val internalEdges = new ListBuffer [VizEdge ]
121- val outgoingEdges = new ListBuffer [VizEdge ]
122- val incomingEdges = new ListBuffer [VizEdge ]
123- edges.foreach { case e : VizEdge =>
124+ val internalEdges = new ListBuffer [RDDOperationEdge ]
125+ val outgoingEdges = new ListBuffer [RDDOperationEdge ]
126+ val incomingEdges = new ListBuffer [RDDOperationEdge ]
127+ edges.foreach { case e : RDDOperationEdge =>
124128 val fromThisGraph = nodes.contains(e.fromId)
125129 val toThisGraph = nodes.contains(e.toId)
126130 (fromThisGraph, toThisGraph) match {
@@ -132,7 +136,7 @@ private[ui] object VizGraph extends Logging {
132136 }
133137 }
134138
135- VizGraph (internalEdges, outgoingEdges, incomingEdges, rootCluster)
139+ RDDOperationGraph (internalEdges, outgoingEdges, incomingEdges, rootCluster)
136140 }
137141
138142 /**
@@ -145,7 +149,7 @@ private[ui] object VizGraph extends Logging {
145149 *
146150 * For the complete DOT specification, see http://www.graphviz.org/Documentation/dotguide.pdf.
147151 */
148- def makeDotFile (graph : VizGraph , forJob : Boolean ): String = {
152+ def makeDotFile (graph : RDDOperationGraph , forJob : Boolean ): String = {
149153 val dotFile = new StringBuilder
150154 dotFile.append(" digraph G {\n "
151155 dotFile.append(makeDotSubgraph(graph.rootCluster, forJob, indent = " "
@@ -159,21 +163,24 @@ private[ui] object VizGraph extends Logging {
159163 }
160164
161165 /**
162- * Return the dot representation of a node.
166+ * Return the dot representation of a node in an RDDOperationGraph .
163167 *
164168 * On the job page, is displayed as a small circle without labels.
165169 * On the stage page, it is displayed as a box with an embedded label.
166170 */
167- private def makeDotNode (node : VizNode , forJob : Boolean ): String = {
171+ private def makeDotNode (node : RDDOperationNode , forJob : Boolean ): String = {
168172 if (forJob) {
169173 s """ ${node.id} [label=" " shape="circle" padding="5" labelStyle="font-size: 0"] """
170174 } else {
171175 s """ ${node.id} [label=" ${node.name} ( ${node.id})"] """
172176 }
173177 }
174178
175- /** Return the dot representation of a subgraph. */
176- private def makeDotSubgraph (scope : VizCluster , forJob : Boolean , indent : String ): String = {
179+ /** Return the dot representation of a subgraph in an RDDOperationGraph. */
180+ private def makeDotSubgraph (
181+ scope : RDDOperationCluster ,
182+ forJob : Boolean ,
183+ indent : String ): String = {
177184 val subgraph = new StringBuilder
178185 subgraph.append(indent + s " subgraph cluster ${scope.id} { \n " )
179186 subgraph.append(indent + s """ label=" ${scope.name}";\n """ )
0 commit comments