TEZ-4318: Tez debugging tools webapp

pom.xml

@@ -220,6 +220,11 @@
         <artifactId>tez-ext-service-tests</artifactId>
         <version>${project.version}</version>
       </dependency>
+      <dependency>
+        <groupId>org.apache.tez</groupId>
+        <artifactId>tez-job-analyzer</artifactId>
+        <version>${project.version}</version>
+      </dependency>
       <dependency>
         <groupId>org.apache.pig</groupId>
         <artifactId>pig</artifactId>

tez-plugins/tez-history-parser/src/main/java/org/apache/tez/history/parser/ATSFileParser.java

@@ -39,6 +39,7 @@
 import java.io.File;
 import java.io.IOException;
 import java.io.InputStream;
+import java.util.ArrayList;
 import java.util.Enumeration;
 import java.util.Iterator;
 import java.util.List;
@@ -84,6 +85,55 @@ public DagInfo getDAGData(String dagId) throws TezException {
     }
   }
 
+  public List<JSONObject> dumpAllEvents() throws Exception {
+    List<JSONObject> dumpedEvents = new ArrayList<>();
+
+    ZipFile atsZipFileToIterate = new ZipFile(this.atsZipFile);
+    try {
+      Enumeration<? extends ZipEntry> zipEntries = atsZipFileToIterate.entries();
+      while (zipEntries.hasMoreElements()) {
+        ZipEntry zipEntry = zipEntries.nextElement();
+        InputStream inputStream = atsZipFileToIterate.getInputStream(zipEntry);
+        JSONObject jsonObject = readJson(inputStream);
+
+        //This json can contain dag, vertices, tasks, task_attempts
+        JSONObject dagJson = jsonObject.optJSONObject(Constants.DAG);
+        if (dagJson != null) {
+          dumpedEvents.add(dagJson);
+        }
+
+        JSONArray vertexJson = jsonObject.optJSONArray(Constants.VERTICES);
+        if (vertexJson != null) {
+          for (int i = 0; i < vertexJson.length(); i++) {
+            dumpedEvents.add(vertexJson.getJSONObject(i));
+          }
+        }
+
+        JSONArray taskJson = jsonObject.optJSONArray(Constants.TASKS);
+        if (taskJson != null) {
+          for (int i = 0; i < taskJson.length(); i++) {
+            dumpedEvents.add(taskJson.getJSONObject(i));
+          }
+        }
+
+        JSONArray attemptsJson = jsonObject.optJSONArray(Constants.TASK_ATTEMPTS);
+        if (attemptsJson != null) {
+          for (int i = 0; i < attemptsJson.length(); i++) {
+            dumpedEvents.add(attemptsJson.getJSONObject(i));
+          }
+        }
+
+        JSONObject tezAppJson = jsonObject.optJSONObject(Constants.APPLICATION);
+        if (tezAppJson != null) {
+          dumpedEvents.add(tezAppJson);
+        }
+      }
+    } finally {
+      atsZipFileToIterate.close();
+    }
+    return dumpedEvents;
+  }
+
   /**
    * Parse vertices json
    *

tez-plugins/tez-history-parser/src/main/java/org/apache/tez/history/parser/ProtoHistoryParser.java

@@ -30,6 +30,7 @@
 import org.apache.tez.dag.history.logging.proto.HistoryEventProtoJsonConversion;
 import org.apache.tez.dag.history.logging.proto.HistoryLoggerProtos.HistoryEventProto;
 import org.apache.tez.dag.history.logging.proto.ProtoMessageReader;
+import org.apache.tez.history.parser.SimpleHistoryParser.JSONObjectSource;
 import org.apache.tez.history.parser.datamodel.DagInfo;
 import org.codehaus.jettison.json.JSONException;
 import org.codehaus.jettison.json.JSONObject;
@@ -76,7 +77,8 @@ private void parseContents(String dagId)
     parse(dagId, source);
   }
 
-  private JSONObjectSource getJsonSource() throws IOException {
+  @Override
+  protected JSONObjectSource getJsonSource() throws IOException {
     final TezConfiguration conf = new TezConfiguration();
 
     Iterator<File> fileIt = protoFiles.iterator();

tez-plugins/tez-history-parser/src/main/java/org/apache/tez/history/parser/SimpleHistoryParser.java

@@ -20,6 +20,7 @@
 import java.io.File;
 import java.io.FileNotFoundException;
 import java.io.IOException;
+import java.util.ArrayList;
 import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
@@ -93,6 +94,21 @@ public DagInfo getDAGData(String dagId) throws TezException {
     }
   }
 
+  public List<JSONObject> dumpAllEvents() throws Exception {
+    List<JSONObject> dumpedEvents = new ArrayList<>();
+    try {
+      JSONObjectSource source = getJsonSource();
+      while (source.hasNext()) {
+        dumpedEvents.add(source.next());
+      }
+      source.close();
+    } catch (Exception e) {
+      throw new TezException(e);
+    }
+
+    return dumpedEvents;
+  }
+
   private void populateOtherInfo(JSONObject source, JSONObject destination) throws JSONException {
     if (source == null || destination == null) {
       return;
@@ -118,7 +134,7 @@ private void parseContents(File historyFile, String dagId)
     parse(dagId, source);
   }
 
-  private JSONObjectSource getJsonSource() throws FileNotFoundException {
+  protected JSONObjectSource getJsonSource() throws JSONException, FileNotFoundException, IOException {
     final Scanner scanner = new Scanner(historyFile, UTF8);
     scanner.useDelimiter(SimpleHistoryLoggingService.RECORD_SEPARATOR);
 

tez-tools/analyzers/job-analyzer/src/main/java/org/apache/tez/analyzer/plugins/CriticalPathAnalyzer.java

@@ -118,6 +118,7 @@ public CriticalPathAnalyzer() {
 
   public CriticalPathAnalyzer(Configuration conf) {
     super(conf);
+    this.configProperties.add(DRAW_SVG);
   }
 
   @Override 

tez-tools/analyzers/job-analyzer/src/main/java/org/apache/tez/analyzer/plugins/LocalityAnalyzer.java

@@ -57,6 +57,7 @@ public class LocalityAnalyzer extends TezAnalyzerBase implements Analyzer {
   public LocalityAnalyzer(Configuration config) {
     super(config);
     csvResult = new CSVResult(headers);
+    this.configProperties.add(DATA_LOCAL_RATIO);
   }
 
   @Override

tez-tools/analyzers/job-analyzer/src/main/java/org/apache/tez/analyzer/plugins/ShuffleTimeAnalyzer.java

@@ -76,6 +76,8 @@ public ShuffleTimeAnalyzer(Configuration config) {
     realWorkDoneRatio = config.getFloat
         (REAL_WORK_DONE_RATIO, REAL_WORK_DONE_RATIO_DEFAULT);
     minShuffleRecords = config.getLong(MIN_SHUFFLE_RECORDS, MIN_SHUFFLE_RECORDS_DEFAULT);
+    this.configProperties.add(REAL_WORK_DONE_RATIO);
+    this.configProperties.add(MIN_SHUFFLE_RECORDS);
   }
 
   @Override

tez-tools/analyzers/job-analyzer/src/main/java/org/apache/tez/analyzer/plugins/SkewAnalyzer.java

@@ -97,6 +97,9 @@ public SkewAnalyzer(Configuration config) {
         ATTEMPT_SHUFFLE_KEY_GROUP_MIN_RATIO_DEFAULT);
     maxShuffleBytesPerSource = config.getLong(SHUFFLE_BYTES_PER_ATTEMPT_PER_SOURCE,
         SHUFFLE_BYTES_PER_ATTEMPT_PER_SOURCE_DEFAULT);
+    this.configProperties.add(SHUFFLE_BYTES_PER_ATTEMPT_PER_SOURCE);
+    this.configProperties.add(ATTEMPT_SHUFFLE_KEY_GROUP_MIN_RATIO);
+    this.configProperties.add(ATTEMPT_SHUFFLE_KEY_GROUP_MAX_RATIO);
   }
 
   @Override

tez-tools/analyzers/job-analyzer/src/main/java/org/apache/tez/analyzer/plugins/SlowTaskIdentifier.java

@@ -54,6 +54,7 @@ public class SlowTaskIdentifier extends TezAnalyzerBase implements Analyzer {
   public SlowTaskIdentifier(Configuration config) {
     super(config);
     this.csvResult = new CSVResult(headers);
+    this.configProperties.add(NO_OF_TASKS);
   }
 
   @Override

tez-tools/analyzers/job-analyzer/src/main/java/org/apache/tez/analyzer/plugins/SlowestVertexAnalyzer.java

@@ -61,7 +61,7 @@ public SlowestVertexAnalyzer(Configuration config) {
     super(config);
     this.vertexRuntimeThreshold = Math.max(1, config.getLong(MAX_VERTEX_RUNTIME,
         MAX_VERTEX_RUNTIME_DEFAULT));
-
+    this.configProperties.add(MAX_VERTEX_RUNTIME);
   }
 
   private long getTaskRuntime(VertexInfo vertexInfo) {

tez-tools/analyzers/job-analyzer/src/main/java/org/apache/tez/analyzer/plugins/SpillAnalyzerImpl.java

@@ -65,6 +65,7 @@ public SpillAnalyzerImpl(Configuration config) {
     minOutputBytesPerTask = Math.max(0, config.getLong(OUTPUT_BYTES_THRESHOLD,
         OUTPUT_BYTES_THRESHOLD_DEFAULT));
     this.csvResult = new CSVResult(headers);
+    this.configProperties.add(OUTPUT_BYTES_THRESHOLD);
   }
 
   @Override

tez-tools/analyzers/job-analyzer/src/main/java/org/apache/tez/analyzer/plugins/TezAnalyzerBase.java

@@ -69,6 +69,8 @@ public abstract class TezAnalyzerBase extends Configured implements Tool, Analyz
   private String outputDir;
   private boolean saveResults = false;
 
+  protected List<String> configProperties = new ArrayList<>();
+
   public TezAnalyzerBase(Configuration config) {
     setConf(config);
   }
@@ -277,4 +279,8 @@ public void printResults() throws TezException {
       LOG.debug(separator);
     }
   }
+
+  public List<String> getConfigProperties(){
+    return configProperties;
+  }
 }

tez-tools/analyzers/job-analyzer/src/main/java/org/apache/tez/analyzer/plugins/VertexLevelCriticalPathAnalyzer.java

@@ -59,6 +59,7 @@ public VertexLevelCriticalPathAnalyzer(Configuration config) {
     super(config);
     this.csvResult = new CSVResult(headers);
     this.dotFileLocation = config.get(DOT_FILE_DIR, DOT_FILE_DIR_DEFAULT);
+    this.configProperties.add(DOT_FILE_DIR);
   }
 
   @Override public void analyze(DagInfo dagInfo) throws TezException {

tez-tools/pom.xml

@@ -29,6 +29,7 @@
   <modules>
     <module>analyzers</module>
     <module>tez-javadoc-tools</module>
+    <module>tez-tools-webapp</module>
   </modules>
 
   <build>

tez-tools/tez-tools-webapp/Dockerfile

@@ -0,0 +1,24 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+FROM openjdk:8-jdk-alpine
+
+#fixing NPE described in https://github.com/docker-library/openjdk/issues/73
+RUN apk add --no-cache ttf-dejavu=2.37-r1
+
+ARG JAR_FILE=target/tez-tools-webapp-*-SNAPSHOT.jar
+COPY ${JAR_FILE} app.jar
+ENTRYPOINT ["java","-jar","/app.jar"]

tez-tools/tez-tools-webapp/README.md

@@ -0,0 +1,36 @@
+<!--
+  Licensed under the Apache License, Version 2.0 (the "License");
+  you may not use this file except in compliance with the License.
+  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License. See accompanying LICENSE file.
+-->
+Tez debugging tools webapp
+=========
+
+This is a post-hoc analysis tool for Apache Tez
+which makes easier to run some tools with a web UI.
+
+You can run it directly via maven...
+
+```bash
+#cd tez-tools/tez-tools-webapp #assuming that you're already here
+mvn clean install -DskipTests spring-boot:run
+```
+
+...or build a docker image and run it (anytime later from your machine):
+
+```bash
+#cd tez-tools/tez-tools-webapp #assuming that you're already here
+mvn clean install -DskipTests
+docker build -t tez-tools-webapp .
+docker run -p 8080:8080 tez-tools-webapp
+```
+
+Then navigate to: <http://localhost:8080>

tez-tools/tez-tools-webapp/findbugs-exclude.xml

@@ -0,0 +1,15 @@
+<!--
+  Licensed under the Apache License, Version 2.0 (the "License");
+  you may not use this file except in compliance with the License.
+  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License. See accompanying LICENSE file.
+-->
+<FindBugsFilter>
+</FindBugsFilter>