diff --git a/pom.xml b/pom.xml index ba65cc6ad6..7eff8bc50f 100644 --- a/pom.xml +++ b/pom.xml @@ -220,6 +220,11 @@ tez-ext-service-tests ${project.version} + + org.apache.tez + tez-job-analyzer + ${project.version} + org.apache.pig pig diff --git a/tez-plugins/tez-history-parser/src/main/java/org/apache/tez/history/parser/ATSFileParser.java b/tez-plugins/tez-history-parser/src/main/java/org/apache/tez/history/parser/ATSFileParser.java index e64fb43f94..fb59dbbbe2 100644 --- a/tez-plugins/tez-history-parser/src/main/java/org/apache/tez/history/parser/ATSFileParser.java +++ b/tez-plugins/tez-history-parser/src/main/java/org/apache/tez/history/parser/ATSFileParser.java @@ -39,6 +39,7 @@ import java.io.File; import java.io.IOException; import java.io.InputStream; +import java.util.ArrayList; import java.util.Enumeration; import java.util.Iterator; import java.util.List; @@ -84,6 +85,55 @@ public DagInfo getDAGData(String dagId) throws TezException { } } + public List dumpAllEvents() throws Exception { + List dumpedEvents = new ArrayList<>(); + + ZipFile atsZipFileToIterate = new ZipFile(this.atsZipFile); + try { + Enumeration zipEntries = atsZipFileToIterate.entries(); + while (zipEntries.hasMoreElements()) { + ZipEntry zipEntry = zipEntries.nextElement(); + InputStream inputStream = atsZipFileToIterate.getInputStream(zipEntry); + JSONObject jsonObject = readJson(inputStream); + + //This json can contain dag, vertices, tasks, task_attempts + JSONObject dagJson = jsonObject.optJSONObject(Constants.DAG); + if (dagJson != null) { + dumpedEvents.add(dagJson); + } + + JSONArray vertexJson = jsonObject.optJSONArray(Constants.VERTICES); + if (vertexJson != null) { + for (int i = 0; i < vertexJson.length(); i++) { + dumpedEvents.add(vertexJson.getJSONObject(i)); + } + } + + JSONArray taskJson = jsonObject.optJSONArray(Constants.TASKS); + if (taskJson != null) { + for (int i = 0; i < taskJson.length(); i++) { + dumpedEvents.add(taskJson.getJSONObject(i)); + } + } + + JSONArray attemptsJson = jsonObject.optJSONArray(Constants.TASK_ATTEMPTS); + if (attemptsJson != null) { + for (int i = 0; i < attemptsJson.length(); i++) { + dumpedEvents.add(attemptsJson.getJSONObject(i)); + } + } + + JSONObject tezAppJson = jsonObject.optJSONObject(Constants.APPLICATION); + if (tezAppJson != null) { + dumpedEvents.add(tezAppJson); + } + } + } finally { + atsZipFileToIterate.close(); + } + return dumpedEvents; + } + /** * Parse vertices json * diff --git a/tez-plugins/tez-history-parser/src/main/java/org/apache/tez/history/parser/ProtoHistoryParser.java b/tez-plugins/tez-history-parser/src/main/java/org/apache/tez/history/parser/ProtoHistoryParser.java index 397a46fde9..c121c2a0ab 100644 --- a/tez-plugins/tez-history-parser/src/main/java/org/apache/tez/history/parser/ProtoHistoryParser.java +++ b/tez-plugins/tez-history-parser/src/main/java/org/apache/tez/history/parser/ProtoHistoryParser.java @@ -30,6 +30,7 @@ import org.apache.tez.dag.history.logging.proto.HistoryEventProtoJsonConversion; import org.apache.tez.dag.history.logging.proto.HistoryLoggerProtos.HistoryEventProto; import org.apache.tez.dag.history.logging.proto.ProtoMessageReader; +import org.apache.tez.history.parser.SimpleHistoryParser.JSONObjectSource; import org.apache.tez.history.parser.datamodel.DagInfo; import org.codehaus.jettison.json.JSONException; import org.codehaus.jettison.json.JSONObject; @@ -76,7 +77,8 @@ private void parseContents(String dagId) parse(dagId, source); } - private JSONObjectSource getJsonSource() throws IOException { + @Override + protected JSONObjectSource getJsonSource() throws IOException { final TezConfiguration conf = new TezConfiguration(); Iterator fileIt = protoFiles.iterator(); diff --git a/tez-plugins/tez-history-parser/src/main/java/org/apache/tez/history/parser/SimpleHistoryParser.java b/tez-plugins/tez-history-parser/src/main/java/org/apache/tez/history/parser/SimpleHistoryParser.java index c1711ce2cb..255ec90fc2 100644 --- a/tez-plugins/tez-history-parser/src/main/java/org/apache/tez/history/parser/SimpleHistoryParser.java +++ b/tez-plugins/tez-history-parser/src/main/java/org/apache/tez/history/parser/SimpleHistoryParser.java @@ -20,6 +20,7 @@ import java.io.File; import java.io.FileNotFoundException; import java.io.IOException; +import java.util.ArrayList; import java.util.Iterator; import java.util.List; import java.util.Map; @@ -93,6 +94,21 @@ public DagInfo getDAGData(String dagId) throws TezException { } } + public List dumpAllEvents() throws Exception { + List dumpedEvents = new ArrayList<>(); + try { + JSONObjectSource source = getJsonSource(); + while (source.hasNext()) { + dumpedEvents.add(source.next()); + } + source.close(); + } catch (Exception e) { + throw new TezException(e); + } + + return dumpedEvents; + } + private void populateOtherInfo(JSONObject source, JSONObject destination) throws JSONException { if (source == null || destination == null) { return; @@ -118,7 +134,7 @@ private void parseContents(File historyFile, String dagId) parse(dagId, source); } - private JSONObjectSource getJsonSource() throws FileNotFoundException { + protected JSONObjectSource getJsonSource() throws JSONException, FileNotFoundException, IOException { final Scanner scanner = new Scanner(historyFile, UTF8); scanner.useDelimiter(SimpleHistoryLoggingService.RECORD_SEPARATOR); diff --git a/tez-tools/analyzers/job-analyzer/src/main/java/org/apache/tez/analyzer/plugins/CriticalPathAnalyzer.java b/tez-tools/analyzers/job-analyzer/src/main/java/org/apache/tez/analyzer/plugins/CriticalPathAnalyzer.java index 3f5e3004b8..b86abae592 100644 --- a/tez-tools/analyzers/job-analyzer/src/main/java/org/apache/tez/analyzer/plugins/CriticalPathAnalyzer.java +++ b/tez-tools/analyzers/job-analyzer/src/main/java/org/apache/tez/analyzer/plugins/CriticalPathAnalyzer.java @@ -118,6 +118,7 @@ public CriticalPathAnalyzer() { public CriticalPathAnalyzer(Configuration conf) { super(conf); + this.configProperties.add(DRAW_SVG); } @Override diff --git a/tez-tools/analyzers/job-analyzer/src/main/java/org/apache/tez/analyzer/plugins/LocalityAnalyzer.java b/tez-tools/analyzers/job-analyzer/src/main/java/org/apache/tez/analyzer/plugins/LocalityAnalyzer.java index d640704f93..7b7e0da821 100644 --- a/tez-tools/analyzers/job-analyzer/src/main/java/org/apache/tez/analyzer/plugins/LocalityAnalyzer.java +++ b/tez-tools/analyzers/job-analyzer/src/main/java/org/apache/tez/analyzer/plugins/LocalityAnalyzer.java @@ -57,6 +57,7 @@ public class LocalityAnalyzer extends TezAnalyzerBase implements Analyzer { public LocalityAnalyzer(Configuration config) { super(config); csvResult = new CSVResult(headers); + this.configProperties.add(DATA_LOCAL_RATIO); } @Override diff --git a/tez-tools/analyzers/job-analyzer/src/main/java/org/apache/tez/analyzer/plugins/ShuffleTimeAnalyzer.java b/tez-tools/analyzers/job-analyzer/src/main/java/org/apache/tez/analyzer/plugins/ShuffleTimeAnalyzer.java index f8f9112bb7..ff39302f65 100644 --- a/tez-tools/analyzers/job-analyzer/src/main/java/org/apache/tez/analyzer/plugins/ShuffleTimeAnalyzer.java +++ b/tez-tools/analyzers/job-analyzer/src/main/java/org/apache/tez/analyzer/plugins/ShuffleTimeAnalyzer.java @@ -76,6 +76,8 @@ public ShuffleTimeAnalyzer(Configuration config) { realWorkDoneRatio = config.getFloat (REAL_WORK_DONE_RATIO, REAL_WORK_DONE_RATIO_DEFAULT); minShuffleRecords = config.getLong(MIN_SHUFFLE_RECORDS, MIN_SHUFFLE_RECORDS_DEFAULT); + this.configProperties.add(REAL_WORK_DONE_RATIO); + this.configProperties.add(MIN_SHUFFLE_RECORDS); } @Override diff --git a/tez-tools/analyzers/job-analyzer/src/main/java/org/apache/tez/analyzer/plugins/SkewAnalyzer.java b/tez-tools/analyzers/job-analyzer/src/main/java/org/apache/tez/analyzer/plugins/SkewAnalyzer.java index a7d14fae25..d93729e0ad 100644 --- a/tez-tools/analyzers/job-analyzer/src/main/java/org/apache/tez/analyzer/plugins/SkewAnalyzer.java +++ b/tez-tools/analyzers/job-analyzer/src/main/java/org/apache/tez/analyzer/plugins/SkewAnalyzer.java @@ -97,6 +97,9 @@ public SkewAnalyzer(Configuration config) { ATTEMPT_SHUFFLE_KEY_GROUP_MIN_RATIO_DEFAULT); maxShuffleBytesPerSource = config.getLong(SHUFFLE_BYTES_PER_ATTEMPT_PER_SOURCE, SHUFFLE_BYTES_PER_ATTEMPT_PER_SOURCE_DEFAULT); + this.configProperties.add(SHUFFLE_BYTES_PER_ATTEMPT_PER_SOURCE); + this.configProperties.add(ATTEMPT_SHUFFLE_KEY_GROUP_MIN_RATIO); + this.configProperties.add(ATTEMPT_SHUFFLE_KEY_GROUP_MAX_RATIO); } @Override diff --git a/tez-tools/analyzers/job-analyzer/src/main/java/org/apache/tez/analyzer/plugins/SlowTaskIdentifier.java b/tez-tools/analyzers/job-analyzer/src/main/java/org/apache/tez/analyzer/plugins/SlowTaskIdentifier.java index 7c9958b250..f100cd5d3b 100644 --- a/tez-tools/analyzers/job-analyzer/src/main/java/org/apache/tez/analyzer/plugins/SlowTaskIdentifier.java +++ b/tez-tools/analyzers/job-analyzer/src/main/java/org/apache/tez/analyzer/plugins/SlowTaskIdentifier.java @@ -54,6 +54,7 @@ public class SlowTaskIdentifier extends TezAnalyzerBase implements Analyzer { public SlowTaskIdentifier(Configuration config) { super(config); this.csvResult = new CSVResult(headers); + this.configProperties.add(NO_OF_TASKS); } @Override diff --git a/tez-tools/analyzers/job-analyzer/src/main/java/org/apache/tez/analyzer/plugins/SlowestVertexAnalyzer.java b/tez-tools/analyzers/job-analyzer/src/main/java/org/apache/tez/analyzer/plugins/SlowestVertexAnalyzer.java index efa39a3223..032ee2b7b1 100644 --- a/tez-tools/analyzers/job-analyzer/src/main/java/org/apache/tez/analyzer/plugins/SlowestVertexAnalyzer.java +++ b/tez-tools/analyzers/job-analyzer/src/main/java/org/apache/tez/analyzer/plugins/SlowestVertexAnalyzer.java @@ -61,7 +61,7 @@ public SlowestVertexAnalyzer(Configuration config) { super(config); this.vertexRuntimeThreshold = Math.max(1, config.getLong(MAX_VERTEX_RUNTIME, MAX_VERTEX_RUNTIME_DEFAULT)); - + this.configProperties.add(MAX_VERTEX_RUNTIME); } private long getTaskRuntime(VertexInfo vertexInfo) { diff --git a/tez-tools/analyzers/job-analyzer/src/main/java/org/apache/tez/analyzer/plugins/SpillAnalyzerImpl.java b/tez-tools/analyzers/job-analyzer/src/main/java/org/apache/tez/analyzer/plugins/SpillAnalyzerImpl.java index 026dd1593f..5e47194a21 100644 --- a/tez-tools/analyzers/job-analyzer/src/main/java/org/apache/tez/analyzer/plugins/SpillAnalyzerImpl.java +++ b/tez-tools/analyzers/job-analyzer/src/main/java/org/apache/tez/analyzer/plugins/SpillAnalyzerImpl.java @@ -65,6 +65,7 @@ public SpillAnalyzerImpl(Configuration config) { minOutputBytesPerTask = Math.max(0, config.getLong(OUTPUT_BYTES_THRESHOLD, OUTPUT_BYTES_THRESHOLD_DEFAULT)); this.csvResult = new CSVResult(headers); + this.configProperties.add(OUTPUT_BYTES_THRESHOLD); } @Override diff --git a/tez-tools/analyzers/job-analyzer/src/main/java/org/apache/tez/analyzer/plugins/TezAnalyzerBase.java b/tez-tools/analyzers/job-analyzer/src/main/java/org/apache/tez/analyzer/plugins/TezAnalyzerBase.java index 705c6e9cfb..b79cf2dc88 100644 --- a/tez-tools/analyzers/job-analyzer/src/main/java/org/apache/tez/analyzer/plugins/TezAnalyzerBase.java +++ b/tez-tools/analyzers/job-analyzer/src/main/java/org/apache/tez/analyzer/plugins/TezAnalyzerBase.java @@ -69,6 +69,8 @@ public abstract class TezAnalyzerBase extends Configured implements Tool, Analyz private String outputDir; private boolean saveResults = false; + protected List configProperties = new ArrayList<>(); + public TezAnalyzerBase(Configuration config) { setConf(config); } @@ -277,4 +279,8 @@ public void printResults() throws TezException { LOG.debug(separator); } } + + public List getConfigProperties(){ + return configProperties; + } } diff --git a/tez-tools/analyzers/job-analyzer/src/main/java/org/apache/tez/analyzer/plugins/VertexLevelCriticalPathAnalyzer.java b/tez-tools/analyzers/job-analyzer/src/main/java/org/apache/tez/analyzer/plugins/VertexLevelCriticalPathAnalyzer.java index 78a4d41f38..e16986bdbf 100644 --- a/tez-tools/analyzers/job-analyzer/src/main/java/org/apache/tez/analyzer/plugins/VertexLevelCriticalPathAnalyzer.java +++ b/tez-tools/analyzers/job-analyzer/src/main/java/org/apache/tez/analyzer/plugins/VertexLevelCriticalPathAnalyzer.java @@ -59,6 +59,7 @@ public VertexLevelCriticalPathAnalyzer(Configuration config) { super(config); this.csvResult = new CSVResult(headers); this.dotFileLocation = config.get(DOT_FILE_DIR, DOT_FILE_DIR_DEFAULT); + this.configProperties.add(DOT_FILE_DIR); } @Override public void analyze(DagInfo dagInfo) throws TezException { diff --git a/tez-tools/pom.xml b/tez-tools/pom.xml index bffe9abc5d..1213a8b802 100644 --- a/tez-tools/pom.xml +++ b/tez-tools/pom.xml @@ -29,6 +29,7 @@ analyzers tez-javadoc-tools + tez-tools-webapp diff --git a/tez-tools/tez-tools-webapp/Dockerfile b/tez-tools/tez-tools-webapp/Dockerfile new file mode 100644 index 0000000000..93cada7b8b --- /dev/null +++ b/tez-tools/tez-tools-webapp/Dockerfile @@ -0,0 +1,24 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +FROM openjdk:8-jdk-alpine + +#fixing NPE described in https://github.com/docker-library/openjdk/issues/73 +RUN apk add --no-cache ttf-dejavu=2.37-r1 + +ARG JAR_FILE=target/tez-tools-webapp-*-SNAPSHOT.jar +COPY ${JAR_FILE} app.jar +ENTRYPOINT ["java","-jar","/app.jar"] \ No newline at end of file diff --git a/tez-tools/tez-tools-webapp/README.md b/tez-tools/tez-tools-webapp/README.md new file mode 100644 index 0000000000..a81c7b0432 --- /dev/null +++ b/tez-tools/tez-tools-webapp/README.md @@ -0,0 +1,36 @@ + +Tez debugging tools webapp +========= + +This is a post-hoc analysis tool for Apache Tez +which makes easier to run some tools with a web UI. + +You can run it directly via maven... + +```bash +#cd tez-tools/tez-tools-webapp #assuming that you're already here +mvn clean install -DskipTests spring-boot:run +``` + +...or build a docker image and run it (anytime later from your machine): + +```bash +#cd tez-tools/tez-tools-webapp #assuming that you're already here +mvn clean install -DskipTests +docker build -t tez-tools-webapp . +docker run -p 8080:8080 tez-tools-webapp +``` + +Then navigate to: diff --git a/tez-tools/tez-tools-webapp/findbugs-exclude.xml b/tez-tools/tez-tools-webapp/findbugs-exclude.xml new file mode 100644 index 0000000000..6c46525807 --- /dev/null +++ b/tez-tools/tez-tools-webapp/findbugs-exclude.xml @@ -0,0 +1,15 @@ + + + diff --git a/tez-tools/tez-tools-webapp/pom.xml b/tez-tools/tez-tools-webapp/pom.xml new file mode 100644 index 0000000000..6d84edf627 --- /dev/null +++ b/tez-tools/tez-tools-webapp/pom.xml @@ -0,0 +1,167 @@ + + + + 4.0.0 + tez-tools-webapp + 0.10.2-SNAPSHOT + tez-tools-webapp + + + org.apache.tez + tez-tools + 0.10.2-SNAPSHOT + + + + 2.5.2 + 5.3.8 + 3.0.0 + 3.0.11.RELEASE + 2.4.1 + 3.1.0 + 0.9.12 + 2.12.3 + 2.8.7 + 5.0.0 + + org.apache.tez.tools.webapp.WebApp + + + + + org.apache.tez + tez-job-analyzer + + + + io.springfox + springfox-boot-starter + ${swagger.version} + + + io.springfox + springfox-swagger-ui + ${swagger.version} + + + + com.fasterxml.jackson.core + jackson-databind + ${jackson.databind.version} + + + com.google.code.gson + gson + ${gson.version} + + + + org.springframework.boot + spring-boot-starter-web + ${spring.boot.version} + + + + ch.qos.logback + logback-classic + + + + + org.springframework + spring-core + ${spring.version} + + + org.springframework + spring-context + ${spring.version} + + + + commons-logging + commons-logging + + + + + org.springframework + spring-webmvc + ${spring.version} + + + org.springframework.boot + spring-boot-starter-thymeleaf + ${spring.boot.version} + + + org.thymeleaf + thymeleaf + ${thymeleaf.version} + + + org.thymeleaf + thymeleaf-spring5 + ${thymeleaf.version} + + + nz.net.ultraq.thymeleaf + thymeleaf-layout-dialect + ${thymeleaf.layout.dialect.version} + + + org.apache.poi + poi + ${poi.version} + + + org.apache.poi + poi-ooxml + ${poi.version} + + + javax.servlet + javax.servlet-api + ${javax.servlet-api.version} + provided + + + org.reflections + reflections + ${reflections.version} + + + + + + org.springframework.boot + spring-boot-maven-plugin + ${spring.boot.version} + + + repackage + + repackage + + + + + ${start-class} + + + + + \ No newline at end of file diff --git a/tez-tools/tez-tools-webapp/src/main/java/org/apache/tez/tools/webapp/SwaggerConfig.java b/tez-tools/tez-tools-webapp/src/main/java/org/apache/tez/tools/webapp/SwaggerConfig.java new file mode 100644 index 0000000000..d2031c07dc --- /dev/null +++ b/tez-tools/tez-tools-webapp/src/main/java/org/apache/tez/tools/webapp/SwaggerConfig.java @@ -0,0 +1,39 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.tez.tools.webapp; + +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +import springfox.documentation.builders.PathSelectors; +import springfox.documentation.builders.RequestHandlerSelectors; +import springfox.documentation.spi.DocumentationType; +import springfox.documentation.spring.web.plugins.Docket; + +@Configuration +public class SwaggerConfig { + + @Bean + public Docket api() { + return new Docket(DocumentationType.SWAGGER_2) + .select() + .apis(RequestHandlerSelectors.any()) + .paths(PathSelectors.any()) + .build(); + } +} diff --git a/tez-tools/tez-tools-webapp/src/main/java/org/apache/tez/tools/webapp/WebApp.java b/tez-tools/tez-tools-webapp/src/main/java/org/apache/tez/tools/webapp/WebApp.java new file mode 100644 index 0000000000..ae6af189c1 --- /dev/null +++ b/tez-tools/tez-tools-webapp/src/main/java/org/apache/tez/tools/webapp/WebApp.java @@ -0,0 +1,56 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.tez.tools.webapp; + +import org.springframework.boot.SpringApplication; +import org.springframework.boot.autoconfigure.SpringBootApplication; +import org.springframework.context.annotation.Bean; +import org.springframework.web.servlet.ViewResolver; +import org.thymeleaf.spring5.ISpringTemplateEngine; +import org.thymeleaf.spring5.SpringTemplateEngine; +import org.thymeleaf.spring5.view.ThymeleafViewResolver; + +import nz.net.ultraq.thymeleaf.LayoutDialect; +import nz.net.ultraq.thymeleaf.decorators.strategies.GroupingStrategy; + +/** + * Java configuration file that is used for web application initialization. + */ +@SpringBootApplication +public class WebApp { + + public static void main(String[] args) throws Exception { + SpringApplication.run(WebApp.class, args); + } + + @Bean + public ViewResolver htmlViewResolver() { + ThymeleafViewResolver resolver = new ThymeleafViewResolver(); + resolver.setTemplateEngine(templateEngine()); + resolver.setContentType("text/html"); + resolver.setCharacterEncoding("UTF-8"); + resolver.setViewNames(new String[] { "*.html" }); + return resolver; + } + + private ISpringTemplateEngine templateEngine() { + SpringTemplateEngine engine = new SpringTemplateEngine(); + engine.addDialect(new LayoutDialect(new GroupingStrategy())); + return engine; + } +} diff --git a/tez-tools/tez-tools-webapp/src/main/java/org/apache/tez/tools/webapp/controller/AnalyzerController.java b/tez-tools/tez-tools-webapp/src/main/java/org/apache/tez/tools/webapp/controller/AnalyzerController.java new file mode 100644 index 0000000000..095d07787d --- /dev/null +++ b/tez-tools/tez-tools-webapp/src/main/java/org/apache/tez/tools/webapp/controller/AnalyzerController.java @@ -0,0 +1,128 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.tez.tools.webapp.controller; + +import java.io.File; +import java.nio.file.Files; +import java.util.ArrayList; +import java.util.List; + +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; + +import org.apache.hadoop.conf.Configuration; +import org.apache.tez.analyzer.plugins.TezAnalyzerBase; +import org.apache.tez.history.parser.datamodel.DagInfo; +import org.apache.tez.tools.webapp.service.AnalyzerService; +import org.apache.tez.tools.webapp.service.AnalyzerService.AnalyzerOutputFormat; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Controller; +import org.springframework.ui.Model; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestMethod; +import org.springframework.web.bind.annotation.ResponseBody; + +@Controller +@RequestMapping(value = "/analyzer") +public class AnalyzerController { + private final Logger LOG = LoggerFactory.getLogger(getClass()); + + private static final String HTML_CHECKBOX_PREFIX = "check_analyzer_"; + private static final String HTML_CONFIG_PREFIX = "config_analyzer_"; + private static final String PARAM_DAGID = "dagId"; + private static final String PARAM_DAGFILES = "dagFiles"; + private static final String PARAM_OUTPUT_FORMAT = "outputFormat"; + + @Autowired + private AnalyzerService analyzerService; + + /** + * Loads the analyzer.html webpage and fills thymeleaf template with analyzer info. + * @param model + * @return the template file name to serve + */ + @RequestMapping(method = RequestMethod.GET) + public String getPage(Model model) { + List analyzers = analyzerService.getAllAnalyzers(); + model.addAttribute("analyzers", analyzers); + return "analyzer.html"; + } + + /** + * Runs analyzers on a given dag. + * @param request + * @param response + * @return a zip file containing output files of analyzers + * @throws Exception + */ + @RequestMapping(method = RequestMethod.POST, produces = "application/zip") + public @ResponseBody byte[] analyze(HttpServletRequest request, HttpServletResponse response) throws Exception { + List files = ControllerUtils.getFiles(request, PARAM_DAGFILES); + Configuration configuration = getConfiguration(request); + List analyzers = getAnalyzers(request, configuration); + DagInfo dagInfo = getDagInfo(request, files); + AnalyzerOutputFormat aof = getOutputFormat(request); + + File zipFile = analyzerService.runAnalyzers(analyzers, dagInfo, aof); + + response.setHeader("Content-Disposition", "attachment; filename=\"" + zipFile.getName() + "\""); + LOG.info("Finished analyzing dag: {}", dagInfo.getDagId()); + return Files.readAllBytes(zipFile.toPath()); + } + + private AnalyzerOutputFormat getOutputFormat(HttpServletRequest request) { + String outputFormat = request.getParameter(PARAM_OUTPUT_FORMAT); + if (outputFormat == null || outputFormat.isEmpty()) { + outputFormat = "csv"; + } + return AnalyzerOutputFormat.valueOf(outputFormat.toUpperCase()); + } + + private Configuration getConfiguration(HttpServletRequest request) { + Configuration configuration = new Configuration(); + configuration.set("output-dir", System.getProperty("java.io.tmpdir")); + + for (String param : request.getParameterMap().keySet()) { + if (param.startsWith(HTML_CONFIG_PREFIX) && !request.getParameter(param).isEmpty()) { + LOG.info("Setting config parameter {} = {}", param.substring(HTML_CONFIG_PREFIX.length()), + request.getParameter(param)); + configuration.set(param, request.getParameter(param)); + } + } + return configuration; + } + + private List getAnalyzers(HttpServletRequest request, Configuration config) throws Exception { + List analyzers = new ArrayList<>(); + for (String param : request.getParameterMap().keySet()) { + if (param.startsWith(HTML_CHECKBOX_PREFIX)) { + String className = AnalyzerService.ANALYZER_PACKAGE + "." + param.substring(HTML_CHECKBOX_PREFIX.length()); + analyzers + .add((TezAnalyzerBase) Class.forName(className).getConstructor(Configuration.class).newInstance(config)); + } + } + return analyzers; + } + + private DagInfo getDagInfo(HttpServletRequest request, List files) throws Exception { + String dagId = request.getParameter(PARAM_DAGID); + return analyzerService.getDagInfo(dagId, files); + } +} diff --git a/tez-tools/tez-tools-webapp/src/main/java/org/apache/tez/tools/webapp/controller/ControllerUtils.java b/tez-tools/tez-tools-webapp/src/main/java/org/apache/tez/tools/webapp/controller/ControllerUtils.java new file mode 100644 index 0000000000..8ffd41b465 --- /dev/null +++ b/tez-tools/tez-tools-webapp/src/main/java/org/apache/tez/tools/webapp/controller/ControllerUtils.java @@ -0,0 +1,50 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.tez.tools.webapp.controller; + +import java.io.File; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +import javax.servlet.http.HttpServletRequest; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.web.multipart.MultipartFile; +import org.springframework.web.multipart.support.StandardMultipartHttpServletRequest; + +public class ControllerUtils { + private static final Logger LOG = LoggerFactory.getLogger(ControllerUtils.class); + + private ControllerUtils() { + } + + public static List getFiles(HttpServletRequest request, String parameterName) throws IOException { + List uploadedFiles = ((StandardMultipartHttpServletRequest) request).getFiles(parameterName); + List files = new ArrayList<>(); + for (MultipartFile uploadedFile : uploadedFiles) { + LOG.info("Input file: " + uploadedFile.getOriginalFilename()); + File tempFile = new File(System.getProperty("java.io.tmpdir"), + String.format("%d_%s", System.currentTimeMillis(), uploadedFile.getOriginalFilename())); + files.add(tempFile); + uploadedFile.transferTo(tempFile); + } + return files; + } +} diff --git a/tez-tools/tez-tools-webapp/src/main/java/org/apache/tez/tools/webapp/controller/EventDumpController.java b/tez-tools/tez-tools-webapp/src/main/java/org/apache/tez/tools/webapp/controller/EventDumpController.java new file mode 100644 index 0000000000..1aeac20202 --- /dev/null +++ b/tez-tools/tez-tools-webapp/src/main/java/org/apache/tez/tools/webapp/controller/EventDumpController.java @@ -0,0 +1,67 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.tez.tools.webapp.controller; + +import java.io.File; +import java.util.List; + +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; + +import org.apache.tez.tools.webapp.service.EventDumpService; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Controller; +import org.springframework.ui.Model; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestMethod; +import org.springframework.web.bind.annotation.ResponseBody; + +@Controller +@RequestMapping(value = "/eventdump") +public class EventDumpController { + private static final String PARAM_DAGID = "dagId"; + private static final String PARAM_DAGFILES = "dagFiles"; + + @Autowired + private EventDumpService eventDumpService; + + /** + * Loads the eventdump.html webpage. + * @param model + * @return the template file name to serve + */ + @RequestMapping(method = RequestMethod.GET) + public String getPage(Model model) { + return "eventdump.html"; + } + + /** + * Dumps all events from an event file and returns a human-readable array of jsons. + * @param request + * @param response + * @return an array of events as JSONObject + * @throws Exception + */ + @RequestMapping(method = RequestMethod.POST, produces = "application/json") + public @ResponseBody String dump(HttpServletRequest request, HttpServletResponse response) throws Exception { + List files = ControllerUtils.getFiles(request, PARAM_DAGFILES); + String dagId = request.getParameter(PARAM_DAGID); + + return eventDumpService.getArrayOfAllEvents(dagId, files).toString(); + } +} diff --git a/tez-tools/tez-tools-webapp/src/main/java/org/apache/tez/tools/webapp/controller/HomeController.java b/tez-tools/tez-tools-webapp/src/main/java/org/apache/tez/tools/webapp/controller/HomeController.java new file mode 100644 index 0000000000..94d71075a8 --- /dev/null +++ b/tez-tools/tez-tools-webapp/src/main/java/org/apache/tez/tools/webapp/controller/HomeController.java @@ -0,0 +1,32 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.tez.tools.webapp.controller; + +import org.springframework.stereotype.Controller; +import org.springframework.ui.Model; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestMethod; +import org.springframework.web.servlet.view.RedirectView; + +@Controller +public class HomeController { + @RequestMapping(value = "/", method = RequestMethod.GET) + public RedirectView getHome(Model model) { + return new RedirectView("/analyzer"); + } +} diff --git a/tez-tools/tez-tools-webapp/src/main/java/org/apache/tez/tools/webapp/controller/package-info.java b/tez-tools/tez-tools-webapp/src/main/java/org/apache/tez/tools/webapp/controller/package-info.java new file mode 100644 index 0000000000..1b93ae53f4 --- /dev/null +++ b/tez-tools/tez-tools-webapp/src/main/java/org/apache/tez/tools/webapp/controller/package-info.java @@ -0,0 +1,22 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * This is the package-info for org.apache.tez.tools.webapp.controller. + */ +package org.apache.tez.tools.webapp.controller; \ No newline at end of file diff --git a/tez-tools/tez-tools-webapp/src/main/java/org/apache/tez/tools/webapp/package-info.java b/tez-tools/tez-tools-webapp/src/main/java/org/apache/tez/tools/webapp/package-info.java new file mode 100644 index 0000000000..5eaa4c1319 --- /dev/null +++ b/tez-tools/tez-tools-webapp/src/main/java/org/apache/tez/tools/webapp/package-info.java @@ -0,0 +1,22 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * This is the package-info for org.apache.tez.tools.webapp. + */ +package org.apache.tez.tools.webapp; \ No newline at end of file diff --git a/tez-tools/tez-tools-webapp/src/main/java/org/apache/tez/tools/webapp/service/AnalyzerService.java b/tez-tools/tez-tools-webapp/src/main/java/org/apache/tez/tools/webapp/service/AnalyzerService.java new file mode 100644 index 0000000000..9d46769536 --- /dev/null +++ b/tez-tools/tez-tools-webapp/src/main/java/org/apache/tez/tools/webapp/service/AnalyzerService.java @@ -0,0 +1,211 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.tez.tools.webapp.service; + +import java.io.BufferedReader; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileNotFoundException; +import java.io.FileOutputStream; +import java.io.FileReader; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Comparator; +import java.util.Iterator; +import java.util.List; +import java.util.Set; +import java.util.zip.ZipEntry; +import java.util.zip.ZipOutputStream; + +import org.apache.hadoop.conf.Configuration; +import org.apache.poi.ss.usermodel.Cell; +import org.apache.poi.ss.usermodel.Row; +import org.apache.poi.ss.usermodel.Sheet; +import org.apache.poi.ss.usermodel.Workbook; +import org.apache.poi.xssf.usermodel.XSSFWorkbook; +import org.apache.tez.analyzer.CSVResult; +import org.apache.tez.analyzer.Result; +import org.apache.tez.analyzer.plugins.TezAnalyzerBase; +import org.apache.tez.dag.api.TezException; +import org.apache.tez.history.parser.ATSFileParser; +import org.apache.tez.history.parser.ProtoHistoryParser; +import org.apache.tez.history.parser.SimpleHistoryParser; +import org.apache.tez.history.parser.datamodel.DagInfo; +import org.reflections.Reflections; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.stereotype.Service; + +import com.google.common.io.Files; + +@Service +public class AnalyzerService { + public enum AnalyzerOutputFormat{ + CSV, XLSX + } + + public static final String ANALYZER_PACKAGE = "org.apache.tez.analyzer.plugins"; + + private final Logger LOG = LoggerFactory.getLogger(getClass()); + + private List analyzers; + + public List getAllAnalyzers() { + if (this.analyzers == null) { + this.analyzers = new ArrayList<>(); + + Reflections reflections = new Reflections("org.apache.tez"); + Set> subTypes = reflections.getSubTypesOf(TezAnalyzerBase.class); + + Configuration configuration = new Configuration(); + subTypes.forEach(c -> { + try { + TezAnalyzerBase analyzer = c.getConstructor(Configuration.class).newInstance(configuration); + analyzers.add(analyzer); + } catch (Exception e) { + throw new RuntimeException(e); + } + }); + + Collections.sort(analyzers, new Comparator() { + @Override + public int compare(TezAnalyzerBase arg0, TezAnalyzerBase arg1) { + return arg0.getClass().getSimpleName().compareTo(arg1.getClass().getSimpleName()); + } + }); + } + return this.analyzers; + } + + public DagInfo getDagInfo(String dagId, List files) throws Exception { + LOG.info("Parsing history file for dag: {}", dagId); + + // .zip file is typically an ATS history file, let's try + if ("zip".equals(Files.getFileExtension(files.get(0).getName()))) { + LOG.info("Assuming ATS history file: {}", files.get(0)); + ATSFileParser parser = new ATSFileParser(files); + return parser.getDAGData(dagId); + } else { + try { + LOG.info("Trying as proto history files: {}", files); + ProtoHistoryParser parser = new ProtoHistoryParser(files); + return parser.getDAGData(dagId); + } catch (TezException e) { + if (e.getCause() instanceof IOException && e.getCause().getMessage().contains("not a SequenceFile")) { + LOG.info("Trying as simple history files: {}", files); + SimpleHistoryParser parser = new SimpleHistoryParser(files); + return parser.getDAGData(dagId); + } else { + throw new RuntimeException(e); + } + } + } + } + + public File runAnalyzers(List analyzers, DagInfo dagInfo, AnalyzerOutputFormat aof) + throws FileNotFoundException, TezException, IOException, Exception { + File zipFile = new File(System.getProperty("java.io.tmpdir"), + String.format("dag_analyzer_result_%d_%s.zip", System.currentTimeMillis(), dagInfo.getDagId())); + FileOutputStream fos = new FileOutputStream(zipFile); + ZipOutputStream zipOut = new ZipOutputStream(fos); + + for (TezAnalyzerBase analyzer : analyzers) { + LOG.info("Starting {} for dag: {}", analyzer.getClass().getSimpleName(), dagInfo.getDagId()); + + analyzer.analyze(dagInfo); + + Result result = analyzer.getResult(); + if (result instanceof CSVResult) { + String filePath = System.getProperty("java.io.tmpdir") + File.separator + analyzer.getClass().getSimpleName() + + "_" + dagInfo.getDagId() + ".csv"; + ((CSVResult) result).dumpToFile(filePath); + + if (aof.equals(AnalyzerOutputFormat.XLSX)) { + // TODO: convert directly to excel instead of writing csv file first + // (not a prio, analyzers don't create huge files) + filePath = convertCSVFileToExcel(filePath); + } + addFileToZip(new File(filePath), zipOut); + } + } + zipOut.close(); + fos.close(); + return zipFile; + } + + private String convertCSVFileToExcel(String filePath) throws IOException { + try (XSSFWorkbook workBook = new XSSFWorkbook()) { + Sheet sheet = workBook.createSheet(Paths.get(filePath).getFileName().toString()); + String currentLine = null; + int rowNum = 0; + try (BufferedReader br = java.nio.file.Files.newBufferedReader(Paths.get(filePath), StandardCharsets.UTF_8)) { + while ((currentLine = br.readLine()) != null) { + String[] str = currentLine.split(","); + rowNum++; + Row currentRow = sheet.createRow(rowNum); + for (int i = 0; i < str.length; i++) { + currentRow.createCell(i).setCellValue(str[i]); + } + } + } + autoSizeColumns(workBook); + String xlsxFilePath = filePath.substring(0, filePath.lastIndexOf(".csv")) + ".xlsx"; + FileOutputStream fileOutputStream = new FileOutputStream(xlsxFilePath); + workBook.write(fileOutputStream); + fileOutputStream.close(); + return xlsxFilePath; + } + } + + private void autoSizeColumns(Workbook workbook) { + int numberOfSheets = workbook.getNumberOfSheets(); + for (int i = 0; i < numberOfSheets; i++) { + Sheet sheet = workbook.getSheetAt(i); + if (sheet.getPhysicalNumberOfRows() > 0) { + Row row = sheet.getRow(sheet.getFirstRowNum()); + Iterator cellIterator = row.cellIterator(); + while (cellIterator.hasNext()) { + Cell cell = cellIterator.next(); + int columnIndex = cell.getColumnIndex(); + int originalColumnWidth = sheet.getColumnWidth(columnIndex); + sheet.autoSizeColumn(columnIndex); + if (sheet.getColumnWidth(columnIndex) > 20000) { + // don't let e.g. comment columns expand too large, 20000 was picked by manual experience + sheet.setColumnWidth(columnIndex, originalColumnWidth); + } + } + } + } + } + + private void addFileToZip(File fileToZip, ZipOutputStream zipOut) throws Exception { + try (FileInputStream fis = new FileInputStream(fileToZip)) { + ZipEntry zipEntry = new ZipEntry(fileToZip.getName()); + zipOut.putNextEntry(zipEntry); + + byte[] bytes = new byte[1024]; + int length; + while ((length = fis.read(bytes)) >= 0) { + zipOut.write(bytes, 0, length); + } + } + } +} diff --git a/tez-tools/tez-tools-webapp/src/main/java/org/apache/tez/tools/webapp/service/EventDumpService.java b/tez-tools/tez-tools-webapp/src/main/java/org/apache/tez/tools/webapp/service/EventDumpService.java new file mode 100644 index 0000000000..925daa4c85 --- /dev/null +++ b/tez-tools/tez-tools-webapp/src/main/java/org/apache/tez/tools/webapp/service/EventDumpService.java @@ -0,0 +1,58 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.tez.tools.webapp.service; + +import java.io.File; +import java.io.IOException; +import java.util.List; + +import org.apache.tez.dag.api.TezException; +import org.apache.tez.history.parser.ATSFileParser; +import org.apache.tez.history.parser.ProtoHistoryParser; +import org.apache.tez.history.parser.SimpleHistoryParser; +import org.codehaus.jettison.json.JSONObject; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.stereotype.Service; + +import com.google.common.io.Files; + +@Service +public class EventDumpService { + private final Logger LOG = LoggerFactory.getLogger(getClass()); + + public List getArrayOfAllEvents(String dagId, List files) throws Exception { + // .zip file is typically an ATS history file, let's try + if ("zip".equals(Files.getFileExtension(files.get(0).getName()))) { + LOG.info("Assuming ATS history file: {}", files.get(0)); + return new ATSFileParser(files).dumpAllEvents(); + } else { + try { + LOG.info("Trying as proto history files: {}", files); + return new ProtoHistoryParser(files).dumpAllEvents(); + } catch (TezException e) { + if (e.getCause() instanceof IOException && e.getCause().getMessage().contains("not a SequenceFile")) { + LOG.info("Trying as simple history files: {}", files); + return new SimpleHistoryParser(files).dumpAllEvents(); + } else { + throw new RuntimeException(e); + } + } + } + } +} diff --git a/tez-tools/tez-tools-webapp/src/main/java/org/apache/tez/tools/webapp/service/package-info.java b/tez-tools/tez-tools-webapp/src/main/java/org/apache/tez/tools/webapp/service/package-info.java new file mode 100644 index 0000000000..0e5b5fc6dc --- /dev/null +++ b/tez-tools/tez-tools-webapp/src/main/java/org/apache/tez/tools/webapp/service/package-info.java @@ -0,0 +1,22 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * This is the package-info for org.apache.tez.tools.webapp.service. + */ +package org.apache.tez.tools.webapp.service; \ No newline at end of file diff --git a/tez-tools/tez-tools-webapp/src/main/resources/application.properties b/tez-tools/tez-tools-webapp/src/main/resources/application.properties new file mode 100644 index 0000000000..51c7579559 --- /dev/null +++ b/tez-tools/tez-tools-webapp/src/main/resources/application.properties @@ -0,0 +1,13 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +spring.servlet.multipart.max-file-size=100MB +spring.servlet.multipart.max-request-size=100MB diff --git a/tez-tools/tez-tools-webapp/src/main/resources/log4j.xml b/tez-tools/tez-tools-webapp/src/main/resources/log4j.xml new file mode 100644 index 0000000000..e6b42e99a4 --- /dev/null +++ b/tez-tools/tez-tools-webapp/src/main/resources/log4j.xml @@ -0,0 +1,29 @@ + + + + + + + + + + + + + + + + diff --git a/tez-tools/tez-tools-webapp/src/main/resources/static/object-visualizer.css b/tez-tools/tez-tools-webapp/src/main/resources/static/object-visualizer.css new file mode 100644 index 0000000000..34d808ec7d --- /dev/null +++ b/tez-tools/tez-tools-webapp/src/main/resources/static/object-visualizer.css @@ -0,0 +1,79 @@ +/* Licensed under the MIT License (https://opensource.org/licenses/MIT) */ +.object-visualizer { + border-radius: 4px; + overflow-x: auto; + margin: 0; + padding: 10px; + font-family: Menlo; + font-size: 0.8rem; + line-height: 1.4; + /* background-color: hsl(0, 0%, 13%); */ +} + +.array > .value, +.object > .value { + display: flex; + flex-direction: column; + margin-left: 2rem; +} + +.key { + /* color: hsl(300, 60%, 65%); */ +} + +.string > .value { + color: hsl(15, 100%, 70%); +} + +.boolean > .value, +.number > .value { + color: hsl(250, 70%, 65%); +} + +.null > .value, +.undefined > .value { + color: hsl(0, 0%, 40%); +} + +.separator { + font-size: 0.8rem; + color: hsl(0, 0%, 80%); +} + +.array > .separator, +.object > .separator { + cursor: pointer; +} + +.indicator { + cursor: pointer; + font-size: 0.8rem; + padding-right: 0.3rem; + user-select: none; + vertical-align: text-bottom; + color: hsl(0, 0%, 50%); +} + +.array > .key, +.object > .key { + cursor: pointer; +} + +.value > .array, +.value > .object { + position: relative; + left: -0.8rem; +} + +.count, +.preview, +.quotes { + font-size: 0.8rem; + color: hsl(0, 0%, 80%); +} + +.count, +.preview { + user-select: none; + cursor: pointer; +} diff --git a/tez-tools/tez-tools-webapp/src/main/resources/templates/analyzer.html b/tez-tools/tez-tools-webapp/src/main/resources/templates/analyzer.html new file mode 100644 index 0000000000..edcddbd301 --- /dev/null +++ b/tez-tools/tez-tools-webapp/src/main/resources/templates/analyzer.html @@ -0,0 +1,193 @@ + + + + + + + + + + + +Tez analyzers + + + + + +

+
+
...
+
+
+

Tez analyzers

+ +
+ +
+ History + file(s) - simple / proto / ATS zip +
+ +
+ DAG id to + look for +
+ +
+ +
+ +
+
+ +
+
+
+
+ + +
+ + +
+ +
+
+
+ +
+
+
+
+
+ + + + + diff --git a/tez-tools/tez-tools-webapp/src/main/resources/templates/eventdump.html b/tez-tools/tez-tools-webapp/src/main/resources/templates/eventdump.html new file mode 100644 index 0000000000..160b3a0b4c --- /dev/null +++ b/tez-tools/tez-tools-webapp/src/main/resources/templates/eventdump.html @@ -0,0 +1,153 @@ + + + + + + + + + + + + + + +Tez history event dump + + + + + +
+
+
...
+
+
+

Tez history event dump

+ +
+ +
+ History + file(s) - simple / proto / ATS zip +
+ +
+ + DAG id + parsed from the file (isn't needed for dumping events, no need + to edit) +
+ + +
+
+ +
+
+
+
+
+
+ + + diff --git a/tez-tools/tez-tools-webapp/src/main/resources/templates/fragments/menu.html b/tez-tools/tez-tools-webapp/src/main/resources/templates/fragments/menu.html new file mode 100644 index 0000000000..cbc0fbb511 --- /dev/null +++ b/tez-tools/tez-tools-webapp/src/main/resources/templates/fragments/menu.html @@ -0,0 +1,42 @@ + +
+
+ + Menu + + + +
+
+
\ No newline at end of file