Skip to content

Commit

Permalink
Fix GCC 9 Code coverage by adding its JSON format
Browse files Browse the repository at this point in the history
GCC 9 changed the intermediate format to a JSON based format and with it
changed the meaning of the `-i` flag. Because of these changes it was not
possible to generate code coverage with GCC 9. This patch addresses that
by adding its format in parallel to the existing GCov parser

Addresses: bazelbuild#9406
  • Loading branch information
helaan authored and Henk van der Laan committed Jun 2, 2020
1 parent 8cd2222 commit 848e91c
Show file tree
Hide file tree
Showing 5 changed files with 173 additions and 6 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,7 @@ java_library(
srcs = glob(["*.java"]),
deps = [
"//third_party:auto_value",
"//third_party:gson",
"//third_party:guava",
"//third_party:jsr305",
"//third_party/java/jcommander",
Expand Down Expand Up @@ -117,6 +118,21 @@ java_library(
],
)

java_library(
name = "GcovJsonParser",
srcs = [
"GcovJsonParser.java",
"Parser.java",
],
deps = [
":BranchCoverage",
":Constants",
":LineCoverage",
":SourceFileCoverage",
"//third_party:gson",
],
)

java_library(
name = "LcovParser",
srcs = [
Expand Down Expand Up @@ -159,6 +175,7 @@ java_library(
":Constants",
":Coverage",
":GcovParser",
":GcovJsonParser",
":LcovMergerFlags",
":LcovParser",
":LcovPrinter",
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,7 @@ class Constants {
static final String TAKEN = "-";
static final String TRACEFILE_EXTENSION = ".dat";
static final String GCOV_EXTENSION = ".gcov";
static final String GCOV_JSON_EXTENSION = ".gcov.json.gz";
static final String PROFDATA_EXTENSION = ".profdata";
static final String DELIMITER = ",";
static final String GCOV_VERSION_MARKER = "version:";
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,121 @@
// Copyright 2020 The Bazel Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package com.google.devtools.coverageoutputgenerator;

import java.io.BufferedReader;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.util.ArrayList;
import java.util.List;
import java.util.logging.Level;
import java.util.logging.Logger;
import java.util.zip.GZIPInputStream;
import com.google.gson.Gson;
import com.google.gson.annotations.SerializedName;

/**
* A {@link Parser} for gcov intermediate json format introduced in GCC 9.1. See the flag {@code --intermediate-format} in <a
* href="https://gcc.gnu.org/onlinedocs/gcc/Invoking-gcov.html">gcov documentation</a>.
*/
public class GcovJsonParser {
private static final Logger logger = Logger.getLogger(GcovParser.class.getName());
private final InputStream inputStream;

private GcovJsonParser(InputStream inputStream) {
this.inputStream = inputStream;
}

public static List<SourceFileCoverage> parse(InputStream inputStream) throws IOException {
return new GcovJsonParser(inputStream).parse();
}

private List<SourceFileCoverage> parse() throws IOException {
ArrayList<SourceFileCoverage> allSourceFiles = new ArrayList<>();
try (InputStream gzipStream = new GZIPInputStream(inputStream)) {
ByteArrayOutputStream contents = new ByteArrayOutputStream();
byte[] buffer = new byte[1024];
int length;
while ((length = gzipStream.read(buffer)) != -1) {
contents.write(buffer, 0, length);
}
Gson gson = new Gson();
//logger.log(Level.INFO, contents.toString());
GcovJsonFormat document = gson.fromJson(contents.toString(), GcovJsonFormat.class);
if (!document.format_version.equals("1")) {
logger.log(Level.WARNING, "Expect GCov JSON format version 1, got format version " + document.format_version);
}
for (GcovJsonFile file : document.files) {
SourceFileCoverage currentFileCoverage = new SourceFileCoverage(file.file);
for (GcovJsonFunction function : file.functions) {
currentFileCoverage.addLineNumber(function.name, function.start_line);
currentFileCoverage.addFunctionExecution(function.name, function.execution_count);
}
for (GcovJsonLine line : file.lines) {
currentFileCoverage.addLine(line.line_number, LineCoverage.create(line.line_number, line.count, null));
for (GcovJsonBranch branch : line.branches) {
currentFileCoverage.addBranch(line.line_number, BranchCoverage.create(line.line_number, branch.count));
}
}
allSourceFiles.add(currentFileCoverage);
}
}

return allSourceFiles;
}
}

// Classes for the Gson data mapper representing the structure of the GCov JSON format

class GcovJsonFormat {
String gcc_version;
GcovJsonFile[] files;
String format_version;
String current_working_directory;
String data_file;
}

class GcovJsonFile {
String file;
GcovJsonFunction[] functions;
GcovJsonLine[] lines;
}

class GcovJsonFunction {
int blocks;
int end_column;
int start_line;
String name;
int blocks_executed;
int execution_count;
String demangled_name;
int start_column;
int end_line;
}

class GcovJsonLine {
GcovJsonBranch[] branches;
int count;
int line_number;
boolean unexecuted_block;
String function_name;
}

class GcovJsonBranch {
boolean fallthrough;
int count;
@SerializedName("throw")
boolean _throw;
}
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@
package com.google.devtools.coverageoutputgenerator;

import static com.google.devtools.coverageoutputgenerator.Constants.GCOV_EXTENSION;
import static com.google.devtools.coverageoutputgenerator.Constants.GCOV_JSON_EXTENSION;
import static com.google.devtools.coverageoutputgenerator.Constants.PROFDATA_EXTENSION;
import static com.google.devtools.coverageoutputgenerator.Constants.TRACEFILE_EXTENSION;
import static java.nio.charset.StandardCharsets.UTF_8;
Expand Down Expand Up @@ -52,6 +53,7 @@ public static void main(String... args) {
int exitCode = runWithArgs(args);
System.exit(exitCode);
} catch (Exception e) {
e.printStackTrace();
logger.log(Level.SEVERE, "Unhandled exception on lcov tool: " + e.getMessage());
System.exit(1);
}
Expand All @@ -78,8 +80,13 @@ static int runWithArgs(String... args) throws ExecutionException, InterruptedExc
getTracefiles(flags, filesInCoverageDir),
LcovParser::parse,
flags.parseParallelism()),
parseFiles(
getGcovInfoFiles(filesInCoverageDir), GcovParser::parse, flags.parseParallelism()));
Coverage.merge(
parseFiles(
getGcovInfoFiles(filesInCoverageDir), GcovParser::parse, flags.parseParallelism()),
parseFiles(
getGcovJsonInfoFiles(filesInCoverageDir),
GcovJsonParser::parse,
flags.parseParallelism())));

if (flags.sourcesToReplaceFile() != null) {
coverage.maybeReplaceSourceFileNames(getMapFromFile(flags.sourcesToReplaceFile()));
Expand Down Expand Up @@ -221,6 +228,16 @@ private static List<File> getGcovInfoFiles(List<File> filesInCoverageDir) {
return gcovFiles;
}

private static List<File> getGcovJsonInfoFiles(List<File> filesInCoverageDir) {
List<File> gcovJsonFiles = getFilesWithExtension(filesInCoverageDir, GCOV_JSON_EXTENSION);
if (gcovJsonFiles.isEmpty()) {
logger.log(Level.INFO, "No gcov json file found.");
} else {
logger.log(Level.INFO, "Found " + gcovJsonFiles.size() + " gcov json files.");
}
return gcovJsonFiles;
}

/**
* Returns a .profdata file from the given files or null if none or more profdata files were
* found.
Expand Down Expand Up @@ -352,6 +369,7 @@ static List<File> getCoverageFilesInDir(String dir) {
p ->
p.toString().endsWith(TRACEFILE_EXTENSION)
|| p.toString().endsWith(GCOV_EXTENSION)
|| p.toString().endsWith(GCOV_JSON_EXTENSION)
|| p.toString().endsWith(PROFDATA_EXTENSION))
.map(path -> path.toFile())
.collect(Collectors.toList());
Expand Down
18 changes: 14 additions & 4 deletions tools/test/collect_cc_coverage.sh
Original file line number Diff line number Diff line change
Expand Up @@ -125,10 +125,20 @@ function gcov_coverage() {
# https://gcc.gnu.org/bugzilla/show_bug.cgi?id=84879).
"${GCOV}" -i $COVERAGE_GCOV_OPTIONS -o "$(dirname ${gcda})" "${gcda}"

# Append all .gcov files in the current directory to the output file.
cat *.gcov >> "$output_file"
# Delete the .gcov files.
rm *.gcov
gcov_version=$("${GCOV}" --version | sed -n -r -e 's/^.*\s([0-9]\.[0-9]\.[0-9])\s?.*$/\1/p')

# Check the gcov version so we can process the data correctly
if [ "$(printf '%s\n%s' "9.1.0" "$gcov_version" | sort -V | head -n 1)" != "$gcov_version" ]; then
# GCOV 9.1.0 or higher generate a JSON based coverage format
# The output is generated into multiple files: "$(basename ${gcda}).gcov.json.gz"
# Concatenating JSON documents does not yield a valid document, so they are moved individually
mv -- *.gcov.json.gz "$(dirname "$output_file")"
else
# Append all .gcov files in the current directory to the output file.
cat -- *.gcov >> "$output_file"
# Delete the .gcov files.
rm -- *.gcov
fi
fi
fi
done < "${COVERAGE_MANIFEST}"
Expand Down

0 comments on commit 848e91c

Please sign in to comment.